diff -r 6db40a9955dc -r 0d413f60cc23 .bzrignore --- a/.bzrignore Sun Jan 24 22:15:20 2016 -0800 +++ b/.bzrignore Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -.purify +´.purify autom4te.cache config.log config.cache @@ -33,6 +33,7 @@ Modules/config.c Modules/ld_so_aix Parser/pgen +Parser/pgen.stamp Lib/test/data/* Lib/lib2to3/Grammar*.pickle Lib/lib2to3/PatternGrammar*.pickle diff -r 6db40a9955dc -r 0d413f60cc23 .gitignore --- a/.gitignore Sun Jan 24 22:15:20 2016 -0800 +++ b/.gitignore Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,3 @@ -# Two-trick pony for OSX and other case insensitive file systems: -# Ignore ./python binary on Unix but still look into ./Python/ directory. -/python -!/Python/** *.cover *.o *.orig @@ -9,87 +5,46 @@ *.pyd *.pyo *.rej -*.swp *~ -*.gc?? -*.profclang? -*.profraw -*.dyn -.gdb_history Doc/build/ -Doc/venv/ -Lib/distutils/command/*.pdb +Doc/tools/docutils/ +Doc/tools/jinja2/ +Doc/tools/pygments/ +Doc/tools/sphinx/ Lib/lib2to3/*.pickle -Lib/test/data/* Lib/_sysconfigdata.py -Lib/plat-mac/errors.rsrc.df.rsrc Makefile Makefile.pre Misc/python.pc -Misc/python-config.sh Modules/Setup Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix -Programs/_freeze_importlib -Programs/_testembed -PC/python_nt*.h -PC/pythonnt_rc*.h -PC/*/*.exe -PC/*/*.exp -PC/*/*.lib -PC/*/*.bsc -PC/*/*.dll -PC/*/*.pdb -PC/*/*.user -PC/*/*.ncb -PC/*/*.suo -PC/*/Win32-temp-* -PC/*/x64-temp-* -PC/*/amd64 -PCbuild/*.user -PCbuild/*.suo -PCbuild/*.*sdf -PCbuild/*-pgi -PCbuild/*-pgo -PCbuild/.vs/ -PCbuild/amd64/ -PCbuild/obj/ -PCBuild/win32/ -.purify +Modules/_testembed +PCbuild/*.bsc +PCbuild/*.dll +PCbuild/*.exe +PCbuild/*.exp +PCbuild/*.lib +PCbuild/*.ncb +PCbuild/*.o +PCbuild/*.pdb +PCbuild/Win32-temp-* Parser/pgen +Parser/pgen.stamp __pycache__ autom4te.cache build/ -buildno -config.cache -config.log -config.status -config.status.lineno -core -db_home config.log config.status libpython*.a libpython*.so* -platform pybuilddir.txt pyconfig.h -python-config -python-config.py -python.bat -python.exe +python python-gdb.py -python.exe-gdb.py -reflog.txt -.svn/ tags -TAGS .coverage coverage/ -externals/ htmlcov/ -Tools/msi/obj -Tools/ssl/amd64 -Tools/ssl/win32 diff -r 6db40a9955dc -r 0d413f60cc23 .hgeol --- a/.hgeol Sun Jan 24 22:15:20 2016 -0800 +++ b/.hgeol Mon Jan 25 17:05:13 2016 +0100 @@ -10,8 +10,6 @@ **.vsprops = BIN **.aif = BIN -**.aifc = BIN -**.aiff = BIN **.au = BIN **.bmp = BIN **.db = BIN @@ -25,26 +23,14 @@ **.png = BIN **.psd = BIN **.tar = BIN -**.wav = BIN -**.whl = BIN **.xar = BIN **.zip = BIN +Lib/email/test/data/msg_26.txt = BIN Lib/test/cjkencodings/* = BIN Lib/test/decimaltestdata/*.decTest = BIN Lib/test/sndhdrdata/sndhdr.* = BIN Lib/test/test_email/data/msg_26.txt = BIN -Lib/test/xmltestdata/* = BIN - -Lib/venv/scripts/nt/* = BIN - -Lib/test/coding20731.py = BIN - -# Windows batch files work best with CRLF, there can be subtle problems with LF -**.bat = CRLF - -# The Windows readme is likely to be read in Notepad, so make it readable -PCbuild/readme.txt = CRLF # All other files (which presumably are human-editable) are "native". # This must be the last rule! diff -r 6db40a9955dc -r 0d413f60cc23 .hgignore --- a/.hgignore Sun Jan 24 22:15:20 2016 -0800 +++ b/.hgignore Mon Jan 25 17:05:13 2016 +0100 @@ -1,15 +1,12 @@ .gdb_history .purify .svn/ -^.idea/ -.DS_Store Makefile$ Makefile.pre$ TAGS$ autom4te.cache$ ^build/ ^Doc/build/ -^Doc/venv/ buildno$ config.cache config.log @@ -19,22 +16,24 @@ platform$ pyconfig.h$ python$ -python.bat$ python.exe$ -python-config$ -python-config.py$ reflog.txt$ tags$ Lib/plat-mac/errors.rsrc.df.rsrc +Doc/tools/sphinx/ +Doc/tools/docutils/ +Doc/tools/jinja/ +Doc/tools/jinja2/ +Doc/tools/pygments/ Misc/python.pc -Misc/python-config.sh$ Modules/Setup$ Modules/Setup.config Modules/Setup.local Modules/config.c Modules/ld_so_aix$ Parser/pgen$ -^lcov-report/ +Parser/pgen.stamp$ +PCbuild/amd64/ ^core ^python-gdb.py ^python.exe-gdb.py @@ -50,51 +49,28 @@ *.pyd *.cover *~ -*.gc?? -*.profclang? -*.profraw -*.dyn -Lib/distutils/command/*.pdb +Lib/_sysconfigdata.py Lib/lib2to3/*.pickle Lib/test/data/* Misc/*.wpu PC/python_nt*.h PC/pythonnt_rc*.h -PC/*/*.exe -PC/*/*.exp -PC/*/*.lib -PC/*/*.bsc -PC/*/*.dll -PC/*/*.pdb -PC/*/*.user -PC/*/*.ncb -PC/*/*.suo -PC/*/Win32-temp-* -PC/*/x64-temp-* -PC/*/amd64 +PC/*.obj +PC/*.exe +PCbuild/*.exe +PCbuild/*.dll +PCbuild/*.pdb +PCbuild/*.lib +PCbuild/*.exp +PCbuild/*.o +PCbuild/*.ncb +PCbuild/*.bsc PCbuild/*.user PCbuild/*.suo -PCbuild/*.*sdf -PCbuild/*-pgi -PCbuild/*-pgo -PCbuild/.vs -PCbuild/amd64 -PCbuild/obj -PCbuild/win32 -Tools/unicode/build/ -Tools/unicode/MAPPINGS/ -BuildLog.htm +PCbuild/Win32-temp-* +PCbuild/x64-temp-* __pycache__ -Programs/_freeze_importlib -Programs/_testembed +Modules/_testembed .coverage coverage/ -externals/ htmlcov/ -*.gcda -*.gcno -*.gcov -coverage.info -Tools/msi/obj -Tools/ssl/amd64 -Tools/ssl/win32 diff -r 6db40a9955dc -r 0d413f60cc23 .hgtags --- a/.hgtags Sun Jan 24 22:15:20 2016 -0800 +++ b/.hgtags Mon Jan 25 17:05:13 2016 +0100 @@ -78,8 +78,6 @@ 32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1 c918ec9f3a76d6afedfbb5d455004de880443a3d v3.1.4 ee26aca3219cf4bb0b93352e83edcc9cb28c7802 v3.1.5rc1 -75db2bc69fc9a3e4801e94e3e19801cb096208d8 v3.1.5rc2 -7395330e495ec3316862ca1f6ce0aaf7bdf6785b v3.1.5 b37b7834757492d009b99cf0ca4d42d2153d7fac v3.2a1 56d4373cecb73c8b45126ba7b045b3c7b3f94b0b v3.2a2 da012d9a2c23d144e399d2e01a55b8a83ad94573 v3.2a3 @@ -98,66 +96,4 @@ 137e45f15c0bd262c9ad4c032d97425bc0589456 v3.2.2 7085403daf439adb3f9e70ef13f6bedb1c447376 v3.2.3rc1 428f05cb7277e1d42bb9dd8d1af6b6270ebc6112 v3.2.3rc2 -3d0686d90f55a78f96d9403da2c52dc2411419d0 v3.2.3 -b2cb7bc1edb8493c0a78f9331eae3e8fba6a881d v3.2.4rc1 -1e10bdeabe3de02f038a63c001911561ac1d13a7 v3.2.4 -cef745775b6583446572cffad704100983db2bea v3.2.5 -51382a5598ec96119cb84594572901c9c964dc3c v3.2.6rc1 -0bd5f4f14de965ca8e44c6e3965fee106176cfc4 v3.2.6 f1a9a6505731714f0e157453ff850e3b71615c45 v3.3.0a1 -2f69db52d6de306cdaef0a0cc00cc823fb350b01 v3.3.0a2 -0b53b70a40a00013505eb35e3660057b62be77be v3.3.0a3 -7c51388a3aa7ce76a8541bbbdfc05d2d259a162c v3.3.0a4 -e15c554cd43eb23bc0a528a4e8741da9bbec9607 v3.3.0b1 -4972a8f1b2aa3d7cdd64dc96aa7fa112fe1ea343 v3.3.0b2 -8bb5c7bc46ba43804480f3e328e1fa956672c885 v3.3.0rc1 -88a0792e8ba3e4916b24c7e7a522c277d326d66e v3.3.0rc2 -c191d21cefafb3832c45570e84854e309aa62eaa v3.3.0rc3 -bd8afb90ebf28ba4edc901d4a235f75e7bbc79fd v3.3.0 -92c2cfb924055ce68c4f78f836dcfe688437ceb8 v3.3.1rc1 -d9893d13c6289aa03d33559ec67f97dcbf5c9e3c v3.3.1 -d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2 -fd53c500f8b80f54f3ecedec9da2e8c7e52a6888 v3.3.3rc1 -d32442c0e60dfbd71234e807d3d1dedd227495a9 v3.3.3rc2 -c3896275c0f61b2510a6c7e6c458a750359a91b8 v3.3.3 -fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1 -7ff62415e4263c432c8acf6e424224209211eadb v3.3.4 -9ec811df548ed154a9bf9815383a916d6df31b98 v3.3.5rc1 -ca5635efe090f78806188ac2758f9948596aa8b2 v3.3.5rc2 -62cf4e77f78564714e7ea3d4bf1479ca1fbd0758 v3.3.5 -51317c9786f54267975abf2e9c502e6aaaa4a249 v3.3.6rc1 -971fec30da1fc5bf2b9fb28e09812a5127014211 v3.3.6 -46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1 -9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2 -dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3 -e245b0d7209bb6d0e19316e1e2af1aa9c2139104 v3.4.0a4 -3405dc9a6afaa0a06dd1f6f182ec5c998dce6f5f v3.4.0b1 -ba32913eb13ec545a46dd0ce18035b6c416f0d78 v3.4.0b2 -a97ce3ecc96af79bd2e1ac66ce48d9138e0ca749 v3.4.0b3 -5e088cea8660677969113741c1313d570d977e02 v3.4.0rc1 -a300712ed38c9a242b736c44e806caea25a6dc05 v3.4.0rc2 -8a81cdab3e9d521daaef989fade94b16455fc3b8 v3.4.0rc3 -04f714765c13824c3bc2835d7b008908862e083a v3.4.0 -c67a19e11a7191baf30f313bf55e2e0b6c6f574e v3.4.1rc1 -c0e311e010fcb5bae8d87ca22051cd0845ea0ca0 v3.4.1 -8711a09513848cfc48c689d983495ee64f4668ca v3.4.2rc1 -ab2c023a9432f16652e89c404bbc84aa91bf55af v3.4.2 -69dd528ca6255a66c37cc5cf680e8357d108b036 v3.4.3rc1 -b4cbecbc0781e89a309d03b60a1f75f8499250e6 v3.4.3 -04f3f725896c6961212c3a12e8ac25be6958f4fa v3.4.4rc1 -737efcadf5a678b184e0fa431aae11276bf06648 v3.4.4 -5d4b6a57d5fd7564bf73f3db0e46fe5eeb00bcd8 v3.5.0a1 -0337bd7ebcb6559d69679bc7025059ad1ce4f432 v3.5.0a2 -82656e28b5e5c4ae48d8dd8b5f0d7968908a82b6 v3.5.0a3 -413e0e0004f4f954331cb8122aa55fe208984955 v3.5.0a4 -071fefbb5e3db770c6c19fba9994699f121b1cea v3.5.0b1 -7a088af5615bf04024e9912068f4bd8f43ed3917 v3.5.0b2 -0035fcd9b9243ae52c2e830204fd9c1f7d528534 v3.5.0b3 -c0d64105463581f85d0e368e8d6e59b7fd8f12b1 v3.5.0b4 -1a58b1227501e046eee13d90f113417b60843301 v3.5.0rc1 -cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2 -66ed52375df802f9d0a34480daaa8ce79fc41313 v3.5.0rc3 -2d033fedfa7f1e325fd14ccdaa9cb42155da206f v3.5.0rc4 -374f501f4567b7595f2ad7798aa09afa2456bb28 v3.5.0 -948ef16a69513ba1ff15c9d7d0b012b949df4c80 v3.5.1rc1 -37a07cee5969e6d3672583187a73cf636ff28e1b v3.5.1 diff -r 6db40a9955dc -r 0d413f60cc23 .hgtouch --- a/.hgtouch Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -# -*- Makefile -*- -# Define dependencies of generated files that are checked into hg. -# The syntax of this file uses make rule dependencies, without actions - -Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c - -Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py - -Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py -Python/Python-ast.c: Include/Python-ast.h - -Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py - -Objects/typeslots.inc: Include/typeslots.h Objects/typeslots.py - -Include/graminit.h: Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c -Python/graminit.c: Include/graminit.h Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c diff -r 6db40a9955dc -r 0d413f60cc23 Doc/ACKS.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/ACKS.txt Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,238 @@ +Contributors to the Python Documentation +---------------------------------------- + +This section lists people who have contributed in some way to the Python +documentation. It is probably not complete -- if you feel that you or +anyone else should be on this list, please let us know (send email to +docs@python.org), and we'll be glad to correct the problem. + +.. acks:: + + * Aahz + * Michael Abbott + * Steve Alexander + * Jim Ahlstrom + * Fred Allen + * A. Amoroso + * Pehr Anderson + * Oliver Andrich + * Heidi Annexstad + * Jesús Cea Avión + * Manuel Balsera + * Daniel Barclay + * Chris Barker + * Don Bashford + * Anthony Baxter + * Alexander Belopolsky + * Bennett Benson + * Jonathan Black + * Robin Boerdijk + * Michal Bozon + * Aaron Brancotti + * Georg Brandl + * Keith Briggs + * Ian Bruntlett + * Lee Busby + * Arnaud Calmettes + * Lorenzo M. Catucci + * Carl Cerecke + * Mauro Cicognini + * Gilles Civario + * Mike Clarkson + * Steve Clift + * Dave Cole + * Matthew Cowles + * Jeremy Craven + * Andrew Dalke + * Ben Darnell + * L. Peter Deutsch + * Robert Donohue + * Fred L. Drake, Jr. + * Jacques Ducasse + * Josip Dzolonga + * Jeff Epler + * Michael Ernst + * Blame Andy Eskilsson + * Carey Evans + * Martijn Faassen + * Carl Feynman + * Dan Finnie + * Hernán Martínez Foffani + * Michael Foord + * Stefan Franke + * Jim Fulton + * Peter Funk + * Ethan Furman + * Lele Gaifax + * Matthew Gallagher + * Gabriel Genellina + * Ben Gertzfield + * Nadim Ghaznavi + * Jonathan Giddy + * Matt Giuca + * Shelley Gooch + * Nathaniel Gray + * Grant Griffin + * Thomas Guettler + * Anders Hammarquist + * Mark Hammond + * Harald Hanche-Olsen + * Manus Hand + * Gerhard Häring + * Travis B. Hartwell + * Tim Hatch + * Janko Hauser + * Ben Hayden + * Thomas Heller + * Bernhard Herzog + * Magnus L. Hetland + * Konrad Hinsen + * Stefan Hoffmeister + * Albert Hofkamp + * Gregor Hoffleit + * Steve Holden + * Thomas Holenstein + * Gerrit Holl + * Rob Hooft + * Brian Hooper + * Randall Hopper + * Michael Hudson + * Eric Huss + * Jeremy Hylton + * Roger Irwin + * Jack Jansen + * Philip H. Jensen + * Pedro Diaz Jimenez + * Kent Johnson + * Lucas de Jonge + * Andreas Jung + * Robert Kern + * Jim Kerr + * Jan Kim + * Kamil Kisiel + * Greg Kochanski + * Guido Kollerie + * Peter A. Koren + * Daniel Kozan + * Andrew M. Kuchling + * Dave Kuhlman + * Erno Kuusela + * Ross Lagerwall + * Thomas Lamb + * Detlef Lannert + * Piers Lauder + * Glyph Lefkowitz + * Robert Lehmann + * Marc-André Lemburg + * Ross Light + * Gediminas Liktaras + * Ulf A. Lindgren + * Everett Lipman + * Mirko Liss + * Martin von Löwis + * Fredrik Lundh + * Jeff MacDonald + * John Machin + * Andrew MacIntyre + * Vladimir Marangozov + * Vincent Marchetti + * Westley Martínez + * Laura Matson + * Daniel May + * Rebecca McCreary + * Doug Mennella + * Paolo Milani + * Skip Montanaro + * Paul Moore + * Ross Moore + * Sjoerd Mullender + * Dale Nagata + * Trent Nelson + * Michal Nowikowski + * Steffen Daode Nurpmeso + * Ng Pheng Siong + * Koray Oner + * Tomas Oppelstrup + * Denis S. Otkidach + * Zooko O'Whielacronx + * Shriphani Palakodety + * William Park + * Joonas Paalasmaa + * Harri Pasanen + * Bo Peng + * Tim Peters + * Benjamin Peterson + * Christopher Petrilli + * Justin D. Pettit + * Chris Phoenix + * François Pinard + * Paul Prescod + * Eric S. Raymond + * Edward K. Ream + * Terry J. Reedy + * Sean Reifschneider + * Bernhard Reiter + * Armin Rigo + * Wes Rishel + * Armin Ronacher + * Jim Roskind + * Guido van Rossum + * Donald Wallace Rouse II + * Mark Russell + * Nick Russo + * Chris Ryland + * Constantina S. + * Hugh Sasse + * Bob Savage + * Scott Schram + * Neil Schemenauer + * Barry Scott + * Joakim Sernbrant + * Justin Sheehy + * Charlie Shepherd + * Yue Shuaijie + * SilentGhost + * Michael Simcich + * Ionel Simionescu + * Michael Sloan + * Gregory P. Smith + * Roy Smith + * Clay Spence + * Nicholas Spies + * Tage Stabell-Kulo + * Frank Stajano + * Anthony Starks + * Greg Stein + * Peter Stoehr + * Mark Summerfield + * Reuben Sumner + * Kalle Svensson + * Jim Tittsler + * David Turner + * Sandro Tosi + * Ville Vainio + * Nadeem Vawda + * Martijn Vries + * Charles G. Waldman + * Greg Ward + * Barry Warsaw + * Corran Webster + * Glyn Webster + * Bob Weiner + * Eddy Welbourne + * Jeff Wheeler + * Mats Wichmann + * Gerry Wiener + * Timothy Wild + * Paul Winkler + * Collin Winter + * Blake Winton + * Dan Wolfe + * Adam Woodbeck + * Steven Work + * Thomas Wouters + * Ka-Ping Yee + * Rory Yorke + * Moshe Zadka + * Milan Zamazal + * Cheng Zhang diff -r 6db40a9955dc -r 0d413f60cc23 Doc/Makefile --- a/Doc/Makefile Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/Makefile Mon Jan 25 17:05:13 2016 +0100 @@ -5,24 +5,24 @@ # You can set these variables from the command line. PYTHON = python -SPHINXBUILD = sphinx-build +SVNROOT = http://svn.python.org/projects +SPHINXOPTS = PAPER = SOURCES = -DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) +DISTVERSION = $(shell $(PYTHON) tools/sphinxext/patchlevel.py) ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) -.PHONY: help build html htmlhelp latex text changes linkcheck \ +.PHONY: help checkout update build html htmlhelp latex text changes linkcheck \ suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ - autobuild-dev autobuild-stable venv + autobuild-dev autobuild-stable help: @echo "Please use \`make ' where is one of" @echo " clean to remove build files" - @echo " venv to create a venv with necessary tools" + @echo " update to update build tools" @echo " html to make standalone HTML files" - @echo " htmlview to open the index page built by the html target in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" @@ -37,8 +37,30 @@ @echo " check to run a check for frequent markup errors" @echo " serve to serve the documentation on the localhost (8000)" -build: - $(SPHINXBUILD) $(ALLSPHINXOPTS) +# Note: if you update versions here, do the same in make.bat and README.txt +checkout: + @if [ ! -d tools/sphinx ]; then \ + echo "Checking out Sphinx..."; \ + svn checkout $(SVNROOT)/external/Sphinx-1.0.7/sphinx tools/sphinx; \ + fi + @if [ ! -d tools/docutils ]; then \ + echo "Checking out Docutils..."; \ + svn checkout $(SVNROOT)/external/docutils-0.6/docutils tools/docutils; \ + fi + @if [ ! -d tools/jinja2 ]; then \ + echo "Checking out Jinja..."; \ + svn checkout $(SVNROOT)/external/Jinja-2.3.1/jinja2 tools/jinja2; \ + fi + @if [ ! -d tools/pygments ]; then \ + echo "Checking out Pygments..."; \ + svn checkout $(SVNROOT)/external/Pygments-1.3.1/pygments tools/pygments; \ + fi + +update: clean checkout + +build: checkout + mkdir -p build/$(BUILDER) build/doctrees + $(PYTHON) tools/sphinx-build.py $(ALLSPHINXOPTS) @echo html: BUILDER = html @@ -69,45 +91,39 @@ @echo "The overview file is in build/changes." linkcheck: BUILDER = linkcheck -linkcheck: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Link check complete; look for any errors in the above output" \ - "or in build/$(BUILDER)/output.txt"; \ - false; } +linkcheck: build + @echo "Link check complete; look for any errors in the above output" \ + "or in build/$(BUILDER)/output.txt" suspicious: BUILDER = suspicious -suspicious: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Suspicious check complete; look for any errors in the above output" \ - "or in build/$(BUILDER)/suspicious.csv. If all issues are false" \ - "positives, append that file to tools/susp-ignored.csv."; \ - false; } +suspicious: build + @echo "Suspicious check complete; look for any errors in the above output" \ + "or in build/$(BUILDER)/suspicious.csv. If all issues are false" \ + "positives, append that file to tools/sphinxext/susp-ignored.csv." coverage: BUILDER = coverage coverage: build @echo "Coverage finished; see c.txt and python.txt in build/coverage" doctest: BUILDER = doctest -doctest: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Testing of doctests in the sources finished, look at the" \ - "results in build/doctest/output.txt"; \ - false; } +doctest: build + @echo "Testing of doctests in the sources finished, look at the" \ + "results in build/doctest/output.txt" pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build - @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + @echo "Building finished; now copy build/pydoc-topics/topics.py" \ + "to ../Lib/pydoc_data/topics.py" htmlview: html $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" clean: - -rm -rf build/* venv/* - -venv: - $(PYTHON) -m venv venv - ./venv/bin/python3 -m pip install -U Sphinx + -rm -rf build/* + -rm -rf tools/sphinx + -rm -rf tools/pygments + -rm -rf tools/jinja2 + -rm -rf tools/docutils dist: rm -rf dist @@ -147,10 +163,16 @@ cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - # copy the epub build + # archive the epub build rm -rf build/epub make epub - cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + mkdir -p dist/python-$(DISTVERSION)-docs-epub + cp -pPR build/epub/*.epub dist/python-$(DISTVERSION)-docs-epub/ + tar -C dist -cf dist/python-$(DISTVERSION)-docs-epub.tar python-$(DISTVERSION)-docs-epub + bzip2 -9 -k dist/python-$(DISTVERSION)-docs-epub.tar + (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-epub.zip python-$(DISTVERSION)-docs-epub) + rm -r dist/python-$(DISTVERSION)-docs-epub + rm dist/python-$(DISTVERSION)-docs-epub.tar check: $(PYTHON) tools/rstlint.py -i tools @@ -162,17 +184,12 @@ # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='-A daily=1 -A versionswitcher=1' - -make suspicious + make update + make dist SPHINXOPTS='-A daily=1' -# for quick rebuilds (HTML only) -autobuild-html: - make html SPHINXOPTS='-A daily=1 -A versionswitcher=1' - -# for stable releases: only build if not in pre-release stage (alpha, beta) -# release candidate downloads are okay, since the stable tree can be in that stage +# for stable releases: only build if not in pre-release stage (alpha, beta, rc) autobuild-stable: - @case $(DISTVERSION) in *[ab]*) \ + @case $(DISTVERSION) in *[abc]*) \ echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac diff -r 6db40a9955dc -r 0d413f60cc23 Doc/README.txt --- a/Doc/README.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/README.txt Mon Jan 25 17:05:13 2016 +0100 @@ -3,119 +3,150 @@ This directory contains the reStructuredText (reST) sources to the Python documentation. You don't need to build them yourself, prebuilt versions are -available at . +available at http://docs.python.org/download/. -Documentation on authoring Python documentation, including information about +Documentation on the authoring Python documentation, including information about both style and markup, is available in the "Documenting Python" chapter of the -developers guide . +documentation. There's also a chapter intended to point out differences to +those familiar with the previous docs written in LaTeX. Building the docs ================= -You need to have Sphinx installed; it is the toolset -used to build the docs. It is not included in this tree, but maintained -separately and available from PyPI . +You need to have Python 2.4 or higher installed; the toolset used to build the +docs is written in Python. It is called *Sphinx*, it is not included in this +tree, but maintained separately. Also needed are the docutils, supplying the +base markup that Sphinx uses, Jinja, a templating engine, and optionally +Pygments, a code highlighter. Using make ---------- -A Makefile has been prepared so that on Unix, provided you have installed -Sphinx, you can just run :: +Luckily, a Makefile has been prepared so that on Unix, provided you have +installed Python and Subversion, you can just run :: make html -to build the HTML output files. - -On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. +to check out the necessary toolset in the `tools/` subdirectory and build the +HTML output files. To view the generated HTML, point your favorite browser at +the top-level index `build/html/index.html` after running "make". To use a Python interpreter that's not called ``python``, use the standard way to set Makefile variables, using e.g. :: - make html PYTHON=python3 - -On Windows, set the PYTHON environment variable instead. - -To use a specific sphinx-build (something other than ``sphinx-build``), set -the SPHINXBUILD variable. + make html PYTHON=/usr/bin/python2.5 Available make targets are: -* "clean", which removes all build files. + * "html", which builds standalone HTML files for offline viewing. -* "html", which builds standalone HTML files for offline viewing. + * "htmlhelp", which builds HTML files and a HTML Help project file usable to + convert them into a single Compiled HTML (.chm) file -- these are popular + under Microsoft Windows, but very handy on every platform. -* "htmlview", which re-uses the "html" builder, but then opens the main page - in your default web browser. + To create the CHM file, you need to run the Microsoft HTML Help Workshop over + the generated project (.hhp) file. -* "htmlhelp", which builds HTML files and a HTML Help project file usable to - convert them into a single Compiled HTML (.chm) file -- these are popular - under Microsoft Windows, but very handy on every platform. + * "latex", which builds LaTeX source files as input to "pdflatex" to produce + PDF documents. - To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for - you on Windows. + * "text", which builds a plain text file for each source file. -* "latex", which builds LaTeX source files as input to "pdflatex" to produce - PDF documents. + * "epub", which builds an EPUB document, suitable to be viewed on e-book + readers. -* "text", which builds a plain text file for each source file. + * "linkcheck", which checks all external references to see whether they are + broken, redirected or malformed, and outputs this information to stdout as + well as a plain-text (.txt) file. -* "epub", which builds an EPUB document, suitable to be viewed on e-book - readers. + * "changes", which builds an overview over all versionadded/versionchanged/ + deprecated items in the current version. This is meant as a help for the + writer of the "What's New" document. -* "linkcheck", which checks all external references to see whether they are - broken, redirected or malformed, and outputs this information to stdout as - well as a plain-text (.txt) file. + * "coverage", which builds a coverage overview for standard library modules and + C API. -* "changes", which builds an overview over all versionadded/versionchanged/ - deprecated items in the current version. This is meant as a help for the - writer of the "What's New" document. + * "pydoc-topics", which builds a Python module containing a dictionary with + plain text documentation for the labels defined in + `tools/sphinxext/pyspecific.py` -- pydoc needs these to show topic and + keyword help. -* "coverage", which builds a coverage overview for standard library modules and - C API. - -* "pydoc-topics", which builds a Python module containing a dictionary with - plain text documentation for the labels defined in - `tools/pyspecific.py` -- pydoc needs these to show topic and keyword help. - -* "suspicious", which checks the parsed markup for text that looks like - malformed and thus unconverted reST. - -* "check", which checks for frequent markup errors. - -* "serve", which serves the build/html directory on port 8000. - -* "dist", (Unix only) which creates distributable archives of HTML, text, - PDF, and EPUB builds. +A "make update" updates the Subversion checkouts in `tools/`. Without make ------------ -Install the Sphinx package and its dependencies from PyPI. +You'll need to install the Sphinx package, either by checking it out via :: -Then, from the ``Doc`` directory, run :: + svn co http://svn.python.org/projects/external/Sphinx-1.0.7/sphinx tools/sphinx - sphinx-build -b . build/ +or by installing it from PyPI. -where ```` is one of html, text, latex, or htmlhelp (for explanations -see the make targets above). +Then, you need to install Docutils, either by checking it out via :: + + svn co http://svn.python.org/projects/external/docutils-0.6/docutils tools/docutils + +or by installing it from http://docutils.sf.net/. + +You also need Jinja2, either by checking it out via :: + + svn co http://svn.python.org/projects/external/Jinja-2.3.1/jinja2 tools/jinja2 + +or by installing it from PyPI. + +You can optionally also install Pygments, either as a checkout via :: + + svn co http://svn.python.org/projects/external/Pygments-1.3.1/pygments tools/pygments + +or from PyPI at http://pypi.python.org/pypi/Pygments. + + +Then, make an output directory, e.g. under `build/`, and run :: + + python tools/sphinx-build.py -b . build/ + +where `` is one of html, text, latex, or htmlhelp (for explanations see +the make targets above). Contributing ============ Bugs in the content should be reported to the Python bug tracker at -https://bugs.python.org. +http://bugs.python.org. Bugs in the toolset should be reported in the Sphinx bug tracker at -https://www.bitbucket.org/birkenfeld/sphinx/issues/. +http://www.bitbucket.org/birkenfeld/sphinx/issues/. You can also send a mail to the Python Documentation Team at docs@python.org, and we will process your request as soon as possible. If you want to help the Documentation Team, you are always welcome. Just send a mail to docs@python.org. + + +Copyright notice +================ + +The Python source is copyrighted, but you can freely use and copy it +as long as you don't change or remove the copyright notice: + +---------------------------------------------------------------------- +Copyright (c) 2000-2012 Python Software Foundation. +All rights reserved. + +Copyright (c) 2000 BeOpen.com. +All rights reserved. + +Copyright (c) 1995-2000 Corporation for National Research Initiatives. +All rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. +All rights reserved. + +See the file "license.rst" for information on usage and redistribution +of this file, and for a DISCLAIMER OF ALL WARRANTIES. +---------------------------------------------------------------------- diff -r 6db40a9955dc -r 0d413f60cc23 Doc/about.rst --- a/Doc/about.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/about.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,34 +6,31 @@ These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a document processor specifically written for the Python documentation. -.. _reStructuredText: http://docutils.sourceforge.net/rst.html -.. _Sphinx: http://sphinx-doc.org/ +.. _reStructuredText: http://docutils.sf.net/rst.html +.. _Sphinx: http://sphinx.pocoo.org/ .. In the online version of these documents, you can submit comments and suggest changes directly on the documentation pages. -Development of the documentation and its toolchain is an entirely volunteer -effort, just like Python itself. If you want to contribute, please take a -look at the :ref:`reporting-bugs` page for information on how to do so. New -volunteers are always welcome! +Development of the documentation and its toolchain takes place on the +docs@python.org mailing list. We're always looking for volunteers wanting +to help with the docs, so feel free to send a mail there! Many thanks go to: * Fred L. Drake, Jr., the creator of the original Python documentation toolset and writer of much of the content; -* the `Docutils `_ project for creating +* the `Docutils `_ project for creating reStructuredText and the Docutils suite; * Fredrik Lundh for his `Alternative Python Reference `_ project from which Sphinx got many good ideas. +See :ref:`reporting-bugs` for information how to report bugs in this +documentation, or Python itself. -Contributors to the Python Documentation ----------------------------------------- - -Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +.. including the ACKS file here so that it can be maintained separately +.. include:: ACKS.txt It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff -r 6db40a9955dc -r 0d413f60cc23 Doc/bugs.rst --- a/Doc/bugs.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/bugs.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,24 +13,21 @@ ================== If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you -have a suggestion how to fix it, include that as well. +please send an e-mail to docs@python.org describing the bug and where you found +it. If you have a suggestion how to fix it, include that as well. -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. +docs@python.org is a mailing list run by volunteers; your request will be +noticed, even if it takes a while to be processed. -.. seealso:: - `Documentation bugs`_ on the Python issue tracker +Of course, if you want a more persistent record of your issue, you can use the +issue tracker for documentation bugs as well. -.. _using-the-tracker: Using the Python issue tracker ============================== Bug reports for Python itself should be submitted via the Python Bug Tracker -(https://bugs.python.org/). The bug tracker offers a Web form which allows +(http://bugs.python.org/). The bug tracker offers a Web form which allows pertinent information to be entered and submitted to the developers. The first step in filing a report is to determine whether the problem has @@ -65,24 +62,14 @@ .. seealso:: + `Python Developer's Guide `_ + Detailed description of the issue workflow and developers tools. + `How to Report Bugs Effectively `_ Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Writing Guidelines `_ + `Bug Writing Guidelines `_ Information about writing a good bug report. Some of this is specific to the Mozilla project, but describes general good practices. - -Getting started contributing to Python yourself -=============================================== - -Beyond just reporting bugs that you find, you are also welcome to submit -patches to fix them. You can find more information on how to get started -patching Python in the `Python Developer's Guide`_. If you have questions, -the `core-mentorship mailing list`_ is a friendly place to get answers to -any and all questions pertaining to the process of fixing issues in Python. - -.. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity -.. _Python Developer's Guide: https://docs.python.org/devguide/ -.. _core-mentorship mailing list: https://mail.python.org/mailman/listinfo/core-mentorship/ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/allocation.rst --- a/Doc/c-api/allocation.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/allocation.rst Mon Jan 25 17:05:13 2016 +0100 @@ -32,7 +32,7 @@ Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of + the memory allocation is determined from the :attr:`tp_basicsize` field of the type object. @@ -41,7 +41,7 @@ Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of + plus *size* fields of the size given by the :attr:`tp_itemsize` field of *type*. This is useful for implementing objects like tuples, which are able to determine their size at construction time. Embedding the array of fields into the same allocation decreases the number of allocations, @@ -52,7 +52,7 @@ Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of + :attr:`tp_dealloc` handler specified in the object's type. The fields of the object should not be accessed after this call as the memory is no longer a valid Python object. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/apiabiversion.rst --- a/Doc/c-api/apiabiversion.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,39 +0,0 @@ -.. highlightlang:: c - -.. _apiabiversion: - -*********************** -API and ABI Versioning -*********************** - -``PY_VERSION_HEX`` is the Python version number encoded in a single integer. - -For example if the ``PY_VERSION_HEX`` is set to ``0x030401a2``, the underlying -version information can be found by treating it as a 32 bit number in -the following manner: - - +-------+-------------------------+------------------------------------------------+ - | Bytes | Bits (big endian order) | Meaning | - +=======+=========================+================================================+ - | ``1`` | ``1-8`` | ``PY_MAJOR_VERSION`` (the ``3`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``2`` | ``9-16`` | ``PY_MINOR_VERSION`` (the ``4`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``3`` | ``17-24`` | ``PY_MICRO_VERSION`` (the ``1`` in | - | | | ``3.4.1a2``) | - +-------+-------------------------+------------------------------------------------+ - | ``4`` | ``25-28`` | ``PY_RELEASE_LEVEL`` (``0xA`` for alpha, | - | | | ``0xB`` for beta, ``0xC`` for release | - | | | candidate and ``0xF`` for final), in this | - | | | case it is alpha. | - +-------+-------------------------+------------------------------------------------+ - | | ``29-32`` | ``PY_RELEASE_SERIAL`` (the ``2`` in | - | | | ``3.4.1a2``, zero for final releases) | - +-------+-------------------------+------------------------------------------------+ - -Thus ``3.4.1a2`` is hexversion ``0x030401a2``. - -All the given macros are defined in :source:`Include/patchlevel.h`. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/arg.rst --- a/Doc/c-api/arg.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/arg.rst Mon Jan 25 17:05:13 2016 +0100 @@ -45,7 +45,6 @@ Unless otherwise stated, buffers are not NUL-terminated. .. note:: - For all ``#`` variants of formats (``s#``, ``y#``, etc.), the type of the length argument (int or :c:type:`Py_ssize_t`) is controlled by defining the macro :c:macro:`PY_SSIZE_T_CLEAN` before including @@ -65,20 +64,20 @@ :exc:`UnicodeError` is raised. .. note:: - This format does not accept :term:`bytes-like objects - `. If you want to accept + This format does not accept bytes-like objects. If you want to accept filesystem paths and convert them to C character strings, it is preferable to use the ``O&`` format with :c:func:`PyUnicode_FSConverter` as *converter*. -``s*`` (:class:`str` or :term:`bytes-like object`) [Py_buffer] - This format accepts Unicode objects as well as bytes-like objects. +``s*`` (:class:`str`, :class:`bytes`, :class:`bytearray` or buffer compatible object) [Py_buffer] + This format accepts Unicode objects as well as objects supporting the + buffer protocol. It fills a :c:type:`Py_buffer` structure provided by the caller. In this case the resulting C string may contain embedded NUL bytes. Unicode objects are converted to C strings using ``'utf-8'`` encoding. -``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`] - Like ``s*``, except that it doesn't accept mutable bytes-like objects +``s#`` (:class:`str`, :class:`bytes` or read-only buffer compatible object) [const char \*, int or :c:type:`Py_ssize_t`] + Like ``s*``, except that it doesn't accept mutable buffer-like objects such as :class:`bytearray`. The result is stored into two C variables, the first one a pointer to a C string, the second one its length. The string may contain embedded null bytes. Unicode objects are converted @@ -88,26 +87,26 @@ Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to *NULL*. -``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] +``z*`` (:class:`str`, :class:`bytes`, :class:`bytearray`, buffer compatible object or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to *NULL*. -``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int] +``z#`` (:class:`str`, :class:`bytes`, read-only buffer compatible object or ``None``) [const char \*, int] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to *NULL*. -``y`` (read-only :term:`bytes-like object`) [const char \*] +``y`` (:class:`bytes`) [const char \*] This format converts a bytes-like object to a C pointer to a character string; it does not accept Unicode objects. The bytes buffer must not contain embedded NUL bytes; if it does, a :exc:`TypeError` exception is raised. -``y*`` (:term:`bytes-like object`) [Py_buffer] - This variant on ``s*`` doesn't accept Unicode objects, only - bytes-like objects. **This is the recommended way to accept +``y*`` (:class:`bytes`, :class:`bytearray` or buffer compatible object) [Py_buffer] + This variant on ``s*`` doesn't accept Unicode objects, only objects + supporting the buffer protocol. **This is the recommended way to accept binary data.** -``y#`` (read-only :term:`bytes-like object`) [const char \*, int] +``y#`` (:class:`bytes`) [const char \*, int] This variant on ``s#`` doesn't accept Unicode objects, only bytes-like objects. @@ -152,7 +151,7 @@ any conversion. Raises :exc:`TypeError` if the object is not a Unicode object. The C variable may also be declared as :c:type:`PyObject\*`. -``w*`` (read-write :term:`bytes-like object`) [Py_buffer] +``w*`` (:class:`bytearray` or read-write byte-oriented buffer) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. The buffer may contain embedded null bytes. The caller have to call @@ -265,8 +264,7 @@ Convert a Python byte, represented as a :class:`bytes` or :class:`bytearray` object of length 1, to a C :c:type:`char`. - .. versionchanged:: 3.3 - Allow :class:`bytearray` objects. + .. versionchanged:: 3.3 Allow :class:`bytearray` objects ``C`` (:class:`str` of length 1) [int] Convert a Python character, represented as a :class:`str` object of @@ -296,8 +294,6 @@ the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. -.. _o_ampersand: - ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C @@ -321,15 +317,6 @@ .. versionchanged:: 3.1 ``Py_CLEANUP_SUPPORTED`` was added. -``p`` (:class:`bool`) [int] - Tests the value passed in for truth (a boolean **p**\ redicate) and converts - the result to its equivalent C true/false integer value. - Sets the int to 1 if the expression was true and 0 if it was false. - This accepts any valid Python value. See :ref:`truth` for more - information about how Python tests values for truth. - - .. versionadded:: 3.3 - ``(items)`` (:class:`tuple`) [*matching-items*] The object must be a Python sequence whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in @@ -430,11 +417,10 @@ Function used to deconstruct the argument lists of "old-style" functions --- these are functions which use the :const:`METH_OLDARGS` parameter parsing - method, which has been removed in Python 3. This is not recommended for use - in parameter parsing in new code, and most code in the standard interpreter - has been modified to no longer use this for that purpose. It does remain a - convenient way to decompose other tuples, however, and may continue to be - used for that purpose. + method. This is not recommended for use in parameter parsing in new code, and + most code in the standard interpreter has been modified to no longer use this + for that purpose. It does remain a convenient way to decompose other tuples, + however, and may continue to be used for that purpose. .. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) @@ -518,7 +504,7 @@ ``None`` is returned. ``y`` (:class:`bytes`) [char \*] - This converts a C string to a Python :class:`bytes` object. If the C + This converts a C string to a Python :func:`bytes` object. If the C string pointer is *NULL*, ``None`` is returned. ``y#`` (:class:`bytes`) [char \*, int] diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/buffer.rst --- a/Doc/c-api/buffer.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/buffer.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,10 +1,5 @@ .. highlightlang:: c -.. index:: - single: buffer protocol - single: buffer interface; (see buffer protocol) - single: buffer object; (see buffer protocol) - .. _bufferobjects: Buffer Protocol @@ -15,6 +10,9 @@ .. sectionauthor:: Stefan Krah +.. index:: + single: buffer interface + Certain objects available in Python wrap access to an underlying memory array or *buffer*. Such objects include the built-in :class:`bytes` and :class:`bytearray`, and some extension types like :class:`array.array`. @@ -26,8 +24,8 @@ then desirable, in some situations, to access that buffer directly and without intermediate copying. -Python provides such a facility at the C level in the form of the :ref:`buffer -protocol `. This protocol has two sides: +Python provides such a facility at the C level in the form of the *buffer +protocol*. This protocol has two sides: .. index:: single: PyBufferProcs @@ -40,7 +38,7 @@ Simple objects such as :class:`bytes` and :class:`bytearray` expose their underlying buffer in byte-oriented form. Other forms are possible; for example, -the elements exposed by an :class:`array.array` can be multi-byte values. +the elements exposed by a :class:`array.array` can be multi-byte values. An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write` method of file objects: any object that can export a series of bytes through @@ -89,16 +87,6 @@ .. c:type:: Py_buffer - .. c:member:: void \*buf - - A pointer to the start of the logical structure described by the buffer - fields. This can be any location within the underlying physical memory - block of the exporter. For example, with negative :c:member:`~Py_buffer.strides` - the value may point to the end of the memory block. - - For :term:`contiguous` arrays, the value points to the beginning of - the memory block. - .. c:member:: void \*obj A new reference to the exporting object. The reference is owned by @@ -111,6 +99,16 @@ this field is *NULL*. In general, exporting objects MUST NOT use this scheme. + .. c:member:: void \*buf + + A pointer to the start of the logical structure described by the buffer + fields. This can be any location within the underlying physical memory + block of the exporter. For example, with negative :c:member:`~Py_buffer.strides` + the value may point to the end of the memory block. + + For contiguous arrays, the value points to the beginning of the memory + block. + .. c:member:: Py_ssize_t len ``product(shape) * itemsize``. For contiguous arrays, this is the length @@ -133,15 +131,15 @@ called on non-NULL :c:member:`~Py_buffer.format` values. Important exception: If a consumer requests a buffer without the - :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_buffer.format` will + :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_Buffer.format` will be set to *NULL*, but :c:member:`~Py_buffer.itemsize` still has the value for the original format. - If :c:member:`~Py_buffer.shape` is present, the equality + If :c:member:`~Py_Buffer.shape` is present, the equality ``product(shape) * itemsize == len`` still holds and the consumer can use :c:member:`~Py_buffer.itemsize` to navigate the buffer. - If :c:member:`~Py_buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE` + If :c:member:`~Py_Buffer.shape` is *NULL* as a result of a :c:macro:`PyBUF_SIMPLE` or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. @@ -156,7 +154,7 @@ .. c:member:: int ndim The number of dimensions the memory represents as an n-dimensional array. - If it is 0, :c:member:`~Py_buffer.buf` points to a single item representing + If it is 0, :c:member:`~Py_Buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be *NULL*. @@ -198,9 +196,6 @@ indicates that no de-referencing should occur (striding in a contiguous memory block). - If all suboffsets are negative (i.e. no de-referencing is needed, then - this field must be NULL (the default value). - This type of array representation is used by the Python Imaging Library (PIL). See `complex arrays`_ for further information how to access elements of such an array. @@ -266,7 +261,6 @@ in decreasing order of complexity. Note that each flag contains all bits of the flags below it. -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l| +-----------------------------+-------+---------+------------+ | Request | shape | strides | suboffsets | @@ -281,16 +275,11 @@ +-----------------------------+-------+---------+------------+ -.. index:: contiguous, C-contiguous, Fortran contiguous - contiguity requests ~~~~~~~~~~~~~~~~~~~ -C or Fortran :term:`contiguity ` can be explicitly requested, -with and without stride information. Without stride information, the buffer -must be C-contiguous. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l| +C or Fortran contiguity can be explicitly requested, with and without stride +information. Without stride information, the buffer must be C-contiguous. +-----------------------------------+-------+---------+------------+--------+ | Request | shape | strides | suboffsets | contig | @@ -315,7 +304,7 @@ In the following table *U* stands for undefined contiguity. The consumer would have to call :c:func:`PyBuffer_IsContiguous` to determine contiguity. -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l|l|l| + +-------------------------------+-------+---------+------------+--------+----------+--------+ | Request | shape | strides | suboffsets | contig | readonly | format | @@ -469,13 +458,13 @@ .. c:function:: int PyBuffer_IsContiguous(Py_buffer *view, char order) Return 1 if the memory defined by the *view* is C-style (*order* is - ``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one + ``'C'``) or Fortran-style (*order* is ``'F'``) contiguous or either one (*order* is ``'A'``). Return 0 otherwise. .. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char order) - Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if + Fill the *strides* array with byte-strides of a contiguous (C-style if *order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the given shape with the given number of bytes per element. @@ -495,8 +484,8 @@ :c:member:`view->obj` to *NULL* and return -1; If this function is used as part of a :ref:`getbufferproc `, - *exporter* MUST be set to the exporting object and *flags* must be passed - unmodified. Otherwise, *exporter* MUST be NULL. + *exporter* MUST be set to the exporting object. Otherwise, *exporter* MUST + be NULL. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/bytearray.rst --- a/Doc/c-api/bytearray.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/bytearray.rst Mon Jan 25 17:05:13 2016 +0100 @@ -40,7 +40,7 @@ .. c:function:: PyObject* PyByteArray_FromObject(PyObject *o) Return a new bytearray object from any object, *o*, that implements the - :ref:`buffer protocol `. + buffer protocol. .. XXX expand about the buffer protocol, at least somewhere @@ -64,8 +64,7 @@ .. c:function:: char* PyByteArray_AsString(PyObject *bytearray) Return the contents of *bytearray* as a char array after checking for a - *NULL* pointer. The returned array always has an extra - null byte appended. + *NULL* pointer. .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/bytes.rst --- a/Doc/c-api/bytes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/bytes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -62,15 +62,13 @@ .. % because not all compilers support the %z width modifier -- we fake it .. % when necessary via interpolating PY_FORMAT_SIZE_T. - .. tabularcolumns:: |l|l|L| - +-------------------+---------------+--------------------------------+ | Format Characters | Type | Comment | +===================+===============+================================+ | :attr:`%%` | *n/a* | The literal % character. | +-------------------+---------------+--------------------------------+ - | :attr:`%c` | int | A single byte, | - | | | represented as a C int. | + | :attr:`%c` | int | A single character, | + | | | represented as an C int. | +-------------------+---------------+--------------------------------+ | :attr:`%d` | int | Exactly equivalent to | | | | ``printf("%d")``. | @@ -109,7 +107,7 @@ +-------------------+---------------+--------------------------------+ An unrecognized format character causes all the rest of the format string to be - copied as-is to the result object, and any extra arguments discarded. + copied as-is to the result string, and any extra arguments discarded. .. c:function:: PyObject* PyBytes_FromFormatV(const char *format, va_list vargs) @@ -136,13 +134,11 @@ .. c:function:: char* PyBytes_AsString(PyObject *o) - Return a pointer to the contents of *o*. The pointer - refers to the internal buffer of *o*, which consists of ``len(o) + 1`` - bytes. The last byte in the buffer is always null, regardless of - whether there are any other null bytes. The data must not be - modified in any way, unless the object was just created using + Return a NUL-terminated representation of the contents of *o*. The pointer + refers to the internal buffer of *o*, not a copy. The data must not be + modified in any way, unless the string was just created using ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns *NULL* + *o* is not a string object at all, :c:func:`PyBytes_AsString` returns *NULL* and raises :exc:`TypeError`. @@ -153,18 +149,16 @@ .. c:function:: int PyBytes_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) - Return the null-terminated contents of the object *obj* + Return a NUL-terminated representation of the contents of the object *obj* through the output variables *buffer* and *length*. - If *length* is *NULL*, the bytes object - may not contain embedded null bytes; + If *length* is *NULL*, the resulting buffer may not contain NUL characters; if it does, the function returns ``-1`` and a :exc:`TypeError` is raised. - The buffer refers to an internal buffer of *obj*, which includes an - additional null byte at the end (not counted in *length*). The data - must not be modified in any way, unless the object was just created using + The buffer refers to an internal string buffer of *obj*, not a copy. The data + must not be modified in any way, unless the string was just created using ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *obj* is not a bytes object at all, :c:func:`PyBytes_AsStringAndSize` + *string* is not a string object at all, :c:func:`PyBytes_AsStringAndSize` returns ``-1`` and raises :exc:`TypeError`. @@ -172,14 +166,14 @@ Create a new bytes object in *\*bytes* containing the contents of *newpart* appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be + the old value of *bytes* will be stolen. If the new string cannot be created, the old reference to *bytes* will still be discarded and the value of *\*bytes* will be set to *NULL*; the appropriate exception will be set. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) - Create a new bytes object in *\*bytes* containing the contents of *newpart* + Create a new string object in *\*bytes* containing the contents of *newpart* appended to *bytes*. This version decrements the reference count of *newpart*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/code.rst --- a/Doc/c-api/code.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/code.rst Mon Jan 25 17:05:13 2016 +0100 @@ -29,13 +29,13 @@ .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. + Return true if *co* is a :class:`code` object -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: int PyCode_GetNumFree(PyObject *co) Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) +.. c:function:: PyCodeObject *PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab) Return a new code object. If you need a dummy code object to create a frame, use :c:func:`PyCode_NewEmpty` instead. Calling @@ -43,7 +43,7 @@ version since the definition of the bytecode changes often. -.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) +.. c:function:: int PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) Return a new empty code object with the specified filename, function name, and first line number. It is illegal to diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/codec.rst --- a/Doc/c-api/codec.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/codec.rst Mon Jan 25 17:05:13 2016 +0100 @@ -52,19 +52,19 @@ .. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) - Get an :class:`~codecs.IncrementalEncoder` object for the given *encoding*. + Get an :class:`IncrementalEncoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) - Get an :class:`~codecs.IncrementalDecoder` object for the given *encoding*. + Get an :class:`IncrementalDecoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`~codecs.StreamReader` factory function for the given *encoding*. + Get a :class:`StreamReader` factory function for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`~codecs.StreamWriter` factory function for the given *encoding*. + Get a :class:`StreamWriter` factory function for the given *encoding*. Registry API for Unicode encoding error handlers @@ -116,8 +116,3 @@ Replace the unicode encode error with backslash escapes (``\x``, ``\u`` and ``\U``). -.. c:function:: PyObject* PyCodec_NameReplaceErrors(PyObject *exc) - - Replace the unicode encode error with ``\N{...}`` escapes. - - .. versionadded:: 3.5 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/concrete.rst --- a/Doc/c-api/concrete.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/concrete.rst Mon Jan 25 17:05:13 2016 +0100 @@ -74,35 +74,26 @@ .. _mapobjects: -Container Objects -================= +Mapping Objects +=============== .. index:: object: mapping .. toctree:: dict.rst - set.rst .. _otherobjects: -Function Objects -================ - -.. toctree:: - - function.rst - method.rst - cell.rst - code.rst - - Other Objects ============= .. toctree:: + set.rst + function.rst + method.rst file.rst module.rst iterator.rst @@ -111,7 +102,7 @@ memoryview.rst weakref.rst capsule.rst + cell.rst gen.rst - coro.rst datetime.rst - + code.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/conversion.rst --- a/Doc/c-api/conversion.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/conversion.rst Mon Jan 25 17:05:13 2016 +0100 @@ -119,13 +119,13 @@ .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: char* PyOS_stricmp(char *s1, char *s2) Case insensitive comparison of strings. The function works almost identically to :c:func:`strcmp` except that it ignores the case. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:function:: char* PyOS_strnicmp(char *s1, char *s2, Py_ssize_t size) Case insensitive comparison of strings. The function works almost identically to :c:func:`strncmp` except that it ignores the case. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/coro.rst --- a/Doc/c-api/coro.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -.. highlightlang:: c - -.. _coro-objects: - -Coroutine Objects ------------------ - -.. versionadded:: 3.5 - -Coroutine objects are what functions declared with an ``async`` keyword -return. - - -.. c:type:: PyCoroObject - - The C structure used for coroutine objects. - - -.. c:var:: PyTypeObject PyCoro_Type - - The type object corresponding to coroutine objects. - - -.. c:function:: int PyCoro_CheckExact(PyObject *ob) - - Return true if *ob*'s type is *PyCoro_Type*; *ob* must not be *NULL*. - - -.. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new coroutine object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/dict.rst --- a/Doc/c-api/dict.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/dict.rst Mon Jan 25 17:05:13 2016 +0100 @@ -36,11 +36,11 @@ Return a new empty dictionary, or *NULL* on failure. -.. c:function:: PyObject* PyDictProxy_New(PyObject *mapping) +.. c:function:: PyObject* PyDictProxy_New(PyObject *dict) - Return a :class:`types.MappingProxyType` object for a mapping which - enforces read-only behavior. This is normally used to create a view to - prevent modification of the dictionary for non-dynamic class types. + Return a proxy object for a mapping which enforces read-only behavior. + This is normally used to create a proxy to prevent modification of the + dictionary for non-dynamic class types. .. c:function:: void PyDict_Clear(PyObject *p) @@ -84,7 +84,7 @@ on failure. -.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) +.. c:function:: int PyDict_DelItemString(PyObject *p, char *key) Remove the entry in dictionary *p* which has a key specified by the string *key*. Return ``0`` on success or ``-1`` on failure. @@ -110,15 +110,6 @@ :c:type:`char\*`, rather than a :c:type:`PyObject\*`. -.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default) - - This is the same as the Python-level :meth:`dict.setdefault`. If present, it - returns the value corresponding to *key* from the dictionary *p*. If the key - is not in the dict, it is inserted with value *defaultobj* and *defaultobj* - is returned. This function evaluates the hash function of *key* only once, - instead of evaluating it independently for the lookup and the insertion. - - .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. @@ -201,11 +192,8 @@ .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) - This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to - ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall - back to the iterating over a sequence of key value pairs if the second - argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an - exception was raised. + This is the same as ``PyDict_Merge(a, b, 1)`` in C, or ``a.update(b)`` in + Python. Return ``0`` on success or ``-1`` if an exception was raised. .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/exceptions.rst --- a/Doc/c-api/exceptions.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/exceptions.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,19 +9,13 @@ The functions described in this chapter will let you handle and raise Python exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: +exception handling. It works somewhat like the Unix :c:data:`errno` variable: there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually *NULL* if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the :c:func:`PyArg_\*` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be NULL if non-set (although some combinations are -forbidden, for example you can't have a non-NULL traceback if the exception -type is NULL). +functions don't clear this on success, but will set it to indicate the cause of +the error on failure. Most functions also return an error indicator, usually +*NULL* if they are supposed to return a pointer, or ``-1`` if they return an +integer (exception: the :c:func:`PyArg_\*` functions return ``1`` for success and +``0`` for failure). When a function must fail because some function it called failed, it generally doesn't set the error indicator; the function it called already set it. It is @@ -33,21 +27,12 @@ propagated, additional calls into the Python/C API may not behave as intended and may fail in mysterious ways. -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). +The error indicator consists of three Python objects corresponding to the result +of ``sys.exc_info()``. API functions exist to interact with the error indicator +in various ways. There is a separate error indicator for each thread. - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. +.. XXX Order of these should be more thoughtful. + Either alphabetical or some kind of structure. .. c:function:: void PyErr_PrintEx(int set_sys_last_vars) @@ -66,24 +51,82 @@ Alias for ``PyErr_PrintEx(1)``. -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) +.. c:function:: PyObject* PyErr_Occurred() - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. + Test whether the error indicator is set. If set, return the exception *type* + (the first argument to the last call to one of the :c:func:`PyErr_Set\*` + functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not + own a reference to the return value, so you do not need to :c:func:`Py_DECREF` + it. - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. The repr of *obj* will be printed in - the warning message. + .. note:: + Do not compare the return value to a specific exception; use + :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could + easily fail since the exception may be an instance instead of a class, in the + case of a class exception, or it may the a subclass of the expected exception.) -Raising exceptions -================== -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -NULL pointer for use in a ``return`` statement. +.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) + + Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This + should only be called when an exception is actually set; a memory access + violation will occur if no exception has been raised. + + +.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) + + Return true if the *given* exception matches the exception in *exc*. If + *exc* is a class object, this also returns true when *given* is an instance + of a subclass. If *exc* is a tuple, all exceptions in the tuple (and + recursively in subtuples) are searched for a match. + + +.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) + + Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below + can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance. + + +.. c:function:: void PyErr_Clear() + + Clear the error indicator. If the error indicator is not set, there is no + effect. + + +.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to *NULL*. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be *NULL* even when the type object is not. + + .. note:: + + This function is normally only used by code that needs to handle exceptions or + by code that needs to save and restore the error indicator temporarily. + + +.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) + + Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are *NULL*, the error + indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + .. note:: + + This function is normally only used by code that needs to save and restore the + error indicator temporarily; use :c:func:`PyErr_Fetch` to save the current + exception state. .. c:function:: void PyErr_SetString(PyObject *type, const char *message) @@ -109,14 +152,6 @@ string. -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - .. c:function:: void PyErr_SetNone(PyObject *type) This is a shorthand for ``PyErr_SetObject(type, Py_None)``. @@ -152,29 +187,14 @@ when the system call returns an error. -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not *NULL*, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the filesystem encoding - (:func:`os.fsdecode`). + *filename* is not *NULL*, it is passed to the constructor of *type* as a third + parameter. In the case of exceptions such as :exc:`IOError` and :exc:`OSError`, + this is used to define the :attr:`filename` attribute of the exception instance. + *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -197,64 +217,33 @@ .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). Availability: Windows. + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior that + if *filename* is not *NULL*, it is passed to the constructor of + :exc:`WindowsError` as a third parameter. *filename* is decoded from the + filesystem encoding (:func:`sys.getfilesystemencoding`). Availability: + Windows. -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. - Availability: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - Availability: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, char *filename) Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional parameter specifying the exception type to be raised. Availability: Windows. -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) +.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset) Set file, line, and offset information for the current exception. If the current exception is not a :exc:`SyntaxError`, then it sets additional attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. + is a :exc:`SyntaxError`. *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). - .. versionadded:: 3.4 +.. versionadded:: 3.2 -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) +.. c:function:: void PyErr_SyntaxLocation(char *filename, int lineno) - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is + Like :c:func:`PyErr_SyntaxLocationExc`, but the col_offset parameter is omitted. @@ -266,23 +255,7 @@ use. -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) +.. c:function:: int PyErr_WarnEx(PyObject *category, char *message, int stack_level) Issue a warning message. The *category* argument is a warning category (see below) or *NULL*; the *message* argument is an UTF-8 encoded string. *stack_level* is a @@ -291,6 +264,18 @@ is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. + This function normally prints a warning message to *sys.stderr*; however, it is + also possible that the user has specified that warnings are to be turned into + errors, and in that case this will raise an exception. It is also possible that + the function raises an exception because of a problem with the warning machinery + (the implementation imports the :mod:`warnings` module to do the heavy lifting). + The return value is ``0`` if no exception is raised, or ``-1`` if an exception + is raised. (It is not possible to determine whether a warning message is + actually printed, nor what the reason is for the exception; this is + intentional.) If an exception is raised, the caller should do its normal + exception handling (for example, :c:func:`Py_DECREF` owned references and return + an error value). + Warning categories must be subclasses of :c:data:`Warning`; the default warning category is :c:data:`RuntimeWarning`. The standard Python warning categories are available as global variables whose names are ``PyExc_`` followed by the Python @@ -307,22 +292,15 @@ documentation. There is no C API for warning control. -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) Issue a warning message with explicit control over all warning attributes. This is a straightforward wrapper around the Python function :func:`warnings.warn_explicit`, see there for more information. The *module* and *registry* arguments may be set to *NULL* to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - filesystem encoding (:func:`os.fsdecode`). + described there. *message* and *module* are UTF-8 encoded strings, + *filename* is decoded from the filesystem encoding + (:func:`sys.getfilesystemencoding`). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) @@ -333,140 +311,6 @@ .. versionadded:: 3.2 - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return *NULL*. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to *NULL*. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be *NULL* even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject **type, **value, **traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are *NULL*, the error - indicator is cleared. Do not pass a *NULL* type and non-*NULL* value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. Returns new references for the three objects, any of which - may be *NULL*. Does not modify the exception info state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass *NULL* for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - -Signal Handling -=============== - - .. c:function:: int PyErr_CheckSignals() .. index:: @@ -500,23 +344,15 @@ .. c:function:: int PySignal_SetWakeupFd(int fd) - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. + This utility function specifies a file descriptor to which a ``'\0'`` byte will + be written whenever a signal is received. It returns the previous such file + descriptor. The value ``-1`` disables the feature; this is the initial state. This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any error checking. *fd* should be a valid file descriptor. The function should only be called from the main thread. - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) +.. c:function:: PyObject* PyErr_NewException(char *name, PyObject *base, PyObject *dict) This utility function creates and returns a new exception class. The *name* argument must be the name of the new exception, a C string of the form @@ -531,7 +367,7 @@ argument can be used to specify a dictionary of class variables and methods. -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) +.. c:function:: PyObject* PyErr_NewExceptionWithDoc(char *name, char *doc, PyObject *base, PyObject *dict) Same as :c:func:`PyErr_NewException`, except that the new exception class can easily be given a docstring: If *doc* is non-*NULL*, it will be used as the @@ -540,6 +376,18 @@ .. versionadded:: 3.2 +.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + + This utility function prints a warning message to ``sys.stderr`` when an + exception has been set but it is impossible for the interpreter to actually + raise the exception. It is used, for example, when an exception occurs in an + :meth:`__del__` method. + + The function is called with a single argument *obj* that identifies the context + in which the unraisable exception occurred. The repr of *obj* will be printed in + the warning message. + + Exception Objects ================= @@ -577,14 +425,20 @@ set by ``raise ... from ...``) associated with the exception as a new reference, as accessible from Python through :attr:`__cause__`. + If there is no cause associated, this returns *NULL* (from Python + ``__cause__ is Ellipsis``). If the cause is :const:`None`, the default + exception display routines stop showing the context chain. -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - Set the cause associated with the exception to *cause*. Use *NULL* to clear - it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. +.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *ctx) - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + Set the cause associated with the exception to *ctx*. Use *NULL* to clear + it. There is no type check to make sure that *ctx* is either an exception + instance or :const:`None`. This steals a reference to *ctx*. + + If the cause is set to :const:`None` the default exception display + routines will not display this exception's context, and will not follow the + chain any further. .. _unicodeexceptions: @@ -674,7 +528,7 @@ recursive code does not necessarily invoke Python code (which tracks its recursion depth automatically). -.. c:function:: int Py_EnterRecursiveCall(const char *where) +.. c:function:: int Py_EnterRecursiveCall(char *where) Marks a point where a recursive C-level call is about to be performed. @@ -683,40 +537,40 @@ sets a :exc:`MemoryError` and returns a nonzero value. The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. + case, a :exc:`RuntimeError` is set and a nonzero value is returned. Otherwise, zero is returned. *where* should be a string such as ``" in instance check"`` to be - concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. + concatenated to the :exc:`RuntimeError` message caused by the recursion depth + limit. .. c:function:: void Py_LeaveRecursiveCall() Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each *successful* invocation of :c:func:`Py_EnterRecursiveCall`. -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires +Properly implementing :attr:`tp_repr` for container types requires special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The +:attr:`tp_repr` also needs to track objects to prevent cycles. The following two functions facilitate this functionality. Effectively, these are the C equivalent to :func:`reprlib.recursive_repr`. .. c:function:: int Py_ReprEnter(PyObject *object) - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to + Called at the beginning of the :attr:`tp_repr` implementation to detect cycles. If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation + positive integer. In that case the :attr:`tp_repr` implementation should return a string object indicating a cycle. As examples, :class:`dict` objects return ``{...}`` and :class:`list` objects return ``[...]``. The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should + is reached. In that case the :attr:`tp_repr` implementation should typically return ``NULL``. - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` + Otherwise, the function returns zero and the :attr:`tp_repr` implementation can continue normally. .. c:function:: void Py_ReprLeave(PyObject *object) @@ -800,8 +654,6 @@ +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | +-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | @@ -831,9 +683,6 @@ :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. -.. versionadded:: 3.5 - :c:data:`PyExc_RecursionError`. - These are compatibility aliases to :c:data:`PyExc_OSError`: @@ -882,7 +731,6 @@ single: PyExc_OverflowError single: PyExc_PermissionError single: PyExc_ProcessLookupError - single: PyExc_RecursionError single: PyExc_ReferenceError single: PyExc_RuntimeError single: PyExc_SyntaxError diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/file.rst --- a/Doc/c-api/file.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/file.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ the :mod:`io` APIs instead. -.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) +.. c:function:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *errors, char *newline, int closefd) Create a Python file object from the file descriptor of an already opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* @@ -40,9 +40,9 @@ Return the file descriptor associated with *p* as an :c:type:`int`. If the object is an integer, its value is returned. If not, the - object's :meth:`~io.IOBase.fileno` method is called if it exists; the - method must return an integer, which is returned as the file descriptor - value. Sets an exception and returns ``-1`` on failure. + object's :meth:`fileno` method is called if it exists; the method must return + an integer, which is returned as the file descriptor value. Sets an + exception and returns ``-1`` on failure. .. c:function:: PyObject* PyFile_GetLine(PyObject *p, int n) @@ -50,8 +50,7 @@ .. index:: single: EOFError (built-in exception) Equivalent to ``p.readline([n])``, this function reads one line from the - object *p*. *p* may be a file object or any object with a - :meth:`~io.IOBase.readline` + object *p*. *p* may be a file object or any object with a :meth:`readline` method. If *n* is ``0``, exactly one line is read, regardless of the length of the line. If *n* is greater than ``0``, no more than *n* bytes will be read from the file; a partial line can be returned. In both cases, an empty string diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/gcsupport.rst --- a/Doc/c-api/gcsupport.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/gcsupport.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,10 +12,10 @@ or strings), do not need to provide any explicit support for garbage collection. -To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must +To create a container type, the :attr:`tp_flags` field of the type object must include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the -:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a -:c:member:`~PyTypeObject.tp_clear` implementation must also be provided. +:attr:`tp_traverse` handler. If instances of the type are mutable, a +:attr:`tp_clear` implementation must also be provided. .. data:: Py_TPFLAGS_HAVE_GC @@ -57,7 +57,7 @@ Adds the object *op* to the set of container objects tracked by the collector. The collector can run at unexpected times so objects must be valid while being tracked. This should be called once all the fields - followed by the :c:member:`~PyTypeObject.tp_traverse` handler become valid, usually near the + followed by the :attr:`tp_traverse` handler become valid, usually near the end of the constructor. @@ -86,8 +86,8 @@ Remove the object *op* from the set of container objects tracked by the collector. Note that :c:func:`PyObject_GC_Track` can be called again on this object to add it back to the set of tracked objects. The deallocator - (:c:member:`~PyTypeObject.tp_dealloc` handler) should call this for the object before any of - the fields used by the :c:member:`~PyTypeObject.tp_traverse` handler become invalid. + (:attr:`tp_dealloc` handler) should call this for the object before any of + the fields used by the :attr:`tp_traverse` handler become invalid. .. c:function:: void _PyObject_GC_UNTRACK(PyObject *op) @@ -95,19 +95,19 @@ A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for extension modules. -The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: +The :attr:`tp_traverse` handler accepts a function parameter of this type: .. c:type:: int (*visitproc)(PyObject *object, void *arg) - Type of the visitor function passed to the :c:member:`~PyTypeObject.tp_traverse` handler. + Type of the visitor function passed to the :attr:`tp_traverse` handler. The function should be called with an object to traverse as *object* and - the third parameter to the :c:member:`~PyTypeObject.tp_traverse` handler as *arg*. The + the third parameter to the :attr:`tp_traverse` handler as *arg*. The Python core uses several visitor functions to implement cyclic garbage detection; it's not expected that users will need to write their own visitor functions. -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: +The :attr:`tp_traverse` handler must have the following type: .. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) @@ -119,15 +119,15 @@ object argument. If *visit* returns a non-zero value that value should be returned immediately. -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation +To simplify writing :attr:`tp_traverse` handlers, a :c:func:`Py_VISIT` macro is +provided. In order to use this macro, the :attr:`tp_traverse` implementation must name its arguments exactly *visit* and *arg*: .. c:function:: void Py_VISIT(PyObject *o) Call the *visit* callback, with arguments *o* and *arg*. If *visit* returns - a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` + a non-zero value, then return it. Using this macro, :attr:`tp_traverse` handlers look like:: static int @@ -138,7 +138,7 @@ return 0; } -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +The :attr:`tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* if the object is immutable. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/gen.rst --- a/Doc/c-api/gen.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/gen.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,7 +7,7 @@ Generator objects are what Python uses to implement generator iterators. They are normally created by iterating over a function that yields values, rather -than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. +than explicitly calling :c:func:`PyGen_New`. .. c:type:: PyGenObject @@ -17,28 +17,22 @@ .. c:var:: PyTypeObject PyGen_Type - The type object corresponding to generator objects. + The type object corresponding to generator objects -.. c:function:: int PyGen_Check(PyObject *ob) +.. c:function:: int PyGen_Check(ob) Return true if *ob* is a generator object; *ob* must not be *NULL*. -.. c:function:: int PyGen_CheckExact(PyObject *ob) +.. c:function:: int PyGen_CheckExact(ob) - Return true if *ob*'s type is *PyGen_Type*; *ob* must not be *NULL*. + Return true if *ob*'s type is *PyGen_Type* is a generator object; *ob* must not + be *NULL*. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) - Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be + Create and return a new generator object based on the *frame* object. A + reference to *frame* is stolen by this function. The parameter must not be *NULL*. - -.. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new generator object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be *NULL*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/import.rst --- a/Doc/c-api/import.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/import.rst Mon Jan 25 17:05:13 2016 +0100 @@ -30,21 +30,22 @@ .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. + This version of :c:func:`PyImport_ImportModule` does not block. It's intended + to be used in C functions that import other modules to execute a function. + The import may block if another thread holds the import lock. The function + :c:func:`PyImport_ImportModuleNoBlock` never blocks. It first tries to fetch + the module from sys.modules and falls back to :c:func:`PyImport_ImportModule` + unless the lock is held, in which case the function will raise an + :exc:`ImportError`. - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - -.. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) +.. c:function:: PyObject* PyImport_ImportModuleEx(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) .. index:: builtin: __import__ Import a module. This is best described by referring to the built-in Python - function :func:`__import__`. + function :func:`__import__`, as the standard :func:`__import__` function calls + this function directly. The return value is a new reference to the imported module or top-level package, or *NULL* with an exception set on failure. Like for @@ -70,14 +71,11 @@ .. versionadded:: 3.3 -.. c:function:: PyObject* PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) +.. c:function:: PyObject* PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is an UTF-8 encoded string instead of a Unicode object. - .. versionchanged:: 3.3 - Negative values for *level* are no longer accepted. - .. c:function:: PyObject* PyImport_Import(PyObject *name) This is a higher-level interface that calls the current "import hook @@ -118,7 +116,7 @@ encoded string instead of a Unicode object. -.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) +.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co) .. index:: builtin: compile @@ -132,14 +130,8 @@ such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`SourceFileLoader` otherwise. - The module's :attr:`__file__` attribute will be set to the code object's - :c:member:`co_filename`. If applicable, :attr:`__cached__` will also - be set. + :c:member:`co_filename`. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -151,7 +143,7 @@ :c:func:`PyImport_ExecCodeModuleWithPathnames`. -.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) +.. c:function:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of the module object is set to *pathname* if it is non-``NULL``. @@ -168,34 +160,25 @@ .. versionadded:: 3.3 -.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) +.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and - *cpathname* are UTF-8 encoded strings. Attempts are also made to figure out - what the value for *pathname* should be from *cpathname* if the former is - set to ``NULL``. + *cpathname* are UTF-8 encoded strings. .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Uses :func:`imp.source_from_cache()` in calculating the source path if - only the bytecode path is provided. .. c:function:: long PyImport_GetMagicNumber() - Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` file). - The magic number should be present in the first four bytes of the bytecode - file, in little-endian byte order. Returns -1 on error. - - .. versionchanged:: 3.3 - Return value of -1 upon failure. + Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` and + :file:`.pyo` files). The magic number should be present in the first four bytes + of the bytecode file, in little-endian byte order. .. c:function:: const char * PyImport_GetMagicTag() Return the magic tag string for :pep:`3147` format Python bytecode file - names. Keep in mind that the value at ``sys.implementation.cache_tag`` is - authoritative and should be used instead of this function. + names. .. versionadded:: 3.2 @@ -236,6 +219,11 @@ For internal use only. +.. c:function:: PyObject* _PyImport_FixupExtension(char *, char *) + + For internal use only. + + .. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) Load a frozen module named *name*. Return ``1`` for success, ``0`` if the @@ -246,11 +234,8 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.4 - The ``__file__`` attribute is no longer set on the module. - -.. c:function:: int PyImport_ImportFrozenModule(const char *name) +.. c:function:: int PyImport_ImportFrozenModule(char *name) Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a UTF-8 encoded string instead of a Unicode object. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/index.rst --- a/Doc/c-api/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,9 @@ Python/C API Reference Manual ################################## +:Release: |version| +:Date: |today| + This manual documents the API used by C and C++ programmers who want to write extension modules or embed Python. It is a companion to :ref:`extending-index`, which describes the general principles of extension writing but does not @@ -13,7 +16,6 @@ :maxdepth: 2 intro.rst - stable.rst veryhigh.rst refcounting.rst exceptions.rst @@ -23,4 +25,3 @@ init.rst memory.rst objimpl.rst - apiabiversion.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/init.rst --- a/Doc/c-api/init.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/init.rst Mon Jan 25 17:05:13 2016 +0100 @@ -25,7 +25,7 @@ triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() - single: Py_FinalizeEx() + single: Py_Finalize() Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; with the @@ -34,7 +34,7 @@ modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a + (without calling :c:func:`Py_Finalize` first). There is no return value; it is a fatal error if the initialization fails. @@ -48,20 +48,19 @@ .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + (zero) if not. After :c:func:`Py_Finalize` is called, this returns false until :c:func:`Py_Initialize` is called again. -.. c:function:: int Py_FinalizeEx() +.. c:function:: void Py_Finalize() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is 0. If there were errors during finalization - (flushing buffered data), -1 is returned. + time (without calling :c:func:`Py_Initialize` again first). There is no return + value; errors during finalization are ignored. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -80,51 +79,13 @@ freed. Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. + :c:func:`Py_Finalize` more than once. Process-wide parameters ======================= -.. c:function:: int Py_SetStandardStreamEncoding(const char *encoding, const char *errors) - - .. index:: - single: Py_Initialize() - single: main() - triple: stdin; stdout; sdterr - - This function should be called before :c:func:`Py_Initialize`, if it is - called at all. It specifies which encoding and error handling to use - with standard IO, with the same meanings as in :func:`str.encode`. - - It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code - to control IO encoding when the environment variable does not work. - - ``encoding`` and/or ``errors`` may be NULL to use - :envvar:`PYTHONIOENCODING` and/or default values (depending on other - settings). - - Note that :data:`sys.stderr` always uses the "backslashreplace" error - handler, regardless of this (or any other) setting. - - If :c:func:`Py_FinalizeEx` is called, this function will need to be called - again in order to affect subsequent calls to :c:func:`Py_Initialize`. - - Returns 0 if successful, a nonzero value on error (e.g. calling after the - interpreter has already been initialized). - - .. versionadded:: 3.4 - - .. c:function:: void Py_SetProgramName(wchar_t *name) .. index:: @@ -143,9 +104,6 @@ change for the duration of the program's execution. No code in the Python interpreter will change the contents of this storage. - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - .. c:function:: wchar* Py_GetProgramName() @@ -248,21 +206,13 @@ :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a default search path but uses the one provided instead. This is useful if Python is embedded by an application that has full knowledge of the location - of all modules. The path components should be separated by the platform - dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` - on Windows. + of all modules. The path components should be separated by semicolons. This also causes :data:`sys.executable` to be set only to the raw program name (see :c:func:`Py_SetProgramName`) and for :data:`sys.prefix` and :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these if required after calling :c:func:`Py_Initialize`. - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - The path argument is copied internally, so the caller may free it after the - call completes. - .. c:function:: const char* Py_GetVersion() @@ -359,9 +309,6 @@ :data:`sys.path`, which is the same as prepending the current working directory (``"."``). - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - .. note:: It is recommended that applications embedding the Python interpreter for purposes other than executing a single script pass 0 as *updatepath*, @@ -382,14 +329,7 @@ .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to 1 unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set to 1. .. c:function:: void Py_SetPythonHome(wchar_t *home) @@ -403,9 +343,6 @@ execution. No code in the Python interpreter will change the contents of this storage. - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:type:`wchar_*` string. - .. c:function:: w_char* Py_GetPythonHome() @@ -505,9 +442,6 @@ standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when compressing or hashing data. - -.. _gilstate: - Non-Python created threads -------------------------- @@ -611,7 +545,6 @@ .. index:: module: _thread .. note:: - When only the main thread exists, no GIL operations are needed. This is a common situation (most Python programs do not use threads), and the lock operations slow the interpreter down a bit. Therefore, the lock is not @@ -713,7 +646,7 @@ :c:func:`PyGILState_Release` on the same thread. -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() +.. c:function:: PyThreadState PyGILState_GetThisThreadState() Get the current thread state for this thread. May return ``NULL`` if no GILState API has been used on the current thread. Note that the main thread @@ -721,20 +654,6 @@ made on the main thread. This is mainly a helper/diagnostic function. -.. c:function:: int PyGILState_Check() - - Return 1 if the current thread is holding the GIL and 0 otherwise. - This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the GIL will it return 1. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the GIL is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. versionadded:: 3.4 - - The following macros are normally used without a trailing semicolon; look for example usage in the Python source distribution. @@ -882,8 +801,6 @@ instead. -.. _sub-interpreter-support: - Sub-interpreter support ======================= @@ -927,7 +844,7 @@ entry.) .. index:: - single: Py_FinalizeEx() + single: Py_Finalize() single: Py_Initialize() Extension modules are shared between (sub-)interpreters as follows: the first @@ -937,7 +854,7 @@ and filled with the contents of this copy; the extension's ``init`` function is not called. Note that this is different from what happens when an extension is imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's + :c:func:`Py_Finalize` and :c:func:`Py_Initialize`; in that case, the extension's ``initmodule`` function *is* called again. .. index:: single: close() (in module os) @@ -945,14 +862,14 @@ .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_FinalizeEx() + .. index:: single: Py_Finalize() Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread states below. When the call returns, the current thread state is *NULL*. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + when it returns.) :c:func:`Py_Finalize` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. @@ -988,44 +905,42 @@ A mechanism is provided to make asynchronous notifications to the main interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. +pointer and a void argument. +.. index:: single: setcheckinterval() (in module sys) + +Every check interval, when the global interpreter lock is released and +reacquired, Python will also call any such provided functions. This can be used +for example by asynchronous IO handlers. The notification can be scheduled from +a worker thread and the actual call than made at the earliest convenience by the +main thread where it has possession of the global interpreter lock and can +perform any Python API calls. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) .. index:: single: Py_AddPendingCall() - Schedule a function to be called from the main interpreter thread. On - success, 0 is returned and *func* is queued for being called in the - main thread. On failure, -1 is returned without setting any exception. + Post a notification to the Python main thread. If successful, *func* will be + called with the argument *arg* at the earliest convenience. *func* will be + called having the global interpreter lock held and can thus use the full + Python API and can take any action such as setting object attributes to + signal IO completion. It must return 0 on success, or -1 signalling an + exception. The notification function won't be interrupted to perform another + asynchronous notification recursively, but it can still be interrupted to + switch threads if the global interpreter lock is released, for example, if it + calls back into Python code. - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: + This function returns 0 on success in which case the notification has been + scheduled. Otherwise, for example if the notification buffer is full, it + returns -1 without setting any exception. - * on a :term:`bytecode` boundary; - * with the main thread holding the :term:`global interpreter lock` - (*func* can therefore use the full C API). - - *func* must return 0 on success, or -1 on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the global interpreter lock is released. - - This function doesn't need a current thread state to run, and it doesn't - need the global interpreter lock. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. + This function can be called on any thread, be it a Python thread or some + other system thread. If it is a Python thread, it doesn't matter if it holds + the global interpreter lock or not. .. versionadded:: 3.1 + .. _profiling: Profiling and Tracing @@ -1208,7 +1123,7 @@ .. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - Return the pointer to the first :c:type:`PyThreadState` object in the list of + Return the a pointer to the first :c:type:`PyThreadState` object in the list of threads associated with the interpreter *interp*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/intro.rst --- a/Doc/c-api/intro.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/intro.rst Mon Jan 25 17:05:13 2016 +0100 @@ -210,7 +210,7 @@ t = PyTuple_New(3); PyTuple_SetItem(t, 0, PyLong_FromLong(1L)); PyTuple_SetItem(t, 1, PyLong_FromLong(2L)); - PyTuple_SetItem(t, 2, PyUnicode_FromString("three")); + PyTuple_SetItem(t, 2, PyString_FromString("three")); Here, :c:func:`PyLong_FromLong` returns a new reference which is immediately stolen by :c:func:`PyTuple_SetItem`. When you want to keep using an object @@ -433,7 +433,7 @@ .. index:: single: sum_sequence() A simple example of detecting exceptions and passing them on is shown in the -:c:func:`sum_sequence` example above. It so happens that this example doesn't +:c:func:`sum_sequence` example above. It so happens that that example doesn't need to clean up any owned references when it detects an error. The following example function shows some error cleanup. First, to remind you why you like Python, we show the equivalent Python code:: @@ -578,9 +578,9 @@ application may want to start over (make another call to :c:func:`Py_Initialize`) or the application is simply done with its use of Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns +by calling :c:func:`Py_Finalize`. The function :c:func:`Py_IsInitialized` returns true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` +these functions is given in a later chapter. Notice that :c:func:`Py_Finalize` does *not* free all memory allocated by the Python interpreter, e.g. memory allocated by extension modules currently cannot be released. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/iter.rst --- a/Doc/c-api/iter.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/iter.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,7 @@ Iterator Protocol ================= -There are two functions specifically for working with iterators. +There are only a couple of functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) @@ -14,10 +14,11 @@ .. c:function:: PyObject* PyIter_Next(PyObject *o) - Return the next value from the iteration *o*. The object must be an iterator - (it is up to the caller to check this). If there are no remaining values, - returns *NULL* with no exception set. If an error occurs while retrieving - the item, returns *NULL* and passes along the exception. + Return the next value from the iteration *o*. If the object is an iterator, + this retrieves the next value from the iteration, and returns *NULL* with no + exception set if there are no remaining items. If the object is not an + iterator, :exc:`TypeError` is raised, or if there is an error in retrieving the + item, returns *NULL* and passes along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/long.rst --- a/Doc/c-api/long.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/long.rst Mon Jan 25 17:05:13 2016 +0100 @@ -80,7 +80,7 @@ *NULL* on failure. -.. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) +.. c:function:: PyObject* PyLong_FromString(char *str, char **pend, int base) Return a new :c:type:`PyLongObject` based on the string value in *str*, which is interpreted according to the radix in *base*. If *pend* is non-*NULL*, @@ -122,55 +122,37 @@ .. XXX alias PyLong_AS_LONG (for now) -.. c:function:: long PyLong_AsLong(PyObject *obj) +.. c:function:: long PyLong_AsLong(PyObject *pylong) .. index:: single: LONG_MAX single: OverflowError (built-in exception) - Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. + Return a C :c:type:`long` representation of the contents of *pylong*. If + *pylong* is greater than :const:`LONG_MAX`, raise an :exc:`OverflowError`, + and return -1. Convert non-long objects automatically to long first, + and return -1 if that raises exceptions. - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. +.. c:function:: long PyLong_AsLongAndOverflow(PyObject *pylong, int *overflow) + Return a C :c:type:`long` representation of the contents of + *pylong*. If *pylong* is greater than :const:`LONG_MAX` or less + than :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, + respectively, and return ``-1``; otherwise, set *\*overflow* to + ``0``. If any other exception occurs (for example a TypeError or + MemoryError), then ``-1`` will be returned and *\*overflow* will + be ``0``. -.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) - Return a C :c:type:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. +.. c:function:: PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject *pylong, int *overflow) - If the value of *obj* is greater than :const:`LONG_MAX` or less than - :const:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and - return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception - occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - - -.. c:function:: PY_LONG_LONG PyLong_AsLongLong(PyObject *obj) - - .. index:: - single: OverflowError (built-in exception) - - Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:type:`long`. - - -.. c:function:: PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) - - Return a C :c:type:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`__int__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is greater than :const:`PY_LLONG_MAX` or less than - :const:`PY_LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, - and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other - exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. + Return a C :c:type:`long long` representation of the contents of + *pylong*. If *pylong* is greater than :const:`PY_LLONG_MAX` or less + than :const:`PY_LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, + respectively, and return ``-1``; otherwise, set *\*overflow* to + ``0``. If any other exception occurs (for example a TypeError or + MemoryError), then ``-1`` will be returned and *\*overflow* will + be ``0``. .. versionadded:: 3.2 @@ -181,11 +163,9 @@ single: PY_SSIZE_T_MAX single: OverflowError (built-in exception) - Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must - be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`Py_ssize_t`. + Return a C :c:type:`Py_ssize_t` representation of the contents of *pylong*. + If *pylong* is greater than :const:`PY_SSIZE_T_MAX`, an :exc:`OverflowError` + is raised and ``-1`` will be returned. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) @@ -194,20 +174,26 @@ single: ULONG_MAX single: OverflowError (built-in exception) - Return a C :c:type:`unsigned long` representation of *pylong*. *pylong* - must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`unsigned long`. + Return a C :c:type:`unsigned long` representation of the contents of *pylong*. + If *pylong* is greater than :const:`ULONG_MAX`, an :exc:`OverflowError` is + raised. .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) - Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. + Return a :c:type:`size_t` representation of the contents of *pylong*. If + *pylong* is greater than the maximum value for a :c:type:`size_t`, an + :exc:`OverflowError` is raised. - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`size_t`. + +.. c:function:: PY_LONG_LONG PyLong_AsLongLong(PyObject *pylong) + + .. index:: + single: OverflowError (built-in exception) + + Return a C :c:type:`long long` from a Python integer. If *pylong* + cannot be represented as a :c:type:`long long`, an + :exc:`OverflowError` is raised and ``-1`` is returned. .. c:function:: unsigned PY_LONG_LONG PyLong_AsUnsignedLongLong(PyObject *pylong) @@ -215,43 +201,32 @@ .. index:: single: OverflowError (built-in exception) - Return a C :c:type:`unsigned PY_LONG_LONG` representation of *pylong*. - *pylong* must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for an - :c:type:`unsigned PY_LONG_LONG`. + Return a C :c:type:`unsigned long long` from a Python integer. If + *pylong* cannot be represented as an :c:type:`unsigned long long`, + an :exc:`OverflowError` is raised and ``(unsigned long long)-1`` is + returned. .. versionchanged:: 3.1 A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`. -.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) +.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *io) - Return a C :c:type:`unsigned long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its :meth:`__int__` - method (if present) to convert it to a :c:type:`PyLongObject`. + Return a C :c:type:`unsigned long` from a Python integer, without checking for + overflow. - If the value of *obj* is out of range for an :c:type:`unsigned long`, - return the reduction of that value modulo :const:`ULONG_MAX + 1`. +.. c:function:: unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject *io) -.. c:function:: unsigned PY_LONG_LONG PyLong_AsUnsignedLongLongMask(PyObject *obj) - - Return a C :c:type:`unsigned long long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its :meth:`__int__` - method (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is out of range for an :c:type:`unsigned long long`, - return the reduction of that value modulo :const:`PY_ULLONG_MAX + 1`. + Return a C :c:type:`unsigned long long` from a Python integer, without + checking for overflow. .. c:function:: double PyLong_AsDouble(PyObject *pylong) - Return a C :c:type:`double` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`double`. + Return a C :c:type:`double` representation of the contents of *pylong*. If + *pylong* cannot be approximately represented as a :c:type:`double`, an + :exc:`OverflowError` exception is raised and ``-1.0`` will be returned. .. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/mapping.rst --- a/Doc/c-api/mapping.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/mapping.rst Mon Jan 25 17:05:13 2016 +0100 @@ -22,7 +22,7 @@ expression ``len(o)``. -.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) +.. c:function:: int PyMapping_DelItemString(PyObject *o, char *key) Remove the mapping for object *key* from the object *o*. Return ``-1`` on failure. This is equivalent to the Python statement ``del o[key]``. @@ -34,7 +34,7 @@ failure. This is equivalent to the Python statement ``del o[key]``. -.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) +.. c:function:: int PyMapping_HasKeyString(PyObject *o, char *key) On success, return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This is equivalent to the Python expression ``key in o``. @@ -50,30 +50,30 @@ .. c:function:: PyObject* PyMapping_Keys(PyObject *o) - On success, return a list, a tuple or a dictionary view in case of a dict, - of the keys in object *o*. On failure, return *NULL*. + On success, return a list of the keys in object *o*. On failure, return *NULL*. + This is equivalent to the Python expression ``list(o.keys())``. .. c:function:: PyObject* PyMapping_Values(PyObject *o) - On success, return a list, a tuple or a dictionary view in case of a dict, of - the values in object *o*. On failure, return *NULL*. + On success, return a list of the values in object *o*. On failure, return + *NULL*. This is equivalent to the Python expression ``list(o.values())``. .. c:function:: PyObject* PyMapping_Items(PyObject *o) - On success, return a list, a tuple or a dictionary view in case of a dict, of - the items in object *o*, where each item is a tuple containing a key-value - pair. On failure, return *NULL*. + On success, return a list of the items in object *o*, where each item is a tuple + containing a key-value pair. On failure, return *NULL*. This is equivalent to + the Python expression ``list(o.items())``. -.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) +.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, char *key) Return element of *o* corresponding to the object *key* or *NULL* on failure. This is the equivalent of the Python expression ``o[key]``. -.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) +.. c:function:: int PyMapping_SetItemString(PyObject *o, char *key, PyObject *v) Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[key] = v``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/marshal.rst --- a/Doc/c-api/marshal.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/marshal.rst Mon Jan 25 17:05:13 2016 +0100 @@ -53,8 +53,6 @@ for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:type:`long`. - On error, raise an exception and return ``-1``. - .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) @@ -62,15 +60,11 @@ for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:type:`short`. - On error, raise an exception and return ``-1``. - .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) Return a Python object from the data stream in a :c:type:`FILE\*` opened for - reading. - - On error, sets the appropriate exception (:exc:`EOFError` or + reading. On error, sets the appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. @@ -82,17 +76,14 @@ aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the file. Only use these variant if you are certain that you won't be reading - anything else from the file. + anything else from the file. On error, sets the appropriate exception + (:exc:`EOFError` or :exc:`TypeError`) and returns *NULL*. - On error, sets the appropriate exception (:exc:`EOFError` or - :exc:`TypeError`) and returns *NULL*. - -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len) +.. c:function:: PyObject* PyMarshal_ReadObjectFromString(char *string, Py_ssize_t len) Return a Python object from the data stream in a character buffer - containing *len* bytes pointed to by *string*. + containing *len* bytes pointed to by *string*. On error, sets the + appropriate exception (:exc:`EOFError` or :exc:`TypeError`) and returns + *NULL*. - On error, sets the appropriate exception (:exc:`EOFError` or - :exc:`TypeError`) and returns *NULL*. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/memory.rst --- a/Doc/c-api/memory.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/memory.rst Mon Jan 25 17:05:13 2016 +0100 @@ -61,7 +61,7 @@ if (buf == NULL) return PyErr_NoMemory(); ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); + res = PyString_FromString(buf); free(buf); /* malloc'ed */ return res; @@ -84,59 +84,6 @@ for the I/O buffer escapes completely the Python memory manager. -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The default raw memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the - allocated memory, or *NULL* if the request fails. Requesting zero bytes - returns a distinct non-*NULL* pointer if possible, as if - ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the - request fails. The memory is initialized to zeros. Requesting zero elements - or elements of size zero bytes returns a distinct non-*NULL* pointer if - possible, as if ``PyMem_RawCalloc(1, 1)`` had been called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*, - the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to - zero, the memory block is resized but is not freed, and the returned pointer - is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a - previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If - the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. - Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined - behavior occurs. If *p* is *NULL*, no operation is performed. - - .. _memoryinterface: Memory Interface @@ -144,41 +91,22 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. +memory from the Python heap: -The default memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the request fails. Requesting zero bytes returns - a distinct non-*NULL* pointer if possible, as if ``PyMem_Malloc(1)`` had + a distinct non-*NULL* pointer if possible, as if :c:func:`PyMem_Malloc(1)` had been called instead. The memory will not have been initialized in any way. -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the - request fails. The memory is initialized to zeros. Requesting zero elements - or elements of size zero bytes returns a distinct non-*NULL* pointer if - possible, as if ``PyMem_Calloc(1, 1)`` had been called instead. - - .. versionadded:: 3.5 - - .. c:function:: void* PyMem_Realloc(void *p, size_t n) Resizes the memory block pointed to by *p* to *n* bytes. The contents will be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*, the - call is equivalent to ``PyMem_Malloc(n)``; else if *n* is equal to zero, + call is equivalent to :c:func:`PyMem_Malloc(n)`; else if *n* is equal to zero, the memory block is resized but is not freed, and the returned pointer is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a previous call to :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. If the request fails, @@ -190,7 +118,7 @@ Frees the memory block pointed to by *p*, which must have been returned by a previous call to :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. Otherwise, or - if ``PyMem_Free(p)`` has been called before, undefined behavior occurs. If + if :c:func:`PyMem_Free(p)` has been called before, undefined behavior occurs. If *p* is *NULL*, no operation is performed. The following type-oriented macros are provided for convenience. Note that @@ -227,133 +155,6 @@ :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`. -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - four fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` - * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`, - :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free` - * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`, - :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free` - - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-NULL pointer when requesting - zero bytes. - - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup hooks to detect bugs in the following Python memory allocator - functions: - - - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`, - :c:func:`PyMem_RawFree` - - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free` - - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`, - :c:func:`PyObject_Free` - - Newly allocated memory is filled with the byte ``0xCB``, freed memory is - filled with the byte ``0xDB``. Additional checks: - - - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - detect write before the start of the buffer (buffer underflow) - - detect write after the end of the buffer (buffer overflow) - - The function does nothing if Python is not compiled is debug mode. - - -Customize PyObject Arena Allocator -================================== - -Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This -allocator is optimized for small objects with a short lifetime. It uses memory -mappings called "arenas" with a fixed size of 256 KB. It falls back to -:c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger -than 512 bytes. *pymalloc* is the default allocator used by -:c:func:`PyObject_Malloc`. - -The default arena allocator uses the following functions: - -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - .. _memoryexamples: Examples @@ -368,7 +169,7 @@ if (buf == NULL) return PyErr_NoMemory(); /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); + res = PyString_FromString(buf); PyMem_Free(buf); /* allocated with PyMem_Malloc */ return res; @@ -380,7 +181,7 @@ if (buf == NULL) return PyErr_NoMemory(); /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); + res = PyString_FromString(buf); PyMem_Del(buf); /* allocated with PyMem_New */ return res; diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/memoryview.rst --- a/Doc/c-api/memoryview.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/memoryview.rst Mon Jan 25 17:05:13 2016 +0100 @@ -35,7 +35,7 @@ .. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) - Create a memoryview object to a :term:`contiguous` chunk of memory (in either + Create a memoryview object to a contiguous chunk of memory (in either 'C' or 'F'ortran *order*) from an object that defines the buffer interface. If memory is contiguous, the memoryview object points to the original memory. Otherwise, a copy is made and the memoryview points to a diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/module.rst --- a/Doc/c-api/module.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/module.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,6 +7,8 @@ .. index:: object: module +There are only a few functions special to module objects. + .. c:var:: PyTypeObject PyModule_Type @@ -33,20 +35,13 @@ single: __name__ (module attribute) single: __doc__ (module attribute) single: __file__ (module attribute) - single: __package__ (module attribute) - single: __loader__ (module attribute) Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. + Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in; + the caller is responsible for providing a :attr:`__file__` attribute. .. versionadded:: 3.3 - .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. - .. c:function:: PyObject* PyModule_New(const char *name) @@ -82,18 +77,6 @@ Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or *NULL*. See - :c:member:`PyModuleDef.m_size`. - - -.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) - - Return a pointer to the :c:type:`PyModuleDef` struct from which the module was - created, or *NULL* if the module wasn't created from a definition. - .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -119,25 +102,50 @@ unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _initializing-modules: +.. c:function:: void* PyModule_GetState(PyObject *module) + + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or *NULL*. See + :c:member:`PyModuleDef.m_size`. + + +.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) + + Return a pointer to the :c:type:`PyModuleDef` struct from which the module was + created, or *NULL* if the module wasn't created with + :c:func:`PyModule_Create`. + Initializing C modules ^^^^^^^^^^^^^^^^^^^^^^ -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. +These functions are usually used in the module initialization function. -The initialization function can either pass pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. +.. c:function:: PyObject* PyModule_Create(PyModuleDef *module) + + Create a new module object, given the definition in *module*. This behaves + like :c:func:`PyModule_Create2` with *module_api_version* set to + :const:`PYTHON_API_VERSION`. + + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *module, int module_api_version) + + Create a new module object, given the definition in *module*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. + .. c:type:: PyModuleDef - The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. + This struct holds all information that is needed to create a module object. + There is usually only one static variable of that type for each module, which + is statically initialized and then passed to :c:func:`PyModule_Create` in the + module initialization function. .. c:member:: PyModuleDef_Base m_base @@ -154,41 +162,24 @@ .. c:member:: Py_ssize_t m_size - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. + If the module object needs additional memory, this should be set to the + number of bytes to allocate; a pointer to the block of memory can be + retrieved with :c:func:`PyModule_GetState`. If no memory is needed, set + this to ``-1``. - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`m_free` function has been called, if present. - - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. - - See :PEP:`3121` for more details. + This memory should be used, rather than static globals, to hold per-module + state, since it is then safe for use in multiple sub-interpreters. It is + freed when the module object is deallocated, after the :c:member:`m_free` + function has been called, if present. .. c:member:: PyMethodDef* m_methods A pointer to a table of module-level functions, described by :c:type:`PyMethodDef` values. Can be *NULL* if no functions are present. - .. c:member:: PyModuleDef_Slot* m_slots + .. c:member:: inquiry m_reload - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be *NULL*. - - .. versionchanged:: 3.5 - - Prior to version 3.5, this member was always set to *NULL*, - and was defined as: - - .. c:member:: inquiry m_reload + Currently unused, should be *NULL*. .. c:member:: traverseproc m_traverse @@ -205,204 +196,6 @@ A function to call during deallocation of the module object, or *NULL* if not needed. -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: - -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :const:`PYTHON_API_VERSION`. - - -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. - -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObject`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`__new__` and :py:meth:`__init__` methods -of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`__dict__` or individual classes created with :c:func:`PyType_FromSpec`). - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or *NULL* if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: - -.. c:type:: PyModuleDef_Slot - - .. c:member:: int slot - - A slot ID, chosen from the available values explained below. - - .. c:member:: void* value - - Value of the slot, whose meaning depends on the slot ID. - - .. versionadded:: 3.5 - -The *m_slots* array must be terminated by a slot with id 0. - -The available slot types are: - -.. c:var:: Py_mod_create - - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: - - .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error - and return *NULL*. - - This function should be kept minimal. In particular, it should not - call arbitrary Python code, as trying to import the same module again may - result in an infinite loop. - - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - - If ``Py_mod_create`` is not specified, the import machinery will create - a normal module object using :c:func:`PyModule_New`. The name is taken from - *spec*, not the definition, to allow extension modules to dynamically adjust - to their place in the module hierarchy and be imported under different - names through symlinks, all while sharing a single module definition. - - There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-*NULL* ``m_methods``, ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. - -.. c:var:: Py_mod_exec - - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. - The signature of the function is: - - .. c:function:: int exec_module(PyObject* module) - - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. - -See :PEP:`489` for more details on multi-phase initialization. - -Low-level module creation functions -................................... - -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. - -.. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - - Create a new module object, given the definition in *module* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :const:`PYTHON_API_VERSION`. - - .. versionadded:: 3.5 - -.. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) - - Create a new module object, given the definition in *module* and the - ModuleSpec *spec*, assuming the API version *module_api_version*. - If that version does not match the version of the running interpreter, - a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` - instead; only use this if you are sure you need it. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) - - Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) - - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) - - Add the functions from the *NULL* terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -Support functions -................. - -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) @@ -410,6 +203,7 @@ be used from the module's initialization function. This steals a reference to *value*. Return ``-1`` on error, ``0`` on success. + .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be @@ -421,7 +215,7 @@ Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - *NULL*-terminated. Return ``-1`` on error, ``0`` on success. + null-terminated. Return ``-1`` on error, ``0`` on success. .. c:function:: int PyModule_AddIntMacro(PyObject *module, macro) @@ -435,36 +229,3 @@ .. c:function:: int PyModule_AddStringMacro(PyObject *module, macro) Add a string constant to *module*. - - -Module lookup -^^^^^^^^^^^^^ - -Single-phase initialization creates singleton modules that can be looked up -in the context of the current interpreter. This allows the module object to be -retrieved later with only a reference to the module definition. - -These functions will not work on modules created using multi-phase initialization, -since multiple such modules can be created from a single definition. - -.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def) - - Returns the module object that was created from *def* for the current interpreter. - This method requires that the module object has been attached to the interpreter state with - :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not - found or has not been attached to the interpreter state yet, it returns *NULL*. - -.. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def) - - Attaches the module object passed to the function to the interpreter state. This allows - the module object to be accessible via :c:func:`PyState_FindModule`. - - Only effective on modules created using single-phase initialization. - - .. versionadded:: 3.3 - -.. c:function:: int PyState_RemoveModule(PyModuleDef *def) - - Removes the module object created from *def* from the interpreter state. - - .. versionadded:: 3.3 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/number.rst --- a/Doc/c-api/number.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/number.rst Mon Jan 25 17:05:13 2016 +0100 @@ -30,14 +30,6 @@ the equivalent of the Python expression ``o1 * o2``. -.. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on - failure. This is the equivalent of the Python expression ``o1 @ o2``. - - .. versionadded:: 3.5 - - .. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) Return the floor of *o1* divided by *o2*, or *NULL* on failure. This is @@ -154,15 +146,6 @@ the Python statement ``o1 *= o2``. -.. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or *NULL* on - failure. The operation is done *in-place* when *o1* supports it. This is - the equivalent of the Python statement ``o1 @= o2``. - - .. versionadded:: 3.5 - - .. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) Returns the mathematical floor of dividing *o1* by *o2*, or *NULL* on failure. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/object.rst --- a/Doc/c-api/object.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/object.rst Mon Jan 25 17:05:13 2016 +0100 @@ -60,43 +60,33 @@ Generic attribute getter function that is meant to be put into a type object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary of classes in the object's MRO as well as an attribute in the object's - :attr:`~object.__dict__` (if present). As outlined in :ref:`descriptors`, - data descriptors take preference over instance attributes, while non-data + :attr:`__dict__` (if present). As outlined in :ref:`descriptors`, data + descriptors take preference over instance attributes, while non-data descriptors don't. Otherwise, an :exc:`AttributeError` is raised. .. c:function:: int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v) Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement + *v*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is - deprecated in favour of using :c:func:`PyObject_DelAttr`. - .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement + *v*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o.attr_name = v``. - If *v* is *NULL*, the attribute is deleted, however this feature is - deprecated in favour of using :c:func:`PyObject_DelAttrString`. - .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) - Generic attribute setter and deleter function that is meant - to be put into a type object's :c:member:`~PyTypeObject.tp_setattro` - slot. It looks for a data descriptor in the + Generic attribute setter function that is meant to be put into a type + object's ``tp_setattro`` slot. It looks for a data descriptor in the dictionary of classes in the object's MRO, and if found it takes preference - over setting or deleting the attribute in the instance dictionary. Otherwise, the - attribute is set or deleted in the object's :attr:`~object.__dict__` (if present). - On success, ``0`` is returned, otherwise an :exc:`AttributeError` - is raised and ``-1`` is returned. + over setting the attribute in the instance dictionary. Otherwise, the + attribute is set in the object's :attr:`__dict__` (if present). Otherwise, + an :exc:`AttributeError` is raised and ``-1`` is returned. .. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) @@ -111,7 +101,7 @@ This is the equivalent of the Python statement ``del o.attr_name``. -.. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) +.. c:function:: PyObject* PyType_GenericGetDict(PyObject *o, void *context) A generic implementation for the getter of a ``__dict__`` descriptor. It creates the dictionary if necessary. @@ -119,7 +109,7 @@ .. versionadded:: 3.3 -.. c:function:: int PyObject_GenericSetDict(PyObject *o, void *context) +.. c:function:: int PyType_GenericSetDict(PyObject *o, void *context) A generic implementation for the setter of a ``__dict__`` descriptor. This implementation does not allow the dictionary to be deleted. @@ -159,9 +149,6 @@ representation on success, *NULL* on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. .. c:function:: PyObject* PyObject_ASCII(PyObject *o) @@ -173,20 +160,16 @@ a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. Called by the :func:`ascii` built-in function. - .. index:: string; PyObject_Str (C function) - .. c:function:: PyObject* PyObject_Str(PyObject *o) + .. index:: builtin: str + Compute a string representation of object *o*. Returns the string representation on success, *NULL* on failure. This is the equivalent of the Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - .. c:function:: PyObject* PyObject_Bytes(PyObject *o) .. index:: builtin: bytes @@ -197,45 +180,40 @@ a TypeError is raised when *o* is an integer instead of a zero-initialized bytes object. +.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) + + Returns ``1`` if *inst* is an instance of the class *cls* or a subclass of + *cls*, or ``0`` if not. On error, returns ``-1`` and sets an exception. If + *cls* is a type object rather than a class object, :c:func:`PyObject_IsInstance` + returns ``1`` if *inst* is of type *cls*. If *cls* is a tuple, the check will + be done against every entry in *cls*. The result will be ``1`` when at least one + of the checks returns ``1``, otherwise it will be ``0``. If *inst* is not a + class instance and *cls* is neither a type object, nor a class object, nor a + tuple, *inst* must have a :attr:`__class__` attribute --- the class relationship + of the value of that attribute with *cls* will be used to determine the result + of this function. + + +Subclass determination is done in a fairly straightforward way, but includes a +wrinkle that implementors of extensions to the class system may want to be aware +of. If :class:`A` and :class:`B` are class objects, :class:`B` is a subclass of +:class:`A` if it inherits from :class:`A` either directly or indirectly. If +either is not a class object, a more general mechanism is used to determine the +class relationship of the two objects. When testing if *B* is a subclass of +*A*, if *A* is *B*, :c:func:`PyObject_IsSubclass` returns true. If *A* and *B* +are different objects, *B*'s :attr:`__bases__` attribute is searched in a +depth-first fashion for *A* --- the presence of the :attr:`__bases__` attribute +is considered sufficient for this determination. + .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) - Return ``1`` if the class *derived* is identical to or derived from the class - *cls*, otherwise return ``0``. In case of an error, return ``-1``. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, - *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. - - Normally only class objects, i.e. instances of :class:`type` or a derived - class, are considered classes. However, objects can override this by having - a :attr:`__bases__` attribute (which must be a tuple of base classes). - - -.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) - - Return ``1`` if *inst* is an instance of the class *cls* or a subclass of - *cls*, or ``0`` if not. On error, returns ``-1`` and sets an exception. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, *inst* - is an instance of *cls* if its class is a subclass of *cls*. - - An instance *inst* can override what is considered its class by having a - :attr:`__class__` attribute. - - An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`__bases__` attribute (which must be a tuple - of base classes). + Returns ``1`` if the class *derived* is identical to or derived from the class + *cls*, otherwise returns ``0``. In case of an error, returns ``-1``. If *cls* + is a tuple, the check will be done against every entry in *cls*. The result will + be ``1`` when at least one of the checks returns ``1``, otherwise it will be + ``0``. If either *derived* or *cls* is not an actual class object (or tuple), + this function uses the generic algorithm described above. .. c:function:: int PyCallable_Check(PyObject *o) @@ -262,7 +240,7 @@ of the Python expression ``callable_object(*args)``. -.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) +.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, char *format, ...) Call a callable Python object *callable*, with a variable number of C arguments. The C arguments are described using a :c:func:`Py_BuildValue` style format @@ -272,11 +250,8 @@ pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. - .. versionchanged:: 3.4 - The type of *format* was changed from ``char *``. - -.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, const char *method, const char *format, ...) +.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, char *method, char *format, ...) Call the method named *method* of object *o* with a variable number of C arguments. The C arguments are described by a :c:func:`Py_BuildValue` format @@ -286,9 +261,6 @@ Note that if you only pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. - .. versionchanged:: 3.4 - The types of *method* and *format* were changed from ``char *``. - .. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) @@ -370,16 +342,6 @@ returned. This is the equivalent to the Python expression ``len(o)``. -.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t default) - - Return an estimated length for the object *o*. First try to return its - actual length, then an estimate using :meth:`~object.__length_hint__`, and - finally return the default value. On error return ``-1``. This is the - equivalent to the Python expression ``operator.length_hint(o, default)``. - - .. versionadded:: 3.4 - - .. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) Return element of *o* corresponding to the object *key* or *NULL* on failure. @@ -388,8 +350,7 @@ .. c:function:: int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v) - Map the object *key* to the value *v*. Raise an exception and - return ``-1`` on failure; return ``0`` on success. This is the + Map the object *key* to the value *v*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[key] = v``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/sequence.rst --- a/Doc/c-api/sequence.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/sequence.rst Mon Jan 25 17:05:13 2016 +0100 @@ -62,14 +62,10 @@ .. c:function:: int PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v) - Assign object *v* to the *i*\ th element of *o*. Raise an exception - and return ``-1`` on failure; return ``0`` on success. This + Assign object *v* to the *i*\ th element of *o*. Returns ``-1`` on failure. This is the equivalent of the Python statement ``o[i] = v``. This function *does not* steal a reference to *v*. - If *v* is *NULL*, the element is deleted, however this feature is - deprecated in favour of using :c:func:`PySequence_DelItem`. - .. c:function:: int PySequence_DelItem(PyObject *o, Py_ssize_t i) @@ -111,9 +107,8 @@ .. c:function:: PyObject* PySequence_List(PyObject *o) - Return a list object with the same contents as the sequence or iterable *o*, - or *NULL* on failure. The returned list is guaranteed to be new. This is - equivalent to the Python expression ``list(o)``. + Return a list object with the same contents as the arbitrary sequence *o*. The + returned list is guaranteed to be new. .. c:function:: PyObject* PySequence_Tuple(PyObject *o) @@ -128,10 +123,10 @@ .. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - Return the sequence *o* as a list, unless it is already a tuple or list, in - which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access - the members of the result. Returns *NULL* on failure. If the object is not - a sequence, raises :exc:`TypeError` with *m* as the message text. + Returns the sequence *o* as a tuple, unless it is already a tuple or list, in + which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access the + members of the result. Returns *NULL* on failure. If the object is not a + sequence, raises :exc:`TypeError` with *m* as the message text. .. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/set.rst --- a/Doc/c-api/set.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/set.rst Mon Jan 25 17:05:13 2016 +0100 @@ -140,7 +140,7 @@ Return 1 if found and removed, 0 if not found (no action taken), and -1 if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is an not an instance of :class:`set` or its subtype. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/stable.rst --- a/Doc/c-api/stable.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,38 +0,0 @@ -.. highlightlang:: c - -.. _stable: - -*********************************** -Stable Application Binary Interface -*********************************** - -Traditionally, the C API of Python will change with every release. Most changes -will be source-compatible, typically by only adding API, rather than changing -existing API or removing API (although some interfaces do get removed after -being deprecated first). - -Unfortunately, the API compatibility does not extend to binary compatibility -(the ABI). The reason is primarily the evolution of struct definitions, where -addition of a new field, or changing the type of a field, might not break the -API, but can break the ABI. As a consequence, extension modules need to be -recompiled for every Python release (although an exception is possible on Unix -when none of the affected interfaces are used). In addition, on Windows, -extension modules link with a specific pythonXY.dll and need to be recompiled to -link with a newer one. - -Since Python 3.2, a subset of the API has been declared to guarantee a stable -ABI. Extension modules wishing to use this API (called "limited API") need to -define ``Py_LIMITED_API``. A number of interpreter details then become hidden -from the extension module; in return, a module is built that works on any 3.x -version (x>=2) without recompilation. - -In some cases, the stable ABI needs to be extended with new functions. -Extension modules wishing to use these new APIs need to set ``Py_LIMITED_API`` -to the ``PY_VERSION_HEX`` value (see :ref:`apiabiversion`) of the minimum Python -version they want to support (e.g. ``0x03030000`` for Python 3.3). Such modules -will work on all subsequent Python releases, but fail to load (because of -missing symbols) on the older releases. - -As of Python 3.2, the set of functions available to the limited API is -documented in PEP 384. In the C API documentation, API elements that are not -part of the limited API are marked as "Not part of the limited API." diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/structures.rst --- a/Doc/c-api/structures.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/structures.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,66 +21,53 @@ All object types are extensions of this type. This is a type which contains the information Python needs to treat a pointer to an object as an object. In a normal "release" build, it contains only the object's - reference count and a pointer to the corresponding type object. - Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:type:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + reference count and a pointer to the corresponding type object. It + corresponds to the fields defined by the expansion of the ``PyObject_HEAD`` + macro. .. c:type:: PyVarObject This is an extension of :c:type:`PyObject` that adds the :attr:`ob_size` field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + This type does not often appear in the Python/C API. It corresponds to the + fields defined by the expansion of the ``PyObject_VAR_HEAD`` macro. +These macros are used in the definition of :c:type:`PyObject` and +:c:type:`PyVarObject`: + +.. XXX need to document PEP 3123 changes here .. c:macro:: PyObject_HEAD - This is a macro used when declaring new types which represent objects - without a varying length. The PyObject_HEAD macro expands to:: + This is a macro which expands to the declarations of the fields of the + :c:type:`PyObject` type; it is used when declaring new types which represent + objects without a varying length. The specific fields it expands to depend + on the definition of :c:macro:`Py_TRACE_REFS`. By default, that macro is + not defined, and :c:macro:`PyObject_HEAD` expands to:: - PyObject ob_base; + Py_ssize_t ob_refcnt; + PyTypeObject *ob_type; - See documentation of :c:type:`PyObject` above. + When :c:macro:`Py_TRACE_REFS` is defined, it expands to:: + + PyObject *_ob_next, *_ob_prev; + Py_ssize_t ob_refcnt; + PyTypeObject *ob_type; .. c:macro:: PyObject_VAR_HEAD - This is a macro used when declaring new types which represent objects - with a length that varies from instance to instance. - The PyObject_VAR_HEAD macro expands to:: + This is a macro which expands to the declarations of the fields of the + :c:type:`PyVarObject` type; it is used when declaring new types which + represent objects with a length that varies from instance to instance. + This macro always expands to:: - PyVarObject ob_base; + PyObject_HEAD + Py_ssize_t ob_size; - See documentation of :c:type:`PyVarObject` above. - - -.. c:macro:: Py_TYPE(o) - - This macro is used to access the :attr:`ob_type` member of a Python object. - It expands to:: - - (((PyObject*)(o))->ob_type) - - -.. c:macro:: Py_REFCNT(o) - - This macro is used to access the :attr:`ob_refcnt` member of a Python - object. - It expands to:: - - (((PyObject*)(o))->ob_refcnt) - - -.. c:macro:: Py_SIZE(o) - - This macro is used to access the :attr:`ob_size` member of a Python object. - It expands to:: - - (((PyVarObject*)(o))->ob_size) + Note that :c:macro:`PyObject_HEAD` is part of the expansion, and that its own + expansion varies depending on the definition of :c:macro:`Py_TRACE_REFS`. .. c:macro:: PyObject_HEAD_INIT(type) @@ -144,7 +131,7 @@ types, but they always return :c:type:`PyObject\*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as -:c:type:`PyObject\*`, it is common that the method implementation uses the +:c:type:`PyObject\*`, it is common that the method implementation uses a the specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/sys.rst --- a/Doc/c-api/sys.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/sys.rst Mon Jan 25 17:05:13 2016 +0100 @@ -47,60 +47,6 @@ not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:type:`void (\*)(int)`. -.. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) - - Decode a byte string from the locale encoding with the :ref:`surrogateescape - error handler `: undecodable bytes are decoded as - characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a - surrogate character, escape the bytes using the surrogateescape error - handler instead of decoding them. - - Return a pointer to a newly allocated wide character string, use - :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write - the number of wide characters excluding the null character into ``*size`` - - Return ``NULL`` on decoding error or memory allocation error. If *size* is - not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to - ``(size_t)-2`` on decoding error. - - Decoding errors should never happen, unless there is a bug in the C - library. - - Use the :c:func:`Py_EncodeLocale` function to encode the character string - back to a byte string. - - .. seealso:: - - The :c:func:`PyUnicode_DecodeFSDefaultAndSize` and - :c:func:`PyUnicode_DecodeLocaleAndSize` functions. - - .. versionadded:: 3.5 - - -.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - - Encode a wide character string to the locale encoding with the - :ref:`surrogateescape error handler `: surrogate characters - in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF. - - Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` - to free the memory. Return ``NULL`` on encoding error or memory allocation - error - - If error_pos is not ``NULL``, ``*error_pos`` is set to the index of the - invalid character on encoding error, or set to ``(size_t)-1`` otherwise. - - Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back - to a wide character string. - - .. seealso:: - - The :c:func:`PyUnicode_EncodeFSDefault` and - :c:func:`PyUnicode_EncodeLocale` functions. - - .. versionadded:: 3.5 - - .. _systemfunctions: System Functions @@ -110,12 +56,18 @@ accessible to C code. They all work with the current interpreter thread's :mod:`sys` module's dict, which is contained in the internal thread state structure. -.. c:function:: PyObject *PySys_GetObject(const char *name) +.. c:function:: PyObject *PySys_GetObject(char *name) Return the object *name* from the :mod:`sys` module or *NULL* if it does not exist, without setting an exception. -.. c:function:: int PySys_SetObject(const char *name, PyObject *v) +.. c:function:: FILE *PySys_GetFile(char *name, FILE *def) + + Return the :c:type:`FILE*` associated with the object *name* in the + :mod:`sys` module, or *def* if *name* is not in the module or is not associated + with a :c:type:`FILE*`. + +.. c:function:: int PySys_SetObject(char *name, PyObject *v) Set *name* in the :mod:`sys` module to *v* unless *v* is *NULL*, in which case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` @@ -212,24 +164,20 @@ .. c:function:: void Py_Exit(int status) .. index:: - single: Py_FinalizeEx() + single: Py_Finalize() single: exit() - Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the - standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` - indicates an error, the exit status is set to 120. - - .. versionchanged:: 3.6 - Errors from finalization no longer ignored. + Exit the current process. This calls :c:func:`Py_Finalize` and then calls the + standard C library function ``exit(status)``. .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_FinalizeEx() + single: Py_Finalize() single: cleanup functions - Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup + Register a cleanup function to be called by :c:func:`Py_Finalize`. The cleanup function will be called with no arguments and should return no value. At most 32 cleanup functions can be registered. When the registration is successful, :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/tuple.rst --- a/Doc/c-api/tuple.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/tuple.rst Mon Jan 25 17:05:13 2016 +0100 @@ -108,111 +108,3 @@ .. c:function:: int PyTuple_ClearFreeList() Clear the free list. Return the total number of freed items. - - -Struct Sequence Objects ------------------------ - -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. -To create a struct sequence, you first have to create a specific struct sequence -type. - -.. c:function:: PyTypeObject* PyStructSequence_NewType(PyStructSequence_Desc *desc) - - Create a new struct sequence type from the data in *desc*, described below. Instances - of the resulting type can be created with :c:func:`PyStructSequence_New`. - - -.. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) - - Initializes a struct sequence type *type* from *desc* in place. - - -.. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. - - .. versionadded:: 3.4 - - -.. c:type:: PyStructSequence_Desc - - Contains the meta information of a struct sequence type to create. - - +-------------------+------------------------------+------------------------------------+ - | Field | C Type | Meaning | - +===================+==============================+====================================+ - | ``name`` | ``char *`` | name of the struct sequence type | - +-------------------+------------------------------+------------------------------------+ - | ``doc`` | ``char *`` | pointer to docstring for the type | - | | | or NULL to omit | - +-------------------+------------------------------+------------------------------------+ - | ``fields`` | ``PyStructSequence_Field *`` | pointer to *NULL*-terminated array | - | | | with field names of the new type | - +-------------------+------------------------------+------------------------------------+ - | ``n_in_sequence`` | ``int`` | number of fields visible to the | - | | | Python side (if used as tuple) | - +-------------------+------------------------------+------------------------------------+ - - -.. c:type:: PyStructSequence_Field - - Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:type:`PyObject\*`. The index in the - :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which - field of the struct sequence is described. - - +-----------+---------------+--------------------------------------+ - | Field | C Type | Meaning | - +===========+===============+======================================+ - | ``name`` | ``char *`` | name for the field or *NULL* to end | - | | | the list of named fields, set to | - | | | PyStructSequence_UnnamedField to | - | | | leave unnamed | - +-----------+---------------+--------------------------------------+ - | ``doc`` | ``char *`` | field docstring or *NULL* to omit | - +-----------+---------------+--------------------------------------+ - - -.. c:var:: char* PyStructSequence_UnnamedField - - Special value for a field name to leave it unnamed. - - -.. c:function:: PyObject* PyStructSequence_New(PyTypeObject *type) - - Creates an instance of *type*, which must have been created with - :c:func:`PyStructSequence_NewType`. - - -.. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) - - Return the object at position *pos* in the struct sequence pointed to by *p*. - No bounds checking is performed. - - -.. c:function:: PyObject* PyStructSequence_GET_ITEM(PyObject *p, Py_ssize_t pos) - - Macro equivalent of :c:func:`PyStructSequence_GetItem`. - - -.. c:function:: void PyStructSequence_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) - - Sets the field at index *pos* of the struct sequence *p* to value *o*. Like - :c:func:`PyTuple_SET_ITEM`, this should only be used to fill in brand new - instances. - - .. note:: - - This function "steals" a reference to *o*. - - -.. c:function:: PyObject* PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) - - Macro equivalent of :c:func:`PyStructSequence_SetItem`. - - .. note:: - - This function "steals" a reference to *o*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/type.rst --- a/Doc/c-api/type.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/type.rst Mon Jan 25 17:05:13 2016 +0100 @@ -37,14 +37,13 @@ .. c:function:: long PyType_GetFlags(PyTypeObject* type) - Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily + Return the :attr:`tp_flags` member of *type*. This function is primarily meant for use with `Py_LIMITED_API`; the individual flag bits are guaranteed to be stable across Python releases, but access to - :c:member:`~PyTypeObject.tp_flags` itself is not part of the limited API. + :attr:`tp_flags` itself is not part of the limited API. .. versionadded:: 3.2 - .. c:function:: void PyType_Modified(PyTypeObject *type) Invalidate the internal lookup cache for the type and all of its @@ -52,13 +51,13 @@ modification of the attributes or base classes of the type. -.. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) +.. c:function:: int PyType_HasFeature(PyObject *o, int feature) Return true if the type object *o* sets the feature *feature*. Type features are denoted by single bit flags. -.. c:function:: int PyType_IS_GC(PyTypeObject *o) +.. c:function:: int PyType_IS_GC(PyObject *o) Return true if the type object includes support for the cycle detector; this tests the type flag :const:`Py_TPFLAGS_HAVE_GC`. @@ -68,22 +67,16 @@ Return true if *a* is a subtype of *b*. - This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call - :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` - would do. - .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to *NULL*. + XXX: Document. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :attr:`tp_new` slot of a type object. Initialize + all instance variables to *NULL*. .. c:function:: int PyType_Ready(PyTypeObject *type) @@ -91,25 +84,3 @@ their initialization. This function is responsible for adding inherited slots from a type's base class. Return ``0`` on success, or return ``-1`` and sets an exception on error. - -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - - Creates and returns a heap type object from the *spec* passed to the function. - -.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - - Creates and returns a heap type object from the *spec*. In addition to that, - the created heap type contains all types contained by the *bases* tuple as base - types. This allows the caller to reference other heap types as base types. - - .. versionadded:: 3.3 - -.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) - - Return the function pointer stored in the given slot. If the - result is *NULL*, this indicates that either the slot is *NULL*, - or that the function was called with invalid parameters. - Callers will typically cast the result pointer into the appropriate - function type. - - .. versionadded:: 3.4 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/typeobj.rst --- a/Doc/c-api/typeobj.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/typeobj.rst Mon Jan 25 17:05:13 2016 +0100 @@ -35,7 +35,7 @@ The type object structure extends the :c:type:`PyVarObject` structure. The :attr:`ob_size` field is used for dynamic types (created by :func:`type_new`, usually called from a class statement). Note that :c:data:`PyType_Type` (the -metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. +metatype) initializes :attr:`tp_itemsize`, which means that its instances (i.e. type objects) *must* have the :attr:`ob_size` field. @@ -94,7 +94,7 @@ This field is not inherited by subtypes. -.. c:member:: const char* PyTypeObject.tp_name +.. c:member:: char* PyTypeObject.tp_name Pointer to a NUL-terminated string containing the name of the type. For types that are accessible as module globals, the string should be the full module @@ -102,7 +102,7 @@ should be just the type name. If the module is a submodule of a package, the full package name is part of the full module name. For example, a type named :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` - should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. + should have the :attr:`tp_name` initializer ``"P.Q.M.T"``. For dynamically allocated type objects, this should just be the type name, and the module name explicitly stored in the type dict as the value for key @@ -113,7 +113,7 @@ attribute, and everything after the last dot is made accessible as the :attr:`__name__` attribute. - If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the + If no dot is present, the entire :attr:`tp_name` field is made accessible as the :attr:`__name__` attribute, and the :attr:`__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. @@ -127,13 +127,13 @@ These fields allow calculating the size in bytes of instances of the type. There are two kinds of types: types with fixed-length instances have a zero - :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero - :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. + :attr:`tp_itemsize` field, types with variable-length instances have a non-zero + :attr:`tp_itemsize` field. For a type with fixed-length instances, all + instances have the same size, given in :attr:`tp_basicsize`. For a type with variable-length instances, the instances must have an - :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N - times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of + :attr:`ob_size` field, and the instance size is :attr:`tp_basicsize` plus N + times :attr:`tp_itemsize`, where N is the "length" of the object. The value of N is typically stored in the instance's :attr:`ob_size` field. There are exceptions: for example, ints use a negative :attr:`ob_size` to indicate a negative number, and N is ``abs(ob_size)`` there. Also, the presence of an @@ -146,20 +146,20 @@ :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to declare the instance struct) and this in turn includes the :attr:`_ob_prev` and :attr:`_ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the + way to get an initializer for the :attr:`tp_basicsize` is to use the ``sizeof`` operator on the struct used to declare the instance layout. The basic size does not include the GC header size. These fields are inherited separately by subtypes. If the base type has a - non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set - :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this + non-zero :attr:`tp_itemsize`, it is generally not safe to set + :attr:`tp_itemsize` to a different non-zero value in a subtype (though this depends on the implementation of the base type). A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: - suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is + this should be taken care of by the value of :attr:`tp_basicsize`. Example: + suppose a type implements an array of ``double``. :attr:`tp_itemsize` is ``sizeof(double)``. It is the programmer's responsibility that - :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the + :attr:`tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the alignment requirement for ``double``). @@ -175,10 +175,10 @@ destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not + last action) call the type's :attr:`tp_free` function. If the type is not subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the + :attr:`tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or :c:func:`PyObject_GC_Del` if the instance was allocated using @@ -189,7 +189,31 @@ .. c:member:: printfunc PyTypeObject.tp_print - Reserved slot, formerly used for print formatting in Python 2.x. + An optional pointer to the instance print function. + + The print function is only called when the instance is printed to a *real* file; + when it is printed to a pseudo-file (like a :class:`StringIO` instance), the + instance's :attr:`tp_repr` or :attr:`tp_str` function is called to convert it to + a string. These are also called when the type's :attr:`tp_print` field is + *NULL*. A type should never implement :attr:`tp_print` in a way that produces + different output than :attr:`tp_repr` or :attr:`tp_str` would. + + The print function is called with the same signature as :c:func:`PyObject_Print`: + ``int tp_print(PyObject *self, FILE *file, int flags)``. The *self* argument is + the instance to be printed. The *file* argument is the stdio file to which it + is to be printed. The *flags* argument is composed of flag bits. The only flag + bit currently defined is :const:`Py_PRINT_RAW`. When the :const:`Py_PRINT_RAW` + flag bit is set, the instance should be printed the same way as :attr:`tp_str` + would format it; when the :const:`Py_PRINT_RAW` flag bit is clear, the instance + should be printed the same was as :attr:`tp_repr` would format it. It should + return ``-1`` and set an exception condition when an error occurred during the + comparison. + + It is possible that the :attr:`tp_print` field will be deprecated. In any case, + it is recommended not to define :attr:`tp_print`, but instead to rely on + :attr:`tp_repr` and :attr:`tp_str` for printing. + + This field is inherited by subtypes. .. c:member:: getattrfunc PyTypeObject.tp_getattr @@ -197,38 +221,32 @@ An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_getattro` function, but taking a C string + that acts the same as the :attr:`tp_getattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is the same as for :c:func:`PyObject_GetAttrString`. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :attr:`tp_getattro`: a subtype + inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when + the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. .. c:member:: setattrfunc PyTypeObject.tp_setattr - An optional pointer to the function for setting and deleting attributes. + An optional pointer to the set-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_setattro` function, but taking a C string + that acts the same as the :attr:`tp_setattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is - the same as for :c:func:`PyObject_SetAttrString`, but setting - *v* to *NULL* to delete an attribute must be supported. + the same as for :c:func:`PyObject_SetAttrString`. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :attr:`tp_setattro`: a subtype + inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when + the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. -.. c:member:: PyAsyncMethods* tp_as_async +.. c:member:: void* PyTypeObject.tp_reserved - Pointer to an additional structure that contains fields relevant only to - objects which implement :term:`awaitable` and :term:`asynchronous iterator` - protocols at the C-level. See :ref:`async-structs` for details. - - .. versionadded:: 3.5 - Formerly known as ``tp_compare`` and ``tp_reserved``. + Reserved slot, formerly known as tp_compare. .. c:member:: reprfunc PyTypeObject.tp_repr @@ -257,7 +275,7 @@ objects which implement the number protocol. These fields are documented in :ref:`number-structs`. - The :c:member:`~PyTypeObject.tp_as_number` field is not inherited, but the contained fields are + The :attr:`tp_as_number` field is not inherited, but the contained fields are inherited individually. @@ -267,7 +285,7 @@ objects which implement the sequence protocol. These fields are documented in :ref:`sequence-structs`. - The :c:member:`~PyTypeObject.tp_as_sequence` field is not inherited, but the contained fields + The :attr:`tp_as_sequence` field is not inherited, but the contained fields are inherited individually. @@ -277,7 +295,7 @@ objects which implement the mapping protocol. These fields are documented in :ref:`mapping-structs`. - The :c:member:`~PyTypeObject.tp_as_mapping` field is not inherited, but the contained fields + The :attr:`tp_as_mapping` field is not inherited, but the contained fields are inherited individually. @@ -305,9 +323,9 @@ object raises :exc:`TypeError`. This field is inherited by subtypes together with - :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both *NULL*. + :attr:`tp_richcompare`: a subtype inherits both of + :attr:`tp_richcompare` and :attr:`tp_hash`, when the subtype's + :attr:`tp_richcompare` and :attr:`tp_hash` are both *NULL*. .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -345,23 +363,22 @@ convenient to set this field to :c:func:`PyObject_GenericGetAttr`, which implements the normal way of looking for object attributes. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :attr:`tp_getattr`: a subtype + inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when + the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. .. c:member:: setattrofunc PyTypeObject.tp_setattro - An optional pointer to the function for setting and deleting attributes. + An optional pointer to the set-attribute function. - The signature is the same as for :c:func:`PyObject_SetAttr`, but setting - *v* to *NULL* to delete an attribute must be supported. It is usually + The signature is the same as for :c:func:`PyObject_SetAttr`. It is usually convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which implements the normal way of setting object attributes. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :attr:`tp_setattr`: a subtype + inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when + the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. .. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer @@ -370,17 +387,17 @@ which implement the buffer interface. These fields are documented in :ref:`buffer-structs`. - The :c:member:`~PyTypeObject.tp_as_buffer` field is not inherited, but the contained fields are + The :attr:`tp_as_buffer` field is not inherited, but the contained fields are inherited individually. -.. c:member:: unsigned long PyTypeObject.tp_flags +.. c:member:: long PyTypeObject.tp_flags This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via - :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and - :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if + :attr:`tp_as_number`, :attr:`tp_as_sequence`, :attr:`tp_as_mapping`, and + :attr:`tp_as_buffer`) that were historically not always present are valid; if such a flag bit is clear, the type fields it guards must not be accessed and must be considered to have a zero or *NULL* value instead. @@ -390,13 +407,13 @@ inherited if the extension structure is inherited, i.e. the base type's value of the flag bit is copied into the subtype together with a pointer to the extension structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with - the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the + the :attr:`tp_traverse` and :attr:`tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the - :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have + :attr:`tp_traverse` and :attr:`tp_clear` fields in the subtype exist and have *NULL* values. The following bit masks are currently defined; these can be ORed together using - the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro + the ``|`` operator to form the value of the :attr:`tp_flags` field. The macro :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and checks whether ``tp->tp_flags & f`` is non-zero. @@ -436,7 +453,7 @@ is set, instances must be created using :c:func:`PyObject_GC_New` and destroyed using :c:func:`PyObject_GC_Del`. More information in section :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in + GC-related fields :attr:`tp_traverse` and :attr:`tp_clear` are present in the type object. @@ -448,33 +465,7 @@ :const:`Py_TPFLAGS_HAVE_VERSION_TAG`. - .. data:: Py_TPFLAGS_LONG_SUBCLASS - .. data:: Py_TPFLAGS_LIST_SUBCLASS - .. data:: Py_TPFLAGS_TUPLE_SUBCLASS - .. data:: Py_TPFLAGS_BYTES_SUBCLASS - .. data:: Py_TPFLAGS_UNICODE_SUBCLASS - .. data:: Py_TPFLAGS_DICT_SUBCLASS - .. data:: Py_TPFLAGS_BASE_EXC_SUBCLASS - .. data:: Py_TPFLAGS_TYPE_SUBCLASS - - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass - of a built-in type; such specific checks are faster than a generic - check, like :c:func:`PyObject_IsInstance`. Custom types that inherit - from built-ins should have their :c:member:`~PyTypeObject.tp_flags` - set appropriately, or the code that interacts with such types - will behave differently depending on what kind of check is used. - - - .. data:: Py_TPFLAGS_HAVE_FINALIZE - - This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the - type structure. - - .. versionadded:: 3.4 - - -.. c:member:: const char* PyTypeObject.tp_doc +.. c:member:: char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`__doc__` attribute on the type and @@ -490,8 +481,8 @@ about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function + The :attr:`tp_traverse` pointer is used by the garbage collector to detect + reference cycles. A typical implementation of a :attr:`tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python objects. For example, this is function :c:func:`local_traverse` from the :mod:`_thread` extension module:: @@ -511,15 +502,15 @@ On the other hand, even if you know a member can never be part of a cycle, as a debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. + :func:`get_referents` function will include it. Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just anything. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :attr:`tp_clear` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and + :attr:`tp_clear` are all inherited from the base type if they are all zero in the subtype. @@ -528,17 +519,17 @@ An optional pointer to a clear function for the garbage collector. This is only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` + The :attr:`tp_clear` member function is used to break reference cycles in cyclic + garbage detected by the garbage collector. Taken together, all :attr:`tp_clear` functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's + subtle, and if in any doubt supply a :attr:`tp_clear` function. For example, + the tuple type does not implement a :attr:`tp_clear` function, because it's possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to + Therefore the :attr:`tp_clear` functions of other types must be sufficient to break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + rarely a good reason to avoid implementing :attr:`tp_clear`. - Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to + Implementations of :attr:`tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those members to *NULL*, as in the following example:: @@ -563,27 +554,25 @@ so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, + Because the goal of :attr:`tp_clear` functions is to break reference cycles, it's not necessary to clear contained objects like Python strings or Python integers, which can't participate in reference cycles. On the other hand, it may be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. + :attr:`tp_dealloc` function to invoke :attr:`tp_clear`. More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :attr:`tp_traverse` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and + :attr:`tp_clear` are all inherited from the base type if they are all zero in the subtype. .. c:member:: richcmpfunc PyTypeObject.tp_richcompare An optional pointer to the rich comparison function, whose signature is - ``PyObject *tp_richcompare(PyObject *a, PyObject *b, int op)``. The first - parameter is guaranteed to be an instance of the type that is defined - by :c:type:`PyTypeObject`. + ``PyObject *tp_richcompare(PyObject *a, PyObject *b, int op)``. The function should return the result of the comparison (usually ``Py_True`` or ``Py_False``). If the comparison is undefined, it must return @@ -596,13 +585,13 @@ comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and friends), directly raise :exc:`TypeError` in the rich comparison function. - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: - a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when - the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both + This field is inherited by subtypes together with :attr:`tp_hash`: + a subtype inherits :attr:`tp_richcompare` and :attr:`tp_hash` when + the subtype's :attr:`tp_richcompare` and :attr:`tp_hash` are both *NULL*. The following constants are defined to be used as the third argument for - :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: + :attr:`tp_richcompare` and for :c:func:`PyObject_RichCompare`: +----------------+------------+ | Constant | Comparison | @@ -621,7 +610,7 @@ +----------------+------------+ -.. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset +.. c:member:: long PyTypeObject.tp_weaklistoffset If the instances of this type are weakly referenceable, this field is greater than zero and contains the offset in the instance structure of the weak @@ -630,26 +619,26 @@ instance structure needs to include a field of type :c:type:`PyObject\*` which is initialized to *NULL*. - Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for + Do not confuse this field with :attr:`tp_weaklist`; that is the list head for weak references to the type object itself. This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype uses a different weak reference list head than the base type. Since the list head is always found via - :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. + :attr:`tp_weaklistoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, + When a type defined by a class statement has no :attr:`__slots__` declaration, and none of its base types are weakly referenceable, the type is made weakly referenceable by adding a weak reference list head slot to the instance layout - and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. + and setting the :attr:`tp_weaklistoffset` of that slot's offset. When a type's :attr:`__slots__` declaration contains a slot named :attr:`__weakref__`, that slot becomes the weak reference list head for instances of the type, and the slot's offset is stored in the type's - :c:member:`~PyTypeObject.tp_weaklistoffset`. + :attr:`tp_weaklistoffset`. When a type's :attr:`__slots__` declaration does not contain a slot named - :attr:`__weakref__`, the type inherits its :c:member:`~PyTypeObject.tp_weaklistoffset` from its + :attr:`__weakref__`, the type inherits its :attr:`tp_weaklistoffset` from its base type. .. c:member:: getiterfunc PyTypeObject.tp_iter @@ -671,7 +660,7 @@ *NULL* too. Its presence signals that the instances of this type are iterators. - Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that + Iterator types should also define the :attr:`tp_iter` function, and that function should return the iterator instance itself (not a new iterator instance). @@ -686,7 +675,7 @@ structures, declaring regular methods of this type. For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a method descriptor. + :attr:`tp_dict` below) containing a method descriptor. This field is not inherited by subtypes (methods are inherited through a different mechanism). @@ -699,7 +688,7 @@ this type. For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a member descriptor. + :attr:`tp_dict` below) containing a member descriptor. This field is not inherited by subtypes (members are inherited through a different mechanism). @@ -711,7 +700,7 @@ structures, declaring computed attributes of instances of this type. For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a getset descriptor. + :attr:`tp_dict` below) containing a getset descriptor. This field is not inherited by subtypes (computed attributes are inherited through a different mechanism). @@ -726,7 +715,7 @@ typedef struct PyGetSetDef { char *name; /* attribute name */ getter get; /* C function to get the attribute */ - setter set; /* C function to set or delete the attribute */ + setter set; /* C function to set the attribute */ char *doc; /* optional doc string */ void *closure; /* optional additional data for getter and setter */ } PyGetSetDef; @@ -759,7 +748,7 @@ .. warning:: It is not safe to use :c:func:`PyDict_SetItem` on or otherwise modify - :c:member:`~PyTypeObject.tp_dict` with the dictionary C-API. + :attr:`tp_dict` with the dictionary C-API. .. c:member:: descrgetfunc PyTypeObject.tp_descr_get @@ -777,27 +766,25 @@ .. c:member:: descrsetfunc PyTypeObject.tp_descr_set - An optional pointer to a function for setting and deleting - a descriptor's value. + An optional pointer to a "descriptor set" function. The function signature is :: int tp_descr_set(PyObject *self, PyObject *obj, PyObject *value); - The *value* argument is set to *NULL* to delete the value. This field is inherited by subtypes. .. XXX explain. -.. c:member:: Py_ssize_t PyTypeObject.tp_dictoffset +.. c:member:: long PyTypeObject.tp_dictoffset If the instances of this type have a dictionary containing instance variables, this field is non-zero and contains the offset in the instances of the type of the instance variable dictionary; this offset is used by :c:func:`PyObject_GenericGetAttr`. - Do not confuse this field with :c:member:`~PyTypeObject.tp_dict`; that is the dictionary for + Do not confuse this field with :attr:`tp_dict`; that is the dictionary for attributes of the type object itself. If the value of this field is greater than zero, it specifies the offset from @@ -806,20 +793,20 @@ offset is more expensive to use, and should only be used when the instance structure contains a variable-length part. This is used for example to add an instance variable dictionary to subtypes of :class:`str` or :class:`tuple`. Note - that the :c:member:`~PyTypeObject.tp_basicsize` field should account for the dictionary added to + that the :attr:`tp_basicsize` field should account for the dictionary added to the end in that case, even though the dictionary is not included in the basic object layout. On a system with a pointer size of 4 bytes, - :c:member:`~PyTypeObject.tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is + :attr:`tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is at the very end of the structure. The real dictionary offset in an instance can be computed from a negative - :c:member:`~PyTypeObject.tp_dictoffset` as follows:: + :attr:`tp_dictoffset` as follows:: dictoffset = tp_basicsize + abs(ob_size)*tp_itemsize + tp_dictoffset if dictoffset is not aligned on sizeof(void*): round up to sizeof(void*) - where :c:member:`~PyTypeObject.tp_basicsize`, :c:member:`~PyTypeObject.tp_itemsize` and :c:member:`~PyTypeObject.tp_dictoffset` are + where :attr:`tp_basicsize`, :attr:`tp_itemsize` and :attr:`tp_dictoffset` are taken from the type object, and :attr:`ob_size` is taken from the instance. The absolute value is taken because ints use the sign of :attr:`ob_size` to store the sign of the number. (There's never a need to do this calculation @@ -828,17 +815,17 @@ This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype instances store the dictionary at a difference offset than the base type. Since the dictionary is - always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. + always found via :attr:`tp_dictoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, + When a type defined by a class statement has no :attr:`__slots__` declaration, and none of its base types has an instance variable dictionary, a dictionary - slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to + slot is added to the instance layout and the :attr:`tp_dictoffset` is set to that slot's offset. When a type defined by a class statement has a :attr:`__slots__` declaration, - the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. + the type inherits its :attr:`tp_dictoffset` from its base type. - (Adding a slot named :attr:`~object.__dict__` to the :attr:`__slots__` declaration does + (Adding a slot named :attr:`__dict__` to the :attr:`__slots__` declaration does not have the expected effect, it just causes confusion. Maybe this should be added as a feature just like :attr:`__weakref__` though.) @@ -860,12 +847,12 @@ arguments represent positional and keyword arguments of the call to :meth:`__init__`. - The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is - created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function - has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an + The :attr:`tp_init` function, if not *NULL*, is called when an instance is + created normally by calling its type, after the type's :attr:`tp_new` function + has returned an instance of the type. If the :attr:`tp_new` function returns an instance of some other type that is not a subtype of the original type, no - :c:member:`~PyTypeObject.tp_init` function is called; if :c:member:`~PyTypeObject.tp_new` returns an instance of a - subtype of the original type, the subtype's :c:member:`~PyTypeObject.tp_init` is called. + :attr:`tp_init` function is called; if :attr:`tp_new` returns an instance of a + subtype of the original type, the subtype's :attr:`tp_init` is called. This field is inherited by subtypes. @@ -882,14 +869,14 @@ initialization. It should return a pointer to a block of memory of adequate length for the instance, suitably aligned, and initialized to zeros, but with :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If - the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field + the type's :attr:`tp_itemsize` is non-zero, the object's :attr:`ob_size` field should be initialized to *nitems* and the length of the allocated memory block should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block - should be :c:member:`~PyTypeObject.tp_basicsize`. + should be :attr:`tp_basicsize`. Do not use this function to do any other instance initialization, not even to - allocate additional memory; that should be done by :c:member:`~PyTypeObject.tp_new`. + allocate additional memory; that should be done by :attr:`tp_new`. This field is inherited by static subtypes, but not by dynamic subtypes (subtypes created by a class statement); in the latter, this field is always set @@ -911,20 +898,20 @@ The subtype argument is the type of the object being created; the *args* and *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that subtype doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` + type. Note that subtype doesn't have to equal the type whose :attr:`tp_new` function is called; it may be a subtype of that type (but not an unrelated type). - The :c:member:`~PyTypeObject.tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` + The :attr:`tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` to allocate space for the object, and then do only as much further initialization as is absolutely necessary. Initialization that can safely be - ignored or repeated should be placed in the :c:member:`~PyTypeObject.tp_init` handler. A good + ignored or repeated should be placed in the :attr:`tp_init` handler. A good rule of thumb is that for immutable types, all initialization should take place - in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be - deferred to :c:member:`~PyTypeObject.tp_init`. + in :attr:`tp_new`, while for mutable types, most initialization should be + deferred to :attr:`tp_init`. This field is inherited by subtypes, except it is not inherited by static types - whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. + whose :attr:`tp_base` is *NULL* or ``&PyBaseObject_Type``. .. c:member:: destructor PyTypeObject.tp_free @@ -948,7 +935,7 @@ The garbage collector needs to know whether a particular object is collectible or not. Normally, it is sufficient to look at the object's type's - :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But + :attr:`tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But some types have a mixture of statically and dynamically allocated instances, and the statically allocated instances are not collectible. Such types should define this function; it should return ``1`` for a collectible instance, and @@ -981,47 +968,6 @@ This field is not inherited; it is calculated fresh by :c:func:`PyType_Ready`. -.. c:member:: destructor PyTypeObject.tp_finalize - - An optional pointer to an instance finalization function. Its signature is - :c:type:`destructor`:: - - void tp_finalize(PyObject *) - - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. - - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: - - static void - local_finalize(PyObject *self) - { - PyObject *error_type, *error_value, *error_traceback; - - /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - /* ... */ - - /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); - } - - For this field to be taken into account (even through inheritance), - you must also set the :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit. - - This field is inherited by subtypes. - - .. versionadded:: 3.4 - - .. seealso:: "Safe object finalization" (:pep:`442`) - - .. c:member:: PyObject* PyTypeObject.tp_cache Unused. Not inherited. Internal use only. @@ -1060,7 +1006,7 @@ .. c:member:: PyTypeObject* PyTypeObject.tp_next - Pointer to the next type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. + Pointer to the next type object with a non-zero :attr:`tp_allocs` field. Also, note that, in a garbage collected Python, tp_dealloc may be called from any Python thread, not just the thread which created the object (if the object @@ -1127,9 +1073,6 @@ binaryfunc nb_inplace_true_divide; unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; } PyNumberMethods; .. note:: @@ -1175,11 +1118,9 @@ .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript - This function is used by :c:func:`PyObject_SetItem` and - :c:func:`PyObject_DelItem`. It has the same signature as - :c:func:`PyObject_SetItem`, but *v* can also be set to *NULL* to delete - an item. If this slot is *NULL*, the object does not support item - assignment and deletion. + This function is used by :c:func:`PyObject_SetItem` and has the same + signature. If this slot is *NULL*, the object does not support item + assignment. .. _sequence-structs: @@ -1204,14 +1145,13 @@ This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric - addition via the :c:member:`~PyTypeObject.tp_as_number.nb_add` slot. + addition via the :attr:`tp_as_number.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_multiply` - slot. + multiplication via the :attr:`tp_as_number.nb_mul` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item @@ -1228,7 +1168,7 @@ This function is used by :c:func:`PySequence_SetItem` and has the same signature. This slot may be left to *NULL* if the object does not support - item assignment and deletion. + item assignment. .. c:member:: objobjproc PySequenceMethods.sq_contains @@ -1343,58 +1283,3 @@ :c:func:`PyBuffer_Release` is the interface for the consumer that wraps this function. - - -.. _async-structs: - - -Async Object Structures -======================= - -.. sectionauthor:: Yury Selivanov - -.. versionadded:: 3.5 - -.. c:type:: PyAsyncMethods - - This structure holds pointers to the functions required to implement - :term:`awaitable` and :term:`asynchronous iterator` objects. - - Here is the structure definition:: - - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - } PyAsyncMethods; - -.. c:member:: unaryfunc PyAsyncMethods.am_await - - The signature of this function is:: - - PyObject *am_await(PyObject *self) - - The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must - return ``1`` for it. - - This slot may be set to *NULL* if an object is not an :term:`awaitable`. - -.. c:member:: unaryfunc PyAsyncMethods.am_aiter - - The signature of this function is:: - - PyObject *am_aiter(PyObject *self) - - Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - - This slot may be set to *NULL* if an object does not implement - asynchronous iteration protocol. - -.. c:member:: unaryfunc PyAsyncMethods.am_anext - - The signature of this function is:: - - PyObject *am_anext(PyObject *self) - - Must return an :term:`awaitable` object. See :meth:`__anext__` for details. - This slot may be set to *NULL*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/unicode.rst --- a/Doc/c-api/unicode.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/unicode.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,7 @@ Unicode Objects and Codecs -------------------------- -.. sectionauthor:: Marc-André Lemburg +.. sectionauthor:: Marc-Andre Lemburg .. sectionauthor:: Georg Brandl Unicode Objects @@ -227,10 +227,7 @@ const char* PyUnicode_AS_DATA(PyObject *o) Return a pointer to a :c:type:`Py_UNICODE` representation of the object. The - returned buffer is always terminated with an extra null code point. It - may also contain embedded null code points, which would cause the string - to be truncated when used in most C functions. The ``AS_DATA`` form - casts the pointer to :c:type:`const char *`. The *o* argument has to be + ``AS_DATA`` form casts the pointer to :c:type:`const char *`. *o* has to be a Unicode object (not checked). .. versionchanged:: 3.3 @@ -442,8 +439,6 @@ .. % Similar comments apply to the %ll width modifier and .. % PY_FORMAT_LONG_LONG. - .. tabularcolumns:: |l|l|L| - +-------------------+---------------------+--------------------------------+ | Format Characters | Type | Comment | +===================+=====================+================================+ @@ -529,23 +524,12 @@ The `"%lld"` and `"%llu"` format specifiers are only available when :const:`HAVE_LONG_LONG` is defined. - .. note:: - The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and - ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of - characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` - (if the ``PyObject*`` argument is not NULL). - .. versionchanged:: 3.2 Support for ``"%lld"`` and ``"%llu"`` added. .. versionchanged:: 3.3 Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added. - .. versionchanged:: 3.4 - Support width and precision formatter for ``"%s"``, ``"%A"``, ``"%U"``, - ``"%V"``, ``"%S"``, ``"%R"`` added. - .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -559,8 +543,7 @@ Coerce an encoded object *obj* to an Unicode object and return a reference with incremented refcount. - :class:`bytes`, :class:`bytearray` and other - :term:`bytes-like objects ` + :class:`bytes`, :class:`bytearray` and other char buffer compatible objects are decoded according to the given *encoding* and using the error handling defined by *errors*. Both can be *NULL* to have the interface use the default values (see the next section for details). @@ -580,7 +563,7 @@ .. c:function:: int PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start, \ - PyObject *from, Py_ssize_t from_start, Py_ssize_t how_many) + PyObject *to, Py_ssize_t from_start, Py_ssize_t how_many) Copy characters from one Unicode object into another. This function performs character conversion when necessary and falls back to :c:func:`memcpy` if @@ -653,8 +636,7 @@ Copy the string *u* into a new UCS4 buffer that is allocated using :c:func:`PyMem_Malloc`. If this fails, *NULL* is returned with a - :exc:`MemoryError` set. The returned buffer always has an extra - null code point appended. + :exc:`MemoryError` set. .. versionadded:: 3.3 @@ -693,9 +675,8 @@ Return a read-only pointer to the Unicode object's internal :c:type:`Py_UNICODE` buffer, or *NULL* on error. This will create the :c:type:`Py_UNICODE*` representation of the object if it is not yet - available. The buffer is always terminated with an extra null code point. - Note that the resulting :c:type:`Py_UNICODE` string may also contain - embedded null code points, which would cause the string to be truncated when + available. Note that the resulting :c:type:`Py_UNICODE` string may contain + embedded null characters, which would cause the string to be truncated when used in most C functions. Please migrate to using :c:func:`PyUnicode_AsUCS4`, @@ -713,9 +694,8 @@ .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) Like :c:func:`PyUnicode_AsUnicode`, but also saves the :c:func:`Py_UNICODE` - array length (excluding the extra null terminator) in *size*. - Note that the resulting :c:type:`Py_UNICODE*` string - may contain embedded null code points, which would cause the string to be + array length in *size*. Note that the resulting :c:type:`Py_UNICODE*` string + may contain embedded null characters, which would cause the string to be truncated when used in most C functions. .. versionadded:: 3.3 @@ -723,11 +703,11 @@ .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode) - Create a copy of a Unicode string ending with a null code point. Return *NULL* + Create a copy of a Unicode string ending with a nul character. Return *NULL* and raise a :exc:`MemoryError` exception on memory allocation failure, otherwise return a new allocated buffer (use :c:func:`PyMem_Free` to free the buffer). Note that the resulting :c:type:`Py_UNICODE*` string may - contain embedded null code points, which would cause the string to be + contain embedded null characters, which would cause the string to be truncated when used in most C functions. .. versionadded:: 3.2 @@ -755,28 +735,26 @@ The current locale encoding can be used to decode text from the operating system. -.. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ - const char *errors) - - Decode a string from the current locale encoding. The supported - error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The decoder uses ``"strict"`` error handler if - *errors* is ``NULL``. *str* must end with a null character but - cannot contain embedded null characters. - - Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). +.. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len, int surrogateescape) + + Decode a string from the current locale encoding. The decoder is strict if + *surrogateescape* is equal to zero, otherwise it uses the + ``'surrogateescape'`` error handler (:pep:`383`) to escape undecodable + bytes. If a byte sequence can be decoded as a surrogate character and + *surrogateescape* is not equal to zero, the byte sequence is escaped using + the ``'surrogateescape'`` error handler instead of being decoded. *str* + must end with a null character but cannot contain embedded null characters. .. seealso:: - The :c:func:`Py_DecodeLocale` function. + Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from + :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at + Python startup). .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, int surrogateescape) Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string length using :c:func:`strlen`. @@ -784,21 +762,18 @@ .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) - - Encode a Unicode object to the current locale encoding. The - supported error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The encoder uses ``"strict"`` error handler if - *errors* is ``NULL``. Return a :class:`bytes` object. *str* cannot - contain embedded null characters. - - Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). +.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, int surrogateescape) + + Encode a Unicode object to the current locale encoding. The encoder is + strict if *surrogateescape* is equal to zero, otherwise it uses the + ``'surrogateescape'`` error handler (:pep:`383`). Return a :class:`bytes` + object. *str* cannot contain embedded null characters. .. seealso:: - The :c:func:`Py_EncodeLocale` function. + Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to + :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at + Python startup). .. versionadded:: 3.3 @@ -838,28 +813,26 @@ .. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) Decode a string using :c:data:`Py_FileSystemDefaultEncoding` and the - ``"surrogateescape"`` error handler, or ``"strict"`` on Windows. + ``'surrogateescape'`` error handler, or ``'strict'`` on Windows. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to decode a string - from the current locale encoding, use - :c:func:`PyUnicode_DecodeLocaleAndSize`. - .. seealso:: - The :c:func:`Py_DecodeLocale` function. + :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the + locale encoding and cannot be modified later. If you need to decode a + string from the current locale encoding, use + :c:func:`PyUnicode_DecodeLocaleAndSize`. .. versionchanged:: 3.2 - Use ``"strict"`` error handler on Windows. + Use ``'strict'`` error handler on Windows. .. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) Decode a null-terminated string using :c:data:`Py_FileSystemDefaultEncoding` - and the ``"surrogateescape"`` error handler, or ``"strict"`` on Windows. + and the ``'surrogateescape'`` error handler, or ``'strict'`` on Windows. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. @@ -867,26 +840,25 @@ Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` if you know the string length. .. versionchanged:: 3.2 - Use ``"strict"`` error handler on Windows. + Use ``'strict'`` error handler on Windows. .. c:function:: PyObject* PyUnicode_EncodeFSDefault(PyObject *unicode) Encode a Unicode object to :c:data:`Py_FileSystemDefaultEncoding` with the - ``"surrogateescape"`` error handler, or ``"strict"`` on Windows, and return + ``'surrogateescape'`` error handler, or ``'strict'`` on Windows, and return :class:`bytes`. Note that the resulting :class:`bytes` object may contain null bytes. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to encode a string - to the current locale encoding, use :c:func:`PyUnicode_EncodeLocale`. - .. seealso:: - The :c:func:`Py_EncodeLocale` function. + :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the + locale encoding and cannot be modified later. If you need to encode a + string to the current locale encoding, use + :c:func:`PyUnicode_EncodeLocale`. .. versionadded:: 3.2 @@ -908,10 +880,10 @@ Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing - null termination character). Return the number of :c:type:`wchar_t` characters + 0-termination character). Return the number of :c:type:`wchar_t` characters copied or -1 in case of an error. Note that the resulting :c:type:`wchar_t*` - string may or may not be null-terminated. It is the responsibility of the caller - to make sure that the :c:type:`wchar_t*` string is null-terminated in case this is + string may or may not be 0-terminated. It is the responsibility of the caller + to make sure that the :c:type:`wchar_t*` string is 0-terminated in case this is required by the application. Also, note that the :c:type:`wchar_t*` string might contain null characters, which would cause the string to be truncated when used with most C functions. @@ -920,8 +892,8 @@ .. c:function:: wchar_t* PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size) Convert the Unicode object to a wide character string. The output string - always ends with a null character. If *size* is not *NULL*, write the number - of wide characters (excluding the trailing null termination character) into + always ends with a nul character. If *size* is not *NULL*, write the number + of wide characters (excluding the trailing 0-termination character) into *\*size*. Returns a buffer allocated by :c:func:`PyMem_Alloc` (use @@ -992,7 +964,7 @@ Create a Unicode object by decoding *size* bytes of the encoded string *s*. *encoding* and *errors* have the same meaning as the parameters of the same name - in the :func:`str` built-in function. The codec to be used is looked up + in the :func:`unicode` built-in function. The codec to be used is looked up using the Python codec registry. Return *NULL* if an exception was raised by the codec. @@ -1002,7 +974,7 @@ Encode a Unicode object and return the result as Python bytes object. *encoding* and *errors* have the same meaning as the parameters of the same - name in the Unicode :meth:`~str.encode` method. The codec to be used is looked up + name in the Unicode :meth:`encode` method. The codec to be used is looked up using the Python codec registry. Return *NULL* if an exception was raised by the codec. @@ -1012,7 +984,7 @@ Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python bytes object. *encoding* and *errors* have the same meaning as the - parameters of the same name in the Unicode :meth:`~str.encode` method. The codec + parameters of the same name in the Unicode :meth:`encode` method. The codec to be used is looked up using the Python codec registry. Return *NULL* if an exception was raised by the codec. @@ -1051,11 +1023,9 @@ .. c:function:: char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) - Return a pointer to the UTF-8 encoding of the Unicode object, and - store the size of the encoded representation (in bytes) in *size*. The - *size* argument can be *NULL*; in this case no size will be stored. The - returned buffer always has an extra null byte appended (not included in - *size*), regardless of whether there are any other null code points. + Return a pointer to the default encoding (UTF-8) of the Unicode object, and + store the size of the encoded representation (in bytes) in *size*. *size* + can be *NULL*, in this case no size will be stored. In the case of an error, *NULL* is returned with an exception set and no *size* is stored. @@ -1113,6 +1083,8 @@ After completion, *\*byteorder* is set to the current byte order at the end of input data. + In a narrow build codepoints outside the BMP will be decoded as surrogate pairs. + If *byteorder* is *NULL*, the codec starts in native order mode. Return *NULL* if an exception was raised by the codec. @@ -1149,7 +1121,7 @@ mark (U+FEFF). In the other two modes, no BOM mark is prepended. If *Py_UNICODE_WIDE* is not defined, surrogate pairs will be output - as a single code point. + as a single codepoint. Return *NULL* if an exception was raised by the codec. @@ -1584,7 +1556,7 @@ Unicode string. -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ +.. c:function:: int PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) Return 1 if *substr* matches ``str[start:end]`` at the given tail end @@ -1635,15 +1607,15 @@ respectively. -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) +.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, char *string) Compare a unicode object, *uni*, with *string* and return -1, 0, 1 for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as - ISO-8859-1 if it contains non-ASCII characters. - - -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) + ISO-8859-1 if it contains non-ASCII characters". + + +.. c:function:: int PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) Rich compare two unicode strings and return one of the following: @@ -1662,7 +1634,7 @@ .. c:function:: PyObject* PyUnicode_Format(PyObject *format, PyObject *args) Return a new string object from *format* and *args*; this is analogous to - ``format % args``. + ``format % args``. The *args* argument must be a tuple. .. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/c-api/veryhigh.rst --- a/Doc/c-api/veryhigh.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/c-api/veryhigh.rst Mon Jan 25 17:05:13 2016 +0100 @@ -95,6 +95,12 @@ leaving *closeit* set to ``0`` and *flags* set to *NULL*. +.. c:function:: int PyRun_SimpleFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, + leaving *closeit* set to ``0``. + + .. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, @@ -144,37 +150,6 @@ (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. -.. c:var:: int (*PyOS_InputHook)(void) - - Can be set to point to a function with the prototype - ``int func(void)``. The function will be called when Python's - interpreter prompt is about to become idle and wait for user input - from the terminal. The return value is ignored. Overriding this - hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the - Python source code. - - -.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) - - Can be set to point to a function with the prototype - ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, - overriding the default function used to read a single line of input - at the interpreter's prompt. The function is expected to output - the string *prompt* if it's not *NULL*, and then read a line of - input from the provided standard input file, returning the - resulting string. For example, The :mod:`readline` module sets - this hook to provide line-editing and tab-completion features. - - The result must be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. - - .. versionchanged:: 3.4 - The result must be allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, instead of being allocated by - :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. - - .. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start) This is a simplified interface to @@ -201,7 +176,7 @@ .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) This is a simplified interface to :c:func:`PyParser_SimpleParseFileFlags` below, - leaving *flags* set to ``0``. + leaving *flags* set to ``0`` .. c:function:: struct _node* PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags) @@ -266,15 +241,16 @@ *optimize* set to ``-1``. -.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code object. The start token is given by *start*; this can be used to constrain the code which can be compiled and should be :const:`Py_eval_input`, :const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns *NULL* if the code - cannot be parsed or compiled. + :exc:`SyntaxError` exception messages, it is decoded from the filesystem + encoding (:func:`sys.getfilesystemencoding`). This returns *NULL* if the + code cannot be parsed or compiled. The integer *optimize* specifies the optimization level of the compiler; a value of ``-1`` selects the optimization level of the interpreter as given by @@ -282,16 +258,9 @@ ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). - .. versionadded:: 3.4 + .. versionadded:: 3.2 -.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) - - Like :c:func:`Py_CompileStringExFlags`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). - - .. versionadded:: 3.2 - .. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just @@ -320,11 +289,7 @@ frame *f* is executed, interpreting bytecode and executing calls as needed. The additional *throwflag* parameter can mostly be ignored - if true, then it causes an exception to immediately be thrown; this is used for the - :meth:`~generator.throw` methods of generator objects. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. + :meth:`throw` methods of generator objects. .. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) @@ -379,3 +344,4 @@ This bit can be set in *flags* to cause division operator ``/`` to be interpreted as "true division" according to :pep:`238`. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/conf.py --- a/Doc/conf.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/conf.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,18 +7,26 @@ # that aren't pickleable (module imports are okay, they're removed automatically). import sys, os, time -sys.path.append(os.path.abspath('tools/extensions')) +sys.path.append(os.path.abspath('tools/sphinxext')) # General configuration # --------------------- -extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', - 'pyspecific', 'c_annotations'] +extensions = ['sphinx.ext.refcounting', 'sphinx.ext.coverage', + 'sphinx.ext.doctest', 'pyspecific'] +templates_path = ['tools/sphinxext'] # General substitutions. project = 'Python' copyright = '1990-%s, Python Software Foundation' % time.strftime('%Y') +# The default replacements for |version| and |release|. +# +# The short X.Y version. +# version = '2.6' +# The full version, including alpha/beta/rc tags. +# release = '2.6a0' + # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. import patchlevel @@ -30,33 +38,46 @@ # Else, today_fmt is used as the format for a strftime call. today_fmt = '%B %d, %Y' +# List of files that shouldn't be included in the build. +unused_docs = [ + 'maclib/scrap', + 'library/xmllib', + 'library/xml.etree', +] + +# Ignore .rst in Sphinx its self. +exclude_trees = ['tools/sphinx'] + +# Relative filename of the reference count data file. +refcount_file = 'data/refcounts.dat' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + # By default, highlight as Python 3. highlight_language = 'python3' -# Require Sphinx 1.2 for build. -needs_sphinx = '1.2' - -# Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*'] - # Options for HTML output # ----------------------- -# Use our custom theme. html_theme = 'pydoctheme' -html_theme_path = ['tools'] +html_theme_path = ['tools/sphinxext'] html_theme_options = {'collapsiblesidebar': True} -# Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' -# Path to find HTML templates. -templates_path = ['tools/templates'] +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +html_use_smartypants = True # Custom sidebar templates, filenames relative to this file. html_sidebars = { @@ -70,10 +91,10 @@ } # Output an OpenSearch description file. -html_use_opensearch = 'https://docs.python.org/' + version +html_use_opensearch = 'http://docs.python.org/dev/py3k' # Additional static files. -html_static_path = ['tools/static'] +html_static_path = ['tools/sphinxext/static'] # Output file base name for HTML help builder. htmlhelp_basename = 'python' + release.replace('.', '') @@ -93,15 +114,15 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = r'Guido van Rossum\\and the Python development team' +_stdauthor = r'Guido van Rossum\\Fred L. Drake, Jr., editor' latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', + ('distutils/index', 'distutils.tex', 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', + ('install/index', 'install.tex', 'Installing Python Modules', _stdauthor, 'manual'), ('library/index', 'library.tex', 'The Python Library Reference', _stdauthor, 'manual'), @@ -138,12 +159,6 @@ # Get LaTeX to handle Unicode correctly latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''} -# Options for Epub output -# ----------------------- - -epub_author = 'Python Documentation Authors' -epub_publisher = 'Python Software Foundation' - # Options for the coverage checker # -------------------------------- @@ -179,19 +194,3 @@ coverage_ignore_c_items = { # 'cfunction': [...] } - - -# Options for the link checker -# ---------------------------- - -# Ignore certain URLs. -linkcheck_ignore = [r'https://bugs.python.org/(issue)?\d+', - # Ignore PEPs for now, they all have permanent redirects. - r'http://www.python.org/dev/peps/pep-\d+'] - - -# Options for extensions -# ---------------------- - -# Relative filename of the reference count data file. -refcount_file = 'data/refcounts.dat' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/contents.rst --- a/Doc/contents.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/contents.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,8 +11,8 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst - installing/index.rst + packaging/index.rst + install/index.rst howto/index.rst faq/index.rst glossary.rst @@ -21,11 +21,3 @@ bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - distutils/index.rst - install/index.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/copyright.rst --- a/Doc/copyright.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/copyright.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,7 @@ Python and this documentation is: -Copyright © 2001-2016 Python Software Foundation. All rights reserved. +Copyright © 2001-2012 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/data/refcounts.dat --- a/Doc/data/refcounts.dat Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/data/refcounts.dat Mon Jan 25 17:05:13 2016 +0100 @@ -29,7 +29,7 @@ # reference to the item argument! # The parameter names are as they appear in the API manual, not the source -# code. +# code. PyBool_FromLong:PyObject*::+1: PyBool_FromLong:long:v:0: @@ -210,7 +210,7 @@ PyDict_DelItemString:int::: PyDict_DelItemString:PyObject*:p:0: -PyDict_DelItemString:const char*:key:: +PyDict_DelItemString:char*:key:: PyDict_GetItem:PyObject*::0:0 PyDict_GetItem:PyObject*:p:0: @@ -218,12 +218,7 @@ PyDict_GetItemString:PyObject*::0: PyDict_GetItemString:PyObject*:p:0: -PyDict_GetItemString:const char*:key:: - -PyDict_SetDefault:PyObject*::0: -PyDict_SetDefault:PyObject*:p:0: -PyDict_SetDefault:PyObject*:key:0:conditionally +1 if inserted into the dict -PyDict_SetDefault:PyObject*:default:0:conditionally +1 if inserted into the dict +PyDict_GetItemString:char*:key:: PyDict_Items:PyObject*::+1: PyDict_Items:PyObject*:p:0: @@ -249,7 +244,7 @@ PyDict_SetItemString:int::: PyDict_SetItemString:PyObject*:p:0: -PyDict_SetItemString:const char*:key:: +PyDict_SetItemString:char*:key:: PyDict_SetItemString:PyObject*:val:+1: PyDict_Size:int::: @@ -282,13 +277,13 @@ PyErr_GivenExceptionMatches:PyObject*:exc:0: PyErr_NewException:PyObject*::+1: -PyErr_NewException:const char*:name:: +PyErr_NewException:char*:name:: PyErr_NewException:PyObject*:base:0: PyErr_NewException:PyObject*:dict:0: PyErr_NewExceptionWithDoc:PyObject*::+1: -PyErr_NewExceptionWithDoc:const char*:name:: -PyErr_NewExceptionWithDoc:const char*:doc:: +PyErr_NewExceptionWithDoc:char*:name:: +PyErr_NewExceptionWithDoc:char*:doc:: PyErr_NewExceptionWithDoc:PyObject*:base:0: PyErr_NewExceptionWithDoc:PyObject*:dict:0: @@ -315,21 +310,21 @@ PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null: PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:0: PyErr_SetExcFromWindowsErrWithFilename:int:ierr:: -PyErr_SetExcFromWindowsErrWithFilename:const char*:filename:: +PyErr_SetExcFromWindowsErrWithFilename:char*:filename:: PyErr_SetFromErrno:PyObject*::null: PyErr_SetFromErrno:PyObject*:type:0: PyErr_SetFromErrnoWithFilename:PyObject*::null: PyErr_SetFromErrnoWithFilename:PyObject*:type:0: -PyErr_SetFromErrnoWithFilename:const char*:filename:: +PyErr_SetFromErrnoWithFilename:char*:filename:: PyErr_SetFromWindowsErr:PyObject*::null: PyErr_SetFromWindowsErr:int:ierr:: PyErr_SetFromWindowsErrWithFilename:PyObject*::null: PyErr_SetFromWindowsErrWithFilename:int:ierr:: -PyErr_SetFromWindowsErrWithFilename:const char*:filename:: +PyErr_SetFromWindowsErrWithFilename:char*:filename:: PyErr_SetInterrupt:void::: @@ -342,18 +337,13 @@ PyErr_SetString:void::: PyErr_SetString:PyObject*:type:+1: -PyErr_SetString:const char*:message:: +PyErr_SetString:char*:message:: PyErr_Format:PyObject*::null: PyErr_Format:PyObject*:exception:+1: -PyErr_Format:const char*:format:: +PyErr_Format:char*:format:: PyErr_Format::...:: -PyErr_FormatV:PyObject*::null: -PyErr_FormatV:PyObject*:exception:+1: -PyErr_FormatV:const char*:format:: -PyErr_FormatV:va_list:vargs:: - PyErr_WarnEx:int::: PyErr_WarnEx:PyObject*:category:0: PyErr_WarnEx:const char*:message:: @@ -396,22 +386,22 @@ PyFile_FromFile:PyObject*::+1: PyFile_FromFile:FILE*:fp:: -PyFile_FromFile:const char*:name:: -PyFile_FromFile:const char*:mode:: +PyFile_FromFile:char*:name:: +PyFile_FromFile:char*:mode:: PyFile_FromFile:int(*:close):: PyFile_FromFileEx:PyObject*::+1: PyFile_FromFileEx:FILE*:fp:: -PyFile_FromFileEx:const char*:name:: -PyFile_FromFileEx:const char*:mode:: +PyFile_FromFileEx:char*:name:: +PyFile_FromFileEx:char*:mode:: PyFile_FromFileEx:int(*:close):: PyFile_FromFileEx:int:buffering:: -PyFile_FromFileEx:const char*:encoding:: -PyFile_FromFileEx:const char*:newline:: +PyFile_FromFileEx:char*:encoding:: +PyFile_FromFileEx:char*:newline:: PyFile_FromString:PyObject*::+1: -PyFile_FromString:const char*:name:: -PyFile_FromString:const char*:mode:: +PyFile_FromString:char*:name:: +PyFile_FromString:char*:mode:: PyFile_GetLine:PyObject*::+1: PyFile_GetLine:PyObject*:p:: @@ -491,41 +481,35 @@ PyGen_New:PyObject*::+1: PyGen_New:PyFrameObject*:frame:0: -PyGen_NewWithQualName:PyObject*::+1: -PyGen_NewWithQualName:PyFrameObject*:frame:0: - -PyCoro_New:PyObject*::+1: -PyCoro_New:PyFrameObject*:frame:0: - Py_InitModule:PyObject*::0: -Py_InitModule:const char*:name:: +Py_InitModule:char*:name:: Py_InitModule:PyMethodDef[]:methods:: Py_InitModule3:PyObject*::0: -Py_InitModule3:const char*:name:: +Py_InitModule3:char*:name:: Py_InitModule3:PyMethodDef[]:methods:: -Py_InitModule3:const char*:doc:: +Py_InitModule3:char*:doc:: Py_InitModule4:PyObject*::0: -Py_InitModule4:const char*:name:: +Py_InitModule4:char*:name:: Py_InitModule4:PyMethodDef[]:methods:: -Py_InitModule4:const char*:doc:: +Py_InitModule4:char*:doc:: Py_InitModule4:PyObject*:self:: Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3 PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules -PyImport_AddModule:const char*:name:: +PyImport_AddModule:char*:name:: PyImport_Cleanup:void::: PyImport_ExecCodeModule:PyObject*::+1: -PyImport_ExecCodeModule:const char*:name:: +PyImport_ExecCodeModule:char*:name:: PyImport_ExecCodeModule:PyObject*:co:0: PyImport_ExecCodeModuleEx:PyObject*::+1: -PyImport_ExecCodeModuleEx:const char*:name:: +PyImport_ExecCodeModuleEx:char*:name:: PyImport_ExecCodeModuleEx:PyObject*:co:0: -PyImport_ExecCodeModuleEx:const char*:pathname:: +PyImport_ExecCodeModuleEx:char*:pathname:: PyImport_GetMagicNumber:long::: @@ -535,19 +519,19 @@ PyImport_Import:PyObject*:name:0: PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:const char*::: +PyImport_ImportFrozenModule:char*::: PyImport_ImportModule:PyObject*::+1: -PyImport_ImportModule:const char*:name:: +PyImport_ImportModule:char*:name:: PyImport_ImportModuleEx:PyObject*::+1: -PyImport_ImportModuleEx:const char*:name:: +PyImport_ImportModuleEx:char*:name:: PyImport_ImportModuleEx:PyObject*:globals:0:??? PyImport_ImportModuleEx:PyObject*:locals:0:??? PyImport_ImportModuleEx:PyObject*:fromlist:0:??? PyImport_ImportModuleLevel:PyObject*::+1: -PyImport_ImportModuleLevel:const char*:name:: +PyImport_ImportModuleLevel:char*:name:: PyImport_ImportModuleLevel:PyObject*:globals:0:??? PyImport_ImportModuleLevel:PyObject*:locals:0:??? PyImport_ImportModuleLevel:PyObject*:fromlist:0:??? @@ -684,7 +668,7 @@ PyLong_FromUnsignedLongLong:unsigned long long:v:: PyLong_FromString:PyObject*::+1: -PyLong_FromString:const char*:str:: +PyLong_FromString:char*:str:: PyLong_FromString:char**:pend:: PyLong_FromString:int:base:: @@ -708,11 +692,11 @@ PyMapping_DelItemString:int::: PyMapping_DelItemString:PyObject*:o:0: -PyMapping_DelItemString:const char*:key:: +PyMapping_DelItemString:char*:key:: PyMapping_GetItemString:PyObject*::+1: PyMapping_GetItemString:PyObject*:o:0: -PyMapping_GetItemString:const char*:key:: +PyMapping_GetItemString:char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: @@ -720,7 +704,7 @@ PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: -PyMapping_HasKeyString:const char*:key:: +PyMapping_HasKeyString:char*:key:: PyMapping_Items:PyObject*::+1: PyMapping_Items:PyObject*:o:0: @@ -733,7 +717,7 @@ PyMapping_SetItemString:int::: PyMapping_SetItemString:PyObject*:o:0: -PyMapping_SetItemString:const char*:key:: +PyMapping_SetItemString:char*:key:: PyMapping_SetItemString:PyObject*:v:+1: PyMapping_Values:PyObject*::+1: @@ -746,7 +730,7 @@ PyMarshal_ReadObjectFromFile:FILE*:file:: PyMarshal_ReadObjectFromString:PyObject*::+1: -PyMarshal_ReadObjectFromString:const char*:string:: +PyMarshal_ReadObjectFromString:char*:string:: PyMarshal_ReadObjectFromString:int:len:: PyMarshal_WriteObjectToString:PyObject*::+1: @@ -778,10 +762,10 @@ PyModule_GetDict:PyObject*::0: PyModule_GetDict::PyObject* module:0: -PyModule_GetFilename:const char*::: +PyModule_GetFilename:char*::: PyModule_GetFilename:PyObject*:module:0: -PyModule_GetName:const char*::: +PyModule_GetName:char*::: PyModule_GetName:PyObject*:module:0: PyModule_New:PyObject*::+1: @@ -918,7 +902,7 @@ PyNumber_Xor:PyObject*:o1:0: PyNumber_Xor:PyObject*:o2:0: -PyObject_AsFileDescriptor:int::: +PyObject_AsFileDescriptor:int::: PyObject_AsFileDescriptor:PyObject*:o:0: PyObject_Call:PyObject*::+1: @@ -928,7 +912,7 @@ PyObject_CallFunction:PyObject*::+1: PyObject_CallFunction:PyObject*:callable_object:0: -PyObject_CallFunction:const char*:format:: +PyObject_CallFunction:char*:format:: PyObject_CallFunction::...:: PyObject_CallFunctionObjArgs:PyObject*::+1: @@ -937,13 +921,13 @@ PyObject_CallMethod:PyObject*::+1: PyObject_CallMethod:PyObject*:o:0: -PyObject_CallMethod:const char*:m:: -PyObject_CallMethod:const char*:format:: +PyObject_CallMethod:char*:m:: +PyObject_CallMethod:char*:format:: PyObject_CallMethod::...:: PyObject_CallMethodObjArgs:PyObject*::+1: PyObject_CallMethodObjArgs:PyObject*:o:0: -PyObject_CallMethodObjArgs:PyObject*:name:0: +PyObject_CallMethodObjArgs:char*:name:: PyObject_CallMethodObjArgs::...:: PyObject_CallObject:PyObject*::+1: @@ -965,7 +949,7 @@ PyObject_DelAttrString:int::: PyObject_DelAttrString:PyObject*:o:0: -PyObject_DelAttrString:const char*:attr_name:: +PyObject_DelAttrString:char*:attr_name:: PyObject_DelItem:int::: PyObject_DelItem:PyObject*:o:0: @@ -980,7 +964,7 @@ PyObject_GetAttrString:PyObject*::+1: PyObject_GetAttrString:PyObject*:o:0: -PyObject_GetAttrString:const char*:attr_name:: +PyObject_GetAttrString:char*:attr_name:: PyObject_GetItem:PyObject*::+1: PyObject_GetItem:PyObject*:o:0: @@ -995,7 +979,7 @@ PyObject_HasAttrString:int::: PyObject_HasAttrString:PyObject*:o:0: -PyObject_HasAttrString:const char*:attr_name:0: +PyObject_HasAttrString:char*:attr_name:0: PyObject_Hash:int::: PyObject_Hash:PyObject*:o:0: @@ -1045,7 +1029,7 @@ PyObject_SetAttrString:int::: PyObject_SetAttrString:PyObject*:o:0: -PyObject_SetAttrString:const char*:attr_name:: +PyObject_SetAttrString:char*:attr_name:: PyObject_SetAttrString:PyObject*:v:+1: PyObject_SetItem:int::: @@ -1064,27 +1048,27 @@ PyParser_SimpleParseFile:struct _node*::: PyParser_SimpleParseFile:FILE*:fp:: -PyParser_SimpleParseFile:const char*:filename:: +PyParser_SimpleParseFile:char*:filename:: PyParser_SimpleParseFile:int:start:: PyParser_SimpleParseString:struct _node*::: -PyParser_SimpleParseString:const char*:str:: +PyParser_SimpleParseString:char*:str:: PyParser_SimpleParseString:int:start:: PyRun_AnyFile:int::: PyRun_AnyFile:FILE*:fp:: -PyRun_AnyFile:const char*:filename:: +PyRun_AnyFile:char*:filename:: PyRun_File:PyObject*::+1:??? -- same as eval_code2() PyRun_File:FILE*:fp:: -PyRun_File:const char*:filename:: +PyRun_File:char*:filename:: PyRun_File:int:start:: PyRun_File:PyObject*:globals:0: PyRun_File:PyObject*:locals:0: PyRun_FileEx:PyObject*::+1:??? -- same as eval_code2() PyRun_FileEx:FILE*:fp:: -PyRun_FileEx:const char*:filename:: +PyRun_FileEx:char*:filename:: PyRun_FileEx:int:start:: PyRun_FileEx:PyObject*:globals:0: PyRun_FileEx:PyObject*:locals:0: @@ -1092,7 +1076,7 @@ PyRun_FileFlags:PyObject*::+1:??? -- same as eval_code2() PyRun_FileFlags:FILE*:fp:: -PyRun_FileFlags:const char*:filename:: +PyRun_FileFlags:char*:filename:: PyRun_FileFlags:int:start:: PyRun_FileFlags:PyObject*:globals:0: PyRun_FileFlags:PyObject*:locals:0: @@ -1100,7 +1084,7 @@ PyRun_FileExFlags:PyObject*::+1:??? -- same as eval_code2() PyRun_FileExFlags:FILE*:fp:: -PyRun_FileExFlags:const char*:filename:: +PyRun_FileExFlags:char*:filename:: PyRun_FileExFlags:int:start:: PyRun_FileExFlags:PyObject*:globals:0: PyRun_FileExFlags:PyObject*:locals:0: @@ -1109,27 +1093,27 @@ PyRun_InteractiveLoop:int::: PyRun_InteractiveLoop:FILE*:fp:: -PyRun_InteractiveLoop:const char*:filename:: +PyRun_InteractiveLoop:char*:filename:: PyRun_InteractiveOne:int::: PyRun_InteractiveOne:FILE*:fp:: -PyRun_InteractiveOne:const char*:filename:: +PyRun_InteractiveOne:char*:filename:: PyRun_SimpleFile:int::: PyRun_SimpleFile:FILE*:fp:: -PyRun_SimpleFile:const char*:filename:: +PyRun_SimpleFile:char*:filename:: PyRun_SimpleString:int::: -PyRun_SimpleString:const char*:command:: +PyRun_SimpleString:char*:command:: PyRun_String:PyObject*::+1:??? -- same as eval_code2() -PyRun_String:const char*:str:: +PyRun_String:char*:str:: PyRun_String:int:start:: PyRun_String:PyObject*:globals:0: PyRun_String:PyObject*:locals:0: PyRun_StringFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_StringFlags:const char*:str:: +PyRun_StringFlags:char*:str:: PyRun_StringFlags:int:start:: PyRun_StringFlags:PyObject*:globals:0: PyRun_StringFlags:PyObject*:locals:0: @@ -1245,7 +1229,7 @@ PySlice_New:PyObject*:stop:0: PySlice_New:PyObject*:step:0: -PyString_AS_STRING:const char*::: +PyString_AS_STRING:char*::: PyString_AS_STRING:PyObject*:string:0: PyString_AsDecodedObject:PyObject*::+1: @@ -1258,7 +1242,7 @@ PyString_AsEncodedObject:const char*:encoding:: PyString_AsEncodedObject:const char*:errors:: -PyString_AsString:const char*::: +PyString_AsString:char*::: PyString_AsString:PyObject*:string:0: PyString_AsStringAndSize:int::: @@ -1326,13 +1310,17 @@ PyString_AsEncodedString:const char*:errors:: PySys_AddWarnOption:void::: -PySys_AddWarnOption:const char*:s:: +PySys_AddWarnOption:char*:s:: PySys_AddXOption:void::: PySys_AddXOption:const wchar_t*:s:: +PySys_GetFile:FILE*::: +PySys_GetFile:char*:name:: +PySys_GetFile:FILE*:def:: + PySys_GetObject:PyObject*::0: -PySys_GetObject:const char*:name:: +PySys_GetObject:char*:name:: PySys_GetXOptions:PyObject*::0: @@ -1341,16 +1329,16 @@ PySys_SetArgv:char**:argv:: PySys_SetObject:int::: -PySys_SetObject:const char*:name:: +PySys_SetObject:char*:name:: PySys_SetObject:PyObject*:v:+1: PySys_ResetWarnOptions:void::: PySys_WriteStdout:void::: -PySys_WriteStdout:const char*:format:: +PySys_WriteStdout:char*:format:: PySys_WriteStderr:void::: -PySys_WriteStderr:const char*:format:: +PySys_WriteStderr:char*:format:: PyThreadState_Clear:void::: PyThreadState_Clear:PyThreadState*:tstate:: @@ -1730,16 +1718,16 @@ Py_AtExit:void (*)():func:: Py_BuildValue:PyObject*::+1: -Py_BuildValue:const char*:format:: +Py_BuildValue:char*:format:: Py_CompileString:PyObject*::+1: -Py_CompileString:const char*:str:: -Py_CompileString:const char*:filename:: +Py_CompileString:char*:str:: +Py_CompileString:char*:filename:: Py_CompileString:int:start:: Py_CompileStringFlags:PyObject*::+1: -Py_CompileStringFlags:const char*:str:: -Py_CompileStringFlags:const char*:filename:: +Py_CompileStringFlags:char*:str:: +Py_CompileStringFlags:char*:filename:: Py_CompileStringFlags:int:start:: Py_CompileStringFlags:PyCompilerFlags*:flags:: @@ -1753,33 +1741,33 @@ Py_Exit:int:status:: Py_FatalError:void::: -Py_FatalError:const char*:message:: +Py_FatalError:char*:message:: Py_FdIsInteractive:int::: Py_FdIsInteractive:FILE*:fp:: -Py_FdIsInteractive:const char*:filename:: +Py_FdIsInteractive:char*:filename:: Py_Finalize:void::: -Py_GetBuildInfoconst:const char*::: +Py_GetBuildInfoconst:char*::: -Py_GetCompilerconst:const char*::: +Py_GetCompilerconst:char*::: -Py_GetCopyrightconst:const char*::: +Py_GetCopyrightconst:char*::: -Py_GetExecPrefix:const char*::: +Py_GetExecPrefix:char*::: -Py_GetPath:const char*::: +Py_GetPath:char*::: -Py_GetPlatformconst:const char*::: +Py_GetPlatformconst:char*::: -Py_GetPrefix:const char*::: +Py_GetPrefix:char*::: -Py_GetProgramFullPath:const char*::: +Py_GetProgramFullPath:char*::: -Py_GetProgramName:const char*::: +Py_GetProgramName:char*::: -Py_GetVersionconst:const char*::: +Py_GetVersionconst:char*::: Py_INCREF:void::: Py_INCREF:PyObject*:o:+1: @@ -1791,7 +1779,7 @@ Py_NewInterpreter:PyThreadState*::: Py_SetProgramName:void::: -Py_SetProgramName:const char*:name:: +Py_SetProgramName:char*:name:: Py_XDECREF:void::: Py_XDECREF:PyObject*:o:-1:if o is not NULL @@ -1800,14 +1788,14 @@ Py_XINCREF:PyObject*:o:+1:if o is not NULL _PyImport_FindExtension:PyObject*::0:??? see PyImport_AddModule -_PyImport_FindExtension:const char*::: -_PyImport_FindExtension:const char*::: +_PyImport_FindExtension:char*::: +_PyImport_FindExtension:char*::: _PyImport_Fini:void::: _PyImport_FixupExtension:PyObject*:::??? -_PyImport_FixupExtension:const char*::: -_PyImport_FixupExtension:const char*::: +_PyImport_FixupExtension:char*::: +_PyImport_FixupExtension:char*::: _PyImport_Init:void::: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distributing/index.rst --- a/Doc/distributing/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,170 +0,0 @@ -.. _distributing-index: - -############################### - Distributing Python Modules -############################### - -:Email: distutils-sig@python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide <installing-index>`. - -.. note:: - - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Packaging Index <https://pypi.python.org/pypi>`__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - <https://packaging.python.org/en/latest/future.html>`__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub <https://github.com/pypa>`__ and - `BitBucket <https://bitbucket.org/pypa/>`__. -* :mod:`distutils` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of :mod:`distutils` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for :mod:`distutils` first - published in 2004. Its most notable addition over the unmodified - :mod:`distutils` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to :mod:`distutils` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to :mod:`distutils`/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.pypa.io/en/latest/setuptools.html -.. _wheel: http://wheel.readthedocs.org - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including Mac OS X and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/en/latest/current.html#packaging-tool-recommendations - -Reading the guide -================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Packaging Index`_ - -.. _Project structure: \ - https://packaging.python.org/en/latest/distributing.html#creating-your-own-project -.. _Building and packaging the project: \ - https://packaging.python.org/en/latest/distributing.html#packaging-your-project -.. _Uploading the project to the Python Packaging Index: \ - https://packaging.python.org/en/latest/distributing.html#uploading-your-project-to-pypi - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Packaging Index to see if the name is already in use -* check popular hosting sites like GitHub, BitBucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - <https://packaging.python.org/en/latest/extensions.html>`__ - -.. other topics: - - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/apiref.rst --- a/Doc/distutils/apiref.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/apiref.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,8 +26,6 @@ The setup function takes a large number of arguments. These are laid out in the following table. - .. tabularcolumns:: |l|L|L| - +--------------------+--------------------------------+-------------------------------------------------------------+ | argument name | value | type | +====================+================================+=============================================================+ @@ -50,10 +48,7 @@ +--------------------+--------------------------------+-------------------------------------------------------------+ | *maintainer* | The name of the current | a string | | | maintainer, if different from | | - | | the author. Note that if | | - | | the maintainer is provided, | | - | | distutils will use it as the | | - | | author in :file:`PKG-INFO` | | + | | the author | | +--------------------+--------------------------------+-------------------------------------------------------------+ | *maintainer_email* | The email address of the | a string | | | current maintainer, if | | @@ -78,7 +73,7 @@ | | be built | :class:`distutils.core.Extension` | +--------------------+--------------------------------+-------------------------------------------------------------+ | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI | - | | package | <https://pypi.python.org/pypi?:action=list_classifiers>`_. | + | | package | <http://pypi.python.org/pypi?:action=list_classifiers>`_. | +--------------------+--------------------------------+-------------------------------------------------------------+ | *distclass* | the :class:`Distribution` | a subclass of | | | class to use | :class:`distutils.core.Distribution` | @@ -127,8 +122,6 @@ *stop_after* tells :func:`setup` when to stop processing; possible values: - .. tabularcolumns:: |l|L| - +---------------+---------------------------------------------+ | value | description | +===============+=============================================+ @@ -169,8 +162,6 @@ The Extension class describes a single C or C++extension module in a setup script. It accepts the following keyword arguments in its constructor: - .. tabularcolumns:: |l|L|l| - +------------------------+--------------------------------+---------------------------+ | argument name | value | type | +========================+================================+===========================+ @@ -521,7 +512,7 @@ .. method:: CCompiler.library_option(lib) - Return the compiler option to add *lib* to the list of libraries linked into the + Return the compiler option to add *dir* to the list of libraries linked into the shared library or executable. @@ -734,7 +725,7 @@ .. method:: CCompiler.execute(func, args[, msg=None, level=1]) - Invokes :func:`distutils.util.execute`. This method invokes a Python function + Invokes :func:`distutils.util.execute` This method invokes a Python function *func* with the given arguments *args*, after logging and taking into account the *dry_run* flag. @@ -853,6 +844,17 @@ port of GCC (same as cygwin in no-cygwin mode). +:mod:`distutils.emxccompiler` --- OS/2 EMX Compiler +=================================================== + +.. module:: distutils.emxccompiler + :synopsis: OS/2 EMX Compiler support + + +This module provides the EMXCCompiler class, a subclass of +:class:`UnixCCompiler` that handles the EMX port of the GNU C compiler to OS/2. + + :mod:`distutils.archive_util` --- Archiving utilities ====================================================== @@ -868,31 +870,23 @@ Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of the file to create, minus any format-specific extension; *format* is the - archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or - ``ztar``. *root_dir* is a directory that will be the root directory of the - archive; ie. we typically ``chdir`` into *root_dir* before creating the - archive. *base_dir* is the directory where we start archiving from; ie. - *base_dir* will be the common prefix of all files and directories in the - archive. *root_dir* and *base_dir* both default to the current directory. - Returns the name of the archive file. - - .. versionchanged: 3.5 - Added support for the ``xztar`` format. + archive format: one of ``zip``, ``tar``, ``ztar``, or ``gztar``. *root_dir* is + a directory that will be the root directory of the archive; ie. we typically + ``chdir`` into *root_dir* before creating the archive. *base_dir* is the + directory where we start archiving from; ie. *base_dir* will be the common + prefix of all files and directories in the archive. *root_dir* and *base_dir* + both default to the current directory. Returns the name of the archive file. .. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0]) 'Create an (optional compressed) archive as a tar file from all files in and - under *base_dir*. *compress* must be ``'gzip'`` (the default), - ``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'`` - method the compression utility named by :program:`compress` must be on the - default program search path, so this is probably Unix-specific. The output - tar file will be named :file:`base_dir.tar`, possibly plus the appropriate - compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the - output filename. - - .. versionchanged: 3.5 - Added support for the ``xz`` compression. + under *base_dir*. *compress* must be ``'gzip'`` (the default), ``'compress'``, + ``'bzip2'``, or ``None``. Both :program:`tar` and the compression utility named + by *compress* must be on the default program search path, so this is probably + Unix-specific. The output tar file will be named :file:`base_dir.tar`, + possibly plus the appropriate compression extension (:file:`.gz`, :file:`.bz2` + or :file:`.Z`). Return the output filename. .. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0]) @@ -928,7 +922,7 @@ Walk two filename lists in parallel, testing if each source is newer than its corresponding target. Return a pair of lists (*sources*, *targets*) where - source is newer than target, according to the semantics of :func:`newer`. + source is newer than target, according to the semantics of :func:`newer` .. % % equivalent to a listcomp... @@ -972,7 +966,7 @@ .. function:: create_tree(base_dir, files[, mode=0o777, verbose=0, dry_run=0]) Create all the empty directories under *base_dir* needed to put *files* there. - *base_dir* is just the name of a directory which doesn't necessarily exist + *base_dir* is just the a name of a directory which doesn't necessarily exist yet; *files* is a list of filenames to be interpreted relative to *base_dir*. *base_dir* + the directory portion of every file in *files* will be created if it doesn't already exist. *mode*, *verbose* and *dry_run* flags are as for @@ -991,20 +985,13 @@ simply the list of all files under *src*, with the names changed to be under *dst*. - *preserve_mode* and *preserve_times* are the same as for - :func:`distutils.file_util.copy_file`; note that they only apply to - regular files, not to + *preserve_mode* and *preserve_times* are the same as for :func:`copy_file` in + :mod:`distutils.file_util`; note that they only apply to regular files, not to directories. If *preserve_symlinks* is true, symlinks will be copied as symlinks (on platforms that support them!); otherwise (the default), the destination of the symlink will be copied. *update* and *verbose* are the same as for :func:`copy_file`. - Files in *src* that begin with :file:`.nfs` are skipped (more information on - these files is available in answer D2 of the `NFS FAQ page - <http://nfs.sourceforge.net/#section_d>`_). - - .. versionchanged:: 3.3.1 - NFS files are ignored. .. function:: remove_tree(directory[, verbose=0, dry_run=0]) @@ -1107,13 +1094,13 @@ during the build of Python), not the OS version of the current system. For universal binary builds on Mac OS X the architecture value reflects - the universal binary status instead of the architecture of the current + the univeral binary status instead of the architecture of the current processor. For 32-bit universal binaries the architecture is ``fat``, for 64-bit universal binaries the architecture is ``fat64``, and for 4-way universal binaries the architecture is ``universal``. Starting from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for - a universal build with the i386 and x86_64 architectures + a univeral build with the i386 and x86_64 architectures Examples of returned values on Mac OS X: @@ -1168,6 +1155,15 @@ underscore. No { } or ( ) style quoting is available. +.. function:: grok_environment_error(exc[, prefix='error: ']) + + Generate a useful error message from an :exc:`OSError` exception object. + Handles Python 1.5.1 and later styles, and does what it can to deal with + exception objects that don't have a filename (which happens when the error + is due to a two-file operation, such as :func:`rename` or :func:`link`). + Returns the error message as a string prefixed with *prefix*. + + .. function:: split_quoted(s) Split a string up according to Unix shell-like rules for quotes and backslashes. @@ -1201,12 +1197,12 @@ .. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None]) - Byte-compile a collection of Python source files to :file:`.pyc` files in a - :file:`__pycache__` subdirectory (see :pep:`3147` and :pep:`488`). + Byte-compile a collection of Python source files to either :file:`.pyc` or + :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`). *py_files* is a list of files to compile; any files that don't end in :file:`.py` are silently skipped. *optimize* must be one of the following: - * ``0`` - don't optimize + * ``0`` - don't optimize (generate :file:`.pyc`) * ``1`` - normal optimization (like ``python -O``) * ``2`` - extra optimization (like ``python -OO``) @@ -1230,13 +1226,10 @@ doing, leave it set to ``None``. .. versionchanged:: 3.2.3 - Create ``.pyc`` files with an :func:`import magic tag + Create ``.pyc`` or ``.pyo`` files with an :func:`import magic tag <imp.get_tag>` in their name, in a :file:`__pycache__` subdirectory instead of files without tag in the current directory. - .. versionchanged: 3.5 - Create ``.pyc`` files according to :pep:`488`. - .. function:: rfc822_escape(header) @@ -1257,8 +1250,8 @@ built/installed/distributed -This module provides the :class:`~distutils.core.Distribution` class, which -represents the module distribution being built/installed/distributed. +This module provides the :class:`Distribution` class, which represents the +module distribution being built/installed/distributed. :mod:`distutils.extension` --- The Extension class @@ -1560,8 +1553,6 @@ The options are all boolean, and affect the values returned by :meth:`readline` - .. tabularcolumns:: |l|L|l| - +------------------+--------------------------------+---------+ | option name | description | default | +==================+================================+=========+ @@ -1704,8 +1695,8 @@ options, is the :meth:`run` method, which must also be implemented by every command class. - The class constructor takes a single argument *dist*, a - :class:`~distutils.core.Distribution` instance. + The class constructor takes a single argument *dist*, a :class:`Distribution` + instance. Creating a new Distutils command @@ -1934,12 +1925,8 @@ .. module:: distutils.command.clean :synopsis: Clean a package build area -This command removes the temporary files created by :command:`build` -and its subcommands, like intermediary compiled object files. With -the ``--all`` option, the complete build directory will be removed. - -Extension modules built :ref:`in place <distutils-build-ext-inplace>` -will not be cleaned, as they are not in the build directory. + +.. % todo :mod:`distutils.command.config` --- Perform package configuration diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/builtdist.rst --- a/Doc/distutils/builtdist.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/builtdist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -72,19 +72,13 @@ +-------------+------------------------------+---------+ | Format | Description | Notes | +=============+==============================+=========+ -| ``gztar`` | gzipped tar file | \(1) | +| ``gztar`` | gzipped tar file | (1),(3) | | | (:file:`.tar.gz`) | | +-------------+------------------------------+---------+ -| ``bztar`` | bzipped tar file | | -| | (:file:`.tar.bz2`) | | -+-------------+------------------------------+---------+ -| ``xztar`` | xzipped tar file | | -| | (:file:`.tar.xz`) | | -+-------------+------------------------------+---------+ | ``ztar`` | compressed tar file | \(3) | | | (:file:`.tar.Z`) | | +-------------+------------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | | +| ``tar`` | tar file (:file:`.tar`) | \(3) | +-------------+------------------------------+---------+ | ``zip`` | zip file (:file:`.zip`) | (2),(4) | +-------------+------------------------------+---------+ @@ -100,9 +94,6 @@ | ``msi`` | Microsoft Installer. | | +-------------+------------------------------+---------+ -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - Notes: @@ -113,7 +104,8 @@ default on Windows (3) - requires external :program:`compress` utility. + requires external utilities: :program:`tar` and possibly one of :program:`gzip`, + :program:`bzip2`, or :program:`compress` (4) requires either external :program:`zip` utility or :mod:`zipfile` module (part @@ -127,22 +119,21 @@ option; you can also use the command that directly implements the format you're interested in. Some of these :command:`bdist` "sub-commands" actually generate several similar formats; for instance, the :command:`bdist_dumb` command -generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``, -``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both -binary and source RPMs. The :command:`bdist` sub-commands, and the formats -generated by each, are: +generates all the "dumb" archive formats (``tar``, ``ztar``, ``gztar``, and +``zip``), and :command:`bdist_rpm` generates both binary and source RPMs. The +:command:`bdist` sub-commands, and the formats generated by each, are: -+--------------------------+-------------------------------------+ -| Command | Formats | -+==========================+=====================================+ -| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip | -+--------------------------+-------------------------------------+ -| :command:`bdist_rpm` | rpm, srpm | -+--------------------------+-------------------------------------+ -| :command:`bdist_wininst` | wininst | -+--------------------------+-------------------------------------+ -| :command:`bdist_msi` | msi | -+--------------------------+-------------------------------------+ ++--------------------------+-----------------------+ +| Command | Formats | ++==========================+=======================+ +| :command:`bdist_dumb` | tar, ztar, gztar, zip | ++--------------------------+-----------------------+ +| :command:`bdist_rpm` | rpm, srpm | ++--------------------------+-----------------------+ +| :command:`bdist_wininst` | wininst | ++--------------------------+-----------------------+ +| :command:`bdist_msi` | msi | ++--------------------------+-----------------------+ The following sections give details on the individual :command:`bdist_\*` commands. @@ -195,21 +186,21 @@ +------------------------------------------+----------------------------------------------+ | RPM :file:`.spec` file option or section | Distutils setup script option | +==========================================+==============================================+ -| Name | ``name`` | +| Name | :option:`name` | +------------------------------------------+----------------------------------------------+ -| Summary (in preamble) | ``description`` | +| Summary (in preamble) | :option:`description` | +------------------------------------------+----------------------------------------------+ -| Version | ``version`` | +| Version | :option:`version` | +------------------------------------------+----------------------------------------------+ -| Vendor | ``author`` and ``author_email``, | -| | or --- & ``maintainer`` and | -| | ``maintainer_email`` | +| Vendor | :option:`author` and :option:`author_email`, | +| | or --- & :option:`maintainer` and | +| | :option:`maintainer_email` | +------------------------------------------+----------------------------------------------+ -| Copyright | ``license`` | +| Copyright | :option:`license` | +------------------------------------------+----------------------------------------------+ -| Url | ``url`` | +| Url | :option:`url` | +------------------------------------------+----------------------------------------------+ -| %description (section) | ``long_description`` | +| %description (section) | :option:`long_description` | +------------------------------------------+----------------------------------------------+ Additionally, there are many options in :file:`.spec` files that don't have @@ -220,27 +211,27 @@ | RPM :file:`.spec` file option | :command:`bdist_rpm` option | default value | | or section | | | +===============================+=============================+=========================+ -| Release | ``release`` | "1" | +| Release | :option:`release` | "1" | +-------------------------------+-----------------------------+-------------------------+ -| Group | ``group`` | "Development/Libraries" | +| Group | :option:`group` | "Development/Libraries" | +-------------------------------+-----------------------------+-------------------------+ -| Vendor | ``vendor`` | (see above) | +| Vendor | :option:`vendor` | (see above) | +-------------------------------+-----------------------------+-------------------------+ -| Packager | ``packager`` | (none) | +| Packager | :option:`packager` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Provides | ``provides`` | (none) | +| Provides | :option:`provides` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Requires | ``requires`` | (none) | +| Requires | :option:`requires` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Conflicts | ``conflicts`` | (none) | +| Conflicts | :option:`conflicts` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Obsoletes | ``obsoletes`` | (none) | +| Obsoletes | :option:`obsoletes` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Distribution | ``distribution_name`` | (none) | +| Distribution | :option:`distribution_name` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| BuildRequires | ``build_requires`` | (none) | +| BuildRequires | :option:`build_requires` | (none) | +-------------------------------+-----------------------------+-------------------------+ -| Icon | ``icon`` | (none) | +| Icon | :option:`icon` | (none) | +-------------------------------+-----------------------------+-------------------------+ Obviously, supplying even a few of these options on the command-line would be @@ -248,8 +239,7 @@ configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If you distribute or package many Python module distributions, you might want to put options that apply to all of them in your personal Distutils configuration -file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable -this file, you can pass the :option:`--no-user-cfg` option to :file:`setup.py`. +file (:file:`~/.pydistutils.cfg`). There are three steps to building a binary RPM package, all of which are handled automatically by the Distutils: @@ -364,7 +354,7 @@ would create a 64bit installation executable on your 32bit version of Windows. To cross-compile, you must download the Python source code and cross-compile -Python itself for the platform you are targeting - it is not possible from a +Python itself for the platform you are targetting - it is not possible from a binary installation of Python (as the .lib etc file for other platforms are not included.) In practice, this means the user of a 32 bit operating system will need to use Visual Studio 2008 to open the diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/configfile.rst --- a/Doc/distutils/configfile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/configfile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -67,9 +67,7 @@ [...] Note that an option spelled :option:`--foo-bar` on the command-line is spelled -``foo_bar`` in configuration files. - -.. _distutils-build-ext-inplace: +:option:`foo_bar` in configuration files. For example, say you want your extensions to be built "in-place"---that is, you have an extension :mod:`pkg.ext`, and you want the compiled extension file @@ -114,7 +112,7 @@ doc/ examples/ -Note that the ``doc_files`` option is simply a whitespace-separated string +Note that the :option:`doc_files` option is simply a whitespace-separated string split across multiple lines for readability. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/examples.rst --- a/Doc/distutils/examples.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/examples.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,7 +11,7 @@ .. seealso:: - `Distutils Cookbook <https://wiki.python.org/moin/Distutils/Cookbook>`_ + `Distutils Cookbook <http://wiki.python.org/moin/Distutils/Cookbook>`_ Collection of recipes showing how to achieve more control over distutils. @@ -22,7 +22,7 @@ If you're just distributing a couple of modules, especially if they don't live in a particular package, you can specify them individually using the -``py_modules`` option in the setup script. +:option:`py_modules` option in the setup script. In the simplest case, you'll have two files to worry about: a setup script and the single module you're distributing, :file:`foo.py` in this example:: @@ -41,12 +41,12 @@ ) Note that the name of the distribution is specified independently with the -``name`` option, and there's no rule that says it has to be the same as +:option:`name` option, and there's no rule that says it has to be the same as the name of the sole module in the distribution (although that's probably a good convention to follow). However, the distribution name is used to generate filenames, so you should stick to letters, digits, underscores, and hyphens. -Since ``py_modules`` is a list, you can of course specify multiple +Since :option:`py_modules` is a list, you can of course specify multiple modules, eg. if you're distributing modules :mod:`foo` and :mod:`bar`, your setup might look like this:: @@ -130,7 +130,7 @@ ) If you want to put modules in directories not named for their package, then you -need to use the ``package_dir`` option again. For example, if the +need to use the :option:`package_dir` option again. For example, if the :file:`src` directory holds modules in the :mod:`foobar` package:: <root>/ @@ -169,8 +169,8 @@ (The empty string also stands for the current directory.) -If you have sub-packages, they must be explicitly listed in ``packages``, -but any entries in ``package_dir`` automatically extend to sub-packages. +If you have sub-packages, they must be explicitly listed in :option:`packages`, +but any entries in :option:`package_dir` automatically extend to sub-packages. (In other words, the Distutils does *not* scan your source tree, trying to figure out which directories correspond to Python packages by looking for :file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: @@ -193,14 +193,17 @@ packages=['foobar', 'foobar.subfoo'], ) +(Again, the empty string in :option:`package_dir` stands for the current +directory.) + .. _single-ext: Single extension module ======================= -Extension modules are specified using the ``ext_modules`` option. -``package_dir`` has no effect on where extension source files are found; +Extension modules are specified using the :option:`ext_modules` option. +:option:`package_dir` has no effect on where extension source files are found; it only affects the source for pure Python modules. The simplest case, a single extension module in a single C source file, is:: @@ -264,7 +267,7 @@ desc = """\ My description - ============== + ============= This is the description of the ``foobar`` package. """ @@ -281,48 +284,6 @@ warning: check: Title underline too short. (line 2) warning: check: Could not finish the parsing. -Reading the metadata -===================== - -The :func:`distutils.core.setup` function provides a command-line interface -that allows you to query the metadata fields of a project through the -``setup.py`` script of a given project:: - - $ python setup.py --name - distribute - -This call reads the ``name`` metadata by running the -:func:`distutils.core.setup` function. Although, when a source or binary -distribution is created with Distutils, the metadata fields are written -in a static file called :file:`PKG-INFO`. When a Distutils-based project is -installed in Python, the :file:`PKG-INFO` file is copied alongside the modules -and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, -where ``NAME`` is the name of the project, ``VERSION`` its version as defined -in the Metadata, and ``pyX.X`` the major and minor version of Python like -``2.7`` or ``3.2``. - -You can read back this static file, by using the -:class:`distutils.dist.DistributionMetadata` class and its -:func:`read_pkg_file` method:: - - >>> from distutils.dist import DistributionMetadata - >>> metadata = DistributionMetadata() - >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) - >>> metadata.name - 'distribute' - >>> metadata.version - '0.6.8' - >>> metadata.description - 'Easily download, build, install, upgrade, and uninstall Python packages' - -Notice that the class can also be instanciated with a metadata file path to -loads its values:: - - >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' - >>> DistributionMetadata(pkg_info_path).name - 'distribute' - - .. % \section{Multiple extension modules} .. % \label{multiple-ext} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/extending.rst --- a/Doc/distutils/extending.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/extending.rst Mon Jan 25 17:05:13 2016 +0100 @@ -61,7 +61,7 @@ requiring modifications to the Python installation. This is expected to allow third-party extensions to provide support for additional packaging systems, but the commands can be used for anything distutils commands can be used for. A new -configuration option, ``command_packages`` (command-line option +configuration option, :option:`command_packages` (command-line option :option:`--command-packages`), can be used to specify additional packages to be searched for modules implementing commands. Like all distutils options, this can be specified on the command line or in a configuration file. This option @@ -75,7 +75,7 @@ packages searched for command implementations; multiple package names should be separated by commas. When not specified, the search is only performed in the :mod:`distutils.command` package. When :file:`setup.py` is run with the option -``--command-packages distcmds,buildcmds``, however, the packages +:option:`--command-packages` :option:`distcmds,buildcmds`, however, the packages :mod:`distutils.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched in that order. New commands are expected to be implemented in modules of the same name as the command by classes sharing the same name. Given the example diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/index.rst --- a/Doc/distutils/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,29 +1,25 @@ .. _distutils-index: -############################################## - Distributing Python Modules (Legacy version) -############################################## +############################### + Distributing Python Modules +############################### :Authors: Greg Ward, Anthony Baxter :Email: distutils-sig@python.org +:Release: |version| +:Date: |today| This document describes the Python Distribution Utilities ("Distutils") from the module developer's point of view, describing how to use the Distutils to make Python modules and extensions easily available to a wider audience with very little overhead for build/release/install mechanics. -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section <https://packaging.python.org/en/latest/current/>`__ - in the Python Packaging User Guide for more information. - +.. deprecated:: 3.3 + :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and + :ref:`packaging-install-index`. .. toctree:: :maxdepth: 2 - :numbered: introduction.rst setupscript.rst @@ -31,7 +27,15 @@ sourcedist.rst builtdist.rst packageindex.rst + uploading.rst examples.rst extending.rst commandref.rst apiref.rst + +Another document describes how to install modules and extensions packaged +following the above guidelines: + +.. toctree:: + + install.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/install.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/distutils/install.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1086 @@ +.. highlightlang:: none + +.. _install-index: + +***************************** + Installing Python Modules +***************************** + +:Author: Greg Ward +:Release: |version| +:Date: |today| + +.. TODO: Fill in XXX comments + +.. The audience for this document includes people who don't know anything + about Python and aren't about to learn the language just in order to + install and maintain it for their users, i.e. system administrators. + Thus, I have to be sure to explain the basics at some point: + sys.path and PYTHONPATH at least. Should probably give pointers to + other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. + + Finally, it might be useful to include all the material from my "Care + and Feeding of a Python Installation" talk in here somewhere. Yow! + +.. topic:: Abstract + + This document describes the Python Distribution Utilities ("Distutils") from the + end-user's point-of-view, describing how to extend the capabilities of a + standard Python installation by building and installing third-party Python + modules and extensions. + + +.. _inst-intro: + +Introduction +============ + +Although Python's extensive standard library covers many programming needs, +there often comes a time when you need to add some new functionality to your +Python installation in the form of third-party modules. This might be necessary +to support your own programming, or to support an application that you want to +use and that happens to be written in Python. + +In the past, there has been little support for adding third-party modules to an +existing Python installation. With the introduction of the Python Distribution +Utilities (Distutils for short) in Python 2.0, this changed. + +This document is aimed primarily at the people who need to install third-party +Python modules: end-users and system administrators who just need to get some +Python application running, and existing Python programmers who want to add some +new goodies to their toolbox. You don't need to know Python to read this +document; there will be some brief forays into using Python's interactive mode +to explore your installation, but that's it. If you're looking for information +on how to distribute your own Python modules so that others may use them, see +the :ref:`distutils-index` manual. + + +.. _inst-trivial-install: + +Best case: trivial installation +------------------------------- + +In the best case, someone will have prepared a special version of the module +distribution you want to install that is targeted specifically at your platform +and is installed just like any other software on your platform. For example, +the module developer might make an executable installer available for Windows +users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, +Mandrake, and many others), a Debian package for users of Debian-based Linux +systems, and so forth. + +In that case, you would download the installer appropriate to your platform and +do the obvious thing with it: run it if it's an executable installer, ``rpm +--install`` it if it's an RPM, etc. You don't need to run Python or a setup +script, you don't need to compile anything---you might not even need to read any +instructions (although it's always a good idea to do so anyway). + +Of course, things will not always be that easy. You might be interested in a +module distribution that doesn't have an easy-to-use installer for your +platform. In that case, you'll have to start with the source distribution +released by the module's author/maintainer. Installing from a source +distribution is not too hard, as long as the modules are packaged in the +standard way. The bulk of this document is about building and installing +modules from standard source distributions. + + +.. _inst-new-standard: + +The new standard: Distutils +--------------------------- + +If you download a module source distribution, you can tell pretty quickly if it +was packaged and distributed in the standard way, i.e. using the Distutils. +First, the distribution's name and version number will be featured prominently +in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or +:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named +directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the +distribution will contain a setup script :file:`setup.py`, and a file named +:file:`README.txt` or possibly just :file:`README`, which should explain that +building and installing the module distribution is a simple matter of running +one command from a terminal:: + + python setup.py install + +For Windows, this command should be run from a command prompt window +(:menuselection:`Start --> Accessories`):: + + setup.py install + +If all these things are true, then you already know how to build and install the +modules you've just downloaded: Run the command above. Unless you need to +install things in a non-standard way or customize the build process, you don't +really need this manual. Or rather, the above command is everything you need to +get out of this manual. + + +.. _inst-standard-install: + +Standard Build and Install +========================== + +As described in section :ref:`inst-new-standard`, building and installing a module +distribution using the Distutils is usually one simple command to run from a +terminal:: + + python setup.py install + + +.. _inst-platform-variations: + +Platform variations +------------------- + +You should always run the setup command from the distribution root directory, +i.e. the top-level subdirectory that the module source distribution unpacks +into. For example, if you've just downloaded a module source distribution +:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: + + gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 + cd foo-1.0 + python setup.py install + +On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the +archive file to :file:`C:\\Temp`, then it would unpack into +:file:`C:\\Temp\\foo-1.0`; you can use either a archive manipulator with a +graphical user interface (such as WinZip) or a command-line tool (such as +:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a +command prompt window and run:: + + cd c:\Temp\foo-1.0 + python setup.py install + + +.. _inst-splitting-up: + +Splitting the job up +-------------------- + +Running ``setup.py install`` builds and installs all modules in one run. If you +prefer to work incrementally---especially useful if you want to customize the +build process, or if things are going wrong---you can use the setup script to do +one thing at a time. This is particularly helpful when the build and install +will be done by different users---for example, you might want to build a module +distribution and hand it off to a system administrator for installation (or do +it yourself, with super-user privileges). + +For example, you can build everything in one step, and then install everything +in a second step, by invoking the setup script twice:: + + python setup.py build + python setup.py install + +If you do this, you will notice that running the :command:`install` command +first runs the :command:`build` command, which---in this case---quickly notices +that it has nothing to do, since everything in the :file:`build` directory is +up-to-date. + +You may not need this ability to break things down often if all you do is +install modules downloaded off the 'net, but it's very handy for more advanced +tasks. If you get into distributing your own Python modules and extensions, +you'll run lots of individual Distutils commands on their own. + + +.. _inst-how-build-works: + +How building works +------------------ + +As implied above, the :command:`build` command is responsible for putting the +files to install into a *build directory*. By default, this is :file:`build` +under the distribution root; if you're excessively concerned with speed, or want +to keep the source tree pristine, you can change the build directory with the +:option:`--build-base` option. For example:: + + python setup.py build --build-base=/tmp/pybuild/foo-1.0 + +(Or you could do this permanently with a directive in your system or personal +Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this +isn't necessary. + +The default layout for the build tree is as follows:: + + --- build/ --- lib/ + or + --- build/ --- lib.<plat>/ + temp.<plat>/ + +where ``<plat>`` expands to a brief description of the current OS/hardware +platform and Python version. The first form, with just a :file:`lib` directory, +is used for "pure module distributions"---that is, module distributions that +include only pure Python modules. If a module distribution contains any +extensions (modules written in C/C++), then the second form, with two ``<plat>`` +directories, is used. In that case, the :file:`temp.{plat}` directory holds +temporary files generated by the compile/link process that don't actually get +installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory +contains all Python modules (pure Python and extensions) that will be installed. + +In the future, more directories will be added to handle Python scripts, +documentation, binary executables, and whatever else is needed to handle the job +of installing Python modules and applications. + + +.. _inst-how-install-works: + +How installation works +---------------------- + +After the :command:`build` command runs (whether you run it explicitly, or the +:command:`install` command does it for you), the work of the :command:`install` +command is relatively simple: all it has to do is copy everything under +:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation +directory. + +If you don't choose an installation directory---i.e., if you just run ``setup.py +install``\ ---then the :command:`install` command installs to the standard +location for third-party Python modules. This location varies by platform and +by how you built/installed Python itself. On Unix (and Mac OS X, which is also +Unix-based), it also depends on whether the module distribution being installed +is pure Python or contains extensions ("non-pure"): + ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Platform | Standard installation location | Default value | Notes | ++=================+=====================================================+==================================================+=======+ +| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ + +Notes: + +(1) + Most Linux distributions include Python as a standard part of the system, so + :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on + Linux. If you build Python yourself on Linux (or any Unix-like system), the + default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. + +(2) + The default installation directory on Windows was :file:`C:\\Program + Files\\Python` under Python 1.6a1, 1.5.2, and earlier. + +:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python +is installed to, and where it finds its libraries at run-time. They are always +the same under Windows, and very often the same under Unix and Mac OS X. You +can find out what your Python installation uses for :file:`{prefix}` and +:file:`{exec-prefix}` by running Python in interactive mode and typing a few +simple commands. Under Unix, just type ``python`` at the shell prompt. Under +Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> +Python (command line)`. Once the interpreter is started, you type Python code +at the prompt. For example, on my Linux system, I type the three Python +statements shown below, and get the output as shown, to find out my +:file:`{prefix}` and :file:`{exec-prefix}`:: + + Python 2.4 (#26, Aug 7 2004, 17:19:02) + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.prefix + '/usr' + >>> sys.exec_prefix + '/usr' + +A few other placeholders are used in this document: :file:`{X.Y}` stands for the +version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by +the value of :data:`sys.abiflags` or the empty string for platforms which don't +define ABI flags; :file:`{distname}` will be replaced by the name of the module +distribution being installed. Dots and capitalization are important in the +paths; for example, a value that uses ``python3.2`` on UNIX will typically use +``Python32`` on Windows. + +If you don't want to install modules to the standard location, or if you don't +have permission to write there, then you need to read about alternate +installations in section :ref:`inst-alt-install`. If you want to customize your +installation directories more heavily, see section :ref:`inst-custom-install` on +custom installations. + + +.. _inst-alt-install: + +Alternate Installation +====================== + +Often, it is necessary or desirable to install modules to a location other than +the standard location for third-party Python modules. For example, on a Unix +system you might not have permission to write to the standard third-party module +directory. Or you might wish to try out a module before making it a standard +part of your local Python installation. This is especially true when upgrading +a distribution already present: you want to make sure your existing base of +scripts still works with the new version before actually upgrading. + +The Distutils :command:`install` command is designed to make installing module +distributions to an alternate location simple and painless. The basic idea is +that you supply a base directory for the installation, and the +:command:`install` command picks a set of directories (called an *installation +scheme*) under this base directory in which to install files. The details +differ across platforms, so read whichever of the following sections applies to +you. + +Note that the various alternate installation schemes are mutually exclusive: you +can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or +``--install-base`` and ``--install-platbase``, but you can't mix from these +groups. + + +.. _inst-alt-install-user: + +Alternate installation: the user scheme +--------------------------------------- + +This scheme is designed to be the most convenient solution for users that don't +have write permission to the global site-packages directory or don't want to +install into it. It is enabled with a simple option:: + + python setup.py install --user + +Files will be installed into subdirectories of :data:`site.USER_BASE` (written +as :file:`{userbase}` hereafter). This scheme installs pure Python modules and +extension modules in the same location (also known as :data:`site.USER_SITE`). +Here are the values for UNIX, including Mac OS X: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{userbase}/lib/python{X.Y}/site-packages` +scripts :file:`{userbase}/bin` +data :file:`{userbase}` +C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` +=============== =========================================================== + +And here are the values used on Windows: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{userbase}\\Python{XY}\\site-packages` +scripts :file:`{userbase}\\Scripts` +data :file:`{userbase}` +C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` +=============== =========================================================== + +The advantage of using this scheme compared to the other ones described below is +that the user site-packages directory is under normal conditions always included +in :data:`sys.path` (see :mod:`site` for more information), which means that +there is no additional step to perform after running the :file:`setup.py` script +to finalize the installation. + +The :command:`build_ext` command also has a ``--user`` option to add +:file:`{userbase}/include` to the compiler search path for header files and +:file:`{userbase}/lib` to the compiler search path for libraries as well as to +the runtime search path for shared C libraries (rpath). + + +.. _inst-alt-install-home: + +Alternate installation: the home scheme +--------------------------------------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the idea of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +This scheme can be used by anyone, regardless of the operating system they +are installing for. + +Installing a new module distribution is as simple as :: + + python setup.py install --home=<dir> + +where you can supply any directory you like for the :option:`--home` option. On +Unix, lazy typists can just type a tilde (``~``); the :command:`install` command +will expand this to your home directory:: + + python setup.py install --home=~ + +To make Python find the distributions installed with this scheme, you may have +to :ref:`modify Python's search path <inst-search-path>` or edit +:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit +:data:`sys.path`. + +The :option:`--home` option defines the installation base directory. Files are +installed to the following directories under the installation base as follows: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{home}/lib/python` +scripts :file:`{home}/bin` +data :file:`{home}` +C headers :file:`{home}/include/python/{distname}` +=============== =========================================================== + +(Mentally replace slashes with backslashes if you're on Windows.) + + +.. _inst-alt-install-prefix-unix: + +Alternate installation: Unix (the prefix scheme) +------------------------------------------------ + +The "prefix scheme" is useful when you wish to use one Python installation to +perform the build/install (i.e., to run the setup script), but install modules +into the third-party module directory of a different Python installation (or +something that looks like a different Python installation). If this sounds a +trifle unusual, it is---that's why the user and home schemes come before. However, +there are at least two known cases where the prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. This can be done with :: + + /usr/bin/python setup.py install --prefix=/usr/local + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could +be done with :: + + /usr/local/bin/python setup.py install --prefix=/mnt/@server/export + +In either case, the :option:`--prefix` option defines the installation base, and +the :option:`--exec-prefix` option defines the platform-specific installation +base, which is used for platform-specific files. (Currently, this just means +non-pure module distributions, but could be expanded to C libraries, binary +executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to +:option:`--prefix`. Files are installed as follows: + +================= ========================================================== +Type of file Installation directory +================= ========================================================== +Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` +extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` +scripts :file:`{prefix}/bin` +data :file:`{prefix}` +C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` +================= ========================================================== + +There is no requirement that :option:`--prefix` or :option:`--exec-prefix` +actually point to an alternate Python installation; if the directories listed +above do not already exist, they are created at installation time. + +Incidentally, the real reason the prefix scheme is important is simply that a +standard Unix installation uses the prefix scheme, but with :option:`--prefix` +and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and +``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, +but every time you run ``python setup.py install`` without any other options, +you're using it. + +Note that installing extensions to an alternate Python installation has no +effect on how those extensions are built: in particular, the Python header files +(:file:`Python.h` and friends) installed with the Python interpreter used to run +the setup script will be used in compiling extensions. It is your +responsibility to ensure that the interpreter used to run extensions installed +in this way is compatible with the interpreter used to build them. The best way +to do this is to ensure that the two interpreters are the same version of Python +(possibly different builds, or possibly copies of the same build). (Of course, +if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an +alternate Python installation, this is immaterial.) + + +.. _inst-alt-install-prefix-windows: + +Alternate installation: Windows (the prefix scheme) +--------------------------------------------------- + +Windows has no concept of a user's home directory, and since the standard Python +installation under Windows is simpler than under Unix, the :option:`--prefix` +option has traditionally been used to install additional packages in separate +locations on Windows. :: + + python setup.py install --prefix="\Temp\Python" + +to install modules to the :file:`\\Temp\\Python` directory on the current drive. + +The installation base is defined by the :option:`--prefix` option; the +:option:`--exec-prefix` option is not supported under Windows, which means that +pure Python modules and extension modules are installed into the same location. +Files are installed as follows: + +=============== ========================================================== +Type of file Installation directory +=============== ========================================================== +modules :file:`{prefix}\\Lib\\site-packages` +scripts :file:`{prefix}\\Scripts` +data :file:`{prefix}` +C headers :file:`{prefix}\\Include\\{distname}` +=============== ========================================================== + + +.. _inst-custom-install: + +Custom Installation +=================== + +Sometimes, the alternate installation schemes described in section +:ref:`inst-alt-install` just don't do what you want. You might want to tweak just +one or two directories while keeping everything under the same base directory, +or you might want to completely redefine the installation scheme. In either +case, you're creating a *custom installation scheme*. + +To create a custom installation scheme, you start with one of the alternate +schemes and override some of the installation directories used for the various +types of files, using these options: + +====================== ======================= +Type of file Override option +====================== ======================= +Python modules ``--install-purelib`` +extension modules ``--install-platlib`` +all modules ``--install-lib`` +scripts ``--install-scripts`` +data ``--install-data`` +C headers ``--install-headers`` +====================== ======================= + +These override options can be relative, absolute, +or explicitly defined in terms of one of the installation base directories. +(There are two installation base directories, and they are normally the same--- +they only differ when you use the Unix "prefix scheme" and supply different +``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will +override values computed or given for ``--install-purelib`` and +``--install-platlib``, and is recommended for schemes that don't make a +difference between Python and extension modules.) + +For example, say you're installing a module distribution to your home directory +under Unix---but you want scripts to go in :file:`~/scripts` rather than +:file:`~/bin`. As you might expect, you can override this directory with the +:option:`--install-scripts` option; in this case, it makes most sense to supply +a relative path, which will be interpreted relative to the installation base +directory (your home directory, in this case):: + + python setup.py install --home=~ --install-scripts=scripts + +Another Unix example: suppose your Python installation was built and installed +with a prefix of :file:`/usr/local/python`, so under a standard installation +scripts will wind up in :file:`/usr/local/python/bin`. If you want them in +:file:`/usr/local/bin` instead, you would supply this absolute directory for the +:option:`--install-scripts` option:: + + python setup.py install --install-scripts=/usr/local/bin + +(This performs an installation using the "prefix scheme," where the prefix is +whatever your Python interpreter was installed with--- :file:`/usr/local/python` +in this case.) + +If you maintain Python on Windows, you might want third-party modules to live in +a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` +itself. This is almost as easy as customizing the script installation directory +---you just have to remember that there are two types of modules to worry about, +Python and extension modules, which can conveniently be both controlled by one +option:: + + python setup.py install --install-lib=Site + +The specified installation directory is relative to :file:`{prefix}`. Of +course, you also have to ensure that this directory is in Python's module +search path, such as by putting a :file:`.pth` file in a site directory (see +:mod:`site`). See section :ref:`inst-search-path` to find out how to modify +Python's search path. + +If you want to define an entire installation scheme, you just have to supply all +of the installation directory options. The recommended way to do this is to +supply relative paths; for example, if you want to maintain all Python +module-related files under :file:`python` in your home directory, and you want a +separate directory for each platform that you use your home directory from, you +might define the following installation scheme:: + + python setup.py install --home=~ \ + --install-purelib=python/lib \ + --install-platlib=python/lib.$PLAT \ + --install-scripts=python/scripts + --install-data=python/data + +or, equivalently, :: + + python setup.py install --home=~/python \ + --install-purelib=lib \ + --install-platlib='lib.$PLAT' \ + --install-scripts=scripts + --install-data=data + +``$PLAT`` is not (necessarily) an environment variable---it will be expanded by +the Distutils as it parses your command line options, just as it does when +parsing your configuration file(s). + +Obviously, specifying the entire installation scheme every time you install a +new module distribution would be very tedious. Thus, you can put these options +into your Distutils config file (see section :ref:`inst-config-files`):: + + [install] + install-base=$HOME + install-purelib=python/lib + install-platlib=python/lib.$PLAT + install-scripts=python/scripts + install-data=python/data + +or, equivalently, :: + + [install] + install-base=$HOME/python + install-purelib=lib + install-platlib=lib.$PLAT + install-scripts=scripts + install-data=data + +Note that these two are *not* equivalent if you supply a different installation +base directory when you run the setup script. For example, :: + + python setup.py install --install-base=/tmp + +would install pure modules to :file:`/tmp/python/lib` in the first case, and +to :file:`/tmp/lib` in the second case. (For the second case, you probably +want to supply an installation base of :file:`/tmp/python`.) + +You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample +configuration file input. These are Distutils configuration variables, which +bear a strong resemblance to environment variables. In fact, you can use +environment variables in config files on platforms that have such a notion but +the Distutils additionally define a few extra variables that may not be in your +environment, such as ``$PLAT``. (And of course, on systems that don't have +environment variables, such as Mac OS 9, the configuration variables supplied by +the Distutils are the only ones you can use.) See section :ref:`inst-config-files` +for details. + +.. XXX need some Windows examples---when would custom installation schemes be + needed on those platforms? + + +.. XXX Move this to Doc/using + +.. _inst-search-path: + +Modifying Python's Search Path +------------------------------ + +When the Python interpreter executes an :keyword:`import` statement, it searches +for both Python code and extension modules along a search path. A default value +for the path is configured into the Python binary when the interpreter is built. +You can determine the path by importing the :mod:`sys` module and printing the +value of ``sys.path``. :: + + $ python + Python 2.2 (#11, Oct 3 2002, 13:31:27) + [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.path + ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', + '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', + '/usr/local/lib/python2.3/site-packages'] + >>> + +The null string in ``sys.path`` represents the current working directory. + +The expected convention for locally installed packages is to put them in the +:file:`{...}/site-packages/` directory, but you may want to install Python +modules into some arbitrary directory. For example, your site may have a +convention of keeping all software related to the web server under :file:`/www`. +Add-on Python modules might then belong in :file:`/www/python`, and in order to +import them, this directory must be added to ``sys.path``. There are several +different ways to add the directory. + +The most convenient way is to add a path configuration file to a directory +that's already on Python's path, usually to the :file:`.../site-packages/` +directory. Path configuration files have an extension of :file:`.pth`, and each +line must contain a single path that will be appended to ``sys.path``. (Because +the new paths are appended to ``sys.path``, modules in the added directories +will not override standard modules. This means you can't use this mechanism for +installing fixed versions of standard modules.) + +Paths can be absolute or relative, in which case they're relative to the +directory containing the :file:`.pth` file. See the documentation of +the :mod:`site` module for more information. + +A slightly less convenient way is to edit the :file:`site.py` file in Python's +standard library, and modify ``sys.path``. :file:`site.py` is automatically +imported when the Python interpreter is executed, unless the :option:`-S` switch +is supplied to suppress this behaviour. So you could simply edit +:file:`site.py` and add two lines to it:: + + import sys + sys.path.append('/www/python/') + +However, if you reinstall the same major version of Python (perhaps when +upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by +the stock version. You'd have to remember that it was modified and save a copy +before doing the installation. + +There are two environment variables that can modify ``sys.path``. +:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python +installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, +the search path will be set to ``['', '/www/python/lib/pythonX.Y/', +'/www/python/lib/pythonX.Y/plat-linux2', ...]``. + +The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be +added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is +set to ``/www/python:/opt/py``, the search path will begin with +``['/www/python', '/opt/py']``. (Note that directories must exist in order to +be added to ``sys.path``; the :mod:`site` module removes paths that don't +exist.) + +Finally, ``sys.path`` is just a regular Python list, so any Python application +can modify it by adding or removing entries. + + +.. _inst-config-files: + +Distutils Configuration Files +============================= + +As mentioned above, you can use Distutils configuration files to record personal +or site preferences for any Distutils options. That is, any option to any +command can be stored in one of two or three (depending on your platform) +configuration files, which will be consulted before the command-line is parsed. +This means that configuration files will override default values, and the +command-line will in turn override configuration files. Furthermore, if +multiple configuration files apply, values from "earlier" files are overridden +by "later" files. + + +.. _inst-config-filenames: + +Location and names of config files +---------------------------------- + +The names and locations of the configuration files vary slightly across +platforms. On Unix and Mac OS X, the three configuration files (in the order +they are processed) are: + ++--------------+----------------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+==========================================================+=======+ +| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | ++--------------+----------------------------------------------------------+-------+ +| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | ++--------------+----------------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+----------------------------------------------------------+-------+ + +And on Windows, the configuration files are: + ++--------------+-------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+=================================================+=======+ +| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | ++--------------+-------------------------------------------------+-------+ +| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | ++--------------+-------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+-------------------------------------------------+-------+ + +On all platforms, the "personal" file can be temporarily disabled by +passing the `--no-user-cfg` option. + +Notes: + +(1) + Strictly speaking, the system-wide configuration file lives in the directory + where the Distutils are installed; under Python 1.6 and later on Unix, this is + as shown. For Python 1.5.2, the Distutils will normally be installed to + :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system + configuration file should be put there under Python 1.5.2. + +(2) + On Unix, if the :envvar:`HOME` environment variable is not defined, the user's + home directory will be determined with the :func:`getpwuid` function from the + standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` + function used by Distutils. + +(3) + I.e., in the current directory (usually the location of the setup script). + +(4) + (See also note (1).) Under Python 1.6 and later, Python's default "installation + prefix" is :file:`C:\\Python`, so the system configuration file is normally + :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the + default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not + part of the standard library---so the system configuration file would be + :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python + 1.5.2 installation under Windows. + +(5) + On Windows, if the :envvar:`HOME` environment variable is not defined, + :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will + be tried. This is done by the :func:`os.path.expanduser` function used + by Distutils. + + +.. _inst-config-syntax: + +Syntax of config files +---------------------- + +The Distutils configuration files all have the same syntax. The config files +are grouped into sections. There is one section for each Distutils command, +plus a ``global`` section for global options that affect every command. Each +section consists of one option per line, specified as ``option=value``. + +For example, the following is a complete config file that just forces all +commands to run quietly by default:: + + [global] + verbose=0 + +If this is installed as the system config file, it will affect all processing of +any Python module distribution by any user on the current system. If it is +installed as your personal config file (on systems that support them), it will +affect only module distributions processed by you. And if it is used as the +:file:`setup.cfg` for a particular module distribution, it affects only that +distribution. + +You could override the default "build base" directory and make the +:command:`build\*` commands always forcibly rebuild all files with the +following:: + + [build] + build-base=blib + force=1 + +which corresponds to the command-line arguments :: + + python setup.py build --build-base=blib --force + +except that including the :command:`build` command on the command-line means +that command will be run. Including a particular command in config files has no +such implication; it only means that if the command is run, the options in the +config file will apply. (Or if other commands that derive values from it are +run, they will use the values in the config file.) + +You can find out the complete list of options for any command using the +:option:`--help` option, e.g.:: + + python setup.py build --help + +and you can find out the complete list of global options by using +:option:`--help` without a command:: + + python setup.py --help + +See also the "Reference" section of the "Distributing Python Modules" manual. + + +.. _inst-building-ext: + +Building Extensions: Tips and Tricks +==================================== + +Whenever possible, the Distutils try to use the configuration information made +available by the Python interpreter used to run the :file:`setup.py` script. +For example, the same compiler and linker flags used to compile Python will also +be used for compiling extensions. Usually this will work well, but in +complicated situations this might be inappropriate. This section discusses how +to override the usual Distutils behaviour. + + +.. _inst-tweak-flags: + +Tweaking compiler/linker flags +------------------------------ + +Compiling a Python extension written in C or C++ will sometimes require +specifying custom flags for the compiler and linker in order to use a particular +library or produce a special kind of object code. This is especially true if the +extension hasn't been tested on your platform, or if you're trying to +cross-compile Python. + +In the most general case, the extension author might have foreseen that +compiling the extensions would be complicated, and provided a :file:`Setup` file +for you to edit. This will likely only be done if the module distribution +contains many separate extension modules, or if they often require elaborate +sets of compiler flags in order to work. + +A :file:`Setup` file, if present, is parsed in order to get a list of extensions +to build. Each line in a :file:`Setup` describes a single module. Lines have +the following structure:: + + module ... [sourcefile ...] [cpparg ...] [library ...] + + +Let's examine each of the fields in turn. + +* *module* is the name of the extension module to be built, and should be a + valid Python identifier. You can't just change this in order to rename a module + (edits to the source code would also be needed), so this should be left alone. + +* *sourcefile* is anything that's likely to be a source code file, at least + judging by the filename. Filenames ending in :file:`.c` are assumed to be + written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are + assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed + to be in Objective C. + +* *cpparg* is an argument for the C preprocessor, and is anything starting with + :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. + +* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or + :option:`-L`. + +If a particular platform requires a special library on your platform, you can +add it by editing the :file:`Setup` file and running ``python setup.py build``. +For example, if the module defined by the line :: + + foo foomodule.c + +must be linked with the math library :file:`libm.a` on your platform, simply add +:option:`-lm` to the line:: + + foo foomodule.c -lm + +Arbitrary switches intended for the compiler or the linker can be supplied with +the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: + + foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm + +The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be +appended to the proper command line, so in the above example the compiler will +be passed the :option:`-o32` option, and the linker will be passed +:option:`-shared`. If a compiler option requires an argument, you'll have to +supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` +the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. + +Compiler flags can also be supplied through setting the :envvar:`CFLAGS` +environment variable. If set, the contents of :envvar:`CFLAGS` will be added to +the compiler flags specified in the :file:`Setup` file. + + +.. _inst-non-ms-compilers: + +Using non-Microsoft compilers on Windows +---------------------------------------- + +.. sectionauthor:: Rene Liebscher <R.Liebscher@gmx.de> + + + +Borland/CodeGear C++ +^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the necessary steps to use Distutils with the Borland +C++ compiler version 5.5. First you have to know that Borland's object file +format (OMF) is different from the format used by the Python version you can +download from the Python or ActiveState Web site. (Python is built with +Microsoft Visual C++, which uses COFF as the object file format.) For this +reason you have to convert Python's library :file:`python25.lib` into the +Borland format. You can do this as follows: + +.. Should we mention that users have to create cfg-files for the compiler? +.. see also http://community.borland.com/article/0,1410,21205,00.html + +:: + + coff2omf python25.lib python25_bcpp.lib + +The :file:`coff2omf` program comes with the Borland compiler. The file +:file:`python25.lib` is in the :file:`Libs` directory of your Python +installation. If your extension uses other libraries (zlib, ...) you have to +convert them too. + +The converted files have to reside in the same directories as the normal +libraries. + +How does Distutils manage to use these libraries with their changed names? If +the extension needs a library (eg. :file:`foo`) Distutils checks first if it +finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then +uses this library. In the case it doesn't find such a special library it uses +the default name (:file:`foo.lib`.) [#]_ + +To let Distutils compile your extension with Borland C++ you now have to type:: + + python setup.py build --compiler=bcpp + +If you want to use the Borland C++ compiler as the default, you could specify +this in your personal or system-wide configuration file for Distutils (see +section :ref:`inst-config-files`.) + + +.. seealso:: + + `C++Builder Compiler <http://www.codegear.com/downloads/free/cppbuilder>`_ + Information about the free C++ compiler from Borland, including links to the + download pages. + + `Creating Python Extensions Using Borland's Free Compiler <http://www.cyberus.ca/~g_will/pyExtenDL.shtml>`_ + Document describing how to use Borland's free command-line C++ compiler to build + Python. + + +GNU C / Cygwin / MinGW +^^^^^^^^^^^^^^^^^^^^^^ + +This section describes the necessary steps to use Distutils with the GNU C/C++ +compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter +that was built with Cygwin, everything should work without any of these +following steps. + +Not all extensions can be built with MinGW or Cygwin, but many can. Extensions +most likely to not work are those that use C++ or depend on Microsoft Visual C +extensions. + +To let Distutils compile your extension with Cygwin you have to type:: + + python setup.py build --compiler=cygwin + +and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: + + python setup.py build --compiler=mingw32 + +If you want to use any of these options/compilers as default, you should +consider writing it in your personal or system-wide configuration file for +Distutils (see section :ref:`inst-config-files`.) + +Older Versions of Python and MinGW +"""""""""""""""""""""""""""""""""" +The following instructions only apply if you're using a version of Python +inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with +binutils-2.13.90-20030111-1). + +These compilers require some special libraries. This task is more complex than +for Borland's C++, because there is no program to convert the library. First +you have to create a list of symbols which the Python DLL exports. (You can find +a good program for this task at +http://www.emmestech.com/software/pexports-0.43/download_pexports.html). + +.. I don't understand what the next line means. --amk +.. (inclusive the references on data structures.) + +:: + + pexports python25.dll >python25.def + +The location of an installed :file:`python25.dll` will depend on the +installation options and the version and language of Windows. In a "just for +me" installation, it will appear in the root of the installation directory. In +a shared installation, it will be located in the system directory. + +Then you can create from these information an import library for gcc. :: + + /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a + +The resulting library has to be placed in the same directory as +:file:`python25.lib`. (Should be the :file:`libs` directory under your Python +installation directory.) + +If your extension uses other libraries (zlib,...) you might have to convert +them too. The converted files have to reside in the same directories as the +normal libraries do. + + +.. seealso:: + + `Building Python modules on MS Windows platform with MinGW <http://www.zope.org/Members/als/tips/win32_mingw_modules>`_ + Information about building the required libraries for the MinGW environment. + + +.. rubric:: Footnotes + +.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries + of the same name. + +.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for more + information + +.. [#] Then you have no POSIX emulation available, but you also don't need + :file:`cygwin1.dll`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/introduction.rst --- a/Doc/distutils/introduction.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/introduction.rst Mon Jan 25 17:05:13 2016 +0100 @@ -156,8 +156,8 @@ pure Python module a module written in Python and contained in a single :file:`.py` file (and - possibly associated :file:`.pyc` files). Sometimes referred to as a - "pure module." + possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes referred + to as a "pure module." extension module a module written in the low-level language of the Python implementation: C/C++ @@ -210,3 +210,5 @@ the top-level directory of your source tree (or source distribution); the directory where :file:`setup.py` exists. Generally :file:`setup.py` will be run from this directory. + + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/packageindex.rst --- a/Doc/distutils/packageindex.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/packageindex.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,64 +1,12 @@ -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - .. _package-index: -******************************* -The Python Package Index (PyPI) -******************************* +********************************** +Registering with the Package Index +********************************** -The `Python Package Index (PyPI)`_ stores :ref:`meta-data <meta-data>` -describing distributions packaged with distutils, as well as package data like -distribution files if a package author wishes. - -Distutils provides the :command:`register` and :command:`upload` commands for -pushing meta-data and distribution files to PyPI, respectively. See -:ref:`package-commands` for information on these commands. - - -PyPI overview -============= - -PyPI lets you submit any number of versions of your distribution to the index. -If you alter the meta-data for a particular version, you can submit it again -and the index will be updated. - -PyPI holds a record for each (name, version) combination submitted. The first -user to submit information for a given name is designated the Owner of that -name. Changes can be submitted through the :command:`register` command or -through the web interface. Owners can designate other users as Owners or -Maintainers. Maintainers can edit the package information, but not designate -new Owners or Maintainers. - -By default PyPI displays only the newest version of a given package. The web -interface lets one change this default behavior and manually select which -versions to display and hide. - -For each version, PyPI displays a home page. The home page is created from -the ``long_description`` which can be submitted via the :command:`register` -command. See :ref:`package-display` for more information. - - -.. _package-commands: - -Distutils commands -================== - -Distutils exposes two commands for submitting package data to PyPI: the -:ref:`register <package-register>` command for submitting meta-data to PyPI -and the :ref:`upload <package-upload>` command for submitting distribution -files. Both commands read configuration data from a special file called a -:ref:`.pypirc file <pypirc>`. - - -.. _package-register: - -The ``register`` command ------------------------- - -The distutils command :command:`register` is used to submit your distribution's -meta-data to an index server. It is invoked as follows:: +The Python Package Index (PyPI) holds meta-data describing distributions +packaged with distutils. The distutils command :command:`register` is used to +submit your distribution's meta-data to the index. It is invoked as follows:: python setup.py register @@ -73,8 +21,7 @@ Your selection [default 1]: Note: if your username and password are saved locally, you will not see this -menu. Also, refer to :ref:`pypirc` for how to store your credentials in a -:file:`.pypirc` file. +menu. If you have not registered with PyPI, then you will need to do so now. You should choose option 2, and enter your details as required. Soon after @@ -85,78 +32,28 @@ prompted for your PyPI username and password, and :command:`register` will then submit your meta-data to the index. -See :ref:`package-cmdoptions` for options to the :command:`register` command. +You may submit any number of versions of your distribution to the index. If you +alter the meta-data for a particular version, you may submit it again and the +index will be updated. +PyPI holds a record for each (name, version) combination submitted. The first +user to submit information for a given name is designated the Owner of that +name. They may submit changes through the :command:`register` command or through +the web interface. They may also designate other users as Owners or Maintainers. +Maintainers may edit the package information, but not designate other Owners or +Maintainers. -.. _package-upload: +By default PyPI will list all versions of a given package. To hide certain +versions, the Hidden property should be set to yes. This must be edited through +the web interface. -The ``upload`` command ----------------------- - -The distutils command :command:`upload` pushes the distribution files to PyPI. - -The command is invoked immediately after building one or more distribution -files. For example, the command :: - - python setup.py sdist bdist_wininst upload - -will cause the source distribution and the Windows installer to be uploaded to -PyPI. Note that these will be uploaded even if they are built using an earlier -invocation of :file:`setup.py`, but that only distributions named on the command -line for the invocation including the :command:`upload` command are uploaded. - -If a :command:`register` command was previously called in the same command, -and if the password was entered in the prompt, :command:`upload` will reuse the -entered password. This is useful if you do not want to store a password in -clear text in a :file:`.pypirc` file. - -You can use the ``--sign`` option to tell :command:`upload` to sign each -uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must -be available for execution on the system :envvar:`PATH`. You can also specify -which key to use for signing using the ``--identity=name`` option. - -See :ref:`package-cmdoptions` for additional options to the :command:`upload` -command. - - -.. _package-cmdoptions: - -Additional command options --------------------------- - -This section describes options common to both the :command:`register` and -:command:`upload` commands. - -The ``--repository`` or ``-r`` option lets you specify a PyPI server -different from the default. For example:: - - python setup.py sdist bdist_wininst upload -r https://example.com/pypi - -For convenience, a name can be used in place of the URL when the -:file:`.pypirc` file is configured to do so. For example:: - - python setup.py register -r other - -See :ref:`pypirc` for more information on defining alternate servers. - -The ``--show-response`` option displays the full response text from the PyPI -server, which is useful when debugging problems with registering and uploading. - - -.. index:: - single: .pypirc file - single: Python Package Index (PyPI); .pypirc file .. _pypirc: -The ``.pypirc`` file --------------------- +The .pypirc file +================ -The :command:`register` and :command:`upload` commands both check for the -existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. -If this file exists, the command uses the username, password, and repository -URL configured in the file. The format of a :file:`.pypirc` file is as -follows:: +The format of the :file:`.pypirc` file is as follows:: [distutils] index-servers = @@ -167,13 +64,13 @@ username: <username> password: <password> -The *distutils* section defines an *index-servers* variable that lists the +The *distutils* section defines a *index-servers* variable that lists the name of all sections describing a repository. Each section describing a repository defines three variables: - *repository*, that defines the url of the PyPI server. Defaults to - ``https://www.python.org/pypi``. + ``http://www.python.org/pypi``. - *username*, which is the registered username on the PyPI server. - *password*, that will be used to authenticate. If omitted the user will be prompt to type it when needed. @@ -192,56 +89,16 @@ password: <password> [other] - repository: https://example.com/pypi + repository: http://example.com/pypi username: <username> password: <password> -This allows the :command:`register` and :command:`upload` commands to be -called with the ``--repository`` option as described in -:ref:`package-cmdoptions`. +:command:`register` can then be called with the -r option to point the +repository to work with:: -Specifically, you might want to add the `PyPI Test Repository -<https://wiki.python.org/moin/TestPyPI>`_ to your ``.pypirc`` to facilitate -testing before doing your first upload to ``PyPI`` itself. + python setup.py register -r http://example.com/pypi +For convenience, the name of the section that describes the repository +may also be used:: -.. _package-display: - -PyPI package display -==================== - -The ``long_description`` field plays a special role at PyPI. It is used by -the server to display a home page for the registered package. - -If you use the `reStructuredText <http://docutils.sourceforge.net/rst.html>`_ -syntax for this field, PyPI will parse it and display an HTML output for -the package home page. - -The ``long_description`` field can be attached to a text file located -in the package:: - - from distutils.core import setup - - with open('README.txt') as file: - long_description = file.read() - - setup(name='Distutils', - long_description=long_description) - -In that case, :file:`README.txt` is a regular reStructuredText text file located -in the root of the package besides :file:`setup.py`. - -To prevent registering broken reStructuredText content, you can use the -:program:`rst2html` program that is provided by the :mod:`docutils` package and -check the ``long_description`` from the command line:: - - $ python setup.py --long-description | rst2html.py > output.html - -:mod:`docutils` will display a warning if there's something wrong with your -syntax. Because PyPI applies additional checks (e.g. by passing ``--no-raw`` -to ``rst2html.py`` in the command above), being able to run the command above -without warnings does not guarantee that PyPI will convert the content -successfully. - - -.. _Python Package Index (PyPI): https://pypi.python.org/pypi + python setup.py register -r other diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/setupscript.rst --- a/Doc/distutils/setupscript.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/setupscript.rst Mon Jan 25 17:05:13 2016 +0100 @@ -28,7 +28,7 @@ description='Python Distribution Utilities', author='Greg Ward', author_email='gward@python.net', - url='https://www.python.org/sigs/distutils-sig/', + url='http://www.python.org/sigs/distutils-sig/', packages=['distutils', 'distutils.command'], ) @@ -62,9 +62,9 @@ Listing whole packages ====================== -The ``packages`` option tells the Distutils to process (build, distribute, +The :option:`packages` option tells the Distutils to process (build, distribute, install, etc.) all pure Python modules found in each package mentioned in the -``packages`` list. In order to do this, of course, there has to be a +:option:`packages` list. In order to do this, of course, there has to be a correspondence between package names and directories in the filesystem. The default correspondence is the most obvious one, i.e. package :mod:`distutils` is found in the directory :file:`distutils` relative to the distribution root. @@ -75,7 +75,7 @@ Distutils will issue a warning but still process the broken package anyway. If you use a different convention to lay out your source directory, that's no -problem: you just have to supply the ``package_dir`` option to tell the +problem: you just have to supply the :option:`package_dir` option to tell the Distutils about your convention. For example, say you keep all Python source under :file:`lib`, so that modules in the "root package" (i.e., not in any package at all) are in :file:`lib`, modules in the :mod:`foo` package are in @@ -94,13 +94,13 @@ package_dir = {'foo': 'lib'} -A ``package: dir`` entry in the ``package_dir`` dictionary implicitly +A ``package: dir`` entry in the :option:`package_dir` dictionary implicitly applies to all packages below *package*, so the :mod:`foo.bar` case is automatically handled here. In this example, having ``packages = ['foo', 'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and -:file:`lib/bar/__init__.py`. (Keep in mind that although ``package_dir`` +:file:`lib/bar/__init__.py`. (Keep in mind that although :option:`package_dir` applies recursively, you must explicitly list all packages in -``packages``: the Distutils will *not* recursively scan your source tree +:option:`packages`: the Distutils will *not* recursively scan your source tree looking for any directory with an :file:`__init__.py` file.) @@ -120,7 +120,7 @@ :mod:`pkg` package. Again, the default package/directory layout implies that these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and that :file:`pkg/__init__.py` exists as well. And again, you can override the -package/directory correspondence using the ``package_dir`` option. +package/directory correspondence using the :option:`package_dir` option. .. _describing-extensions: @@ -138,9 +138,8 @@ .. XXX read over this section All of this is done through another keyword argument to :func:`setup`, the -``ext_modules`` option. ``ext_modules`` is just a list of -:class:`~distutils.core.Extension` instances, each of which describes a -single extension module. +:option:`ext_modules` option. :option:`ext_modules` is just a list of +:class:`Extension` instances, each of which describes a single extension module. Suppose your distribution includes a single extension, called :mod:`foo` and implemented by :file:`foo.c`. If no additional instructions to the compiler/linker are needed, describing this extension is quite simple:: @@ -166,8 +165,8 @@ Extension names and packages ---------------------------- -The first argument to the :class:`~distutils.core.Extension` constructor is -always the name of the extension, including any package names. For example, :: +The first argument to the :class:`Extension` constructor is always the name of +the extension, including any package names. For example, :: Extension('foo', ['src/foo1.c', 'src/foo2.c']) @@ -181,7 +180,7 @@ resulting extension lives. If you have a number of extensions all in the same package (or all under the -same base package), use the ``ext_package`` keyword argument to +same base package), use the :option:`ext_package` keyword argument to :func:`setup`. For example, :: setup(..., @@ -197,8 +196,7 @@ Extension source files ---------------------- -The second argument to the :class:`~distutils.core.Extension` constructor is -a list of source +The second argument to the :class:`Extension` constructor is a list of source files. Since the Distutils currently only support C, C++, and Objective-C extensions, these are normally C/C++/Objective-C source files. (Be sure to use appropriate extensions to distinguish C++\ source files: :file:`.cc` and @@ -234,9 +232,9 @@ Preprocessor options -------------------- -Three optional arguments to :class:`~distutils.core.Extension` will help if -you need to specify include directories to search or preprocessor macros to -define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. +Three optional arguments to :class:`Extension` will help if you need to specify +include directories to search or preprocessor macros to define/undefine: +``include_dirs``, ``define_macros``, and ``undef_macros``. For example, if your extension requires header files in the :file:`include` directory under your distribution root, use the ``include_dirs`` option:: @@ -336,24 +334,24 @@ There are still some other options which can be used to handle special cases. -The ``optional`` option is a boolean; if it is true, +The :option:`optional` option is a boolean; if it is true, a build failure in the extension will not abort the build process, but instead simply not install the failing extension. -The ``extra_objects`` option is a list of object files to be passed to the +The :option:`extra_objects` option is a list of object files to be passed to the linker. These files must not have extensions, as the default extension for the compiler is used. -``extra_compile_args`` and ``extra_link_args`` can be used to +:option:`extra_compile_args` and :option:`extra_link_args` can be used to specify additional command line options for the respective compiler and linker command lines. -``export_symbols`` is only useful on Windows. It can contain a list of +:option:`export_symbols` is only useful on Windows. It can contain a list of symbols (functions or variables) to be exported. This option is not needed when building compiled extensions: Distutils will automatically add ``initmodule`` to the list of exported symbols. -The ``depends`` option is a list of files that the extension depends on +The :option:`depends` option is a list of files that the extension depends on (for example header files). The build command will call the compiler on the sources to rebuild extension if any on this files has been modified since the previous build. @@ -449,7 +447,7 @@ the current interpreter location. The :option:`--executable` (or :option:`-e`) option will allow the interpreter path to be explicitly overridden. -The ``scripts`` option simply is a list of files to be handled in this +The :option:`scripts` option simply is a list of files to be handled in this way. From the PyXML setup script:: setup(..., @@ -514,11 +512,11 @@ Installing Additional Files =========================== -The ``data_files`` option can be used to specify additional files needed +The :option:`data_files` option can be used to specify additional files needed by the module distribution: configuration files, message catalogs, data files, anything which doesn't fit in the previous categories. -``data_files`` specifies a sequence of (*directory*, *files*) pairs in the +:option:`data_files` specifies a sequence of (*directory*, *files*) pairs in the following way:: setup(..., @@ -539,7 +537,7 @@ directory information from *files* is used to determine the final location of the installed file; only the name of the file is used. -You can specify the ``data_files`` options as a simple sequence of files +You can specify the :option:`data_files` options as a simple sequence of files without specifying a target directory, but this is not recommended, and the :command:`install` command will print a warning in this case. To install data files directly in the target directory, an empty string should be given as the @@ -603,18 +601,16 @@ It is recommended that versions take the form *major.minor[.patch[.sub]]*. (3) - Either the author or the maintainer must be identified. If maintainer is - provided, distutils lists it as the author in :file:`PKG-INFO`. + Either the author or the maintainer must be identified. (4) These fields should not be used if your package is to be compatible with Python versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - <https://pypi.python.org/pypi>`_. + <http://pypi.python.org/pypi>`_. (5) - The ``long_description`` field is used by PyPI when you are - :ref:`registering <package-register>` a package, to - :ref:`build its home page <package-display>`. + The ``long_description`` field is used by PyPI when you are registering a + package, to build its home page. (6) The ``license`` field is a text indicating the license covering the @@ -628,7 +624,7 @@ 'long string' Multiple lines of plain text in reStructuredText format (see - http://docutils.sourceforge.net/). + http://docutils.sf.net/). 'list of strings' See below. @@ -650,7 +646,7 @@ 1.0.1a2 the second alpha release of the first patch version of 1.0 -``classifiers`` are specified in a Python list:: +:option:`classifiers` are specified in a Python list:: setup(..., classifiers=[ @@ -671,7 +667,19 @@ ], ) -.. _debug-setup-script: +If you wish to include classifiers in your :file:`setup.py` file and also wish +to remain backwards-compatible with Python releases prior to 2.2.3, then you can +include the following code fragment in your :file:`setup.py` before the +:func:`setup` call. :: + + # patch distutils if it can't cope with the "classifiers" or + # "download_url" keywords + from sys import version + if version < '2.2.3': + from distutils.dist import DistributionMetadata + DistributionMetadata.classifiers = None + DistributionMetadata.download_url = None + Debugging the setup script ========================== @@ -688,8 +696,7 @@ and see that it's a permission problem. On the other hand, this doesn't help the developer to find the cause of the -failure. For this purpose, the :envvar:`DISTUTILS_DEBUG` environment variable can be set +failure. For this purpose, the DISTUTILS_DEBUG environment variable can be set to anything except an empty string, and distutils will now print detailed -information about what it is doing, dump the full traceback when an exception -occurs, and print the whole command line when an external program (like a C -compiler) fails. +information what it is doing, and prints the full traceback in case an exception +occurs. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/sourcedist.rst --- a/Doc/distutils/sourcedist.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/sourcedist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,24 +26,18 @@ +===========+=========================+=========+ | ``zip`` | zip file (:file:`.zip`) | (1),(3) | +-----------+-------------------------+---------+ -| ``gztar`` | gzip'ed tar file | \(2) | +| ``gztar`` | gzip'ed tar file | (2),(4) | | | (:file:`.tar.gz`) | | +-----------+-------------------------+---------+ -| ``bztar`` | bzip2'ed tar file | | +| ``bztar`` | bzip2'ed tar file | \(4) | | | (:file:`.tar.bz2`) | | +-----------+-------------------------+---------+ -| ``xztar`` | xz'ed tar file | | -| | (:file:`.tar.xz`) | | -+-----------+-------------------------+---------+ | ``ztar`` | compressed tar file | \(4) | | | (:file:`.tar.Z`) | | +-----------+-------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | | +| ``tar`` | tar file (:file:`.tar`) | \(4) | +-----------+-------------------------+---------+ -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - Notes: (1) @@ -57,16 +51,8 @@ of the standard Python library since Python 1.6) (4) - requires the :program:`compress` program. Notice that this format is now - pending for deprecation and will be removed in the future versions of Python. - -When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or -``tar``), under Unix you can specify the ``owner`` and ``group`` names -that will be set for each member of the archive. - -For example, if you want all files of the archive to be owned by root:: - - python setup.py sdist --owner=root --group=root + requires external utilities: :program:`tar` and possibly one of :program:`gzip`, + :program:`bzip2`, or :program:`compress` .. _manifest: @@ -78,16 +64,16 @@ generate one), the :command:`sdist` command puts a minimal default set into the source distribution: -* all Python source files implied by the ``py_modules`` and - ``packages`` options +* all Python source files implied by the :option:`py_modules` and + :option:`packages` options -* all C source files mentioned in the ``ext_modules`` or - ``libraries`` options +* all C source files mentioned in the :option:`ext_modules` or + :option:`libraries` options ( .. XXX getting C library sources currently broken---no :meth:`get_source_files` method in :file:`build_clib.py`! -* scripts identified by the ``scripts`` option +* scripts identified by the :option:`scripts` option See :ref:`distutils-installing-scripts`. * anything that looks like a test script: :file:`test/test\*.py` (currently, the @@ -173,7 +159,7 @@ #. include all Python source files in the :file:`distutils` and :file:`distutils/command` subdirectories (because packages corresponding to - those two directories were mentioned in the ``packages`` option in the + those two directories were mentioned in the :option:`packages` option in the setup script---see section :ref:`setup-script`) #. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard diff -r 6db40a9955dc -r 0d413f60cc23 Doc/distutils/uploading.rst --- a/Doc/distutils/uploading.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/distutils/uploading.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,76 @@ -:orphan: +.. _package-upload: *************************************** Uploading Packages to the Package Index *************************************** -The contents of this page have moved to the section :ref:`package-index`. +The Python Package Index (PyPI) not only stores the package info, but also the +package data if the author of the package wishes to. The distutils command +:command:`upload` pushes the distribution files to PyPI. + +The command is invoked immediately after building one or more distribution +files. For example, the command :: + + python setup.py sdist bdist_wininst upload + +will cause the source distribution and the Windows installer to be uploaded to +PyPI. Note that these will be uploaded even if they are built using an earlier +invocation of :file:`setup.py`, but that only distributions named on the command +line for the invocation including the :command:`upload` command are uploaded. + +The :command:`upload` command uses the username, password, and repository URL +from the :file:`$HOME/.pypirc` file (see section :ref:`pypirc` for more on this +file). If a :command:`register` command was previously called in the same command, +and if the password was entered in the prompt, :command:`upload` will reuse the +entered password. This is useful if you do not want to store a clear text +password in the :file:`$HOME/.pypirc` file. + +You can specify another PyPI server with the :option:`--repository=*url*` option:: + + python setup.py sdist bdist_wininst upload -r http://example.com/pypi + +See section :ref:`pypirc` for more on defining several servers. + +You can use the :option:`--sign` option to tell :command:`upload` to sign each +uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must +be available for execution on the system :envvar:`PATH`. You can also specify +which key to use for signing using the :option:`--identity=*name*` option. + +Other :command:`upload` options include :option:`--repository=<url>` or +:option:`--repository=<section>` where *url* is the url of the server and +*section* the name of the section in :file:`$HOME/.pypirc`, and +:option:`--show-response` (which displays the full response text from the PyPI +server for help in debugging upload problems). + +PyPI package display +==================== + +The ``long_description`` field plays a special role at PyPI. It is used by +the server to display a home page for the registered package. + +If you use the `reStructuredText <http://docutils.sourceforge.net/rst.html>`_ +syntax for this field, PyPI will parse it and display an HTML output for +the package home page. + +The ``long_description`` field can be attached to a text file located +in the package:: + + from distutils.core import setup + + with open('README.txt') as file: + long_description = file.read() + + setup(name='Distutils', + long_description=long_description) + +In that case, :file:`README.txt` is a regular reStructuredText text file located +in the root of the package besides :file:`setup.py`. + +To prevent registering broken reStructuredText content, you can use the +:program:`rst2html` program that is provided by the :mod:`docutils` package and +check the ``long_description`` from the command line:: + + $ python setup.py --long-description | rst2html.py > output.html + +:mod:`docutils` will display a warning if there's something wrong with your +syntax. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/building.rst --- a/Doc/extending/building.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/building.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,58 +1,27 @@ .. highlightlang:: c + .. _building: -***************************** -Building C and C++ Extensions -***************************** - -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. - -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using distutils, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully-initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlightlang:: python - -For modules with ASCII-only names, the function must be named -``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See :PEP:`489#multiple-modules-in-one-library` for details. - - -.. highlightlang:: c - +******************************************** Building C and C++ Extensions with distutils -============================================ +******************************************** .. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> -Extension modules can be built using distutils, which is included in Python. -Since distutils also supports creation of binary packages, users don't -necessarily need a compiler and distutils to install the extension. + +Starting in Python 1.4, Python provides, on Unix, a special make file for +building make files for building dynamically-linked extensions and custom +interpreters. Starting with Python 2.0, this mechanism (known as related to +Makefile.pre.in, and Setup files) is no longer supported. Building custom +interpreters was rarely used, and extension modules can be built using +distutils. + +Building an extension module using distutils requires that distutils is +installed on the build machine, which is included in Python 2.x and available +separately for Python 1.5. Since distutils also supports creation of binary +packages, users don't necessarily need a compiler and distutils to install the +extension. A distutils package contains a driver script, :file:`setup.py`. This is a plain Python file, which, in the most simple case, could look like this:: @@ -81,17 +50,16 @@ function. This takes a variable number of keyword arguments, of which the example above uses only a subset. Specifically, the example specifies meta-information to build packages, and it specifies the contents of the -package. Normally, a package will contain additional modules, like Python +package. Normally, a package will contain of addition modules, like Python source modules, documentation, subpackages, etc. Please refer to the distutils documentation in :ref:`distutils-index` to learn more about the features of distutils; this section explains building extension modules only. It is common to pre-compute arguments to :func:`setup`, to better structure the -driver script. In the example above, the ``ext_modules`` argument to +driver script. In the example above, the\ ``ext_modules`` argument to :func:`setup` is a list of extension modules, each of which is an instance of -the :class:`~distutils.extension.Extension`. In the example, the instance -defines an extension named ``demo`` which is build by compiling a single source -file, :file:`demo.c`. +the :class:`Extension`. In the example, the instance defines an extension named +``demo`` which is build by compiling a single source file, :file:`demo.c`. In many cases, building an extension is more complex, since additional preprocessor defines and libraries may be needed. This is demonstrated in the @@ -112,7 +80,7 @@ description = 'This is a demo package', author = 'Martin v. Loewis', author_email = 'martin@v.loewis.de', - url = 'https://docs.python.org/extending/building', + url = 'http://docs.python.org/extending/building', long_description = ''' This is really just a demo package. ''', diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/embedding.rst --- a/Doc/extending/embedding.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/embedding.rst Mon Jan 25 17:05:13 2016 +0100 @@ -58,27 +58,16 @@ int main(int argc, char *argv[]) { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - Py_SetProgramName(program); /* optional but recommended */ - Py_Initialize(); - PyRun_SimpleString("from time import time,ctime\n" - "print('Today is', ctime(time()))\n"); - if (Py_FinalizeEx() < 0) { - exit(120); - } - PyMem_RawFree(program); - return 0; + Py_Initialize(); + PyRun_SimpleString("from time import time,ctime\n" + "print('Today is', ctime(time()))\n"); + Py_Finalize(); + return 0; } -The :c:func:`Py_SetProgramName` function should be called before -:c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time -libraries. Next, the Python interpreter is initialized with +The above code first initializes the Python interpreter with :c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts +that print the date and time. Afterwards, the :c:func:`Py_Finalize` call shuts the interpreter down, followed by the end of the program. In a real program, you may want to get the Python script from another source, perhaps a text-editor routine, a file, or a database. Getting the Python code from a file can better @@ -146,9 +135,7 @@ in ``argv[2]``. Its integer arguments are the other values of the ``argv`` array. If you :ref:`compile and link <compiling>` this program (let's call the finished executable :program:`call`), and use it to execute a Python -script, such as: - -.. code-block:: python +script, such as:: def multiply(a,b): print("Will compute", a, "times", b) @@ -168,13 +155,13 @@ interesting part with respect to embedding Python starts with :: Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); + pName = PyString_FromString(argv[1]); /* Error checking of pName left out */ pModule = PyImport_Import(pName); After initializing the interpreter, the script is loaded using :c:func:`PyImport_Import`. This routine needs a Python string as its argument, -which is constructed using the :c:func:`PyUnicode_FromString` data conversion +which is constructed using the :c:func:`PyString_FromString` data conversion routine. :: pFunc = PyObject_GetAttrString(pModule, argv[2]); @@ -248,9 +235,7 @@ These two lines initialize the ``numargs`` variable, and make the :func:`emb.numargs` function accessible to the embedded Python interpreter. -With these extensions, the Python script can do things like - -.. code-block:: python +With these extensions, the Python script can do things like :: import emb print("Number of arguments", emb.numargs()) @@ -293,14 +278,14 @@ * ``pythonX.Y-config --cflags`` will give you the recommended flags when compiling:: - $ /opt/bin/python3.4-config --cflags - -I/opt/include/python3.4m -I/opt/include/python3.4m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes + $ /opt/bin/python3.2-config --cflags + -I/opt/include/python3.2m -I/opt/include/python3.2m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes * ``pythonX.Y-config --ldflags`` will give you the recommended flags when linking:: - $ /opt/bin/python3.4-config --ldflags - -L/opt/lib/python3.4/config-3.4m -lpthread -ldl -lutil -lm -lpython3.4m -Xlinker -export-dynamic + $ /opt/bin/python3.2-config --ldflags + -I/opt/lib/python3.2/config-3.2m -lpthread -ldl -lutil -lm -lpython3.2m -Xlinker -export-dynamic .. note:: To avoid confusion between several Python installations (and especially @@ -315,13 +300,9 @@ to find its location) and compilation options. In this case, the :mod:`sysconfig` module is a useful tool to programmatically extract the configuration values that you will want to -combine together. For example: - -.. code-block:: python +combine together:: >>> import sysconfig - >>> sysconfig.get_config_var('LIBS') - '-lpthread -ldl -lutil' >>> sysconfig.get_config_var('LINKFORSHARED') '-Xlinker -export-dynamic' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/extending.rst --- a/Doc/extending/extending.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/extending.rst Mon Jan 25 17:05:13 2016 +0100 @@ -20,17 +20,12 @@ The compilation of an extension module depends on its intended use as well as on your system setup; details are given in later chapters. -.. note:: +Do note that if your use case is calling C library functions or system calls, +you should consider using the :mod:`ctypes` module rather than writing custom +C code. Not only does :mod:`ctypes` let you write Python code to interface +with C code, but it is more portable between implementations of Python than +writing and compiling an extension module which typically ties you to CPython. - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - <http://cffi.readthedocs.org>`_ library rather than writing custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. .. _extending-simpleexample: @@ -375,17 +370,11 @@ int main(int argc, char *argv[]) { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - /* Add a built-in module, before Py_Initialize */ PyImport_AppendInittab("spam", PyInit_spam); /* Pass argv[0] to the Python interpreter */ - Py_SetProgramName(program); + Py_SetProgramName(argv[0]); /* Initialize the Python interpreter. Required. */ Py_Initialize(); @@ -395,12 +384,6 @@ imports it. */ PyImport_ImportModule("spam"); - ... - - PyMem_RawFree(program); - return 0; - } - .. note:: Removing entries from ``sys.modules`` or importing compiled modules into @@ -413,13 +396,6 @@ as :file:`Modules/xxmodule.c`. This file may be used as a template or simply read as an example. -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - .. _compilation: @@ -539,7 +515,7 @@ value of the Python function. :c:func:`PyObject_CallObject` is "reference-count-neutral" with respect to its arguments. In the example a new tuple was created to serve as the argument list, which is :c:func:`Py_DECREF`\ --ed immediately after the :c:func:`PyObject_CallObject` call. +-ed immediately after the call. The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand new object, or it is an existing object whose reference count has been @@ -607,7 +583,7 @@ The :c:func:`PyArg_ParseTuple` function is declared as follows:: - int PyArg_ParseTuple(PyObject *arg, const char *format, ...); + int PyArg_ParseTuple(PyObject *arg, char *format, ...); The *arg* argument must be a tuple object containing an argument list passed from Python to a C function. The *format* argument must be a format string, @@ -700,7 +676,7 @@ The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); + char *format, char *kwlist[], ...); The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of @@ -741,7 +717,9 @@ action, voltage); printf("-- Lovely plumage, the %s -- It's %s!\n", type, state); - Py_RETURN_NONE; + Py_INCREF(Py_None); + + return Py_None; } static PyMethodDef keywdarg_methods[] = { @@ -754,18 +732,13 @@ {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods - }; +:: - PyMODINIT_FUNC - PyInit_keywdarg(void) + void + initkeywdarg(void) { - return PyModule_Create(&keywdargmodule); + /* Create the module and add the functions */ + Py_InitModule("keywdarg", keywdarg_methods); } @@ -777,7 +750,7 @@ This function is the counterpart to :c:func:`PyArg_ParseTuple`. It is declared as follows:: - PyObject *Py_BuildValue(const char *format, ...); + PyObject *Py_BuildValue(char *format, ...); It recognizes a set of format units similar to the ones recognized by :c:func:`PyArg_ParseTuple`, but the arguments (which are input to the function, @@ -879,9 +852,12 @@ from the objects in the cycle, even though there are no further references to the cycle itself. -The cycle detector is able to detect garbage cycles and can reclaim them. -The :mod:`gc` module exposes a way to run the detector (the -:func:`~gc.collect` function), as well as configuration +The cycle detector is able to detect garbage cycles and can reclaim them so long +as there are no finalizers implemented in Python (:meth:`__del__` methods). +When there are such finalizers, the detector exposes the cycles through the +:mod:`gc` module (specifically, the +``garbage`` variable in that module). The :mod:`gc` module also exposes a way +to run the detector (the :func:`collect` function), as well as configuration interfaces and the ability to disable the detector at runtime. The cycle detector is considered an optional component; though it is included by default, it can be disabled at build time using the :option:`--without-cycle-gc` option diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/index.rst --- a/Doc/extending/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,13 +4,16 @@ Extending and Embedding the Python Interpreter ################################################## +:Release: |version| +:Date: |today| + This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how -to embed the Python interpreter in another application, for use as an extension -language. Finally, it shows how to compile and link extension modules so that -they can be loaded dynamically (at run time) into the interpreter, if the -underlying operating system supports this feature. +interpreter with new modules. Those modules can define new functions but also +new object types and their methods. The document also describes how to embed +the Python interpreter in another application, for use as an extension language. +Finally, it shows how to compile and link extension modules so that they can be +loaded dynamically (at run time) into the interpreter, if the underlying +operating system supports this feature. This document assumes basic knowledge about Python. For an informal introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` @@ -21,32 +24,6 @@ For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. - -Recommended third party tools -============================= - -This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like Cython, -``cffi``, SWIG and Numba offer both simpler and more sophisticated -approaches to creating C and C++ extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/en/latest/extensions.html>`_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. - - -Creating extensions without third party tools -============================================= - -This section of the guide covers creating C and C++ extensions without -assistance from third party tools. It is intended primarily for creators -of those tools, rather than being a recommended way to create your own -C extensions. - .. toctree:: :maxdepth: 2 :numbered: @@ -55,17 +32,4 @@ newtypes.rst building.rst windows.rst - -Embedding the CPython runtime in a larger application -===================================================== - -Sometimes, rather than creating an extension that runs inside the Python -interpreter as the main application, it is desirable to instead embed -the CPython runtime inside a larger application. This section covers -some of the details involved in doing that successfully. - -.. toctree:: - :maxdepth: 2 - :numbered: - embedding.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/newtypes.rst --- a/Doc/extending/newtypes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/newtypes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -80,7 +80,7 @@ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ - 0, /* tp_as_async */ + 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ @@ -135,11 +135,11 @@ .. note:: If you want your type to be subclassable from Python, and your type has the same - :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple + :attr:`tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's + in its :attr:`__bases__`, or else it will not be able to call your type's :meth:`__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its + ensuring that your type has a larger value for :attr:`tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your base type will be :class:`object`, or else you will be adding data members to your base type, and therefore increasing its size. @@ -157,10 +157,9 @@ Py_TPFLAGS_DEFAULT, /* tp_flags */ All types should include this constant in their flags. It enables all of the -members defined until at least Python 3.3. If you need further members, -you will need to OR the corresponding flags. +members defined by the current version of Python. -We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: +We provide a doc string for the type in :attr:`tp_doc`. :: "Noddy objects", /* tp_doc */ @@ -169,12 +168,12 @@ the module. We'll expand this example later to have more interesting behavior. For now, all we want to be able to do is to create new :class:`Noddy` objects. -To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation. +To enable object creation, we have to provide a :attr:`tp_new` implementation. In this case, we can just use the default implementation provided by the API function :c:func:`PyType_GenericNew`. We'd like to just assign this to the -:c:member:`~PyTypeObject.tp_new` slot, but we can't, for portability sake, On some platforms or +:attr:`tp_new` slot, but we can't, for portability sake, On some platforms or compilers, we can't statically initialize a structure member with a function -defined in another C module, so, instead, we'll assign the :c:member:`~PyTypeObject.tp_new` slot +defined in another C module, so, instead, we'll assign the :attr:`tp_new` slot in the module initialization function just before calling :c:func:`PyType_Ready`:: @@ -269,13 +268,13 @@ Py_TYPE(self)->tp_free((PyObject*)self); } -which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: +which is assigned to the :attr:`tp_dealloc` member:: (destructor)Noddy_dealloc, /*tp_dealloc*/ This method decrements the reference counts of the two Python attributes. We use :c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members -could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type +could be *NULL*. It then calls the :attr:`tp_free` member of the object's type to free the object's memory. Note that the object's type might not be :class:`NoddyType`, because the object may be an instance of a subclass. @@ -289,13 +288,13 @@ self = (Noddy *)type->tp_alloc(type, 0); if (self != NULL) { - self->first = PyUnicode_FromString(""); + self->first = PyString_FromString(""); if (self->first == NULL) { Py_DECREF(self); return NULL; } - self->last = PyUnicode_FromString(""); + self->last = PyString_FromString(""); if (self->last == NULL) { Py_DECREF(self); return NULL; @@ -307,7 +306,7 @@ return (PyObject *)self; } -and install it in the :c:member:`~PyTypeObject.tp_new` member:: +and install it in the :attr:`tp_new` member:: Noddy_new, /* tp_new */ @@ -327,17 +326,17 @@ created. New methods always accept positional and keyword arguments, but they often ignore the arguments, leaving the argument handling to initializer methods. Note that if the type supports subclassing, the type passed may not be -the type being defined. The new method calls the :c:member:`~PyTypeObject.tp_alloc` slot to -allocate memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather +the type being defined. The new method calls the :attr:`tp_alloc` slot to +allocate memory. We don't fill the :attr:`tp_alloc` slot ourselves. Rather :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, which is :class:`object` by default. Most types use the default allocation. .. note:: - If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's - :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method + If you are creating a co-operative :attr:`tp_new` (one that calls a base type's + :attr:`tp_new` or :meth:`__new__`), you must *not* try to determine what method to call using method resolution order at runtime. Always statically determine - what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via + what type you are going to call, and call its :attr:`tp_new` directly, or via ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your type that also inherit from other Python-defined classes may not work correctly. (Specifically, you may not be able to create instances of such subclasses @@ -374,17 +373,16 @@ return 0; } -by filling the :c:member:`~PyTypeObject.tp_init` slot. :: +by filling the :attr:`tp_init` slot. :: (initproc)Noddy_init, /* tp_init */ -The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It +The :attr:`tp_init` slot is exposed in Python as the :meth:`__init__` method. It is used to initialize an object after it's created. Unlike the new method, we can't guarantee that the initializer is called. The initializer isn't called when unpickling objects and it can be overridden. Our initializer accepts arguments to provide initial values for our instance. Initializers always accept -positional and keyword arguments. Initializers should return either 0 on -success or -1 on error. +positional and keyword arguments. Initializers can be called multiple times. Anyone can call the :meth:`__init__` method on our objects. For this reason, we have to be extra careful when @@ -409,7 +407,7 @@ * when we know that deallocation of the object [#]_ will not cause any calls back into our type's code -* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when +* when decrementing a reference count in a :attr:`tp_dealloc` handler when garbage-collections is not supported [#]_ We want to expose our instance variables as attributes. There are a @@ -425,7 +423,7 @@ {NULL} /* Sentinel */ }; -and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: +and put the definitions in the :attr:`tp_members` slot:: Noddy_members, /* tp_members */ @@ -485,7 +483,7 @@ {NULL} /* Sentinel */ }; -and assign them to the :c:member:`~PyTypeObject.tp_methods` slot:: +and assign them to the :attr:`tp_methods` slot:: Noddy_methods, /* tp_methods */ @@ -542,9 +540,9 @@ return -1; } - if (! PyUnicode_Check(value)) { + if (! PyString_Check(value)) { PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a str"); + "The first attribute value must be a string"); return -1; } @@ -580,7 +578,7 @@ {NULL} /* Sentinel */ }; -and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: +and register it in the :attr:`tp_getset` slot:: Noddy_getseters, /* tp_getset */ @@ -597,7 +595,7 @@ {NULL} /* Sentinel */ }; -We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to +We also need to update the :attr:`tp_init` handler to only allow strings [#]_ to be passed:: static int @@ -715,7 +713,7 @@ .. note:: - Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly + Note that the :attr:`tp_traverse` implementation must name its arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage uniformity across these boring implementations. @@ -752,7 +750,7 @@ reference count drops to zero, we might cause code to run that calls back into the object. In addition, because we now support garbage collection, we also have to worry about code being run that triggers garbage collection. If garbage -collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't +collection is run, our :attr:`tp_traverse` handler could get called. We can't take a chance of having :c:func:`Noddy_traverse` called when a member's reference count has dropped to zero and its value hasn't been set to *NULL*. @@ -772,8 +770,8 @@ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ -That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or -:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection. +That's pretty much it. If we had written custom :attr:`tp_alloc` or +:attr:`tp_free` slots, we'd need to modify them for cyclic-garbage collection. Most extensions will use the versions automatically provided. @@ -832,8 +830,8 @@ This pattern is important when writing a type with custom :attr:`new` and :attr:`dealloc` methods. The :attr:`new` method should not actually create the -memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base -class when calling its :c:member:`~PyTypeObject.tp_new`. +memory for the object with :attr:`tp_alloc`, that will be handled by the base +class when calling its :attr:`tp_new`. When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't @@ -859,8 +857,8 @@ } Before calling :c:func:`PyType_Ready`, the type structure must have the -:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not -necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew` +:attr:`tp_base` slot filled in. When we are deriving a new type, it is not +necessary to fill out the :attr:`tp_alloc` slot with :c:func:`PyType_GenericNew` -- the allocate function from the base type will be inherited. After that, calling :c:func:`PyType_Ready` and adding the type object to the @@ -893,20 +891,20 @@ all the fields you need (even if they're initialized to ``0``) and then change the values to suit your new type. :: - const char *tp_name; /* For printing */ + char *tp_name; /* For printing */ The name of the type - as mentioned in the last section, this will appear in various places, almost entirely for diagnostic purposes. Try to choose something that will be helpful in such a situation! :: - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + int tp_basicsize, tp_itemsize; /* For allocation */ These fields tell the runtime how much memory to allocate when new objects of this type are created. Python has some built-in support for variable length -structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field +structures (think: strings, lists) which is where the :attr:`tp_itemsize` field comes in. This will be dealt with later. :: - const char *tp_doc; + char *tp_doc; Here you can put a string (or its address) that you want returned when the Python script references ``obj.__doc__`` to retrieve the doc string. @@ -930,9 +928,8 @@ This function is called when the reference count of the instance of your type is reduced to zero and the Python interpreter wants to reclaim it. If your type -has memory to free or other clean-up to perform, you can put it here. The -object itself needs to be freed here as well. Here is an example of this -function:: +has memory to free or other clean-up to perform, put it here. The object itself +needs to be freed here as well. Here is an example of this function:: static void newdatatype_dealloc(newdatatypeobject * obj) @@ -965,9 +962,10 @@ if (self->my_callback != NULL) { PyObject *err_type, *err_value, *err_traceback; + int have_error = PyErr_Occurred() ? 1 : 0; - /* This saves the current exception state */ - PyErr_Fetch(&err_type, &err_value, &err_traceback); + if (have_error) + PyErr_Fetch(&err_type, &err_value, &err_traceback); cbresult = PyObject_CallObject(self->my_callback, NULL); if (cbresult == NULL) @@ -975,38 +973,22 @@ else Py_DECREF(cbresult); - /* This restores the saved exception state */ - PyErr_Restore(err_type, err_value, err_traceback); + if (have_error) + PyErr_Restore(err_type, err_value, err_traceback); Py_DECREF(self->my_callback); } Py_TYPE(obj)->tp_free((PyObject*)self); } -.. note:: - There are limitations to what you can safely do in a deallocator function. - First, if your type supports garbage collection (using :c:member:`~PyTypeObject.tp_traverse` - and/or :c:member:`~PyTypeObject.tp_clear`), some of the object's members can have been - cleared or finalized by the time :c:member:`~PyTypeObject.tp_dealloc` is called. Second, in - :c:member:`~PyTypeObject.tp_dealloc`, your object is in an unstable state: its reference - count is equal to zero. Any call to a non-trivial object or API (as in the - example above) might end up calling :c:member:`~PyTypeObject.tp_dealloc` again, causing a - double free and a crash. - - Starting with Python 3.4, it is recommended not to put any complex - finalization code in :c:member:`~PyTypeObject.tp_dealloc`, and instead use the new - :c:member:`~PyTypeObject.tp_finalize` type method. - - .. seealso:: - :pep:`442` explains the new finalization scheme. - -.. index:: - single: string; object representation - builtin: repr Object Presentation ------------------- +.. index:: + builtin: repr + builtin: str + In Python, there are two ways to generate a textual representation of an object: the :func:`repr` function, and the :func:`str` function. (The :func:`print` function just calls :func:`str`.) These handlers are both optional. @@ -1016,26 +998,26 @@ reprfunc tp_repr; reprfunc tp_str; -The :c:member:`~PyTypeObject.tp_repr` handler should return a string object containing a +The :attr:`tp_repr` handler should return a string object containing a representation of the instance for which it is called. Here is a simple example:: static PyObject * newdatatype_repr(newdatatypeobject * obj) { - return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:\%d}}", - obj->obj_UnderlyingDatatypePtr->size); + return PyString_FromFormat("Repr-ified_newdatatype{{size:\%d}}", + obj->obj_UnderlyingDatatypePtr->size); } -If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a -representation that uses the type's :c:member:`~PyTypeObject.tp_name` and a uniquely-identifying +If no :attr:`tp_repr` handler is specified, the interpreter will supply a +representation that uses the type's :attr:`tp_name` and a uniquely-identifying value for the object. -The :c:member:`~PyTypeObject.tp_str` handler is to :func:`str` what the :c:member:`~PyTypeObject.tp_repr` handler +The :attr:`tp_str` handler is to :func:`str` what the :attr:`tp_repr` handler described above is to :func:`repr`; that is, it is called when Python code calls :func:`str` on an instance of your object. Its implementation is very similar -to the :c:member:`~PyTypeObject.tp_repr` function, but the resulting string is intended for human -consumption. If :c:member:`~PyTypeObject.tp_str` is not specified, the :c:member:`~PyTypeObject.tp_repr` handler is +to the :attr:`tp_repr` function, but the resulting string is intended for human +consumption. If :attr:`tp_str` is not specified, the :attr:`tp_repr` handler is used instead. Here is a simple example:: @@ -1043,8 +1025,8 @@ static PyObject * newdatatype_str(newdatatypeobject * obj) { - return PyUnicode_FromFormat("Stringified_newdatatype{{size:\%d}}", - obj->obj_UnderlyingDatatypePtr->size); + return PyString_FromFormat("Stringified_newdatatype{{size:\%d}}", + obj->obj_UnderlyingDatatypePtr->size); } @@ -1100,7 +1082,7 @@ type object. Each descriptor controls access to one attribute of the instance object. Each of the tables is optional; if all three are *NULL*, instances of the type will only have attributes that are inherited from their base type, and -should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as +should leave the :attr:`tp_getattro` and :attr:`tp_setattro` fields *NULL* as well, allowing the base type to handle attributes. The tables are declared as three fields of the type object:: @@ -1109,7 +1091,7 @@ struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; -If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of +If :attr:`tp_methods` is not *NULL*, it must refer to an array of :c:type:`PyMethodDef` structures. Each entry in the table is an instance of this structure:: @@ -1165,13 +1147,13 @@ single: WRITE_RESTRICTED single: RESTRICTED -An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build +An interesting advantage of using the :attr:`tp_members` table to build descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value +As with the :attr:`tp_methods` table, a sentinel entry with a :attr:`name` value of *NULL* is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -1195,7 +1177,7 @@ called, so that if you do need to extend their functionality, you'll understand what needs to be done. -The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute +The :attr:`tp_getattr` handler is called when the object requires an attribute look-up. It is called in the same situations where the :meth:`__getattr__` method of a class would be called. @@ -1206,7 +1188,7 @@ { if (strcmp(name, "data") == 0) { - return PyLong_FromLong(obj->data); + return PyInt_FromLong(obj->data); } PyErr_Format(PyExc_AttributeError, @@ -1215,11 +1197,11 @@ return NULL; } -The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or +The :attr:`tp_setattr` handler is called when the :meth:`__setattr__` or :meth:`__delattr__` method of a class instance would be called. When an attribute should be deleted, the third parameter will be *NULL*. Here is an example that simply raises an exception; if this were really all you wanted, the -:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: +:attr:`tp_setattr` handler should be set to *NULL*. :: static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) @@ -1235,7 +1217,7 @@ richcmpfunc tp_richcompare; -The :c:member:`~PyTypeObject.tp_richcompare` handler is called when comparisons are needed. It is +The :attr:`tp_richcompare` handler is called when comparisons are needed. It is analogous to the :ref:`rich comparison methods <richcmpfuncs>`, like :meth:`__lt__`, and also called by :c:func:`PyObject_RichCompare` and :c:func:`PyObject_RichCompareBool`. @@ -1251,7 +1233,7 @@ Here is a sample implementation, for a datatype that is considered equal if the size of an internal pointer is equal:: - static PyObject * + static int newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op) { PyObject *result; @@ -1295,9 +1277,9 @@ bit does not indicate that the slot values are non-*NULL*. The flag may be set to indicate the presence of a slot, but a slot may still be unfilled.) :: - PyNumberMethods *tp_as_number; - PySequenceMethods *tp_as_sequence; - PyMappingMethods *tp_as_mapping; + PyNumberMethods tp_as_number; + PySequenceMethods tp_as_sequence; + PyMappingMethods tp_as_mapping; If you wish your object to be able to act like a number, a sequence, or a mapping object, then you place the address of a structure that implements the C @@ -1326,7 +1308,7 @@ This function is called when an instance of your data type is "called", for example, if ``obj1`` is an instance of your data type and the Python script -contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is invoked. +contains ``obj1('hello')``, the :attr:`tp_call` handler is invoked. This function takes three arguments: @@ -1360,10 +1342,11 @@ if (!PyArg_ParseTuple(args, "sss:call", &arg1, &arg2, &arg3)) { return NULL; } - result = PyUnicode_FromFormat( + result = PyString_FromFormat( "Returning -- value: [\%d] arg1: [\%s] arg2: [\%s] arg3: [\%s]\n", obj->obj_UnderlyingDatatypePtr->size, arg1, arg2, arg3); + printf("\%s", PyString_AS_STRING(result)); return result; } @@ -1413,7 +1396,7 @@ For an object to be weakly referencable, the extension must include a :c:type:`PyObject\*` field in the instance structure for the use of the weak reference mechanism; it must be initialized to *NULL* by the object's -constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the +constructor. It must also set the :attr:`tp_weaklistoffset` field of the corresponding type object to the offset of the field. For example, the instance type is defined with the following structure:: @@ -1454,8 +1437,9 @@ } The only further addition is that the destructor needs to call the weak -reference manager to clear any weak references. This is only required if the -weak reference list is non-*NULL*:: +reference manager to clear any weak references. This should be done before any +other parts of the destruction have occurred, but is only required if the weak +reference list is non-*NULL*:: static void instance_dealloc(PyInstanceObject *inst) @@ -1499,7 +1483,7 @@ .. [#] This is true when we know that the object is a basic type, like a string or a float. -.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our +.. [#] We relied on this in the :attr:`tp_dealloc` handler in this example, because our type doesn't support garbage collection. Even if a type supports garbage collection, there are calls that can be made to "untrack" the object from garbage collection, however, these calls are advanced and not covered here. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/extending/windows.rst --- a/Doc/extending/windows.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/extending/windows.rst Mon Jan 25 17:05:13 2016 +0100 @@ -37,9 +37,150 @@ are on Unix: use the :mod:`distutils` package to control the build process, or do things manually. The distutils approach works well for most extensions; documentation on using :mod:`distutils` to build and package extension modules -is available in :ref:`distutils-index`. If you find you really need to do -things manually, it may be instructive to study the project file for the -:source:`winsound <PCbuild/winsound.vcxproj>` standard library module. +is available in :ref:`distutils-index`. This section describes the manual +approach to building Python extensions written in C or C++. + +To build extensions using these instructions, you need to have a copy of the +Python sources of the same version as your installed Python. You will need +Microsoft Visual C++ "Developer Studio"; project files are supplied for VC++ +version 7.1, but you can use older versions of VC++. Notice that you should use +the same version of VC++that was used to build Python itself. The example files +described here are distributed with the Python sources in the +:file:`PC\\example_nt\\` directory. + +#. **Copy the example files** --- The :file:`example_nt` directory is a + subdirectory of the :file:`PC` directory, in order to keep all the PC-specific + files under the same directory in the source distribution. However, the + :file:`example_nt` directory can't actually be used from this location. You + first need to copy or move it up one level, so that :file:`example_nt` is a + sibling of the :file:`PC` and :file:`Include` directories. Do all your work + from within this new location. + +#. **Open the project** --- From VC++, use the :menuselection:`File --> Open + Solution` dialog (not :menuselection:`File --> Open`!). Navigate to and select + the file :file:`example.sln`, in the *copy* of the :file:`example_nt` directory + you made above. Click Open. + +#. **Build the example DLL** --- In order to check that everything is set up + right, try building: + +#. Select a configuration. This step is optional. Choose + :menuselection:`Build --> Configuration Manager --> Active Solution Configuration` + and select either :guilabel:`Release` or :guilabel:`Debug`. If you skip this + step, VC++ will use the Debug configuration by default. + +#. Build the DLL. Choose :menuselection:`Build --> Build Solution`. This + creates all intermediate and result files in a subdirectory called either + :file:`Debug` or :file:`Release`, depending on which configuration you selected + in the preceding step. + +#. **Testing the debug-mode DLL** --- Once the Debug build has succeeded, bring + up a DOS box, and change to the :file:`example_nt\\Debug` directory. You should + now be able to repeat the following session (``C>`` is the DOS prompt, ``>>>`` + is the Python prompt; note that build information and various debug output from + Python may not match this screen dump exactly):: + + C>..\..\PCbuild\python_d + Adding parser accelerators ... + Done. + Python 2.2 (#28, Dec 19 2001, 23:26:37) [MSC 32 bit (Intel)] on win32 + Type "copyright", "credits" or "license" for more information. + >>> import example + [4897 refs] + >>> example.foo() + Hello, world + [4903 refs] + >>> + + Congratulations! You've successfully built your first Python extension module. + +#. **Creating your own project** --- Choose a name and create a directory for + it. Copy your C sources into it. Note that the module source file name does + not necessarily have to match the module name, but the name of the + initialization function should match the module name --- you can only import a + module :mod:`spam` if its initialization function is called :c:func:`initspam`, + and it should call :c:func:`Py_InitModule` with the string ``"spam"`` as its + first argument (use the minimal :file:`example.c` in this directory as a guide). + By convention, it lives in a file called :file:`spam.c` or :file:`spammodule.c`. + The output file should be called :file:`spam.pyd` (in Release mode) or + :file:`spam_d.pyd` (in Debug mode). The extension :file:`.pyd` was chosen + to avoid confusion with a system library :file:`spam.dll` to which your module + could be a Python interface. + + Now your options are: + +#. Copy :file:`example.sln` and :file:`example.vcproj`, rename them to + :file:`spam.\*`, and edit them by hand, or + +#. Create a brand new project; instructions are below. + + In either case, copy :file:`example_nt\\example.def` to :file:`spam\\spam.def`, + and edit the new :file:`spam.def` so its second line contains the string + '``initspam``'. If you created a new project yourself, add the file + :file:`spam.def` to the project now. (This is an annoying little file with only + two lines. An alternative approach is to forget about the :file:`.def` file, + and add the option :option:`/export:initspam` somewhere to the Link settings, by + manually editing the setting in Project Properties dialog). + +#. **Creating a brand new project** --- Use the :menuselection:`File --> New + --> Project` dialog to create a new Project Workspace. Select :guilabel:`Visual + C++ Projects/Win32/ Win32 Project`, enter the name (``spam``), and make sure the + Location is set to parent of the :file:`spam` directory you have created (which + should be a direct subdirectory of the Python build tree, a sibling of + :file:`Include` and :file:`PC`). Select Win32 as the platform (in my version, + this is the only choice). Make sure the Create new workspace radio button is + selected. Click OK. + + You should now create the file :file:`spam.def` as instructed in the previous + section. Add the source files to the project, using :menuselection:`Project --> + Add Existing Item`. Set the pattern to ``*.*`` and select both :file:`spam.c` + and :file:`spam.def` and click OK. (Inserting them one by one is fine too.) + + Now open the :menuselection:`Project --> spam properties` dialog. You only need + to change a few settings. Make sure :guilabel:`All Configurations` is selected + from the :guilabel:`Settings for:` dropdown list. Select the C/C++ tab. Choose + the General category in the popup menu at the top. Type the following text in + the entry box labeled :guilabel:`Additional Include Directories`:: + + ..\Include,..\PC + + Then, choose the General category in the Linker tab, and enter :: + + ..\PCbuild + + in the text box labelled :guilabel:`Additional library Directories`. + + Now you need to add some mode-specific settings: + + Select :guilabel:`Release` in the :guilabel:`Configuration` dropdown list. + Choose the :guilabel:`Link` tab, choose the :guilabel:`Input` category, and + append ``pythonXY.lib`` to the list in the :guilabel:`Additional Dependencies` + box. + + Select :guilabel:`Debug` in the :guilabel:`Configuration` dropdown list, and + append ``pythonXY_d.lib`` to the list in the :guilabel:`Additional Dependencies` + box. Then click the C/C++ tab, select :guilabel:`Code Generation`, and select + :guilabel:`Multi-threaded Debug DLL` from the :guilabel:`Runtime library` + dropdown list. + + Select :guilabel:`Release` again from the :guilabel:`Configuration` dropdown + list. Select :guilabel:`Multi-threaded DLL` from the :guilabel:`Runtime + library` dropdown list. + +If your module creates a new type, you may have trouble with this line:: + + PyVarObject_HEAD_INIT(&PyType_Type, 0) + +Static type object initializers in extension modules may cause +compiles to fail with an error message like "initializer not a +constant". This shows up when building DLL under MSVC. Change it to:: + + PyVarObject_HEAD_INIT(NULL, 0) + +and add the following to the module initialization function:: + + if (PyType_Ready(&MyObject_Type) < 0) + return NULL; .. _dynamic-linking: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/design.rst --- a/Doc/faq/design.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/design.rst Mon Jan 25 17:05:13 2016 +0100 @@ -43,45 +43,56 @@ See the next question. -Why are floating-point calculations so inaccurate? +Why are floating point calculations so inaccurate? -------------------------------------------------- -Users are often surprised by results like this:: +People are often very surprised by results like this:: - >>> 1.2 - 1.0 - 0.19999999999999996 + >>> 1.2 - 1.0 + 0.199999999999999996 -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. +and think it is a bug in Python. It's not. This has nothing to do with Python, +but with how the underlying C platform handles floating point numbers, and +ultimately with the inaccuracies introduced when writing down numbers as a +string of a fixed number of digits. -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. +The internal representation of floating point numbers uses a fixed number of +binary digits to represent a decimal number. Some decimal numbers can't be +represented exactly in binary, resulting in small roundoff errors. -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: +In decimal math, there are many numbers that can't be represented with a fixed +number of decimal digits, e.g. 1/3 = 0.3333333333....... - >>> x = 1.2 +In base 2, 1/2 = 0.1, 1/4 = 0.01, 1/8 = 0.001, etc. .2 equals 2/10 equals 1/5, +resulting in the binary fractional number 0.001100110011001... -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: +Floating point numbers only have 32 or 64 bits of precision, so the digits are +cut off at some point, and the resulting number is 0.199999999999999996 in +decimal, not 0.2. - 1.0011001100110011001100110011001100110011001100110011 (binary) +A floating point number's ``repr()`` function prints as many digits are +necessary to make ``eval(repr(f)) == f`` true for any float f. The ``str()`` +function prints fewer digits and this often results in the more sensible number +that was probably intended:: -which is exactly:: + >>> 1.1 - 0.9 + 0.20000000000000007 + >>> print(1.1 - 0.9) + 0.2 - 1.1999999999999999555910790149937383830547332763671875 (decimal) +One of the consequences of this is that it is error-prone to compare the result +of some computation to a float with ``==``. Tiny inaccuracies may mean that +``==`` fails. Instead, you have to check that the difference between the two +numbers is less than a certain threshold:: -The typical precision of 53 bits provides Python floats with 15-16 -decimal digits of accuracy. + epsilon = 0.0000000000001 # Tiny allowed error + expected_result = 0.4 -For a fuller explanation, please see the :ref:`floating point arithmetic -<tut-fp-issues>` chapter in the Python tutorial. + if expected_result-epsilon <= computation() <= expected_result+epsilon: + ... + +Please see the chapter on :ref:`floating point arithmetic <tut-fp-issues>` in +the Python tutorial for more information. Why are Python strings immutable? @@ -214,7 +225,7 @@ generic for a group of types and which were intended to work even for objects that didn't have methods at all (e.g. tuples). It is also convenient to have a function that can readily be applied to an amorphous collection of objects when -you use the functional features of Python (``map()``, ``zip()`` et al). +you use the functional features of Python (``map()``, ``apply()`` et al). In fact, implementing ``len()``, ``max()``, ``min()`` as a built-in function is actually less code than implementing them as methods for each type. One can @@ -345,22 +356,25 @@ Answer 2: Fortunately, there is `Stackless Python <http://www.stackless.com>`_, which has a completely redesigned interpreter loop that avoids the C stack. +It's still experimental but looks very promising. Although it is binary +compatible with standard Python, it's still unclear whether Stackless will make +it into the core -- maybe it's just too revolutionary. -Why can't lambda expressions contain statements? ------------------------------------------------- +Why can't lambda forms contain statements? +------------------------------------------ -Python lambda expressions cannot contain statements because Python's syntactic +Python lambda forms cannot contain statements because Python's syntactic framework can't handle statements nested inside expressions. However, in Python, this is not a serious problem. Unlike lambda forms in other languages, where they add functionality, Python lambdas are only a shorthand notation if you're too lazy to define a function. Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a +local scope. Therefore the only advantage of using a lambda form instead of a locally-defined function is that you don't need to invent a name for the function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! +is exactly the same type of object that a lambda form yields) is assigned! Can Python be compiled to machine code, C or some other language? @@ -368,9 +382,9 @@ Practical answer: -`Cython <http://cython.org/>`_ and `Pyrex <http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ +`Cython <http://cython.org/>`_ and `Pyrex <http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_ compile a modified version of Python with optional annotations into C -extensions. `Weave <http://docs.scipy.org/doc/scipy-dev/reference/tutorial/weave.html>`_ makes it easy to +extensions. `Weave <http://www.scipy.org/Weave>`_ makes it easy to intermingle Python and C code in various ways to increase performance. `Nuitka <http://www.nuitka.net/>`_ is an up-and-coming compiler of Python into C++ code, aiming to support the full Python language. @@ -386,13 +400,13 @@ operations like ``x+1``. Several projects described in the Python newsgroup or at past `Python -conferences <https://www.python.org/community/workshops/>`_ have shown that this +conferences <http://python.org/community/workshops/>`_ have shown that this approach is feasible, although the speedups reached so far are only modest (e.g. 2x). Jython uses the same strategy for compiling to Java bytecode. (Jim Hugunin has demonstrated that in combination with whole-program analysis, speedups of 1000x are feasible for small demo programs. See the proceedings from the `1997 Python conference -<http://legacy.python.org/workshops/1997-10/proceedings/>`_ for more information.) +<http://python.org/workshops/1997-10/proceedings/>`_ for more information.) How does Python manage memory? @@ -634,8 +648,7 @@ (ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check whether an instance or a class implements a particular ABC. The :mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. +:class:`Iterable`, :class:`Container`, and :class:`MutableMapping`. For Python, many of the advantages of interface specifications can be obtained by an appropriate test discipline for components. There is also a tool, @@ -664,6 +677,62 @@ sloppy and not write test cases at all. +Why are default values shared between objects? +---------------------------------------------- + +This type of bug commonly bites neophyte programmers. Consider this function:: + + def foo(mydict={}): # Danger: shared reference to one dict for all calls + ... compute something ... + mydict[key] = value + return mydict + +The first time you call this function, ``mydict`` contains a single item. The +second time, ``mydict`` contains two items because when ``foo()`` begins +executing, ``mydict`` starts out with an item already in it. + +It is often expected that a function call creates new objects for default +values. This is not what happens. Default values are created exactly once, when +the function is defined. If that object is changed, like the dictionary in this +example, subsequent calls to the function will refer to this changed object. + +By definition, immutable objects such as numbers, strings, tuples, and ``None``, +are safe from change. Changes to mutable objects such as dictionaries, lists, +and class instances can lead to confusion. + +Because of this feature, it is good programming practice to not use mutable +objects as default values. Instead, use ``None`` as the default value and +inside the function, check if the parameter is ``None`` and create a new +list/dictionary/whatever if it is. For example, don't write:: + + def foo(mydict={}): + ... + +but:: + + def foo(mydict=None): + if mydict is None: + mydict = {} # create a new dict for local namespace + +This feature can be useful. When you have a function that's time-consuming to +compute, a common technique is to cache the parameters and the resulting value +of each call to the function, and return the cached value if the same value is +requested again. This is called "memoizing", and can be implemented like this:: + + # Callers will never provide a third parameter for this function. + def expensive (arg1, arg2, _cache={}): + if (arg1, arg2) in _cache: + return _cache[(arg1, arg2)] + + # Calculate the value + result = ... expensive computation ... + _cache[(arg1, arg2)] = result # Store result in the cache + return result + +You could use a global variable containing a dictionary instead of the default +value; it's a matter of taste. + + Why is there no goto? --------------------- @@ -672,11 +741,11 @@ reasonable uses of the "go" or "goto" constructs of C, Fortran, and other languages. For example:: - class label(Exception): pass # declare a label + class label: pass # declare a label try: ... - if condition: raise label() # goto label + if (condition): raise label() # goto label ... except label: # where to goto pass @@ -807,8 +876,8 @@ When you have a literal value for a list, tuple, or dictionary spread across multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. +remember to add a comma to the previous line. The lines can also be sorted in +your editor without creating a syntax error. Accidentally omitting the comma can lead to errors that are hard to diagnose. For example:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/extending.rst --- a/Doc/faq/extending.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/extending.rst Mon Jan 25 17:05:13 2016 +0100 @@ -2,9 +2,7 @@ Extending/Embedding FAQ ======================= -.. only:: html - - .. contents:: +.. contents:: .. highlight:: c @@ -42,7 +40,7 @@ .. XXX make sure these all work `Cython <http://cython.org>`_ and its relative `Pyrex -<http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers +<http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_ are compilers that accept a slightly modified form of Python and generate the corresponding C code. Cython and Pyrex make it possible to write an extension without having to learn Python's C API. @@ -50,11 +48,11 @@ If you need to interface to some C or C++ library for which no Python extension currently exists, you can try wrapping the library's data types and functions with a tool such as `SWIG <http://www.swig.org>`_. `SIP -<http://www.riverbankcomputing.co.uk/software/sip/intro>`__, `CXX +<http://www.riverbankcomputing.co.uk/software/sip/>`__, `CXX <http://cxx.sourceforge.net/>`_ `Boost <http://www.boost.org/libs/python/doc/index.html>`_, or `Weave -<http://docs.scipy.org/doc/scipy-dev/reference/tutorial/weave.html>`_ are also -alternatives for wrapping C++ libraries. +<http://www.scipy.org/Weave>`_ are also alternatives for wrapping +C++ libraries. How can I execute arbitrary Python statements from C? @@ -84,20 +82,18 @@ index. Lists have similar functions, :c:func:`PyListSize` and :c:func:`PyList_GetItem`. -For bytes, :c:func:`PyBytes_Size` returns its length and -:c:func:`PyBytes_AsStringAndSize` provides a pointer to its value and its -length. Note that Python bytes objects may contain null bytes so C's -:c:func:`strlen` should not be used. +For strings, :c:func:`PyString_Size` returns its length and +:c:func:`PyString_AsString` a pointer to its value. Note that Python strings may +contain null bytes so C's :c:func:`strlen` should not be used. To test the type of an object, first make sure it isn't *NULL*, and then use -:c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. +:c:func:`PyString_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. There is also a high-level API to Python objects which is provided by the so-called 'abstract' interface -- read ``Include/abstract.h`` for further details. It allows interfacing with any kind of Python sequence using calls -like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc. as well -as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et -al.) and mappings in the PyMapping APIs. +like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc.) as well as +many other useful protocols. How do I use Py_BuildValue() to create a tuple of arbitrary length? @@ -115,8 +111,8 @@ argument values:: PyObject * - PyObject_CallMethod(PyObject *object, const char *method_name, - const char *arg_format, ...); + PyObject_CallMethod(PyObject *object, char *method_name, + char *arg_format, ...); This works for any object that has methods -- whether built-in or user-defined. You are responsible for eventually :c:func:`Py_DECREF`\ 'ing the return value. @@ -247,6 +243,20 @@ For Debian, run ``apt-get install python-dev``. +What does "SystemError: _PyImport_FixupExtension: module yourmodule not loaded" mean? +------------------------------------------------------------------------------------- + +This means that you have created an extension module named "yourmodule", but +your module init function does not initialize with that name. + +Every module init function will have a line similar to:: + + module = Py_InitModule("yourmodule", yourmodule_functions); + +If the string passed to this function is not the same name as your extension +module, the :exc:`SystemError` exception will be raised. + + How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ @@ -334,7 +344,7 @@ { line = readline (prompt); - if (NULL == line) /* Ctrl-D pressed */ + if (NULL == line) /* CTRL-D pressed */ { done = 1; } @@ -429,8 +439,8 @@ Can I create an object class with some methods implemented in C and others in Python (e.g. through inheritance)? ---------------------------------------------------------------------------------------------------------------- -Yes, you can inherit from built-in classes such as :class:`int`, :class:`list`, -:class:`dict`, etc. +In Python 2.2, you can inherit from built-in classes such as :class:`int`, +:class:`list`, :class:`dict`, etc. The Boost Python Library (BPL, http://www.boost.org/libs/python/doc/index.html) provides a way of doing this from C++ (i.e. you can inherit from an extension diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/general.rst --- a/Doc/faq/general.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/general.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,7 @@ General Python FAQ ================== -.. only:: html - - .. contents:: - +.. contents:: General Information =================== @@ -25,7 +22,7 @@ Windows 2000 and later. To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_ links to other +Python <http://wiki.python.org/moin/BeginnersGuide>`_ links to other introductory tutorials and resources for learning Python. @@ -36,11 +33,11 @@ holds the copyright on Python versions 2.1 and newer. The PSF's mission is to advance open source technology related to the Python programming language and to publicize the use of Python. The PSF's home page is at -https://www.python.org/psf/. +http://www.python.org/psf/. Donations to the PSF are tax-exempt in the US. If you use Python and find it helpful, please contribute via `the PSF donation page -<https://www.python.org/psf/donations/>`_. +<http://www.python.org/psf/donations/>`_. Are there copyright restrictions on the use of Python? @@ -53,12 +50,12 @@ unmodified), or to sell products that incorporate Python in some form. We would still like to know about all commercial use of Python, of course. -See `the PSF license page <https://www.python.org/psf/license/>`_ to find further +See `the PSF license page <http://python.org/psf/license/>`_ to find further explanations and a link to the full text of the license. The Python logo is trademarked, and in certain cases permission is required to use it. Consult `the Trademark Usage Policy -<https://www.python.org/psf/trademarks/>`__ for more information. +<http://www.python.org/psf/trademarks/>`__ for more information. Why was Python created in the first place? @@ -117,7 +114,7 @@ Python code), and operating system interfaces (system calls, filesystems, TCP/IP sockets). Look at the table of contents for :ref:`library-index` to get an idea of what's available. A wide variety of third-party extensions are also -available. Consult `the Python Package Index <https://pypi.python.org/pypi>`_ to +available. Consult `the Python Package Index <http://pypi.python.org/pypi>`_ to find packages of interest to you. @@ -151,24 +148,23 @@ next minor version, which becomes the "a0" version, e.g. "2.4a0". -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and -:data:`sys.version_info`. +See also the documentation for ``sys.version``, ``sys.hexversion``, and +``sys.version_info``. How do I obtain a copy of the Python source? -------------------------------------------- The latest Python source distribution is always available from python.org, at -https://www.python.org/download/. The latest development sources can be obtained -via anonymous Mercurial access at https://hg.python.org/cpython. +http://www.python.org/download/. The latest development sources can be obtained +via anonymous Mercurial access at http://hg.python.org/cpython. The source distribution is a gzipped tar file containing the complete C source, Sphinx-formatted documentation, Python library modules, example programs, and several useful pieces of freely distributable software. The source will compile and run out of the box on most UNIX platforms. -Consult the `Getting Started section of the Python Developer's Guide -<https://docs.python.org/devguide/setup.html>`__ for more +Consult the `Developer FAQ <http://docs.python.org/devguide/faq>`__ for more information on getting the source code and compiling it. @@ -178,12 +174,12 @@ .. XXX mention py3k The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are -also available at https://docs.python.org/3/download.html. +at http://docs.python.org/. PDF, plain text, and downloadable HTML versions are +also available at http://docs.python.org/download.html. The documentation is written in reStructuredText and processed by `the Sphinx -documentation tool <http://sphinx-doc.org/>`__. The reStructuredText source for -the documentation is part of the Python source distribution. +documentation tool <http://sphinx.pocoo.org/>`__. The reStructuredText source +for the documentation is part of the Python source distribution. I've never programmed before. Is there a Python tutorial? @@ -192,7 +188,7 @@ There are numerous tutorials and books available. The standard documentation includes :ref:`tutorial-index`. -Consult `the Beginner's Guide <https://wiki.python.org/moin/BeginnersGuide>`_ to +Consult `the Beginner's Guide <http://wiki.python.org/moin/BeginnersGuide>`_ to find information for beginning Python programmers, including lists of tutorials. @@ -200,7 +196,7 @@ ------------------------------------------------------- There is a newsgroup, :newsgroup:`comp.lang.python`, and a mailing list, -`python-list <https://mail.python.org/mailman/listinfo/python-list>`_. The +`python-list <http://mail.python.org/mailman/listinfo/python-list>`_. The newsgroup and mailing list are gatewayed into each other -- if you can read news it's unnecessary to subscribe to the mailing list. :newsgroup:`comp.lang.python` is high-traffic, receiving hundreds of postings @@ -209,38 +205,38 @@ Announcements of new software releases and events can be found in comp.lang.python.announce, a low-traffic moderated list that receives about five postings per day. It's available as `the python-announce mailing list -<https://mail.python.org/mailman/listinfo/python-announce-list>`_. +<http://mail.python.org/mailman/listinfo/python-announce-list>`_. More info about other mailing lists and newsgroups -can be found at https://www.python.org/community/lists/. +can be found at http://www.python.org/community/lists/. How do I get a beta test version of Python? ------------------------------------------- -Alpha and beta releases are available from https://www.python.org/download/. All +Alpha and beta releases are available from http://www.python.org/download/. All releases are announced on the comp.lang.python and comp.lang.python.announce -newsgroups and on the Python home page at https://www.python.org/; an RSS feed of +newsgroups and on the Python home page at http://www.python.org/; an RSS feed of news is available. -You can also access the development version of Python through Mercurial. See -https://docs.python.org/devguide/faq.html for details. +You can also access the development version of Python through Subversion. See +http://docs.python.org/devguide/faq for details. How do I submit bug reports and patches for Python? --------------------------------------------------- To report a bug or submit a patch, please use the Roundup installation at -https://bugs.python.org/. +http://bugs.python.org/. You must have a Roundup account to report bugs; this makes it possible for us to contact you if we have follow-up questions. It will also enable Roundup to send you updates as we act on your bug. If you had previously used SourceForge to report bugs to Python, you can obtain your Roundup password through Roundup's -`password reset procedure <https://bugs.python.org/user?@template=forgotten>`_. +`password reset procedure <http://bugs.python.org/user?@template=forgotten>`_. For more information on how Python is developed, consult `the Python Developer's -Guide <https://docs.python.org/devguide/>`_. +Guide <http://docs.python.org/devguide/>`_. Are there any published articles about Python that I can reference? @@ -260,7 +256,7 @@ ------------------------------ Yes, there are many, and more are being published. See the python.org wiki at -https://wiki.python.org/moin/PythonBooks for a list. +http://wiki.python.org/moin/PythonBooks for a list. You can also search online bookstores for "Python" and filter out the Monty Python references; or perhaps search for "Python" and "language". @@ -269,14 +265,9 @@ Where in the world is www.python.org located? --------------------------------------------- -The Python project's infrastructure is located all over the world. -`www.python.org <https://www.python.org>`_ is graciously hosted by `Rackspace -<http://www.rackspace.com>`_, with CDN caching provided by `Fastly -<https://www.fastly.com>`_. `Upfront Systems -<http://www.upfrontsystems.co.za>`_ hosts `bugs.python.org -<https://bugs.python.org>`_. Many other Python services like `the Wiki -<https://wiki.python.org>`_ are hosted by `Oregon State -University Open Source Lab <https://osuosl.org>`_. +It's currently in Amsterdam, graciously hosted by `XS4ALL +<http://www.xs4all.nl>`_. Thanks to Thomas Wouters for his work in arranging +python.org's hosting. Why is it called Python? @@ -284,7 +275,7 @@ When he began implementing Python, Guido van Rossum was also reading the published scripts from `"Monty Python's Flying Circus" -<http://en.wikipedia.org/wiki/Monty_Python>`__, a BBC comedy series from the 1970s. Van Rossum +<http://pythonline.com/>`__, a BBC comedy series from the 1970s. Van Rossum thought he needed a name that was short, unique, and slightly mysterious, so he decided to call the language Python. @@ -313,7 +304,7 @@ releases. The latest stable releases can always be found on the `Python download page -<https://www.python.org/download/>`_. There are two recommended production-ready +<http://python.org/download/>`_. There are two recommended production-ready versions at this point in time, because at the moment there are two branches of stable releases: 2.x and 3.x. Python 3.x may be less useful than 2.x, since currently there is more third party software available for Python 2 than for @@ -337,9 +328,9 @@ Have any significant projects been done in Python? -------------------------------------------------- -See https://www.python.org/about/success for a list of projects that use Python. +See http://python.org/about/success for a list of projects that use Python. Consulting the proceedings for `past Python conferences -<https://www.python.org/community/workshops/>`_ will reveal contributions from many +<http://python.org/community/workshops/>`_ will reveal contributions from many different companies and organizations. High-profile Python projects include `the Mailman mailing list manager @@ -353,14 +344,14 @@ What new developments are expected for Python in the future? ------------------------------------------------------------ -See https://www.python.org/dev/peps/ for the Python Enhancement Proposals +See http://www.python.org/dev/peps/ for the Python Enhancement Proposals (PEPs). PEPs are design documents describing a suggested new feature for Python, providing a concise technical specification and a rationale. Look for a PEP titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been publicly released yet. New development is discussed on `the python-dev mailing list -<https://mail.python.org/mailman/listinfo/python-dev/>`_. +<http://mail.python.org/mailman/listinfo/python-dev/>`_. Is it reasonable to propose incompatible changes to Python? @@ -378,6 +369,43 @@ changes while minimizing disruption for users. +Is Python Y2K (Year 2000) Compliant? +------------------------------------ + +.. remove this question? + +As of August, 2003 no major problems have been reported and Y2K compliance seems +to be a non-issue. + +Python does very few date calculations and for those it does perform relies on +the C library functions. Python generally represents times either as seconds +since 1970 or as a ``(year, month, day, ...)`` tuple where the year is expressed +with four digits, which makes Y2K bugs unlikely. So as long as your C library +is okay, Python should be okay. Of course, it's possible that a particular +application written in Python makes assumptions about 2-digit years. + +Because Python is available free of charge, there are no absolute guarantees. +If there *are* unforeseen problems, liability is the user's problem rather than +the developers', and there is nobody you can sue for damages. The Python +copyright notice contains the following disclaimer: + + 4. PSF is making Python 2.3 available to Licensee on an "AS IS" + basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY + WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY + REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR + PURPOSE OR THAT THE USE OF PYTHON 2.3 WILL NOT INFRINGE ANY THIRD PARTY + RIGHTS. + + 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + 2.3 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.3, + OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +The good news is that *if* you encounter a problem, you have full source +available to track it down and fix it. This is one advantage of an open source +programming environment. + + Is Python a good language for beginning programmers? ---------------------------------------------------- @@ -416,25 +444,14 @@ remember the methods for a list, they can do something like this:: >>> L = [] - >>> dir(L) # doctest: +NORMALIZE_WHITESPACE - ['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', - '__dir__', '__doc__', '__eq__', '__format__', '__ge__', - '__getattribute__', '__getitem__', '__gt__', '__hash__', '__iadd__', - '__imul__', '__init__', '__iter__', '__le__', '__len__', '__lt__', - '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', - '__repr__', '__reversed__', '__rmul__', '__setattr__', '__setitem__', - '__sizeof__', '__str__', '__subclasshook__', 'append', 'clear', - 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', + >>> dir(L) + ['append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort'] - >>> [d for d in dir(L) if '__' not in d] - ['append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort'] - >>> help(L.append) Help on built-in function append: - <BLANKLINE> + append(...) - L.append(object) -> None -- append object to end - <BLANKLINE> + L.append(object) -- append object to end >>> L.append(1) >>> L [1] @@ -447,9 +464,8 @@ Emacs users will be happy to know that there is a very good Python mode for Emacs. All of these programming environments provide syntax highlighting, auto-indenting, and access to the interactive interpreter while coding. Consult -`the Python wiki <https://wiki.python.org/moin/PythonEditors>`_ for a full list -of Python editing environments. +http://www.python.org/editors/ for a full list of Python editing environments. If you want to discuss Python's use in education, you may be interested in joining `the edu-sig mailing list -<https://www.python.org/community/sigs/current/edu-sig>`_. +<http://python.org/community/sigs/current/edu-sig>`_. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/gui.rst --- a/Doc/faq/gui.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/gui.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,9 +4,7 @@ Graphic User Interface FAQ ========================== -.. only:: html - - .. contents:: +.. contents:: .. XXX need review for Python 3. @@ -29,17 +27,17 @@ Standard builds of Python include an object-oriented interface to the Tcl/Tk widget set, called :ref:`tkinter <Tkinter>`. This is probably the easiest to install (since it comes included with most -`binary distributions <https://www.python.org/download/>`_ of Python) and use. +`binary distributions <http://www.python.org/download/>`_ of Python) and use. For more info about Tk, including pointers to the source, see the `Tcl/Tk home page <http://www.tcl.tk>`_. Tcl/Tk is fully portable to the -Mac OS X, Windows, and Unix platforms. +MacOS, Windows, and Unix platforms. wxWidgets --------- wxWidgets (http://www.wxwidgets.org) is a free, portable GUI class library written in C++ that provides a native look and feel on a -number of platforms, with Windows, Mac OS X, GTK, X11, all listed as +number of platforms, with Windows, MacOS X, GTK, X11, all listed as current stable targets. Language bindings are available for a number of languages including Python, Perl, Ruby, etc. @@ -58,14 +56,14 @@ --- There are bindings available for the Qt toolkit (using either `PyQt -<http://www.riverbankcomputing.co.uk/software/pyqt/intro>`_ or `PySide -<http://www.pyside.org/>`_) and for KDE (`PyKDE <https://techbase.kde.org/Development/Languages/Python>`__). +<http://www.riverbankcomputing.co.uk/software/pyqt/>`_ or `PySide +<http://www.pyside.org/>`_) and for KDE (`PyKDE <http://www.riverbankcomputing.co.uk/software/pykde/intro>`__). PyQt is currently more mature than PySide, but you must buy a PyQt license from `Riverbank Computing <http://www.riverbankcomputing.co.uk/software/pyqt/license>`_ if you want to write proprietary applications. PySide is free for all applications. Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses -are available from `The Qt Company <http://www.qt.io/licensing/>`_. +are available from `Nokia <http://qt.nokia.com/>`_. Gtk+ ---- @@ -102,9 +100,13 @@ What platform-specific GUI toolkits exist for Python? ======================================================== +`The Mac port <http://python.org/download/mac>`_ by Jack Jansen has a rich and +ever-growing set of modules that support the native Mac toolbox calls. The port +supports MacOS X's Carbon libraries. + By installing the `PyObjc Objective-C bridge -<https://pythonhosted.org/pyobjc/>`_, Python programs can use Mac OS X's -Cocoa libraries. +<http://pyobjc.sourceforge.net>`_, Python programs can use MacOS X's +Cocoa libraries. See the documentation that comes with the Mac port. :ref:`Pythonwin <windows-faq>` by Mark Hammond includes an interface to the Microsoft Foundation Classes and a Python programming environment @@ -139,11 +141,30 @@ Can I have Tk events handled while waiting for I/O? --------------------------------------------------- -On platforms other than Windows, yes, and you don't even -need threads! But you'll have to restructure your I/O +Yes, and you don't even need threads! But you'll have to restructure your I/O code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you to register a callback function which will be called from the Tk mainloop when -I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. +I/O is possible on a file descriptor. Here's what you need:: + + from Tkinter import tkinter + tkinter.createfilehandler(file, mask, callback) + +The file may be a Python file or socket object (actually, anything with a +fileno() method), or an integer file descriptor. The mask is one of the +constants tkinter.READABLE or tkinter.WRITABLE. The callback is called as +follows:: + + callback(file, mask) + +You must unregister the callback when you're done, using :: + + tkinter.deletefilehandler(file) + +Note: since you don't know *how many bytes* are available for reading, you can't +use the Python file object's read or readline methods, since these will insist +on reading a predefined number of bytes. For sockets, the :meth:`recv` or +:meth:`recvfrom` methods will work fine; for other files, use +``os.read(file.fileno(), maxbytecount)``. I can't get key bindings to work in Tkinter: why? diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/index.rst --- a/Doc/faq/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,10 @@ -.. _faq-index: - ################################### Python Frequently Asked Questions ################################### +:Release: |version| +:Date: |today| + .. toctree:: :maxdepth: 1 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/installed.rst --- a/Doc/faq/installed.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/installed.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,7 +11,7 @@ software developers at places such as Google, NASA, and Lucasfilm Ltd. If you wish to learn more about Python, start with the `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_. +Python <http://wiki.python.org/moin/BeginnersGuide>`_. Why is Python installed on my machine? diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/library.rst --- a/Doc/faq/library.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/library.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,9 +4,7 @@ Library and Extension FAQ ========================= -.. only:: html - - .. contents:: +.. contents:: General Library Questions ========================= @@ -16,10 +14,10 @@ Check :ref:`the Library Reference <library-index>` to see if there's a relevant standard library module. (Eventually you'll learn what's in the standard -library and will be able to skip this step.) +library and will able to skip this step.) For third-party packages, search the `Python Package Index -<https://pypi.python.org/pypi>`_ or try `Google <https://www.google.com>`_ or +<http://pypi.python.org/pypi>`_ or try `Google <http://www.google.com>`_ or another Web search engine. Searching for "Python" plus a keyword or two for your topic of interest will usually find something helpful. @@ -30,7 +28,7 @@ If you can't find a source file for a module it may be a built-in or dynamically loaded module implemented in C, C++ or other compiled language. In this case you may not have the source file or it may be something like -:file:`mathmodule.c`, somewhere in a C source directory (not on the Python Path). +mathmodule.c, somewhere in a C source directory (not on the Python Path). There are (at least) three kinds of modules in Python: @@ -62,18 +60,18 @@ interpreter is installed on your platform. If you would like the script to be independent of where the Python interpreter -lives, you can use the :program:`env` program. Almost all Unix variants support -the following, assuming the Python interpreter is in a directory on the user's -:envvar:`PATH`:: +lives, you can use the "env" program. Almost all Unix variants support the +following, assuming the Python interpreter is in a directory on the user's +$PATH:: #!/usr/bin/env python -*Don't* do this for CGI scripts. The :envvar:`PATH` variable for CGI scripts is -often very minimal, so you need to use the actual absolute pathname of the +*Don't* do this for CGI scripts. The $PATH variable for CGI scripts is often +very minimal, so you need to use the actual absolute pathname of the interpreter. -Occasionally, a user's environment is so full that the :program:`/usr/bin/env` -program fails; or there's no env program at all. In that case, you can try the +Occasionally, a user's environment is so full that the /usr/bin/env program +fails; or there's no env program at all. In that case, you can try the following hack (due to Alex Rezinsky):: #! /bin/sh @@ -94,11 +92,11 @@ .. XXX curses *is* built by default, isn't it? For Unix variants: The standard Python source distribution comes with a curses -module in the :source:`Modules` subdirectory, though it's not compiled by default. -(Note that this is not available in the Windows distribution -- there is no -curses module for Windows.) +module in the ``Modules/`` subdirectory, though it's not compiled by default +(note that this is not available in the Windows distribution -- there is no +curses module for Windows). -The :mod:`curses` module supports basic curses features as well as many additional +The curses module supports basic curses features as well as many additional functions from ncurses and SYSV curses such as colour, alternative character set support, pads, and mouse support. This means the module isn't compatible with operating systems that only have BSD curses, but there don't seem to be any @@ -112,7 +110,7 @@ ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +onexit. Why don't my signal handlers work? @@ -142,8 +140,8 @@ The :mod:`unittest` module is a fancier testing framework modelled on Java and Smalltalk testing frameworks. -To make testing easier, you should use good modular design in your program. -Your program should have almost all functionality +For testing, it helps to write the program so that it may be easily tested by +using good modular design. Your program should have almost all functionality encapsulated in either functions or class methods -- and this sometimes has the surprising and delightful effect of making the program run faster (because local variable accesses are faster than global accesses). Furthermore the program @@ -159,7 +157,7 @@ Once your program is organized as a tractable collection of functions and class behaviours you should write test functions that exercise the behaviours. A test -suite that automates a sequence of tests can be associated with each module. +suite can be associated with each module which automates a sequence of tests. This sounds like a lot of work, but since Python is so terse and flexible it's surprisingly easy. You can make coding much more pleasant and fun by writing your test functions in parallel with the "production code", since this makes it @@ -181,14 +179,14 @@ The :mod:`pydoc` module can create HTML from the doc strings in your Python source code. An alternative for creating API documentation purely from -docstrings is `epydoc <http://epydoc.sourceforge.net/>`_. `Sphinx -<http://sphinx-doc.org>`_ can also include docstring content. +docstrings is `epydoc <http://epydoc.sf.net/>`_. `Sphinx +<http://sphinx.pocoo.org>`_ can also include docstring content. How do I get a single keypress at a time? ----------------------------------------- -For Unix variants there are several solutions. It's straightforward to do this +For Unix variants: There are several solutions. It's straightforward to do this using curses, but curses is a fairly large module to learn. .. XXX this doesn't work out of the box, some IO expert needs to check why @@ -211,7 +209,7 @@ try: c = sys.stdin.read(1) print("Got character", repr(c)) - except OSError: + except IOError: pass finally: termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm) @@ -224,11 +222,7 @@ :func:`termios.tcsetattr` turns off stdin's echoing and disables canonical mode. :func:`fcntl.fnctl` is used to obtain stdin's file descriptor flags and modify them for non-blocking mode. Since reading stdin when it is empty - results in an :exc:`OSError`, this error is caught and ignored. - - .. versionchanged:: 3.3 - *sys.stdin.read* used to raise :exc:`IOError`. Starting from Python 3.3 - :exc:`IOError` is alias for :exc:`OSError`. + results in an :exc:`IOError`, this error is caught and ignored. Threads @@ -281,7 +275,7 @@ time.sleep(10) -Instead of trying to guess a good delay value for :func:`time.sleep`, +Instead of trying to guess how long a :func:`time.sleep` delay will be enough, it's better to use some kind of semaphore mechanism. One idea is to use the :mod:`queue` module to create a queue object, let each thread append a token to the queue when it finishes, and let the main thread read as many tokens from the @@ -297,9 +291,9 @@ Or, if you want fine control over the dispatching algorithm, you can write your own logic manually. Use the :mod:`queue` module to create a queue containing a list of jobs. The :class:`~queue.Queue` class maintains a -list of objects and has a ``.put(obj)`` method that adds items to the queue and -a ``.get()`` method to return them. The class will take care of the locking -necessary to ensure that each job is handed out exactly once. +list of objects with ``.put(obj)`` to add an item to the queue and ``.get()`` +to return an item. The class will take care of the locking necessary to +ensure that each job is handed out exactly once. Here's a trivial example:: @@ -308,7 +302,7 @@ # The worker thread gets jobs off the queue. When the queue is empty, it # assumes there will be no more work and exits. # (Realistically workers will run until terminated.) - def worker(): + def worker (): print('Running worker') time.sleep(0.1) while True: @@ -339,9 +333,7 @@ print('Main thread sleeping') time.sleep(5) -When run, this will produce the following output: - -.. code-block:: none +When run, this will produce the following output:: Running worker Running worker @@ -357,8 +349,8 @@ Worker <Thread(worker 1, started 130283832797456)> running with argument 5 ... -Consult the module's documentation for more details; the :class:`~queue.Queue` -class provides a featureful interface. +Consult the module's documentation for more details; the ``Queue`` class +provides a featureful interface. What kinds of global value mutation are thread-safe? @@ -475,7 +467,7 @@ To truncate a file, open it using ``f = open(filename, "rb+")``, and use ``f.truncate(offset)``; offset defaults to the current seek position. There's also ``os.ftruncate(fd, offset)`` for files opened with :func:`os.open`, where -*fd* is the file descriptor (a small integer). +``fd`` is the file descriptor (a small integer). The :mod:`shutil` module also contains a number of functions to work on files including :func:`~shutil.copyfile`, :func:`~shutil.copytree`, and @@ -509,16 +501,15 @@ "short integer" (2 bytes), and 'l' reads one "long integer" (4 bytes) from the string. -For data that is more regular (e.g. a homogeneous list of ints or floats), +For data that is more regular (e.g. a homogeneous list of ints or thefloats), you can also use the :mod:`array` module. -.. note:: - - To read and write binary data, it is mandatory to open the file in - binary mode (here, passing ``"rb"`` to :func:`open`). If you use - ``"r"`` instead (the default), the file will be open in text mode - and ``f.read()`` will return :class:`str` objects rather than - :class:`bytes` objects. + .. note:: + To read and write binary data, it is mandatory to open the file in + binary mode (here, passing ``"rb"`` to :func:`open`). If you use + ``"r"`` instead (the default), the file will be open in text mode + and ``f.read()`` will return :class:`str` objects rather than + :class:`bytes` objects. I can't seem to use os.read() on a pipe created with os.popen(); why? @@ -527,7 +518,7 @@ :func:`os.read` is a low-level function which takes a file descriptor, a small integer representing the opened file. :func:`os.popen` creates a high-level file object, the same type returned by the built-in :func:`open` function. -Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need to +Thus, to read n bytes from a pipe p created with :func:`os.popen`, you need to use ``p.read(n)``. @@ -547,8 +538,8 @@ Warning: in general it is unwise to do this because you can easily cause a deadlock where your process is blocked waiting for output from the child while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. + because the parent expects the child to output more text than it does, or it + can be caused by data being stuck in stdio buffers due to lack of flushing. The Python parent can of course explicitly flush the data it sends to the child before it reads any output, but if the child is a naive C program it may have been written to never explicitly flush its output, even if it is @@ -570,7 +561,7 @@ get the result back. Unless the amount of data is very large, the easiest way to do this is to write it to a temporary file and run the command with that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: + ``mktemp()`` function to generate unique temporary file names. :: import tempfile import os @@ -607,7 +598,7 @@ "expect" library. A Python extension that interfaces to expect is called "expy" and available from http://expectpy.sourceforge.net. A pure Python solution that works like expect is `pexpect - <https://pypi.python.org/pypi/pexpect/>`_. + <http://pypi.python.org/pypi/pexpect/>`_. How do I access the serial (RS232) port? @@ -663,7 +654,7 @@ .. XXX check if wiki page is still up to date A summary of available frameworks is maintained by Paul Boddie at -https://wiki.python.org/moin/WebProgramming\ . +http://wiki.python.org/moin/WebProgramming . Cameron Laird maintains a useful set of pages about Python web technologies at http://phaseit.net/claird/comp.lang.python/web_python. @@ -687,12 +678,11 @@ ### connect and send the server a path req = urllib.request.urlopen('http://www.some-server.out-there' '/cgi-bin/some-cgi-script', data=qs) - with req: - msg, hdrs = req.read(), req.info() + msg, hdrs = req.read(), req.info() Note that in general for percent-encoded POST operations, query strings must be -quoted using :func:`urllib.parse.urlencode`. For example, to send -``name=Guy Steele, Jr.``:: +quoted using :func:`urllib.parse.urlencode`. For example to send name="Guy Steele, +Jr.":: >>> import urllib.parse >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'}) @@ -706,8 +696,19 @@ .. XXX add modern template languages -You can find a collection of useful links on the `Web Programming wiki page -<https://wiki.python.org/moin/WebProgramming>`_. +There are many different modules available: + +* HTMLgen is a class library of objects corresponding to all the HTML 3.2 markup + tags. It's used when you are writing in Python and wish to synthesize HTML + pages for generating a web or for CGI forms, etc. + +* DocumentTemplate and Zope Page Templates are two different systems that are + part of Zope. + +* Quixote's PTL uses Python syntax to assemble strings of text. + +Consult the `Web Programming wiki pages +<http://wiki.python.org/moin/WebProgramming>`_ for more links. How do I send mail from a Python script? @@ -736,7 +737,7 @@ server.quit() A Unix-only alternative uses sendmail. The location of the sendmail program -varies between systems; sometimes it is ``/usr/lib/sendmail``, sometimes +varies between systems; sometimes it is ``/usr/lib/sendmail``, sometime ``/usr/sbin/sendmail``. The sendmail manual page will help you out. Here's some sample code:: @@ -774,7 +775,7 @@ .. note:: The :mod:`asyncore` module presents a framework-like approach to the problem of writing non-blocking networking code. - The third-party `Twisted <https://twistedmatrix.com/trac/>`_ library is + The third-party `Twisted <http://twistedmatrix.com/>`_ library is a popular and feature-rich alternative. @@ -793,7 +794,7 @@ Support for most relational databases is available. See the `DatabaseProgramming wiki page -<https://wiki.python.org/moin/DatabaseProgramming>`_ for details. +<http://wiki.python.org/moin/DatabaseProgramming>`_ for details. How do you implement persistent objects in Python? @@ -804,6 +805,14 @@ :mod:`shelve` library module uses pickle and (g)dbm to create persistent mappings containing arbitrary Python objects. +A more awkward way of doing things is to use pickle's little sister, marshal. +The :mod:`marshal` module provides very fast ways to store noncircular basic +Python types to files and strings, and back again. Although marshal does not do +fancy things like store instances or handle shared references properly, it does +run extremely fast. For example loading a half megabyte of data may take less +than a third of a second. This often beats doing something more complex and +general such as using gdbm with pickle/shelve. + Mathematics and Numerics ======================== diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/programming.rst --- a/Doc/faq/programming.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/programming.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,9 +4,7 @@ Programming FAQ =============== -.. only:: html - - .. contents:: +.. contents:: General Questions ================= @@ -23,14 +21,15 @@ The IDLE interactive development environment, which is part of the standard Python distribution (normally available as Tools/scripts/idle), includes a -graphical debugger. +graphical debugger. There is documentation for the IDLE debugger at +http://www.python.org/idle/doc/idle2.html#Debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The Pythonwin debugger colors breakpoints and has quite a few cool features such as debugging non-Pythonwin programs. Pythonwin is available as part of the `Python for Windows Extensions <http://sourceforge.net/projects/pywin32/>`__ project and as a part of the ActivePython distribution (see -http://www.activestate.com/activepython\ ). +http://www.activestate.com/Products/ActivePython/index.html). `Boa Constructor <http://boa-constructor.sourceforge.net/>`_ is an IDE and GUI builder that uses wxWidgets. It offers visual frame creation and manipulation, @@ -38,7 +37,7 @@ hierarchies, doc string generated html documentation, an advanced debugger, integrated help, and Zope support. -`Eric <http://eric-ide.python-projects.org/>`_ is an IDE built on PyQt +`Eric <http://www.die-offenbachs.de/eric/index.html>`_ is an IDE built on PyQt and the Scintilla editing component. Pydb is a version of the standard Python debugger pdb, modified for use with DDD @@ -50,8 +49,7 @@ They include: * Wing IDE (http://wingware.com/) -* Komodo IDE (http://komodoide.com/) -* PyCharm (https://www.jetbrains.com/pycharm/) +* Komodo IDE (http://www.activestate.com/Products/Komodo) Is there a tool to help find bugs or perform static analysis? @@ -61,7 +59,7 @@ PyChecker is a static analysis tool that finds bugs in Python source code and warns about code complexity and style. You can get PyChecker from -http://pychecker.sourceforge.net/. +http://pychecker.sf.net. `Pylint <http://www.logilab.org/projects/pylint>`_ is another tool that checks if a module satisfies a coding standard, and also makes it possible to write @@ -69,7 +67,8 @@ PyChecker performs, Pylint offers some additional features such as checking line length, whether variable names are well-formed according to your coding standard, whether declared interfaces are fully implemented, and more. -http://docs.pylint.org/ provides a full list of Pylint's features. +http://www.logilab.org/card/pylint_manual provides a full list of Pylint's +features. How can I create a stand-alone binary from a Python script? @@ -100,7 +99,13 @@ http://www.py2exe.org/ -Another tool is Anthony Tuininga's `cx_Freeze <http://cx-freeze.sourceforge.net/>`_. +Another is Christian Tismer's `SQFREEZE <http://starship.python.net/crew/pirx>`_ +which appends the byte code to a specially-prepared Python interpreter that can +find the byte code in the executable. + +Other tools include Fredrik Lundh's `Squeeze +<http://www.pythonware.com/products/python/squeeze>`_ and Anthony Tuininga's +`cx_Freeze <http://starship.python.net/crew/atuining/cx_Freeze/index.html>`_. Are there coding standards or a style guide for Python programs? @@ -187,8 +192,10 @@ ------------------------------------------------------------ In Python, variables that are only referenced inside a function are implicitly -global. If a variable is assigned a value anywhere within the function's body, -it's assumed to be a local unless explicitly declared as global. +global. If a variable is assigned a new value anywhere within the function's +body, it's assumed to be a local. If a variable is ever assigned a new value +inside the function, the variable is implicitly local, and you need to +explicitly declare it as 'global'. Though a bit surprising at first, a moment's consideration explains this. On one hand, requiring :keyword:`global` for assigned variables provides a bar @@ -199,58 +206,6 @@ declaration for identifying side-effects. -Why do lambdas defined in a loop with different values all return the same result? ----------------------------------------------------------------------------------- - -Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda: x**2) - -This gives you a list that contains 5 lambdas that calculate ``x**2``. You -might expect that, when called, they would return, respectively, ``0``, ``1``, -``4``, ``9``, and ``16``. However, when you actually try you will see that -they all return ``16``:: - - >>> squares[2]() - 16 - >>> squares[4]() - 16 - -This happens because ``x`` is not local to the lambdas, but is defined in -the outer scope, and it is accessed when the lambda is called --- not when it -is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by -changing the value of ``x`` and see how the results of the lambdas change:: - - >>> x = 8 - >>> squares[2]() - 64 - -In order to avoid this, you need to save the values in variables local to the -lambdas, so that they don't rely on the value of the global ``x``:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda n=x: n**2) - -Here, ``n=x`` creates a new variable ``n`` local to the lambda and computed -when the lambda is defined so that it has the same value that ``x`` had at -that point in the loop. This means that the value of ``n`` will be ``0`` -in the first lambda, ``1`` in the second, ``2`` in the third, and so on. -Therefore each lambda will now return the correct result:: - - >>> squares[2]() - 4 - >>> squares[4]() - 16 - -Note that this behaviour is not peculiar to lambdas, but applies to regular -functions too. - - How do I share global variables across modules? ------------------------------------------------ @@ -283,8 +238,9 @@ ----------------------------------------------------------- In general, don't use ``from modulename import *``. Doing so clutters the -importer's namespace, and makes it much harder for linters to detect undefined -names. +importer's namespace. Some people avoid this idiom even with the few modules +that were designed to be imported in this manner. Modules designed in this +manner include :mod:`tkinter`, and :mod:`threading`. Import modules at the top of a file. Doing so makes it clear what other modules your code requires and avoids questions of whether the module name is in scope. @@ -298,6 +254,11 @@ directory) -- e.g. mx.DateTime, ZODB, PIL.Image, etc. 3. locally-developed modules +Never use relative package imports. If you're writing code that's in the +``package.sub.m1`` module and want to import ``package.sub.m2``, do not just +write ``from . import m2``, even though it's legal. Write ``from package.sub +import m2`` instead. See :pep:`328` for details. + It is sometimes necessary to move imports to a function or class to avoid problems with circular imports. Gordon McMillan says: @@ -328,61 +289,13 @@ couple of dictionary lookups. Even if the module name has gone out of scope, the module is probably available in :data:`sys.modules`. - -Why are default values shared between objects? ----------------------------------------------- - -This type of bug commonly bites neophyte programmers. Consider this function:: - - def foo(mydict={}): # Danger: shared reference to one dict for all calls - ... compute something ... - mydict[key] = value - return mydict - -The first time you call this function, ``mydict`` contains a single item. The -second time, ``mydict`` contains two items because when ``foo()`` begins -executing, ``mydict`` starts out with an item already in it. - -It is often expected that a function call creates new objects for default -values. This is not what happens. Default values are created exactly once, when -the function is defined. If that object is changed, like the dictionary in this -example, subsequent calls to the function will refer to this changed object. - -By definition, immutable objects such as numbers, strings, tuples, and ``None``, -are safe from change. Changes to mutable objects such as dictionaries, lists, -and class instances can lead to confusion. - -Because of this feature, it is good programming practice to not use mutable -objects as default values. Instead, use ``None`` as the default value and -inside the function, check if the parameter is ``None`` and create a new -list/dictionary/whatever if it is. For example, don't write:: - - def foo(mydict={}): - ... - -but:: - - def foo(mydict=None): - if mydict is None: - mydict = {} # create a new dict for local namespace - -This feature can be useful. When you have a function that's time-consuming to -compute, a common technique is to cache the parameters and the resulting value -of each call to the function, and return the cached value if the same value is -requested again. This is called "memoizing", and can be implemented like this:: - - # Callers will never provide a third parameter for this function. - def expensive(arg1, arg2, _cache={}): - if (arg1, arg2) in _cache: - return _cache[(arg1, arg2)] - - # Calculate the value - result = ... expensive computation ... - _cache[(arg1, arg2)] = result # Store result in the cache - return result - -You could use a global variable containing a dictionary instead of the default -value; it's a matter of taste. +If only instances of a specific class use a module, then it is reasonable to +import the module in the class's ``__init__`` method and then assign the module +to an instance variable so that the module is always available (via that +instance variable) during the life of the object. Note that to delay an import +until the class is instantiated, the import must be inside a method. Putting +the import inside the class but outside of any method still causes the import to +occur when the module is initialized. How can I pass optional or keyword parameters from one function to another? @@ -400,106 +313,6 @@ g(x, *args, **kwargs) -.. index:: - single: argument; difference from parameter - single: parameter; difference from argument - -.. _faq-argument-vs-parameter: - -What is the difference between arguments and parameters? --------------------------------------------------------- - -:term:`Parameters <parameter>` are defined by the names that appear in a -function definition, whereas :term:`arguments <argument>` are the values -actually passed to a function when calling it. Parameters define what types of -arguments a function can accept. For example, given the function definition:: - - def func(foo, bar=None, **kwargs): - pass - -*foo*, *bar* and *kwargs* are parameters of ``func``. However, when calling -``func``, for example:: - - func(42, bar=314, extra=somevar) - -the values ``42``, ``314``, and ``somevar`` are arguments. - - -Why did changing list 'y' also change list 'x'? ------------------------------------------------- - -If you wrote code like:: - - >>> x = [] - >>> y = x - >>> y.append(10) - >>> y - [10] - >>> x - [10] - -you might be wondering why appending an element to ``y`` changed ``x`` too. - -There are two factors that produce this result: - -1) Variables are simply names that refer to objects. Doing ``y = x`` doesn't - create a copy of the list -- it creates a new variable ``y`` that refers to - the same object ``x`` refers to. This means that there is only one object - (the list), and both ``x`` and ``y`` refer to it. -2) Lists are :term:`mutable`, which means that you can change their content. - -After the call to :meth:`~list.append`, the content of the mutable object has -changed from ``[]`` to ``[10]``. Since both the variables refer to the same -object, using either name accesses the modified value ``[10]``. - -If we instead assign an immutable object to ``x``:: - - >>> x = 5 # ints are immutable - >>> y = x - >>> x = x + 1 # 5 can't be mutated, we are creating a new object here - >>> x - 6 - >>> y - 5 - -we can see that in this case ``x`` and ``y`` are not equal anymore. This is -because integers are :term:`immutable`, and when we do ``x = x + 1`` we are not -mutating the int ``5`` by incrementing its value; instead, we are creating a -new object (the int ``6``) and assigning it to ``x`` (that is, changing which -object ``x`` refers to). After this assignment we have two objects (the ints -``6`` and ``5``) and two variables that refer to them (``x`` now refers to -``6`` but ``y`` still refers to ``5``). - -Some operations (for example ``y.append(10)`` and ``y.sort()``) mutate the -object, whereas superficially similar operations (for example ``y = y + [10]`` -and ``sorted(y)``) create a new object. In general in Python (and in all cases -in the standard library) a method that mutates an object will return ``None`` -to help avoid getting the two types of operations confused. So if you -mistakenly write ``y.sort()`` thinking it will give you a sorted copy of ``y``, -you'll instead end up with ``None``, which will likely cause your program to -generate an easily diagnosed error. - -However, there is one class of operations where the same operation sometimes -has different behaviors with different types: the augmented assignment -operators. For example, ``+=`` mutates lists but not tuples or ints (``a_list -+= [1, 2, 3]`` is equivalent to ``a_list.extend([1, 2, 3])`` and mutates -``a_list``, whereas ``some_tuple += (1, 2, 3)`` and ``some_int += 1`` create -new objects). - -In other words: - -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables - that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, - but operations that transform that value into a new value always return a new - object. - -If you want to know if two variables refer to the same object or not, you can -use the :keyword:`is` operator, or the built-in function :func:`id`. - - How do I write a function with output parameters (call by reference)? --------------------------------------------------------------------- @@ -700,11 +513,11 @@ Since the comma is not an operator, but a separator between expressions the above is evaluated as if you had entered:: - ("a" in "b"), "a" + >>> ("a" in "b"), "a" not:: - "a" in ("b", "a") + >>> "a" in ("b", "a") The same is true of the various assignment operators (``=``, ``+=`` etc). They are not truly operators but syntactic delimiters in assignment statements. @@ -819,7 +632,7 @@ 144`` and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, base)`` takes the base to convert from as a second optional argument, so ``int('0x144', 16) == 324``. If the base is specified as 0, the number is interpreted using Python's -rules: a leading '0o' indicates octal, and '0x' indicates a hex number. +rules: a leading '0' indicates octal, and '0x' indicates a hex number. Do not use the built-in function :func:`eval` if all you need is to convert strings to numbers. :func:`eval` will be significantly slower and it presents a @@ -840,7 +653,7 @@ constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see the :ref:`string-formatting` section, e.g. ``"{:04d}".format(144)`` yields -``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. +``'0144'`` and ``"{:.3f}".format(1/3)`` yields ``'0.333'``. How do I modify a string in place? @@ -849,10 +662,9 @@ You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +unicode data, try using a :class:`io.StringIO` object or the :mod:`array` module:: - >>> import io >>> s = "Hello, world" >>> sio = io.StringIO(s) >>> sio.getvalue() @@ -870,7 +682,7 @@ array('u', 'Hello, world') >>> a[0] = 'y' >>> print(a) - array('u', 'yello, world') + array('u', 'yello world') >>> a.tounicode() 'yello, world' @@ -1010,7 +822,8 @@ may come up with. This is doubly true for primitives written in C, such as builtins and some extension types. For example, be sure to use either the :meth:`list.sort` built-in method or the related :func:`sorted` - function to do sorting (and see the :ref:`sortinghowto` for examples + function to do sorting (and see the + `sorting mini-HOWTO <http://wiki.python.org/moin/HowTo/Sorting>`_ for examples of moderately advanced usage). * Abstractions tend to create indirections and force the interpreter to work @@ -1030,7 +843,7 @@ .. seealso:: The wiki page devoted to `performance tips - <https://wiki.python.org/moin/PythonSpeed/PerformanceTips>`_. + <http://wiki.python.org/moin/PythonSpeed/PerformanceTips>`_. .. _efficient_string_concatenation: @@ -1115,7 +928,7 @@ See the Python Cookbook for a long discussion of many ways to do this: - http://code.activestate.com/recipes/52560/ + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560 If you don't mind reordering the list, sort it and then scan from the end of the list, deleting duplicates as you go:: @@ -1163,14 +976,12 @@ usually a lot slower than using Python lists. -.. _faq-multidimensional-list: - How do I create a multidimensional list? ---------------------------------------- You probably tried to make a multidimensional array like this:: - >>> A = [[None] * 2] * 3 + A = [[None] * 2] * 3 This looks correct if you print it:: @@ -1202,7 +1013,7 @@ A = [[None] * w for i in range(h)] Or, you can use an extension that provides a matrix datatype; `Numeric Python -<http://www.numpy.org/>`_ is the best known. +<http://numpy.scipy.org/>`_ is the best known. How do I apply a method to a sequence of objects? @@ -1212,101 +1023,38 @@ result = [obj.method() for obj in mylist] -.. _faq-augmented-assignment-tuple-error: - -Why does a_tuple[i] += ['item'] raise an exception when the addition works? ---------------------------------------------------------------------------- - -This is because of a combination of the fact that augmented assignment -operators are *assignment* operators, and the difference between mutable and -immutable objects in Python. - -This discussion applies in general when augmented assignment operators are -applied to elements of a tuple that point to mutable objects, but we'll use -a ``list`` and ``+=`` as our exemplar. - -If you wrote:: - - >>> a_tuple = (1, 2) - >>> a_tuple[0] += 1 - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The reason for the exception should be immediately clear: ``1`` is added to the -object ``a_tuple[0]`` points to (``1``), producing the result object, ``2``, -but when we attempt to assign the result of the computation, ``2``, to element -``0`` of the tuple, we get an error because we can't change what an element of -a tuple points to. - -Under the covers, what this augmented assignment statement is doing is -approximately this:: - - >>> result = a_tuple[0] + 1 - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -It is the assignment part of the operation that produces the error, since a -tuple is immutable. - -When you write something like:: - - >>> a_tuple = (['foo'], 'bar') - >>> a_tuple[0] += ['item'] - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The exception is a bit more surprising, and even more surprising is the fact -that even though there was an error, the append worked:: - - >>> a_tuple[0] - ['foo', 'item'] - -To see why this happens, you need to know that (a) if an object implements an -``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment -is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for ``list.extend``:: - - >>> a_list = [] - >>> a_list += [1] - >>> a_list - [1] - -This is equivalent to:: - - >>> result = a_list.__iadd__([1]) - >>> a_list = result - -The object pointed to by a_list has been mutated, and the pointer to the -mutated object is assigned back to ``a_list``. The end result of the -assignment is a no-op, since it is a pointer to the same object that ``a_list`` -was previously pointing to, but the assignment still happens. - -Thus, in our tuple example what is happening is equivalent to:: - - >>> result = a_tuple[0].__iadd__(['item']) - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The ``__iadd__`` succeeds, and thus the list is extended, but even though -``result`` points to the same object that ``a_tuple[0]`` already points to, -that final assignment still results in an error, because tuples are immutable. - Dictionaries ============ -How can I get a dictionary to store and display its keys in a consistent order? -------------------------------------------------------------------------------- +How can I get a dictionary to display its keys in a consistent order? +--------------------------------------------------------------------- -Use :class:`collections.OrderedDict`. +You can't. Dictionaries store their keys in an unpredictable order, so the +display order of a dictionary's elements will be similarly unpredictable. + +This can be frustrating if you want to save a printable version to a file, make +some changes and then compare it with some other printed dictionary. In this +case, use the ``pprint`` module to pretty-print the dictionary; the items will +be presented in order sorted by the key. + +A more complicated solution is to subclass ``dict`` to create a +``SortedDict`` class that prints itself in a predictable order. Here's one +simpleminded implementation of such a class:: + + class SortedDict(dict): + def __repr__(self): + keys = sorted(self.keys()) + result = ("{!r}: {!r}".format(k, self[k]) for k in keys) + return "{{{}}}".format(", ".join(result)) + + __str__ = __repr__ + +This will work for many common situations you might encounter, though it's far +from a perfect solution. The largest flaw is that if some values in the +dictionary are also dictionaries, their values won't be presented in any +particular order. + I want to do a complicated sort: can you do a Schwartzian Transform in Python? ------------------------------------------------------------------------------ @@ -1685,76 +1433,41 @@ keeping a list of weak references to each instance. -Why does the result of ``id()`` appear to be not unique? --------------------------------------------------------- - -The :func:`id` builtin returns an integer that is guaranteed to be unique during -the lifetime of the object. Since in CPython, this is the object's memory -address, it happens frequently that after an object is deleted from memory, the -next freshly created object is allocated at the same position in memory. This -is illustrated by this example: - ->>> id(1000) -13901272 ->>> id(2000) -13901272 - -The two ids belong to different integer objects that are created before, and -deleted immediately after execution of the ``id()`` call. To be sure that -objects whose id you want to examine are still alive, create another reference -to the object: - ->>> a = 1000; b = 2000 ->>> id(a) -13901272 ->>> id(b) -13891296 - - Modules ======= How do I create a .pyc file? ---------------------------- -When a module is imported for the first time (or when the source file has -changed since the current compiled file was created) a ``.pyc`` file containing -the compiled code should be created in a ``__pycache__`` subdirectory of the -directory containing the ``.py`` file. The ``.pyc`` file will have a -filename that starts with the same name as the ``.py`` file, and ends with -``.pyc``, with a middle component that depends on the particular ``python`` -binary that created it. (See :pep:`3147` for details.) +When a module is imported for the first time (or when the source is more recent +than the current compiled file) a ``.pyc`` file containing the compiled code +should be created in the same directory as the ``.py`` file. -One reason that a ``.pyc`` file may not be created is a permissions problem -with the directory containing the source file, meaning that the ``__pycache__`` -subdirectory cannot be created. This can happen, for example, if you develop as -one user but run as another, such as if you are testing with a web server. +One reason that a ``.pyc`` file may not be created is permissions problems with +the directory. This can happen, for example, if you develop as one user but run +as another, such as if you are testing with a web server. Creation of a .pyc +file is automatic if you're importing a module and Python has the ability +(permissions, free space, etc...) to write the compiled module back to the +directory. -Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, -creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` -subdirectory and write the compiled module to that subdirectory. +Running Python on a top level script is not considered an import and no ``.pyc`` +will be created. For example, if you have a top-level module ``abc.py`` that +imports another module ``xyz.py``, when you run abc, ``xyz.pyc`` will be created +since xyz is imported, but no ``abc.pyc`` file will be created since ``abc.py`` +isn't being imported. -Running Python on a top level script is not considered an import and no -``.pyc`` will be created. For example, if you have a top-level module -``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by -typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for -``xyz`` because ``xyz`` is imported, but no ``.pyc`` file will be created for -``foo`` since ``foo.py`` isn't being imported. - -If you need to create a ``.pyc`` file for ``foo`` -- that is, to create a -``.pyc`` file for a module that is not imported -- you can, using the -:mod:`py_compile` and :mod:`compileall` modules. +If you need to create abc.pyc -- that is, to create a .pyc file for a module +that is not imported -- you can, using the :mod:`py_compile` and +:mod:`compileall` modules. The :mod:`py_compile` module can manually compile any module. One way is to use the ``compile()`` function in that module interactively:: >>> import py_compile - >>> py_compile.compile('foo.py') # doctest: +SKIP + >>> py_compile.compile('abc.py') -This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same -location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +This will write the ``.pyc`` to the same location as ``abc.py`` (or you can +override that with the optional parameter ``cfile``). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -1839,10 +1552,19 @@ __import__('x.y.z') returns <module 'x'>; how do I get z? --------------------------------------------------------- -Consider using the convenience function :func:`~importlib.import_module` from -:mod:`importlib` instead:: +Try:: - z = importlib.import_module('x.y.z') + __import__('x.y.z').y.z + +For more realistic situations, you may have to do something like :: + + m = __import__(s) + for i in s.split(".")[1:]: + m = getattr(m, i) + +See :mod:`importlib` for a convenience function called +:func:`~importlib.import_module`. + When I edit an imported module and reimport it, the changes don't show up. Why does this happen? @@ -1851,12 +1573,12 @@ For reasons of efficiency as well as consistency, Python only reads the module file on the first time a module is imported. If it didn't, in a program consisting of many modules where each one imports the same basic module, the -basic module would be parsed and re-parsed many times. To force re-reading of a +basic module would be parsed and re-parsed many times. To force rereading of a changed module, do this:: - import importlib + import imp import modname - importlib.reload(modname) + imp.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules containing statements like :: @@ -1868,10 +1590,10 @@ updated to use the new class definition. This can result in the following paradoxical behaviour: - >>> import importlib + >>> import imp >>> import cls >>> c = cls.C() # Create an instance of C - >>> importlib.reload(cls) + >>> imp.reload(cls) <module 'cls' from 'cls.py'> >>> isinstance(c, cls.C) # isinstance is false?!? False diff -r 6db40a9955dc -r 0d413f60cc23 Doc/faq/windows.rst --- a/Doc/faq/windows.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/faq/windows.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,9 +6,7 @@ Python on Windows FAQ ===================== -.. only:: html - - .. contents:: +.. contents:: .. XXX need review for Python 3. XXX need review for Windows Vista/Seven? @@ -19,7 +17,9 @@ This is not necessarily a straightforward question. If you are already familiar with running programs from the Windows command line then everything will seem -obvious; otherwise, you might need a little more guidance. +obvious; otherwise, you might need a little more guidance. There are also +differences between Windows 95, 98, NT, ME, 2000 and XP which can add to the +confusion. .. sidebar:: |Python Development on XP|_ :subtitle: `Python Development on XP`_ @@ -31,12 +31,12 @@ .. |Python Development on XP| image:: python-video-icon.png .. _`Python Development on XP`: - http://showmedo.com/videotutorials/series?name=pythonOzsvaldPyNewbieSeries + http://www.showmedo.com/videos/series?name=pythonOzsvaldPyNewbieSeries Unless you use some sort of integrated development environment, you will end up *typing* Windows commands into what is variously referred to as a "DOS window" or "Command prompt window". Usually you can create such a window from your -Start menu; under Windows 7 the menu selection is :menuselection:`Start --> +Start menu; under Windows 2000 the menu selection is :menuselection:`Start --> Programs --> Accessories --> Command Prompt`. You should be able to recognize when you have started such a window because you will see a Windows "command prompt", which usually looks like this:: @@ -46,27 +46,23 @@ The letter may be different, and there might be other things after it, so you might just as easily see something like:: - D:\YourName\Projects\Python> + D:\Steve\Projects\Python> depending on how your computer has been set up and what else you have recently done with it. Once you have started such a window, you are well on the way to running Python programs. You need to realize that your Python scripts have to be processed by another -program called the Python *interpreter*. The interpreter reads your script, +program called the Python interpreter. The interpreter reads your script, compiles it into bytecodes, and then executes the bytecodes to run your program. So, how do you arrange for the interpreter to handle your Python? First, you need to make sure that your command window recognises the word "python" as an instruction to start the interpreter. If you have opened a command window, you should try entering the command ``python`` and hitting -return.:: +return. You should then see something like:: - C:\Users\YourName> python - -You should then see something like:: - - Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32 + Python 2.2 (#28, Dec 21 2001, 12:21:22) [MSC 32 bit (Intel)] on win32 Type "help", "copyright", "credits" or "license" for more information. >>> @@ -78,23 +74,24 @@ >>> print("Hello") Hello >>> "Hello" * 3 - 'HelloHelloHello' + HelloHelloHello Many people use the interactive mode as a convenient yet highly programmable -calculator. When you want to end your interactive Python session, hold the :kbd:`Ctrl` -key down while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get back to your +calculator. When you want to end your interactive Python session, hold the Ctrl +key down while you enter a Z, then hit the "Enter" key to get back to your Windows command prompt. You may also find that you have a Start-menu entry such as :menuselection:`Start ---> Programs --> Python 3.3 --> Python (command line)` that results in you +--> Programs --> Python 2.2 --> Python (command line)` that results in you seeing the ``>>>`` prompt in a new window. If so, the window will disappear -after you enter the :kbd:`Ctrl-Z` character; Windows is running a single "python" +after you enter the Ctrl-Z character; Windows is running a single "python" command in the window, and closes it when you terminate the interpreter. If the ``python`` command, instead of displaying the interpreter prompt ``>>>``, gives you a message like:: - 'python' is not recognized as an internal or external command, operable program or batch file. + 'python' is not recognized as an internal or external command, + operable program or batch file. .. sidebar:: |Adding Python to DOS Path|_ :subtitle: `Adding Python to DOS Path`_ @@ -105,7 +102,7 @@ .. |Adding Python to DOS Path| image:: python-video-icon.png .. _`Adding Python to DOS Path`: - http://showmedo.com/videotutorials/video?name=960000&fromSeriesID=96 + http://showmedo.com/videos/video?name=960000&fromSeriesID=96 or:: @@ -123,33 +120,115 @@ dir C:\py* will probably tell you where it is installed; the usual location is something -like ``C:\Python33``. Otherwise you will be reduced to a search of your whole +like ``C:\Python23``. Otherwise you will be reduced to a search of your whole disk ... use :menuselection:`Tools --> Find` or hit the :guilabel:`Search` button and look for "python.exe". Supposing you discover that Python is -installed in the ``C:\Python33`` directory (the default at the time of writing), +installed in the ``C:\Python23`` directory (the default at the time of writing), you should make sure that entering the command :: - c:\Python33\python + c:\Python23\python -starts up the interpreter as above (and don't forget you'll need a ":kbd:`Ctrl-Z`" and -an ":kbd:`Enter`" to get out of it). Once you have verified the directory, you can -add it to the system path to make it easier to start Python by just running -the ``python`` command. This is currently an option in the installer as of -CPython 3.3. +starts up the interpreter as above (and don't forget you'll need a "CTRL-Z" and +an "Enter" to get out of it). Once you have verified the directory, you need to +add it to the start-up routines your computer goes through. For older versions +of Windows the easiest way to do this is to edit the ``C:\AUTOEXEC.BAT`` +file. You would want to add a line like the following to ``AUTOEXEC.BAT``:: -More information about environment variables can be found on the -:ref:`Using Python on Windows <setting-envvars>` page. + PATH C:\Python23;%PATH% + +For Windows NT, 2000 and (I assume) XP, you will need to add a string such as :: + + ;C:\Python23 + +to the current setting for the PATH environment variable, which you will find in +the properties window of "My Computer" under the "Advanced" tab. Note that if +you have sufficient privilege you might get a choice of installing the settings +either for the Current User or for System. The latter is preferred if you want +everybody to be able to run Python on the machine. + +If you aren't confident doing any of these manipulations yourself, ask for help! +At this stage you may want to reboot your system to make absolutely sure the new +setting has taken effect. You probably won't need to reboot for Windows NT, XP +or 2000. You can also avoid it in earlier versions by editing the file +``C:\WINDOWS\COMMAND\CMDINIT.BAT`` instead of ``AUTOEXEC.BAT``. + +You should now be able to start a new command window, enter ``python`` at the +``C:\>`` (or whatever) prompt, and see the ``>>>`` prompt that indicates the +Python interpreter is reading interactive commands. + +Let's suppose you have a program called ``pytest.py`` in directory +``C:\Steve\Projects\Python``. A session to run that program might look like +this:: + + C:\> cd \Steve\Projects\Python + C:\Steve\Projects\Python> python pytest.py + +Because you added a file name to the command to start the interpreter, when it +starts up it reads the Python script in the named file, compiles it, executes +it, and terminates, so you see another ``C:\>`` prompt. You might also have +entered :: + + C:\> python \Steve\Projects\Python\pytest.py + +if you hadn't wanted to change your current directory. + +Under NT, 2000 and XP you may well find that the installation process has also +arranged that the command ``pytest.py`` (or, if the file isn't in the current +directory, ``C:\Steve\Projects\Python\pytest.py``) will automatically recognize +the ".py" extension and run the Python interpreter on the named file. Using this +feature is fine, but *some* versions of Windows have bugs which mean that this +form isn't exactly equivalent to using the interpreter explicitly, so be +careful. + +The important things to remember are: + +1. Start Python from the Start Menu, or make sure the PATH is set correctly so + Windows can find the Python interpreter. :: + + python + + should give you a '>>>' prompt from the Python interpreter. Don't forget the + CTRL-Z and ENTER to terminate the interpreter (and, if you started the window + from the Start Menu, make the window disappear). + +2. Once this works, you run programs with commands:: + + python {program-file} + +3. When you know the commands to use you can build Windows shortcuts to run the + Python interpreter on any of your scripts, naming particular working + directories, and adding them to your menus. Take a look at :: + + python --help + + if your needs are complex. + +4. Interactive mode (where you see the ``>>>`` prompt) is best used for checking + that individual statements and expressions do what you think they will, and + for developing code by experiment. + How do I make Python scripts executable? ---------------------------------------- -On Windows, the standard Python installer already associates the .py +On Windows 2000, the standard Python installer already associates the .py extension with a file type (Python.File) and gives that file type an open command that runs the interpreter (``D:\Program Files\Python\python.exe "%1" %*``). This is enough to make scripts executable from the command prompt as 'foo.py'. If you'd rather be able to execute the script by simple typing 'foo' with no extension you need to add .py to the PATHEXT environment variable. +On Windows NT, the steps taken by the installer as described above allow you to +run a script with 'foo.py', but a longtime bug in the NT command processor +prevents you from redirecting the input or output of any script executed in this +way. This is often important. + +The incantation for making a Python script executable under WinNT is to give the +file an extension of .cmd and add the following as the first line:: + + @setlocal enableextensions & python -x %~f0 %* & goto :EOF + + Why does Python sometimes take so long to start? ------------------------------------------------ @@ -167,23 +246,32 @@ offender. -How do I make an executable from a Python script? -------------------------------------------------- +Where is Freeze for Windows? +---------------------------- -See http://cx-freeze.sourceforge.net/ for a distutils extension that allows you -to create console and GUI executables from Python code. -`py2exe <http://www.py2exe.org/>`_, the most popular extension for building -Python 2.x-based executables, does not yet support Python 3 but a version that -does is in development. +"Freeze" is a program that allows you to ship a Python program as a single +stand-alone executable file. It is *not* a compiler; your programs don't run +any faster, but they are more easily distributable, at least to platforms with +the same OS and CPU. Read the README file of the freeze program for more +disclaimers. + +You can use freeze on Windows, but you must download the source tree (see +http://www.python.org/download/source). The freeze program is in the +``Tools\freeze`` subdirectory of the source tree. + +You need the Microsoft VC++ compiler, and you probably need to build Python. +The required project files are in the PCbuild directory. Is a ``*.pyd`` file the same as a DLL? -------------------------------------- +.. XXX update for py3k (PyInit_foo) + Yes, .pyd files are dll's, but there are a few differences. If you have a DLL -named ``foo.pyd``, then it must have a function ``PyInit_foo()``. You can then +named ``foo.pyd``, then it must have a function ``initfoo()``. You can then write Python "import foo", and Python will search for foo.pyd (as well as -foo.py, foo.pyc) and if it finds it, will attempt to call ``PyInit_foo()`` to +foo.py, foo.pyc) and if it finds it, will attempt to call ``initfoo()`` to initialize it. You do not link your .exe with foo.lib, as that would cause Windows to require the DLL to be present. @@ -204,7 +292,7 @@ be a DLL to handle importing modules that are themselves DLL's. (This is the first key undocumented fact.) Instead, link to :file:`python{NN}.dll`; it is typically installed in ``C:\Windows\System``. *NN* is the Python version, a - number such as "33" for Python 3.3. + number such as "23" for Python 2.3. You can link to Python in two different ways. Load-time linking means linking against :file:`python{NN}.lib`, while run-time linking means linking @@ -249,7 +337,7 @@ ... Py_Initialize(); // Initialize Python. initmyAppc(); // Initialize (import) the helper class. - PyRun_SimpleString("import myApp"); // Import the shadow class. + PyRun_SimpleString("import myApp") ; // Import the shadow class. 5. There are two problems with Python's C API which will become apparent if you use a compiler other than MSVC, the compiler used to build pythonNN.dll. @@ -288,6 +376,47 @@ object that supports read and write, so all you need is a Python object (defined in your extension module) that contains read() and write() methods. + +How do I use Python for CGI? +---------------------------- + +On the Microsoft IIS server or on the Win95 MS Personal Web Server you set up +Python in the same way that you would set up any other scripting engine. + +Run regedt32 and go to:: + + HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\W3SVC\Parameters\ScriptMap + +and enter the following line (making any specific changes that your system may +need):: + + .py :REG_SZ: c:\<path to python>\python.exe -u %s %s + +This line will allow you to call your script with a simple reference like: +``http://yourserver/scripts/yourscript.py`` provided "scripts" is an +"executable" directory for your server (which it usually is by default). The +:option:`-u` flag specifies unbuffered and binary mode for stdin - needed when +working with binary data. + +In addition, it is recommended that using ".py" may not be a good idea for the +file extensions when used in this context (you might want to reserve ``*.py`` +for support modules and use ``*.cgi`` or ``*.cgp`` for "main program" scripts). + +In order to set up Internet Information Services 5 to use Python for CGI +processing, please see the following links: + + http://www.e-coli.net/pyiis_server.html (for Win2k Server) + http://www.e-coli.net/pyiis.html (for Win2k pro) + +Configuring Apache is much simpler. In the Apache configuration file +``httpd.conf``, add the following line at the end of the file:: + + ScriptInterpreterSource Registry + +Then, give your Python CGI-scripts the extension .py and put them in the cgi-bin +directory. + + How do I keep editors from inserting tabs into my Python source? ---------------------------------------------------------------- @@ -327,10 +456,120 @@ return (0 != kernel32.TerminateProcess(handle, 0)) In 2.7 and 3.2, :func:`os.kill` is implemented similar to the above function, -with the additional feature of being able to send :kbd:`Ctrl+C` and :kbd:`Ctrl+Break` +with the additional feature of being able to send CTRL+C and CTRL+BREAK to console subprocesses which are designed to handle those signals. See :func:`os.kill` for further details. + +Why does os.path.isdir() fail on NT shared directories? +------------------------------------------------------- + +The solution appears to be always append the "\\" on the end of shared +drives. + + >>> import os + >>> os.path.isdir( '\\\\rorschach\\public') + 0 + >>> os.path.isdir( '\\\\rorschach\\public\\') + 1 + +It helps to think of share points as being like drive letters. Example:: + + k: is not a directory + k:\ is a directory + k:\media is a directory + k:\media\ is not a directory + +The same rules apply if you substitute "k:" with "\\conky\foo":: + + \\conky\foo is not a directory + \\conky\foo\ is a directory + \\conky\foo\media is a directory + \\conky\foo\media\ is not a directory + + +cgi.py (or other CGI programming) doesn't work sometimes on NT or win95! +------------------------------------------------------------------------ + +Be sure you have the latest python.exe, that you are using python.exe rather +than a GUI version of Python and that you have configured the server to execute +:: + + "...\python.exe -u ..." + +for the CGI execution. The :option:`-u` (unbuffered) option on NT and Win95 +prevents the interpreter from altering newlines in the standard input and +output. Without it post/multipart requests will seem to have the wrong length +and binary (e.g. GIF) responses may get garbled (resulting in broken images, PDF +files, and other binary downloads failing). + + +Why doesn't os.popen() work in PythonWin on NT? +----------------------------------------------- + +The reason that os.popen() doesn't work from within PythonWin is due to a bug in +Microsoft's C Runtime Library (CRT). The CRT assumes you have a Win32 console +attached to the process. + +You should use the win32pipe module's popen() instead which doesn't depend on +having an attached Win32 console. + +Example:: + + import win32pipe + f = win32pipe.popen('dir /c c:\\') + print(f.readlines()) + f.close() + + +Why doesn't os.popen()/win32pipe.popen() work on Win9x? +------------------------------------------------------- + +There is a bug in Win9x that prevents os.popen/win32pipe.popen* from +working. The good news is there is a way to work around this problem. The +Microsoft Knowledge Base article that you need to lookup is: Q150956. You will +find links to the knowledge base at: http://support.microsoft.com/. + + +PyRun_SimpleFile() crashes on Windows but not on Unix; why? +----------------------------------------------------------- + +This is very sensitive to the compiler vendor, version and (perhaps) even +options. If the FILE* structure in your embedding program isn't the same as is +assumed by the Python interpreter it won't work. + +The Python 1.5.* DLLs (``python15.dll``) are all compiled with MS VC++ 5.0 and +with multithreading-DLL options (``/MD``). + +If you can't change compilers or flags, try using :c:func:`Py_RunSimpleString`. +A trick to get it to run an arbitrary file is to construct a call to +:func:`exec` and :func:`open` with the name of your file as argument. + +Also note that you can not mix-and-match Debug and Release versions. If you +wish to use the Debug Multithreaded DLL, then your module *must* have ``_d`` +appended to the base name. + + +Importing _tkinter fails on Windows 95/98: why? +------------------------------------------------ + +Sometimes, the import of _tkinter fails on Windows 95 or 98, complaining with a +message like the following:: + + ImportError: DLL load failed: One of the library files needed + to run this application cannot be found. + +It could be that you haven't installed Tcl/Tk, but if you did install Tcl/Tk, +and the Wish application works correctly, the problem may be that its installer +didn't manage to edit the autoexec.bat file correctly. It tries to add a +statement that changes the PATH environment variable to include the Tcl/Tk 'bin' +subdirectory, but sometimes this edit doesn't quite work. Opening it with +notepad usually reveals what the problem is. + +(One additional hint, noted by David Szafranski: you can't use long filenames +here; e.g. use ``C:\PROGRA~1\Tcl\bin`` instead of ``C:\Program Files\Tcl\bin``.) + + How do I extract the downloaded documentation on Windows? --------------------------------------------------------- @@ -342,3 +581,38 @@ able to handle it. (If your copy of WinZip doesn't, get a newer one from http://www.winzip.com.) + +Missing cw3215mt.dll (or missing cw3215.dll) +-------------------------------------------- + +Sometimes, when using Tkinter on Windows, you get an error that cw3215mt.dll or +cw3215.dll is missing. + +Cause: you have an old Tcl/Tk DLL built with cygwin in your path (probably +``C:\Windows``). You must use the Tcl/Tk DLLs from the standard Tcl/Tk +installation (Python 1.5.2 comes with one). + + +Warning about CTL3D32 version from installer +-------------------------------------------- + +The Python installer issues a warning like this:: + + This version uses CTL3D32.DLL which is not the correct version. + This version is used for windows NT applications only. + +Tim Peters: + + This is a Microsoft DLL, and a notorious source of problems. The message + means what it says: you have the wrong version of this DLL for your operating + system. The Python installation did not cause this -- something else you + installed previous to this overwrote the DLL that came with your OS (probably + older shareware of some sort, but there's no way to tell now). If you search + for "CTL3D32" using any search engine (AltaVista, for example), you'll find + hundreds and hundreds of web pages complaining about the same problem with + all sorts of installation programs. They'll point you to ways to get the + correct version reinstalled on your system (since Python doesn't cause this, + we can't fix it). + +David A Burton has written a little program to fix this. Go to +http://www.burtonsys.com/downloads.html and click on "ctl3dfix.zip". diff -r 6db40a9955dc -r 0d413f60cc23 Doc/glossary.rst --- a/Doc/glossary.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/glossary.rst Mon Jan 25 17:05:13 2016 +0100 @@ -40,90 +40,25 @@ ABCs with the :mod:`abc` module. argument - A value passed to a :term:`function` (or :term:`method`) when calling the - function. There are two kinds of argument: + A value passed to a function or method, assigned to a named local + variable in the function body. A function or method may have both + positional arguments and keyword arguments in its definition. + Positional and keyword arguments may be variable-length: ``*`` accepts + or passes (if in the function definition or call) several positional + arguments in a list, while ``**`` does the same for keyword arguments + in a dictionary. - * :dfn:`keyword argument`: an argument preceded by an identifier (e.g. - ``name=``) in a function call or passed as a value in a dictionary - preceded by ``**``. For example, ``3`` and ``5`` are both keyword - arguments in the following calls to :func:`complex`:: - - complex(real=3, imag=5) - complex(**{'real': 3, 'imag': 5}) - - * :dfn:`positional argument`: an argument that is not a keyword argument. - Positional arguments can appear at the beginning of an argument list - and/or be passed as elements of an :term:`iterable` preceded by ``*``. - For example, ``3`` and ``5`` are both positional arguments in the - following calls:: - - complex(3, 5) - complex(*(3, 5)) - - Arguments are assigned to the named local variables in a function body. - See the :ref:`calls` section for the rules governing this assignment. - Syntactically, any expression can be used to represent an argument; the - evaluated value is assigned to the local variable. - - See also the :term:`parameter` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, and :pep:`362`. - - asynchronous context manager - An object which controls the environment seen in an - :keyword:`async with` statement by defining :meth:`__aenter__` and - :meth:`__aexit__` methods. Introduced by :pep:`492`. - - asynchronous iterable - An object, that can be used in an :keyword:`async for` statement. - Must return an :term:`awaitable` from its :meth:`__aiter__` method, - which should in turn be resolved in an :term:`asynchronous iterator` - object. Introduced by :pep:`492`. - - asynchronous iterator - An object that implements :meth:`__aiter__` and :meth:`__anext__` - methods, that must return :term:`awaitable` objects. - :keyword:`async for` resolves awaitable returned from asynchronous - iterator's :meth:`__anext__` method until it raises - :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + Any expression may be used within the argument list, and the evaluated + value is passed to the local variable. attribute A value associated with an object which is referenced by name using dotted expressions. For example, if an object *o* has an attribute *a* it would be referenced as *o.a*. - awaitable - An object that can be used in an :keyword:`await` expression. Can be - a :term:`coroutine` or an object with an :meth:`__await__` method. - See also :pep:`492`. - BDFL Benevolent Dictator For Life, a.k.a. `Guido van Rossum - <https://www.python.org/~guido/>`_, Python's creator. - - binary file - A :term:`file object` able to read and write - :term:`bytes-like objects <bytes-like object>`. - - .. seealso:: - A :term:`text file` reads and writes :class:`str` objects. - - bytes-like object - An object that supports the :ref:`bufferobjects` and can - export a C-:term:`contiguous` buffer. This includes all :class:`bytes`, - :class:`bytearray`, and :class:`array.array` objects, as well as many - common :class:`memoryview` objects. Bytes-like objects can - be used for various operations that work with binary data; these include - compression, saving to a binary file, and sending over a socket. - - Some operations need the binary data to be mutable. The documentation - often refers to these as "read-write bytes-like objects". Example - mutable buffer objects include :class:`bytearray` and a - :class:`memoryview` of a :class:`bytearray`. - Other operations require the binary data to be stored in - immutable objects ("read-only bytes-like objects"); examples - of these include :class:`bytes` and a :class:`memoryview` - of a :class:`bytes` object. + <http://www.python.org/~guido/>`_, Python's creator. bytecode Python source code is compiled into bytecode, the internal representation @@ -171,35 +106,9 @@ statement by defining :meth:`__enter__` and :meth:`__exit__` methods. See :pep:`343`. - contiguous - .. index:: C-contiguous, Fortran contiguous - - A buffer is considered contiguous exactly if it is either - *C-contiguous* or *Fortran contiguous*. Zero-dimensional buffers are - C and Fortran contiguous. In one-dimensional arrays, the items - must be layed out in memory next to each other, in order of - increasing indexes starting from zero. In multidimensional - C-contiguous arrays, the last index varies the fastest when - visiting items in order of memory address. However, in - Fortran contiguous arrays, the first index varies the fastest. - - coroutine - Coroutines is a more generalized form of subroutines. Subroutines are - entered at one point and exited at another point. Coroutines can be - entered, exited, and resumed at many different points. They can be - implemented with the :keyword:`async def` statement. See also - :pep:`492`. - - coroutine function - A function which returns a :term:`coroutine` object. A coroutine - function may be defined with the :keyword:`async def` statement, - and may contain :keyword:`await`, :keyword:`async for`, and - :keyword:`async with` keywords. These were introduced - by :pep:`492`. - CPython The canonical implementation of the Python programming language, as - distributed on `python.org <https://www.python.org>`_. The term "CPython" + distributed on `python.org <http://python.org>`_. The term "CPython" is used when necessary to distinguish this implementation from others such as Jython or IronPython. @@ -241,14 +150,6 @@ keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. Called a hash in Perl. - dictionary view - The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and - :meth:`dict.items` are called dictionary views. They provide a dynamic - view on the dictionary’s entries, which means that when the dictionary - changes, the view reflects these changes. To force the - dictionary view to become a full list use ``list(dictview)``. See - :ref:`dict-views`. - docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is @@ -293,29 +194,24 @@ An object exposing a file-oriented API (with methods such as :meth:`read()` or :meth:`write()`) to an underlying resource. Depending on the way it was created, a file object can mediate access to a real - on-disk file or to another type of storage or communication device + on-disk file or to another other type of storage or communication device (for example standard input/output, in-memory buffers, sockets, pipes, etc.). File objects are also called :dfn:`file-like objects` or :dfn:`streams`. - There are actually three categories of file objects: raw - :term:`binary files <binary file>`, buffered - :term:`binary files <binary file>` and :term:`text files <text file>`. - Their interfaces are defined in the :mod:`io` module. The canonical - way to create a file object is by using the :func:`open` function. + There are actually three categories of file objects: raw binary files, + buffered binary files and text files. Their interfaces are defined in the + :mod:`io` module. The canonical way to create a file object is by using + the :func:`open` function. file-like object A synonym for :term:`file object`. finder - An object that tries to find the :term:`loader` for a module that is - being imported. - - Since Python 3.3, there are two types of finder: :term:`meta path finders - <meta path finder>` for use with :data:`sys.meta_path`, and :term:`path - entry finders <path entry finder>` for use with :data:`sys.path_hooks`. - - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + An object that tries to find the :term:`loader` for a module. It must + implement a method named :meth:`find_module`. See :pep:`302` for + details and :class:`importlib.abc.Finder` for an + :term:`abstract base class`. floor division Mathematical division that rounds down to nearest integer. The floor @@ -326,19 +222,8 @@ function A series of statements which returns some value to a caller. It can also - be passed zero or more :term:`arguments <argument>` which may be used in - the execution of the body. See also :term:`parameter`, :term:`method`, - and the :ref:`function` section. - - function annotation - An arbitrary metadata value associated with a function parameter or return - value. Its syntax is explained in section :ref:`function`. Annotations - may be accessed via the :attr:`__annotations__` special attribute of a - function object. - - Python itself does not assign any particular meaning to function - annotations. They are intended to be interpreted by third-party libraries - or tools. See :pep:`3107`, which describes some of their potential uses. + be passed zero or more arguments which may be used in the execution of + the body. See also :term:`argument` and :term:`method`. __future__ A pseudo-module which programmers can use to enable new language features @@ -360,23 +245,14 @@ .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. - - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. - - generator iterator - An object created by a :term:`generator` function. - - Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks-up where - it left-off (in contrast to functions which start fresh on every - invocation). + A function which returns an iterator. It looks like a normal function + except that it contains :keyword:`yield` statements for producing a series + a values usable in a for-loop or that can be retrieved one at a time with + the :func:`next` function. Each :keyword:`yield` temporarily suspends + processing, remembering the location execution state (including local + variables and pending try-statements). When the generator resumes, it + picks-up where it left-off (in contrast to functions which start fresh on + every invocation). .. index:: single: generator expression @@ -389,15 +265,6 @@ >>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81 285 - generic function - A function composed of multiple functions implementing the same operation - for different types. Which implementation should be used during a call is - determined by the dispatch algorithm. - - See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. - - GIL See :term:`global interpreter lock`. @@ -434,8 +301,7 @@ All of Python's immutable built-in objects are hashable, while no mutable containers (such as lists or dictionaries) are. Objects which are instances of user-defined classes are hashable by default; they all - compare unequal (except with themselves), and their hash value is derived - from their :func:`id`. + compare unequal, and their hash value is their :func:`id`. IDLE An Integrated Development Environment for Python. IDLE is a basic editor @@ -449,17 +315,6 @@ role in places where a constant hash value is needed, for example as a key in a dictionary. - import path - A list of locations (or :term:`path entries <path entry>`) that are - searched by the :term:`path based finder` for modules to import. During - import, this list of locations usually comes from :data:`sys.path`, but - for subpackages it may also come from the parent package's ``__path__`` - attribute. - - importing - The process by which Python code in one module is made available to - Python code in another module. - importer An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. @@ -481,38 +336,25 @@ than compiled ones, though their programs generally also run more slowly. See also :term:`interactive`. - interpreter shutdown - When asked to shut down, the Python interpreter enters a special phase - where it gradually releases all allocated resources, such as modules - and various critical internal structures. It also makes several calls - to the :term:`garbage collector <garbage collection>`. This can trigger - the execution of code in user-defined destructors or weakref callbacks. - Code executed during the shutdown phase can encounter various - exceptions as the resources it relies on may not function anymore - (common examples are library modules or the warnings machinery). - - The main reason for interpreter shutdown is that the ``__main__`` module - or the script being run has finished executing. - iterable - An object capable of returning its members one at a time. Examples of - iterables include all sequence types (such as :class:`list`, :class:`str`, - and :class:`tuple`) and some non-sequence types like :class:`dict`, - :term:`file objects <file object>`, and objects of any classes you define - with an :meth:`__iter__` or :meth:`__getitem__` method. Iterables can be - used in a :keyword:`for` loop and in many other places where a sequence is - needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed - as an argument to the built-in function :func:`iter`, it returns an - iterator for the object. This iterator is good for one pass over the set - of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` + An object capable of returning its members one at a + time. Examples of iterables include all sequence types (such as + :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence + types like :class:`dict` and :class:`file` and objects of any classes you + define with an :meth:`__iter__` or :meth:`__getitem__` method. Iterables + can be used in a :keyword:`for` loop and in many other places where a + sequence is needed (:func:`zip`, :func:`map`, ...). When an iterable + object is passed as an argument to the built-in function :func:`iter`, it + returns an iterator for the object. This iterator is good for one pass + over the set of values. When using iterables, it is usually not necessary + to call :func:`iter` or deal with iterator objects yourself. The ``for`` statement does that automatically for you, creating a temporary unnamed variable to hold the iterator for the duration of the loop. See also :term:`iterator`, :term:`sequence`, and :term:`generator`. iterator An object representing a stream of data. Repeated calls to the iterator's - :meth:`~iterator.__next__` method (or passing it to the built-in function + :meth:`__next__` method (or passing it to the built-in function :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its @@ -536,21 +378,23 @@ A number of tools in Python accept key functions to control how elements are ordered or grouped. They include :func:`min`, :func:`max`, - :func:`sorted`, :meth:`list.sort`, :func:`heapq.merge`, - :func:`heapq.nsmallest`, :func:`heapq.nlargest`, and - :func:`itertools.groupby`. + :func:`sorted`, :meth:`list.sort`, :func:`heapq.nsmallest`, + :func:`heapq.nlargest`, and :func:`itertools.groupby`. There are several ways to create a key function. For example. the :meth:`str.lower` method can serve as a key function for case insensitive - sorts. Alternatively, a key function can be built from a + sorts. Alternatively, an ad-hoc key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, - the :mod:`operator` module provides three key function constructors: + the :mod:`operator` module provides three key function constuctors: :func:`~operator.attrgetter`, :func:`~operator.itemgetter`, and :func:`~operator.methodcaller`. See the :ref:`Sorting HOW TO <sortinghowto>` for examples of how to create and use key functions. keyword argument - See :term:`argument`. + Arguments which are preceded with a ``variable_name=`` in the call. + The variable name designates the local name in the function to which the + value is assigned. ``**`` is used to accept or pass a dictionary of + keyword arguments. See :term:`argument`. lambda An anonymous inline function consisting of a single :term:`expression` @@ -596,14 +440,6 @@ include :class:`dict`, :class:`collections.defaultdict`, :class:`collections.OrderedDict` and :class:`collections.Counter`. - meta path finder - A :term:`finder` returned by a search of :data:`sys.meta_path`. Meta path - finders are related to, but different from :term:`path entry finders - <path entry finder>`. - - See :class:`importlib.abc.MetaPathFinder` for the methods that meta path - finders implement. - metaclass The class of a class. Class definitions create a class name, a class dictionary, and a list of base classes. The metaclass is responsible for @@ -626,19 +462,7 @@ method resolution order Method Resolution Order is the order in which base classes are searched for a member during lookup. See `The Python 2.3 Method Resolution Order - <https://www.python.org/download/releases/2.3/mro/>`_ for details of the - algorithm used by the Python interpreter since the 2.3 release. - - module - An object that serves as an organizational unit of Python code. Modules - have a namespace containing arbitrary Python objects. Modules are loaded - into Python by the process of :term:`importing`. - - See also :term:`package`. - - module spec - A namespace containing the import-related information used to load a - module. An instance of :class:`importlib.machinery.ModuleSpec`. + <http://www.python.org/download/releases/2.3/mro/>`_. MRO See :term:`method resolution order`. @@ -665,21 +489,13 @@ dictionaries. There are the local, global and built-in namespaces as well as nested namespaces in objects (in methods). Namespaces support modularity by preventing naming conflicts. For instance, the functions - :func:`builtins.open <.open>` and :func:`os.open` are distinguished by - their namespaces. Namespaces also aid readability and maintainability by - making it clear which module implements a function. For instance, writing + :func:`builtins.open` and :func:`os.open` are distinguished by their + namespaces. Namespaces also aid readability and maintainability by making + it clear which module implements a function. For instance, writing :func:`random.seed` or :func:`itertools.islice` makes it clear that those functions are implemented by the :mod:`random` and :mod:`itertools` modules, respectively. - namespace package - A :pep:`420` :term:`package` which serves only as a container for - subpackages. Namespace packages may have no physical representation, - and specifically are not like a :term:`regular package` because they - have no ``__init__.py`` file. - - See also :term:`module`. - nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -692,122 +508,24 @@ new-style class Old name for the flavor of classes now used for all class objects. In earlier Python versions, only new-style classes could use Python's newer, - versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. + versatile features like :attr:`__slots__`, descriptors, properties, + :meth:`__getattribute__`, class methods, and static methods. object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style class`. - package - A Python :term:`module` which can contain submodules or recursively, - subpackages. Technically, a package is a Python module with an - ``__path__`` attribute. - - See also :term:`regular package` and :term:`namespace package`. - - parameter - A named entity in a :term:`function` (or method) definition that - specifies an :term:`argument` (or in some cases, arguments) that the - function can accept. There are five kinds of parameter: - - * :dfn:`positional-or-keyword`: specifies an argument that can be passed - either :term:`positionally <argument>` or as a :term:`keyword argument - <argument>`. This is the default kind of parameter, for example *foo* - and *bar* in the following:: - - def func(foo, bar=None): ... - - * :dfn:`positional-only`: specifies an argument that can be supplied only - by position. Python has no syntax for defining positional-only - parameters. However, some built-in functions have positional-only - parameters (e.g. :func:`abs`). - - .. _keyword-only_parameter: - - * :dfn:`keyword-only`: specifies an argument that can be supplied only - by keyword. Keyword-only parameters can be defined by including a - single var-positional parameter or bare ``*`` in the parameter list - of the function definition before them, for example *kw_only1* and - *kw_only2* in the following:: - - def func(arg, *, kw_only1, kw_only2): ... - - * :dfn:`var-positional`: specifies that an arbitrary sequence of - positional arguments can be provided (in addition to any positional - arguments already accepted by other parameters). Such a parameter can - be defined by prepending the parameter name with ``*``, for example - *args* in the following:: - - def func(*args, **kwargs): ... - - * :dfn:`var-keyword`: specifies that arbitrarily many keyword arguments - can be provided (in addition to any keyword arguments already accepted - by other parameters). Such a parameter can be defined by prepending - the parameter name with ``**``, for example *kwargs* in the example - above. - - Parameters can specify both optional and required arguments, as well as - default values for some optional arguments. - - See also the :term:`argument` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, the :class:`inspect.Parameter` class, the - :ref:`function` section, and :pep:`362`. - - path entry - A single location on the :term:`import path` which the :term:`path - based finder` consults to find modules for importing. - - path entry finder - A :term:`finder` returned by a callable on :data:`sys.path_hooks` - (i.e. a :term:`path entry hook`) which knows how to locate modules given - a :term:`path entry`. - - See :class:`importlib.abc.PathEntryFinder` for the methods that path entry - finders implement. - - path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path - entry finder` if it knows how to find modules on a specific :term:`path - entry`. - - path based finder - One of the default :term:`meta path finders <meta path finder>` which - searches an :term:`import path` for modules. - - portion - A set of files in a single directory (possibly stored in a zip file) - that contribute to a namespace package, as defined in :pep:`420`. - positional argument - See :term:`argument`. - - provisional API - A provisional API is one which has been deliberately excluded from - the standard library's backwards compatibility guarantees. While major - changes to such interfaces are not expected, as long as they are marked - provisional, backwards incompatible changes (up to and including removal - of the interface) may occur if deemed necessary by core developers. Such - changes will not be made gratuitously -- they will occur only if serious - fundamental flaws are uncovered that were missed prior to the inclusion - of the API. - - Even for provisional APIs, backwards incompatible changes are seen as - a "solution of last resort" - every attempt will still be made to find - a backwards compatible resolution to any identified problems. - - This process allows the standard library to continue to evolve over - time, without locking in problematic design errors for extended periods - of time. See :pep:`411` for more details. - - provisional package - See :term:`provisional API`. + The arguments assigned to local names inside a function or method, + determined by the order in which they were given in the call. ``*`` is + used to either accept multiple positional arguments (when in the + definition), or pass several arguments as a list to a function. See + :term:`argument`. Python 3000 - Nickname for the Python 3.x release line (coined long ago when the - release of version 3 was something in the distant future.) This is also + Nickname for the Python 3.x release line (coined long ago when the release + of version 3 was something in the distant future.) This is also abbreviated "Py3k". Pythonic @@ -844,14 +562,6 @@ >>> C.D.meth.__qualname__ 'C.D.meth' - When used to refer to modules, the *fully qualified name* means the - entire dotted path to the module, including any parent packages, - e.g. ``email.mime.text``:: - - >>> import email.mime.text - >>> email.mime.text.__name__ - 'email.mime.text' - reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Reference counting is @@ -860,12 +570,6 @@ :func:`~sys.getrefcount` function that programmers can call to return the reference count for a particular object. - regular package - A traditional :term:`package`, such as a directory containing an - ``__init__.py`` file. - - See also :term:`namespace package`. - __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though @@ -876,25 +580,13 @@ sequence An :term:`iterable` which supports efficient element access using integer indices via the :meth:`__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. + :meth:`len` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also supports :meth:`__getitem__` and :meth:`__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary :term:`immutable` keys rather than integers. - The :class:`collections.abc.Sequence` abstract base class - defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded - interface can be registered explicitly using - :func:`~abc.register`. - - single dispatch - A form of :term:`generic function` dispatch where the implementation is - chosen based on the type of a single argument. - slice An object usually containing a portion of a :term:`sequence`. A slice is created using the subscript notation, ``[]`` with colons between numbers @@ -909,7 +601,7 @@ statement A statement is part of a suite (a "block" of code). A statement is either - an :term:`expression` or one of several constructs with a keyword, such + an :term:`expression` or a one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. struct sequence @@ -920,17 +612,6 @@ :meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences include :data:`sys.float_info` and the return value of :func:`os.stat`. - text encoding - A codec which encodes Unicode strings to bytes. - - text file - A :term:`file object` able to read and write :class:`str` objects. - Often, a text file actually accesses a byte-oriented datastream - and handles the :term:`text encoding` automatically. - - .. seealso:: - A :term:`binary file` reads and write :class:`bytes` objects. - triple-quoted string A string which is bound by three instances of either a quotation mark (") or an apostrophe ('). While they don't provide any functionality @@ -943,23 +624,14 @@ type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with - ``type(obj)``. + :attr:`__class__` attribute or can be retrieved with ``type(obj)``. - universal newlines - A manner of interpreting text streams in which all of the following are - recognized as ending a line: the Unix end-of-line convention ``'\n'``, - the Windows convention ``'\r\n'``, and the old Macintosh convention - ``'\r'``. See :pep:`278` and :pep:`3116`, as well as - :func:`bytes.splitlines` for an additional use. - - virtual environment - A cooperatively isolated runtime environment that allows Python users - and applications to install and upgrade Python distribution packages - without interfering with the behaviour of other Python applications - running on the same system. - - See also :ref:`scripts-pyvenv`. + view + The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and + :meth:`dict.items` are called dictionary views. They are lazy sequences + that will see changes in the underlying dictionary. To force the + dictionary view to become a full list use ``list(dictview)``. See + :ref:`dict-views`. virtual machine A computer defined entirely in software. Python's virtual machine diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/advocacy.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/howto/advocacy.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,355 @@ +************************* + Python Advocacy HOWTO +************************* + +:Author: A.M. Kuchling +:Release: 0.03 + + +.. topic:: Abstract + + It's usually difficult to get your management to accept open source software, + and Python is no exception to this rule. This document discusses reasons to use + Python, strategies for winning acceptance, facts and arguments you can use, and + cases where you *shouldn't* try to use Python. + + +Reasons to Use Python +===================== + +There are several reasons to incorporate a scripting language into your +development process, and this section will discuss them, and why Python has some +properties that make it a particularly good choice. + + +Programmability +--------------- + +Programs are often organized in a modular fashion. Lower-level operations are +grouped together, and called by higher-level functions, which may in turn be +used as basic operations by still further upper levels. + +For example, the lowest level might define a very low-level set of functions for +accessing a hash table. The next level might use hash tables to store the +headers of a mail message, mapping a header name like ``Date`` to a value such +as ``Tue, 13 May 1997 20:00:54 -0400``. A yet higher level may operate on +message objects, without knowing or caring that message headers are stored in a +hash table, and so forth. + +Often, the lowest levels do very simple things; they implement a data structure +such as a binary tree or hash table, or they perform some simple computation, +such as converting a date string to a number. The higher levels then contain +logic connecting these primitive operations. Using the approach, the primitives +can be seen as basic building blocks which are then glued together to produce +the complete product. + +Why is this design approach relevant to Python? Because Python is well suited +to functioning as such a glue language. A common approach is to write a Python +module that implements the lower level operations; for the sake of speed, the +implementation might be in C, Java, or even Fortran. Once the primitives are +available to Python programs, the logic underlying higher level operations is +written in the form of Python code. The high-level logic is then more +understandable, and easier to modify. + +John Ousterhout wrote a paper that explains this idea at greater length, +entitled "Scripting: Higher Level Programming for the 21st Century". I +recommend that you read this paper; see the references for the URL. Ousterhout +is the inventor of the Tcl language, and therefore argues that Tcl should be +used for this purpose; he only briefly refers to other languages such as Python, +Perl, and Lisp/Scheme, but in reality, Ousterhout's argument applies to +scripting languages in general, since you could equally write extensions for any +of the languages mentioned above. + + +Prototyping +----------- + +In *The Mythical Man-Month*, Fredrick Brooks suggests the following rule when +planning software projects: "Plan to throw one away; you will anyway." Brooks +is saying that the first attempt at a software design often turns out to be +wrong; unless the problem is very simple or you're an extremely good designer, +you'll find that new requirements and features become apparent once development +has actually started. If these new requirements can't be cleanly incorporated +into the program's structure, you're presented with two unpleasant choices: +hammer the new features into the program somehow, or scrap everything and write +a new version of the program, taking the new features into account from the +beginning. + +Python provides you with a good environment for quickly developing an initial +prototype. That lets you get the overall program structure and logic right, and +you can fine-tune small details in the fast development cycle that Python +provides. Once you're satisfied with the GUI interface or program output, you +can translate the Python code into C++, Fortran, Java, or some other compiled +language. + +Prototyping means you have to be careful not to use too many Python features +that are hard to implement in your other language. Using ``eval()``, or regular +expressions, or the :mod:`pickle` module, means that you're going to need C or +Java libraries for formula evaluation, regular expressions, and serialization, +for example. But it's not hard to avoid such tricky code, and in the end the +translation usually isn't very difficult. The resulting code can be rapidly +debugged, because any serious logical errors will have been removed from the +prototype, leaving only more minor slip-ups in the translation to track down. + +This strategy builds on the earlier discussion of programmability. Using Python +as glue to connect lower-level components has obvious relevance for constructing +prototype systems. In this way Python can help you with development, even if +end users never come in contact with Python code at all. If the performance of +the Python version is adequate and corporate politics allow it, you may not need +to do a translation into C or Java, but it can still be faster to develop a +prototype and then translate it, instead of attempting to produce the final +version immediately. + +One example of this development strategy is Microsoft Merchant Server. Version +1.0 was written in pure Python, by a company that subsequently was purchased by +Microsoft. Version 2.0 began to translate the code into C++, shipping with some +C++code and some Python code. Version 3.0 didn't contain any Python at all; all +the code had been translated into C++. Even though the product doesn't contain +a Python interpreter, the Python language has still served a useful purpose by +speeding up development. + +This is a very common use for Python. Past conference papers have also +described this approach for developing high-level numerical algorithms; see +David M. Beazley and Peter S. Lomdahl's paper "Feeding a Large-scale Physics +Application to Python" in the references for a good example. If an algorithm's +basic operations are things like "Take the inverse of this 4000x4000 matrix", +and are implemented in some lower-level language, then Python has almost no +additional performance cost; the extra time required for Python to evaluate an +expression like ``m.invert()`` is dwarfed by the cost of the actual computation. +It's particularly good for applications where seemingly endless tweaking is +required to get things right. GUI interfaces and Web sites are prime examples. + +The Python code is also shorter and faster to write (once you're familiar with +Python), so it's easier to throw it away if you decide your approach was wrong; +if you'd spent two weeks working on it instead of just two hours, you might +waste time trying to patch up what you've got out of a natural reluctance to +admit that those two weeks were wasted. Truthfully, those two weeks haven't +been wasted, since you've learnt something about the problem and the technology +you're using to solve it, but it's human nature to view this as a failure of +some sort. + + +Simplicity and Ease of Understanding +------------------------------------ + +Python is definitely *not* a toy language that's only usable for small tasks. +The language features are general and powerful enough to enable it to be used +for many different purposes. It's useful at the small end, for 10- or 20-line +scripts, but it also scales up to larger systems that contain thousands of lines +of code. + +However, this expressiveness doesn't come at the cost of an obscure or tricky +syntax. While Python has some dark corners that can lead to obscure code, there +are relatively few such corners, and proper design can isolate their use to only +a few classes or modules. It's certainly possible to write confusing code by +using too many features with too little concern for clarity, but most Python +code can look a lot like a slightly-formalized version of human-understandable +pseudocode. + +In *The New Hacker's Dictionary*, Eric S. Raymond gives the following definition +for "compact": + +.. epigraph:: + + Compact *adj.* Of a design, describes the valuable property that it can all be + apprehended at once in one's head. This generally means the thing created from + the design can be used with greater facility and fewer errors than an equivalent + tool that is not compact. Compactness does not imply triviality or lack of + power; for example, C is compact and FORTRAN is not, but C is more powerful than + FORTRAN. Designs become non-compact through accreting features and cruft that + don't merge cleanly into the overall design scheme (thus, some fans of Classic C + maintain that ANSI C is no longer compact). + + (From http://www.catb.org/~esr/jargon/html/C/compact.html) + +In this sense of the word, Python is quite compact, because the language has +just a few ideas, which are used in lots of places. Take namespaces, for +example. Import a module with ``import math``, and you create a new namespace +called ``math``. Classes are also namespaces that share many of the properties +of modules, and have a few of their own; for example, you can create instances +of a class. Instances? They're yet another namespace. Namespaces are currently +implemented as Python dictionaries, so they have the same methods as the +standard dictionary data type: .keys() returns all the keys, and so forth. + +This simplicity arises from Python's development history. The language syntax +derives from different sources; ABC, a relatively obscure teaching language, is +one primary influence, and Modula-3 is another. (For more information about ABC +and Modula-3, consult their respective Web sites at http://www.cwi.nl/~steven/abc/ +and http://www.m3.org.) Other features have come from C, Icon, +Algol-68, and even Perl. Python hasn't really innovated very much, but instead +has tried to keep the language small and easy to learn, building on ideas that +have been tried in other languages and found useful. + +Simplicity is a virtue that should not be underestimated. It lets you learn the +language more quickly, and then rapidly write code -- code that often works the +first time you run it. + + +Java Integration +---------------- + +If you're working with Java, Jython (http://www.jython.org/) is definitely worth +your attention. Jython is a re-implementation of Python in Java that compiles +Python code into Java bytecodes. The resulting environment has very tight, +almost seamless, integration with Java. It's trivial to access Java classes +from Python, and you can write Python classes that subclass Java classes. +Jython can be used for prototyping Java applications in much the same way +CPython is used, and it can also be used for test suites for Java code, or +embedded in a Java application to add scripting capabilities. + + +Arguments and Rebuttals +======================= + +Let's say that you've decided upon Python as the best choice for your +application. How can you convince your management, or your fellow developers, +to use Python? This section lists some common arguments against using Python, +and provides some possible rebuttals. + +**Python is freely available software that doesn't cost anything. How good can +it be?** + +Very good, indeed. These days Linux and Apache, two other pieces of open source +software, are becoming more respected as alternatives to commercial software, +but Python hasn't had all the publicity. + +Python has been around for several years, with many users and developers. +Accordingly, the interpreter has been used by many people, and has gotten most +of the bugs shaken out of it. While bugs are still discovered at intervals, +they're usually either quite obscure (they'd have to be, for no one to have run +into them before) or they involve interfaces to external libraries. The +internals of the language itself are quite stable. + +Having the source code should be viewed as making the software available for +peer review; people can examine the code, suggest (and implement) improvements, +and track down bugs. To find out more about the idea of open source code, along +with arguments and case studies supporting it, go to http://www.opensource.org. + +**Who's going to support it?** + +Python has a sizable community of developers, and the number is still growing. +The Internet community surrounding the language is an active one, and is worth +being considered another one of Python's advantages. Most questions posted to +the comp.lang.python newsgroup are quickly answered by someone. + +Should you need to dig into the source code, you'll find it's clear and +well-organized, so it's not very difficult to write extensions and track down +bugs yourself. If you'd prefer to pay for support, there are companies and +individuals who offer commercial support for Python. + +**Who uses Python for serious work?** + +Lots of people; one interesting thing about Python is the surprising diversity +of applications that it's been used for. People are using Python to: + +* Run Web sites + +* Write GUI interfaces + +* Control number-crunching code on supercomputers + +* Make a commercial application scriptable by embedding the Python interpreter + inside it + +* Process large XML data sets + +* Build test suites for C or Java code + +Whatever your application domain is, there's probably someone who's used Python +for something similar. Yet, despite being useable for such high-end +applications, Python's still simple enough to use for little jobs. + +See http://wiki.python.org/moin/OrganizationsUsingPython for a list of some of +the organizations that use Python. + +**What are the restrictions on Python's use?** + +They're practically nonexistent. Consult :ref:`history-and-license` for the full +language, but it boils down to three conditions: + +* You have to leave the copyright notice on the software; if you don't include + the source code in a product, you have to put the copyright notice in the + supporting documentation. + +* Don't claim that the institutions that have developed Python endorse your + product in any way. + +* If something goes wrong, you can't sue for damages. Practically all software + licenses contain this condition. + +Notice that you don't have to provide source code for anything that contains +Python or is built with it. Also, the Python interpreter and accompanying +documentation can be modified and redistributed in any way you like, and you +don't have to pay anyone any licensing fees at all. + +**Why should we use an obscure language like Python instead of well-known +language X?** + +I hope this HOWTO, and the documents listed in the final section, will help +convince you that Python isn't obscure, and has a healthily growing user base. +One word of advice: always present Python's positive advantages, instead of +concentrating on language X's failings. People want to know why a solution is +good, rather than why all the other solutions are bad. So instead of attacking +a competing solution on various grounds, simply show how Python's virtues can +help. + + +Useful Resources +================ + +http://www.pythonology.com/success + The Python Success Stories are a collection of stories from successful users of + Python, with the emphasis on business and corporate users. + +.. http://www.fsbassociates.com/books/pythonchpt1.htm + The first chapter of \emph{Internet Programming with Python} also + examines some of the reasons for using Python. The book is well worth + buying, but the publishers have made the first chapter available on + the Web. + +http://www.tcl.tk/doc/scripting.html + John Ousterhout's white paper on scripting is a good argument for the utility of + scripting languages, though naturally enough, he emphasizes Tcl, the language he + developed. Most of the arguments would apply to any scripting language. + +http://www.python.org/workshops/1997-10/proceedings/beazley.html + The authors, David M. Beazley and Peter S. Lomdahl, describe their use of + Python at Los Alamos National Laboratory. It's another good example of how + Python can help get real work done. This quotation from the paper has been + echoed by many people: + + .. epigraph:: + + Originally developed as a large monolithic application for massively parallel + processing systems, we have used Python to transform our application into a + flexible, highly modular, and extremely powerful system for performing + simulation, data analysis, and visualization. In addition, we describe how + Python has solved a number of important problems related to the development, + debugging, deployment, and maintenance of scientific software. + +http://pythonjournal.cognizor.com/pyj1/Everitt-Feit_interview98-V1.html + This interview with Andy Feit, discussing Infoseek's use of Python, can be used + to show that choosing Python didn't introduce any difficulties into a company's + development process, and provided some substantial benefits. + +.. http://www.python.org/psa/Commercial.html + Robin Friedrich wrote this document on how to support Python's use in + commercial projects. + +http://www.python.org/workshops/1997-10/proceedings/stein.ps + For the 6th Python conference, Greg Stein presented a paper that traced Python's + adoption and usage at a startup called eShop, and later at Microsoft. + +http://www.opensource.org + Management may be doubtful of the reliability and usefulness of software that + wasn't written commercially. This site presents arguments that show how open + source software can have considerable advantages over closed-source software. + +http://www.faqs.org/docs/Linux-mini/Advocacy.html + The Linux Advocacy mini-HOWTO was the inspiration for this document, and is also + well worth reading for general suggestions on winning acceptance for a new + technology, such as Linux or Python. In general, you won't make much progress + by simply attacking existing systems and complaining about their inadequacies; + this often ends up looking like unfocused whining. It's much better to point + out some of the many areas where Python is an improvement over other systems. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/argparse.rst --- a/Doc/howto/argparse.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,765 +0,0 @@ -***************** -Argparse Tutorial -***************** - -:author: Tshepang Lekhonkhobe - -.. _argparse-tutorial: - -This tutorial is intended to be a gentle introduction to :mod:`argparse`, the -recommended command-line parsing module in the Python standard library. - -.. note:: - - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for :c:func:`getopt` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. - - -Concepts -======== - -Let's show the sort of functionality that we are going to explore in this -introductory tutorial by making use of the :command:`ls` command: - -.. code-block:: sh - - $ ls - cpython devguide prog.py pypy rm-unused-function.patch - $ ls pypy - ctypes_configure demo dotviewer include lib_pypy lib-python ... - $ ls -l - total 20 - drwxr-xr-x 19 wena wena 4096 Feb 18 18:51 cpython - drwxr-xr-x 4 wena wena 4096 Feb 8 12:04 devguide - -rwxr-xr-x 1 wena wena 535 Feb 19 00:05 prog.py - drwxr-xr-x 14 wena wena 4096 Feb 7 00:59 pypy - -rw-r--r-- 1 wena wena 741 Feb 18 01:01 rm-unused-function.patch - $ ls --help - Usage: ls [OPTION]... [FILE]... - List information about the FILEs (the current directory by default). - Sort entries alphabetically if none of -cftuvSUX nor --sort is specified. - ... - -A few concepts we can learn from the four commands: - -* The :command:`ls` command is useful when run without any options at all. It defaults - to displaying the contents of the current directory. - -* If we want beyond what it provides by default, we tell it a bit more. In - this case, we want it to display a different directory, ``pypy``. - What we did is specify what is known as a positional argument. It's named so - because the program should know what to do with the value, solely based on - where it appears on the command line. This concept is more relevant - to a command like :command:`cp`, whose most basic usage is ``cp SRC DEST``. - The first position is *what you want copied,* and the second - position is *where you want it copied to*. - -* Now, say we want to change behaviour of the program. In our example, - we display more info for each file instead of just showing the file names. - The ``-l`` in that case is known as an optional argument. - -* That's a snippet of the help text. It's very useful in that you can - come across a program you have never used before, and can figure out - how it works simply by reading its help text. - - -The basics -========== - -Let us start with a very simple example which does (almost) nothing:: - - import argparse - parser = argparse.ArgumentParser() - parser.parse_args() - -Following is a result of running the code: - -.. code-block:: sh - - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] - - optional arguments: - -h, --help show this help message and exit - $ python3 prog.py --verbose - usage: prog.py [-h] - prog.py: error: unrecognized arguments: --verbose - $ python3 prog.py foo - usage: prog.py [-h] - prog.py: error: unrecognized arguments: foo - -Here is what is happening: - -* Running the script without any options results in nothing displayed to - stdout. Not so useful. - -* The second one starts to display the usefulness of the :mod:`argparse` - module. We have done almost nothing, but already we get a nice help message. - -* The ``--help`` option, which can also be shortened to ``-h``, is the only - option we get for free (i.e. no need to specify it). Specifying anything - else results in an error. But even then, we do get a useful usage message, - also for free. - - -Introducing Positional arguments -================================ - -An example:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo") - args = parser.parse_args() - print(args.echo) - -And running the code: - -.. code-block:: sh - - $ python3 prog.py - usage: prog.py [-h] echo - prog.py: error: the following arguments are required: echo - $ python3 prog.py --help - usage: prog.py [-h] echo - - positional arguments: - echo - - optional arguments: - -h, --help show this help message and exit - $ python3 prog.py foo - foo - -Here is what's happening: - -* We've added the :meth:`add_argument` method, which is what we use to specify - which command-line options the program is willing to accept. In this case, - I've named it ``echo`` so that it's in line with its function. - -* Calling our program now requires us to specify an option. - -* The :meth:`parse_args` method actually returns some data from the - options specified, in this case, ``echo``. - -* The variable is some form of 'magic' that :mod:`argparse` performs for free - (i.e. no need to specify which variable that value is stored in). - You will also notice that its name matches the string argument given - to the method, ``echo``. - -Note however that, although the help display looks nice and all, it currently -is not as helpful as it can be. For example we see that we got ``echo`` as a -positional argument, but we don't know what it does, other than by guessing or -by reading the source code. So, let's make it a bit more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo", help="echo the string you use here") - args = parser.parse_args() - print(args.echo) - -And we get: - -.. code-block:: sh - - $ python3 prog.py -h - usage: prog.py [-h] echo - - positional arguments: - echo echo the string you use here - - optional arguments: - -h, --help show this help message and exit - -Now, how about doing something even more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number") - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: sh - - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 5, in <module> - print(args.square**2) - TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int' - -That didn't go so well. That's because :mod:`argparse` treats the options we -give it as strings, unless we tell it otherwise. So, let's tell -:mod:`argparse` to treat that input as an integer:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number", - type=int) - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: sh - - $ python3 prog.py 4 - 16 - $ python3 prog.py four - usage: prog.py [-h] square - prog.py: error: argument square: invalid int value: 'four' - -That went well. The program now even helpfully quits on bad illegal input -before proceeding. - - -Introducing Optional arguments -============================== - -So far we, have been playing with positional arguments. Let us -have a look on how to add optional ones:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbosity", help="increase output verbosity") - args = parser.parse_args() - if args.verbosity: - print("verbosity turned on") - -And the output: - -.. code-block:: sh - - $ python3 prog.py --verbosity 1 - verbosity turned on - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] [--verbosity VERBOSITY] - - optional arguments: - -h, --help show this help message and exit - --verbosity VERBOSITY - increase output verbosity - $ python3 prog.py --verbosity - usage: prog.py [-h] [--verbosity VERBOSITY] - prog.py: error: argument --verbosity: expected one argument - -Here is what is happening: - -* The program is written so as to display something when ``--verbosity`` is - specified and display nothing when not. - -* To show that the option is actually optional, there is no error when running - the program without it. Note that by default, if an optional argument isn't - used, the relevant variable, in this case :attr:`args.verbosity`, is - given ``None`` as a value, which is the reason it fails the truth - test of the :keyword:`if` statement. - -* The help message is a bit different. - -* When using the ``--verbosity`` option, one must also specify some value, - any value. - -The above example accepts arbitrary integer values for ``--verbosity``, but for -our simple program, only two values are actually useful, ``True`` or ``False``. -Let's modify the code accordingly:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And the output: - -.. code-block:: sh - - $ python3 prog.py --verbose - verbosity turned on - $ python3 prog.py --verbose 1 - usage: prog.py [-h] [--verbose] - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py --help - usage: prog.py [-h] [--verbose] - - optional arguments: - -h, --help show this help message and exit - --verbose increase output verbosity - -Here is what is happening: - -* The option is now more of a flag than something that requires a value. - We even changed the name of the option to match that idea. - Note that we now specify a new keyword, ``action``, and give it the value - ``"store_true"``. This means that, if the option is specified, - assign the value ``True`` to :data:`args.verbose`. - Not specifying it implies ``False``. - -* It complains when you specify a value, in true spirit of what flags - actually are. - -* Notice the different help text. - - -Short options -------------- - -If you are familiar with command line usage, -you will notice that I haven't yet touched on the topic of short -versions of the options. It's quite simple:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And here goes: - -.. code-block:: sh - - $ python3 prog.py -v - verbosity turned on - $ python3 prog.py --help - usage: prog.py [-h] [-v] - - optional arguments: - -h, --help show this help message and exit - -v, --verbose increase output verbosity - -Note that the new ability is also reflected in the help text. - - -Combining Positional and Optional arguments -=========================================== - -Our program keeps growing in complexity:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbose", action="store_true", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbose: - print("the square of {} equals {}".format(args.square, answer)) - else: - print(answer) - -And now the output: - -.. code-block:: sh - - $ python3 prog.py - usage: prog.py [-h] [-v] square - prog.py: error: the following arguments are required: square - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 --verbose - the square of 4 equals 16 - $ python3 prog.py --verbose 4 - the square of 4 equals 16 - -* We've brought back a positional argument, hence the complaint. - -* Note that the order does not matter. - -How about we give this program of ours back the ability to have -multiple verbosity values, and actually get to use them:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And the output: - -.. code-block:: sh - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - usage: prog.py [-h] [-v VERBOSITY] square - prog.py: error: argument -v/--verbosity: expected one argument - $ python3 prog.py 4 -v 1 - 4^2 == 16 - $ python3 prog.py 4 -v 2 - the square of 4 equals 16 - $ python3 prog.py 4 -v 3 - 16 - -These all look good except the last one, which exposes a bug in our program. -Let's fix it by restricting the values the ``--verbosity`` option can accept:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, choices=[0, 1, 2], - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And the output: - -.. code-block:: sh - - $ python3 prog.py 4 -v 3 - usage: prog.py [-h] [-v {0,1,2}] square - prog.py: error: argument -v/--verbosity: invalid choice: 3 (choose from 0, 1, 2) - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v {0,1,2}] square - - positional arguments: - square display a square of a given number - - optional arguments: - -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} - increase output verbosity - -Note that the change also reflects both in the error message as well as the -help string. - -Now, let's use a different approach of playing with verbosity, which is pretty -common. It also matches the way the CPython executable handles its own -verbosity argument (check the output of ``python --help``):: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display the square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity == 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -We have introduced another action, "count", -to count the number of occurrences of a specific optional arguments: - -.. code-block:: sh - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - 4^2 == 16 - $ python3 prog.py 4 -vv - the square of 4 equals 16 - $ python3 prog.py 4 --verbosity --verbosity - the square of 4 equals 16 - $ python3 prog.py 4 -v 1 - usage: prog.py [-h] [-v] square - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v] square - - positional arguments: - square display a square of a given number - - optional arguments: - -h, --help show this help message and exit - -v, --verbosity increase output verbosity - $ python3 prog.py 4 -vvv - 16 - -* Yes, it's now more of a flag (similar to ``action="store_true"``) in the - previous version of our script. That should explain the complaint. - -* It also behaves similar to "store_true" action. - -* Now here's a demonstration of what the "count" action gives. You've probably - seen this sort of usage before. - -* And, just like the "store_true" action, if you don't specify the ``-v`` flag, - that flag is considered to have ``None`` value. - -* As should be expected, specifying the long form of the flag, we should get - the same output. - -* Sadly, our help output isn't very informative on the new ability our script - has acquired, but that can always be fixed by improving the documentation for - out script (e.g. via the ``help`` keyword argument). - -* That last output exposes a bug in our program. - - -Let's fix:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - - # bugfix: replace == with >= - if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -And this is what it gives: - -.. code-block:: sh - - $ python3 prog.py 4 -vvv - the square of 4 equals 16 - $ python3 prog.py 4 -vvvv - the square of 4 equals 16 - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 11, in <module> - if args.verbosity >= 2: - TypeError: '>=' not supported between instances of 'NoneType' and 'int' - - -* First output went well, and fixes the bug we had before. - That is, we want any value >= 2 to be as verbose as possible. - -* Third output not so good. - -Let's fix that bug:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", default=0, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity >= 2: - print("the square of {} equals {}".format(args.square, answer)) - elif args.verbosity >= 1: - print("{}^2 == {}".format(args.square, answer)) - else: - print(answer) - -We've just introduced yet another keyword, ``default``. -We've set it to ``0`` in order to make it comparable to the other int values. -Remember that by default, -if an optional argument isn't specified, -it gets the ``None`` value, and that cannot be compared to an int value -(hence the :exc:`TypeError` exception). - -And: - -.. code-block:: sh - - $ python3 prog.py 4 - 16 - -You can go quite far just with what we've learned so far, -and we have only scratched the surface. -The :mod:`argparse` module is very powerful, -and we'll explore a bit more of it before we end this tutorial. - - -Getting a little more advanced -============================== - -What if we wanted to expand our tiny program to perform other powers, -not just squares:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - elif args.verbosity >= 1: - print("{}^{} == {}".format(args.x, args.y, answer)) - else: - print(answer) - -Output: - -.. code-block:: sh - - $ python3 prog.py - usage: prog.py [-h] [-v] x y - prog.py: error: the following arguments are required: x, y - $ python3 prog.py -h - usage: prog.py [-h] [-v] x y - - positional arguments: - x the base - y the exponent - - optional arguments: - -h, --help show this help message and exit - -v, --verbosity - $ python3 prog.py 4 2 -v - 4^2 == 16 - - -Notice that so far we've been using verbosity level to *change* the text -that gets displayed. The following example instead uses verbosity level -to display *more* text instead:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print("Running '{}'".format(__file__)) - if args.verbosity >= 1: - print("{}^{} == ".format(args.x, args.y), end="") - print(answer) - -Output: - -.. code-block:: sh - - $ python3 prog.py 4 2 - 16 - $ python3 prog.py 4 2 -v - 4^2 == 16 - $ python3 prog.py 4 2 -vv - Running 'prog.py' - 4^2 == 16 - - -Conflicting options -------------------- - -So far, we have been working with two methods of an -:class:`argparse.ArgumentParser` instance. Let's introduce a third one, -:meth:`add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program so that -the new functionality makes more sense: -we'll introduce the ``--quiet`` option, -which will be the opposite of the ``--verbose`` one:: - - import argparse - - parser = argparse.ArgumentParser() - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - else: - print("{}^{} == {}".format(args.x, args.y, answer)) - -Our program is now simpler, and we've lost some functionality for the sake of -demonstration. Anyways, here's the output: - -.. code-block:: sh - - $ python3 prog.py 4 2 - 4^2 == 16 - $ python3 prog.py 4 2 -q - 16 - $ python3 prog.py 4 2 -v - 4 to the power 2 equals 16 - $ python3 prog.py 4 2 -vq - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - $ python3 prog.py 4 2 -v --quiet - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - -That should be easy to follow. I've added that last output so you can see the -sort of flexibility you get, i.e. mixing long form options with short form -ones. - -Before we conclude, you probably want to tell your users the main purpose of -your program, just in case they don't know:: - - import argparse - - parser = argparse.ArgumentParser(description="calculate X to the power of Y") - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print("{} to the power {} equals {}".format(args.x, args.y, answer)) - else: - print("{}^{} == {}".format(args.x, args.y, answer)) - -Note that slight difference in the usage text. Note the ``[-v | -q]``, -which tells us that we can either use ``-v`` or ``-q``, -but not both at the same time: - -.. code-block:: sh - - $ python3 prog.py --help - usage: prog.py [-h] [-v | -q] x y - - calculate X to the power of Y - - positional arguments: - x the base - y the exponent - - optional arguments: - -h, --help show this help message and exit - -v, --verbose - -q, --quiet - - -Conclusion -========== - -The :mod:`argparse` module offers a lot more than shown here. -Its docs are quite detailed and thorough, and full of examples. -Having gone through this tutorial, you should easily digest them -without feeling overwhelmed. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/clinic.rst --- a/Doc/howto/clinic.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1683 +0,0 @@ -********************** -Argument Clinic How-To -********************** - -:author: Larry Hastings - - -.. topic:: Abstract - - Argument Clinic is a preprocessor for CPython C files. - Its purpose is to automate all the boilerplate involved - with writing argument parsing code for "builtins". - This document shows you how to convert your first C - function to work with Argument Clinic, and then introduces - some advanced topics on Argument Clinic usage. - - Currently Argument Clinic is considered internal-only - for CPython. Its use is not supported for files outside - CPython, and no guarantees are made regarding backwards - compatibility for future versions. In other words: if you - maintain an external C extension for CPython, you're welcome - to experiment with Argument Clinic in your own code. But the - version of Argument Clinic that ships with CPython 3.5 *could* - be totally incompatible and break all your code. - -The Goals Of Argument Clinic -============================ - -Argument Clinic's primary goal -is to take over responsibility for all argument parsing code -inside CPython. This means that, when you convert a function -to work with Argument Clinic, that function should no longer -do any of its own argument parsing--the code generated by -Argument Clinic should be a "black box" to you, where CPython -calls in at the top, and your code gets called at the bottom, -with ``PyObject *args`` (and maybe ``PyObject *kwargs``) -magically converted into the C variables and types you need. - -In order for Argument Clinic to accomplish its primary goal, -it must be easy to use. Currently, working with CPython's -argument parsing library is a chore, requiring maintaining -redundant information in a surprising number of places. -When you use Argument Clinic, you don't have to repeat yourself. - -Obviously, no one would want to use Argument Clinic unless -it's solving their problem--and without creating new problems of -its own. -So it's paramount that Argument Clinic generate correct code. -It'd be nice if the code was faster, too, but at the very least -it should not introduce a major speed regression. (Eventually Argument -Clinic *should* make a major speedup possible--we could -rewrite its code generator to produce tailor-made argument -parsing code, rather than calling the general-purpose CPython -argument parsing library. That would make for the fastest -argument parsing possible!) - -Additionally, Argument Clinic must be flexible enough to -work with any approach to argument parsing. Python has -some functions with some very strange parsing behaviors; -Argument Clinic's goal is to support all of them. - -Finally, the original motivation for Argument Clinic was -to provide introspection "signatures" for CPython builtins. -It used to be, the introspection query functions would throw -an exception if you passed in a builtin. With Argument -Clinic, that's a thing of the past! - -One idea you should keep in mind, as you work with -Argument Clinic: the more information you give it, the -better job it'll be able to do. -Argument Clinic is admittedly relatively simple right -now. But as it evolves it will get more sophisticated, -and it should be able to do many interesting and smart -things with all the information you give it. - - -Basic Concepts And Usage -======================== - -Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. -If you run that script, specifying a C file as an argument:: - - % python3 Tools/clinic/clinic.py foo.c - -Argument Clinic will scan over the file looking for lines that -look exactly like this:: - - /*[clinic input] - -When it finds one, it reads everything up to a line that looks -exactly like this:: - - [clinic start generated code]*/ - -Everything in between these two lines is input for Argument Clinic. -All of these lines, including the beginning and ending comment -lines, are collectively called an Argument Clinic "block". - -When Argument Clinic parses one of these blocks, it -generates output. This output is rewritten into the C file -immediately after the block, followed by a comment containing a checksum. -The Argument Clinic block now looks like this:: - - /*[clinic input] - ... clinic input goes here ... - [clinic start generated code]*/ - ... clinic output goes here ... - /*[clinic end generated code: checksum=...]*/ - -If you run Argument Clinic on the same file a second time, Argument Clinic -will discard the old output and write out the new output with a fresh checksum -line. However, if the input hasn't changed, the output won't change either. - -You should never modify the output portion of an Argument Clinic block. Instead, -change the input until it produces the output you want. (That's the purpose of the -checksum--to detect if someone changed the output, as these edits would be lost -the next time Argument Clinic writes out fresh output.) - -For the sake of clarity, here's the terminology we'll use with Argument Clinic: - -* The first line of the comment (``/*[clinic input]``) is the *start line*. -* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*. -* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*. -* In between the start line and the end line is the *input*. -* In between the end line and the checksum line is the *output*. -* All the text collectively, from the start line to the checksum line inclusively, - is the *block*. (A block that hasn't been successfully processed by Argument - Clinic yet doesn't have output or a checksum line, but it's still considered - a block.) - - -Converting Your First Function -============================== - -The best way to get a sense of how Argument Clinic works is to -convert a function to work with it. Here, then, are the bare -minimum steps you'd need to follow to convert a function to -work with Argument Clinic. Note that for code you plan to -check in to CPython, you really should take the conversion farther, -using some of the advanced concepts you'll see later on in -the document (like "return converters" and "self converters"). -But we'll keep it simple for this walkthrough so you can learn. - -Let's dive in! - -0. Make sure you're working with a freshly updated checkout - of the CPython trunk. - -1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted - to work with Argument Clinic yet. - For my example I'm using ``_pickle.Pickler.dump()``. - -2. If the call to the ``PyArg_Parse`` function uses any of the - following format units:: - - O& - O! - es - es# - et - et# - - or if it has multiple calls to :c:func:`PyArg_ParseTuple`, - you should choose a different function. Argument Clinic *does* - support all of these scenarios. But these are advanced - topics--let's do something simpler for your first function. - - Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple` - or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different - types for the same argument, or if the function uses something besides - PyArg_Parse functions to parse its arguments, it probably - isn't suitable for conversion to Argument Clinic. Argument Clinic - doesn't support generic functions or polymorphic parameters. - -3. Add the following boilerplate above the function, creating our block:: - - /*[clinic input] - [clinic start generated code]*/ - -4. Cut the docstring and paste it in between the ``[clinic]`` lines, - removing all the junk that makes it a properly quoted C string. - When you're done you should have just the text, based at the left - margin, with no line wider than 80 characters. - (Argument Clinic will preserve indents inside the docstring.) - - If the old docstring had a first line that looked like a function - signature, throw that line away. (The docstring doesn't need it - anymore--when you use ``help()`` on your builtin in the future, - the first line will be built automatically based on the function's - signature.) - - Sample:: - - /*[clinic input] - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -5. If your docstring doesn't have a "summary" line, Argument Clinic will - complain. So let's make sure it has one. The "summary" line should - be a paragraph consisting of a single 80-column line - at the beginning of the docstring. - - (Our example docstring consists solely of a summary line, so the sample - code doesn't have to change for this step.) - -6. Above the docstring, enter the name of the function, followed - by a blank line. This should be the Python name of the function, - and should be the full dotted path - to the function--it should start with the name of the module, - include any sub-modules, and if the function is a method on - a class it should include the class name too. - - Sample:: - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -7. If this is the first time that module or class has been used with Argument - Clinic in this C file, - you must declare the module and/or class. Proper Argument Clinic hygiene - prefers declaring these in a separate block somewhere near the - top of the C file, in the same way that include files and statics go at - the top. (In our sample code we'll just show the two blocks next to - each other.) - - The name of the class and module should be the same as the one - seen by Python. Check the name defined in the :c:type:`PyModuleDef` - or :c:type:`PyTypeObject` as appropriate. - - When you declare a class, you must also specify two aspects of its type - in C: the type declaration you'd use for a pointer to an instance of - this class, and a pointer to the :c:type:`PyTypeObject` for this class. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - - - -8. Declare each of the parameters to the function. Each parameter - should get its own line. All the parameter lines should be - indented from the function name and the docstring. - - The general form of these parameter lines is as follows:: - - name_of_parameter: converter - - If the parameter has a default value, add that after the - converter:: - - name_of_parameter: converter = default_value - - Argument Clinic's support for "default values" is quite sophisticated; - please see :ref:`the section below on default values <default_values>` - for more information. - - Add a blank line below the parameters. - - What's a "converter"? It establishes both the type - of the variable used in C, and the method to convert the Python - value into a C value at runtime. - For now you're going to use what's called a "legacy converter"--a - convenience syntax intended to make porting old code into Argument - Clinic easier. - - For each parameter, copy the "format unit" for that - parameter from the ``PyArg_Parse()`` format argument and - specify *that* as its converter, as a quoted - string. ("format unit" is the formal name for the one-to-three - character substring of the ``format`` parameter that tells - the argument parsing function what the type of the variable - is and how to convert it. For more on format units please - see :ref:`arg-parsing`.) - - For multicharacter format units like ``z#``, use the - entire two-or-three character string. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -9. If your function has ``|`` in the format string, meaning some - parameters have default values, you can ignore it. Argument - Clinic infers which parameters are optional based on whether - or not they have default values. - - If your function has ``$`` in the format string, meaning it - takes keyword-only arguments, specify ``*`` on a line by - itself before the first keyword-only argument, indented the - same as the parameter lines. - - (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.) - - -10. If the existing C function calls :c:func:`PyArg_ParseTuple` - (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its - arguments are positional-only. - - To mark all parameters as positional-only in Argument Clinic, - add a ``/`` on a line by itself after the last parameter, - indented the same as the parameter lines. - - Currently this is all-or-nothing; either all parameters are - positional-only, or none of them are. (In the future Argument - Clinic may relax this restriction.) - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -11. It's helpful to write a per-parameter docstring for each parameter. - But per-parameter docstrings are optional; you can skip this step - if you prefer. - - Here's how to add a per-parameter docstring. The first line - of the per-parameter docstring must be indented further than the - parameter definition. The left margin of this first line establishes - the left margin for the whole per-parameter docstring; all the text - you write will be outdented by this amount. You can write as much - text as you like, across multiple lines if you wish. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -12. Save and close the file, then run ``Tools/clinic/clinic.py`` on it. - With luck everything worked and your block now has output! Reopen - the file in your text editor to see:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - PyDoc_STRVAR(_pickle_Pickler_dump__doc__, - "Write a pickled representation of obj to the open file.\n" - "\n" - ... - static PyObject * - _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ - - Obviously, if Argument Clinic didn't produce any output, it's because - it found an error in your input. Keep fixing your errors and retrying - until Argument Clinic processes your file without complaint. - -13. Double-check that the argument-parsing code Argument Clinic generated - looks basically the same as the existing code. - - First, ensure both places use the same argument-parsing function. - The existing code must call either - :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`; - ensure that the code generated by Argument Clinic calls the - *exact* same function. - - Second, the format string passed in to :c:func:`PyArg_ParseTuple` or - :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same - as the hand-written one in the existing function, up to the colon - or semi-colon. - - (Argument Clinic always generates its format strings - with a ``:`` followed by the name of the function. If the - existing code's format string ends with ``;``, to provide - usage help, this change is harmless--don't worry about it.) - - Third, for parameters whose format units require two arguments - (like a length variable, or an encoding string, or a pointer - to a conversion function), ensure that the second argument is - *exactly* the same between the two invocations. - - Fourth, inside the output portion of the block you'll find a preprocessor - macro defining the appropriate static :c:type:`PyMethodDef` structure for - this builtin:: - - #define __PICKLE_PICKLER_DUMP_METHODDEF \ - {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__}, - - This static structure should be *exactly* the same as the existing static - :c:type:`PyMethodDef` structure for this builtin. - - If any of these items differ in *any way*, - adjust your Argument Clinic function specification and rerun - ``Tools/clinic/clinic.py`` until they *are* the same. - - -14. Notice that the last line of its output is the declaration - of your "impl" function. This is where the builtin's implementation goes. - Delete the existing prototype of the function you're modifying, but leave - the opening curly brace. Now delete its argument parsing code and the - declarations of all the variables it dumps the arguments into. - Notice how the Python arguments are now arguments to this impl function; - if the implementation used different names for these variables, fix it. - - Let's reiterate, just because it's kind of weird. Your code should now - look like this:: - - static return_type - your_function_impl(...) - /*[clinic end generated code: checksum=...]*/ - { - ... - - Argument Clinic generated the checksum line and the function prototype just - above it. You should write the opening (and closing) curly braces for the - function, and the implementation inside. - - Sample:: - - /*[clinic input] - module _pickle - class _pickle.Pickler "PicklerObject *" "&Pickler_Type" - [clinic start generated code]*/ - /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ - - /*[clinic input] - _pickle.Pickler.dump - - obj: 'O' - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - - PyDoc_STRVAR(__pickle_Pickler_dump__doc__, - "Write a pickled representation of obj to the open file.\n" - "\n" - ... - static PyObject * - _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj) - /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/ - { - /* Check whether the Pickler was initialized correctly (issue3664). - Developers often forget to call __init__() in their subclasses, which - would trigger a segfault without this check. */ - if (self->write == NULL) { - PyErr_Format(PicklingError, - "Pickler.__init__() was not called by %s.__init__()", - Py_TYPE(self)->tp_name); - return NULL; - } - - if (_Pickler_ClearBuffer(self) < 0) - return NULL; - - ... - -15. Remember the macro with the :c:type:`PyMethodDef` structure for this - function? Find the existing :c:type:`PyMethodDef` structure for this - function and replace it with a reference to the macro. (If the builtin - is at module scope, this will probably be very near the end of the file; - if the builtin is a class method, this will probably be below but relatively - near to the implementation.) - - Note that the body of the macro contains a trailing comma. So when you - replace the existing static :c:type:`PyMethodDef` structure with the macro, - *don't* add a comma to the end. - - Sample:: - - static struct PyMethodDef Pickler_methods[] = { - __PICKLE_PICKLER_DUMP_METHODDEF - __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF - {NULL, NULL} /* sentinel */ - }; - - -16. Compile, then run the relevant portions of the regression-test suite. - This change should not introduce any new compile-time warnings or errors, - and there should be no externally-visible change to Python's behavior. - - Well, except for one difference: ``inspect.signature()`` run on your function - should now provide a valid signature! - - Congratulations, you've ported your first function to work with Argument Clinic! - -Advanced Topics -=============== - -Now that you've had some experience working with Argument Clinic, it's time -for some advanced topics. - - -Symbolic default values ------------------------ - -The default value you provide for a parameter can't be any arbitrary -expression. Currently the following are explicitly supported: - -* Numeric constants (integer and float) -* String constants -* ``True``, ``False``, and ``None`` -* Simple symbolic constants like ``sys.maxsize``, which must - start with the name of the module - -In case you're curious, this is implemented in ``from_builtin()`` -in ``Lib/inspect.py``. - -(In the future, this may need to get even more elaborate, -to allow full expressions like ``CONSTANT - 1``.) - - -Renaming the C functions and variables generated by Argument Clinic -------------------------------------------------------------------- - -Argument Clinic automatically names the functions it generates for you. -Occasionally this may cause a problem, if the generated name collides with -the name of an existing C function. There's an easy solution: override the names -used for the C functions. Just add the keyword ``"as"`` -to your function declaration line, followed by the function name you wish to use. -Argument Clinic will use that function name for the base (generated) function, -then add ``"_impl"`` to the end and use that for the name of the impl function. - -For example, if we wanted to rename the C function names generated for -``pickle.Pickler.dump``, it'd look like this:: - - /*[clinic input] - pickle.Pickler.dump as pickler_dumper - - ... - -The base function would now be named ``pickler_dumper()``, -and the impl function would now be named ``pickler_dumper_impl()``. - - -Similarly, you may have a problem where you want to give a parameter -a specific Python name, but that name may be inconvenient in C. Argument -Clinic allows you to give a parameter different names in Python and in C, -using the same ``"as"`` syntax:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - file as file_obj: object - protocol: object = NULL - * - fix_imports: bool = True - -Here, the name used in Python (in the signature and the ``keywords`` -array) would be ``file``, but the C variable would be named ``file_obj``. - -You can use this to rename the ``self`` parameter too! - - -Converting functions using PyArg_UnpackTuple --------------------------------------------- - -To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, -simply write out all the arguments, specifying each as an ``object``. You -may specify the ``type`` argument to cast the type as appropriate. All -arguments should be marked positional-only (add a ``/`` on a line by itself -after the last argument). - -Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this -will change soon. - -Optional Groups ---------------- - -Some legacy functions have a tricky approach to parsing their arguments: -they count the number of positional arguments, then use a ``switch`` statement -to call one of several different :c:func:`PyArg_ParseTuple` calls depending on -how many positional arguments there are. (These functions cannot accept -keyword-only arguments.) This approach was used to simulate optional -arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created. - -While functions using this approach can often be converted to -use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values, -it's not always possible. Some of these legacy functions have -behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support. -The most obvious example is the builtin function ``range()``, which has -an optional argument on the *left* side of its required argument! -Another example is ``curses.window.addch()``, which has a group of two -arguments that must always be specified together. (The arguments are -called ``x`` and ``y``; if you call the function passing in ``x``, -you must also pass in ``y``--and if you don't pass in ``x`` you may not -pass in ``y`` either.) - -In any case, the goal of Argument Clinic is to support argument parsing -for all existing CPython builtins without changing their semantics. -Therefore Argument Clinic supports -this alternate approach to parsing, using what are called *optional groups*. -Optional groups are groups of arguments that must all be passed in together. -They can be to the left or the right of the required arguments. They -can *only* be used with positional-only parameters. - -.. note:: Optional groups are *only* intended for use when converting - functions that make multiple calls to :c:func:`PyArg_ParseTuple`! - Functions that use *any* other approach for parsing arguments - should *almost never* be converted to Argument Clinic using - optional groups. Functions using optional groups currently - cannot have accurate sigantures in Python, because Python just - doesn't understand the concept. Please avoid using optional - groups wherever possible. - -To specify an optional group, add a ``[`` on a line by itself before -the parameters you wish to group together, and a ``]`` on a line by itself -after these parameters. As an example, here's how ``curses.window.addch`` -uses optional groups to make the first two parameters and the last -parameter optional:: - - /*[clinic input] - - curses.window.addch - - [ - x: int - X-coordinate. - y: int - Y-coordinate. - ] - - ch: object - Character to add. - - [ - attr: long - Attributes for the character. - ] - / - - ... - - -Notes: - -* For every optional group, one additional parameter will be passed into the - impl function representing the group. The parameter will be an int named - ``group_{direction}_{number}``, - where ``{direction}`` is either ``right`` or ``left`` depending on whether the group - is before or after the required parameters, and ``{number}`` is a monotonically - increasing number (starting at 1) indicating how far away the group is from - the required parameters. When the impl is called, this parameter will be set - to zero if this group was unused, and set to non-zero if this group was used. - (By used or unused, I mean whether or not the parameters received arguments - in this invocation.) - -* If there are no required arguments, the optional groups will behave - as if they're to the right of the required arguments. - -* In the case of ambiguity, the argument parsing code - favors parameters on the left (before the required parameters). - -* Optional groups can only contain positional-only parameters. - -* Optional groups are *only* intended for legacy code. Please do not - use optional groups for new code. - - -Using real Argument Clinic converters, instead of "legacy converters" ---------------------------------------------------------------------- - -To save time, and to minimize how much you need to learn -to achieve your first port to Argument Clinic, the walkthrough above tells -you to use "legacy converters". "Legacy converters" are a convenience, -designed explicitly to make porting existing code to Argument Clinic -easier. And to be clear, their use is acceptable when porting code for -Python 3.4. - -However, in the long term we probably want all our blocks to -use Argument Clinic's real syntax for converters. Why? A couple -reasons: - -* The proper converters are far easier to read and clearer in their intent. -* There are some format units that are unsupported as "legacy converters", - because they require arguments, and the legacy converter syntax doesn't - support specifying arguments. -* In the future we may have a new argument parsing library that isn't - restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility - won't be available to parameters using legacy converters. - -Therefore, if you don't mind a little extra effort, please use the normal -converters instead of legacy converters. - -In a nutshell, the syntax for Argument Clinic (non-legacy) converters -looks like a Python function call. However, if there are no explicit -arguments to the function (all functions take their default values), -you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly -the same converters. - -All arguments to Argument Clinic converters are keyword-only. -All Argument Clinic converters accept the following arguments: - - ``c_default`` - The default value for this parameter when defined in C. - Specifically, this will be the initializer for the variable declared - in the "parse function". See :ref:`the section on default values <default_values>` - for how to use this. - Specified as a string. - - ``annotation`` - The annotation value for this parameter. Not currently supported, - because PEP 8 mandates that the Python library may not use - annotations. - -In addition, some converters accept additional arguments. Here is a list -of these arguments, along with their meanings: - - ``accept`` - A set of Python types (and possibly pseudo-types); - this restricts the allowable Python argument to values of these types. - (This is not a general-purpose facility; as a rule it only supports - specific lists of types as shown in the legacy converter table.) - - To accept ``None``, add ``NoneType`` to this set. - - ``bitwise`` - Only supported for unsigned integers. The native integer value of this - Python argument will be written to the parameter without any range checking, - even for negative values. - - ``converter`` - Only supported by the ``object`` converter. Specifies the name of a - :ref:`C "converter function" <o_ampersand>` - to use to convert this object to a native type. - - ``encoding`` - Only supported for strings. Specifies the encoding to use when converting - this string from a Python str (Unicode) value into a C ``char *`` value. - - - ``subclass_of`` - Only supported for the ``object`` converter. Requires that the Python - value be a subclass of a Python type, as expressed in C. - - ``type`` - Only supported for the ``object`` and ``self`` converters. Specifies - the C type that will be used to declare the variable. Default value is - ``"PyObject *"``. - - ``zeroes`` - Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are - permitted inside the value. The length of the string will be passed in - to the impl function, just after the string parameter, as a parameter named - ``<parameter_name>_length``. - -Please note, not every possible combination of arguments will work. -Usually these arguments are implemented by specific ``PyArg_ParseTuple`` -*format units*, with specific behavior. For example, currently you cannot -call ``unsigned_short`` without also specifying ``bitwise=True``. -Although it's perfectly reasonable to think this would work, these semantics don't -map to any existing format unit. So Argument Clinic doesn't support it. (Or, at -least, not yet.) - -Below is a table showing the mapping of legacy converters into real -Argument Clinic converters. On the left is the legacy converter, -on the right is the text you'd replace it with. - -========= ================================================================================= -``'B'`` ``unsigned_char(bitwise=True)`` -``'b'`` ``unsigned_char`` -``'c'`` ``char`` -``'C'`` ``int(accept={str})`` -``'d'`` ``double`` -``'D'`` ``Py_complex`` -``'es'`` ``str(encoding='name_of_encoding')`` -``'es#'`` ``str(encoding='name_of_encoding', zeroes=True)`` -``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})`` -``'et#'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str}, zeroes=True)`` -``'f'`` ``float`` -``'h'`` ``short`` -``'H'`` ``unsigned_short(bitwise=True)`` -``'i'`` ``int`` -``'I'`` ``unsigned_int(bitwise=True)`` -``'k'`` ``unsigned_long(bitwise=True)`` -``'K'`` ``unsigned_PY_LONG_LONG(bitwise=True)`` -``'l'`` ``long`` -``'L'`` ``PY_LONG_LONG`` -``'n'`` ``Py_ssize_t`` -``'O'`` ``object`` -``'O!'`` ``object(subclass_of='&PySomething_Type')`` -``'O&'`` ``object(converter='name_of_c_function')`` -``'p'`` ``bool`` -``'S'`` ``PyBytesObject`` -``'s'`` ``str`` -``'s#'`` ``str(zeroes=True)`` -``'s*'`` ``Py_buffer(accept={buffer, str})`` -``'U'`` ``unicode`` -``'u'`` ``Py_UNICODE`` -``'u#'`` ``Py_UNICODE(zeroes=True)`` -``'w*'`` ``Py_buffer(accept={rwbuffer})`` -``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(accept={bytes})`` -``'y#'`` ``str(accept={robuffer}, zeroes=True)`` -``'y*'`` ``Py_buffer`` -``'Z'`` ``Py_UNICODE(accept={str, NoneType})`` -``'Z#'`` ``Py_UNICODE(accept={str, NoneType}, zeroes=True)`` -``'z'`` ``str(accept={str, NoneType})`` -``'z#'`` ``str(accept={str, NoneType}, zeroes=True)`` -``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})`` -========= ================================================================================= - -As an example, here's our sample ``pickle.Pickler.dump`` using the proper -converter:: - - /*[clinic input] - pickle.Pickler.dump - - obj: object - The object to be pickled. - / - - Write a pickled representation of obj to the open file. - [clinic start generated code]*/ - -Argument Clinic will show you all the converters it has -available. For each converter it'll show you all the parameters -it accepts, along with the default value for each parameter. -Just run ``Tools/clinic/clinic.py --converters`` to see the full list. - -Py_buffer ---------- - -When using the ``Py_buffer`` converter -(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters), -you *must* not call :c:func:`PyBuffer_Release` on the provided buffer. -Argument Clinic generates code that does it for you (in the parsing function). - - - -Advanced converters -------------------- - -Remember those format units you skipped for your first -time because they were advanced? Here's how to handle those too. - -The trick is, all those format units take arguments--either -conversion functions, or types, or strings specifying an encoding. -(But "legacy converters" don't support arguments. That's why we -skipped them for your first function.) The argument you specified -to the format unit is now an argument to the converter; this -argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``), -or ``encoding`` (for all the format units that start with ``e``). - -When using ``subclass_of``, you may also want to use the other -custom argument for ``object()``: ``type``, which lets you set the type -actually used for the parameter. For example, if you want to ensure -that the object is a subclass of ``PyUnicode_Type``, you probably want -to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``. - -One possible problem with using Argument Clinic: it takes away some possible -flexibility for the format units starting with ``e``. When writing a -``PyArg_Parse`` call by hand, you could theoretically decide at runtime what -encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must -be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate; -it made supporting this format unit much easier, and may allow for future optimizations. -This restriction doesn't seem unreasonable; CPython itself always passes in static -hard-coded encoding strings for parameters whose format units start with ``e``. - - -.. _default_values: - -Parameter default values ------------------------- - -Default values for parameters can be any of a number of values. -At their simplest, they can be string, int, or float literals:: - - foo: str = "abc" - bar: int = 123 - bat: float = 45.6 - -They can also use any of Python's built-in constants:: - - yep: bool = True - nope: bool = False - nada: object = None - -There's also special support for a default value of ``NULL``, and -for simple expressions, documented in the following sections. - - -The ``NULL`` default value --------------------------- - -For string and object parameters, you can set them to ``None`` to indicate -that there's no default. However, that means the C variable will be -initialized to ``Py_None``. For convenience's sakes, there's a special -value called ``NULL`` for just this reason: from Python's perspective it -behaves like a default value of ``None``, but the C variable is initialized -with ``NULL``. - -Expressions specified as default values ---------------------------------------- - -The default value for a parameter can be more than just a literal value. -It can be an entire expression, using math operators and looking up attributes -on objects. However, this support isn't exactly simple, because of some -non-obvious semantics. - -Consider the following example:: - - foo: Py_ssize_t = sys.maxsize - 1 - -``sys.maxsize`` can have different values on different platforms. Therefore -Argument Clinic can't simply evaluate that expression locally and hard-code it -in C. So it stores the default in such a way that it will get evaluated at -runtime, when the user asks for the function's signature. - -What namespace is available when the expression is evaluated? It's evaluated -in the context of the module the builtin came from. So, if your module has an -attribute called "``max_widgets``", you may simply use it:: - - foo: Py_ssize_t = max_widgets - -If the symbol isn't found in the current module, it fails over to looking in -``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you -don't know in advance what modules the user will load into their interpreter, -it's best to restrict yourself to modules that are preloaded by Python itself.) - -Evaluating default values only at runtime means Argument Clinic can't compute -the correct equivalent C default value. So you need to tell it explicitly. -When you use an expression, you must also specify the equivalent expression -in C, using the ``c_default`` parameter to the converter:: - - foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 - -Another complication: Argument Clinic can't know in advance whether or not the -expression you supply is valid. It parses it to make sure it looks legal, but -it can't *actually* know. You must be very careful when using expressions to -specify values that are guaranteed to be valid at runtime! - -Finally, because expressions must be representable as static C values, there -are many restrictions on legal expressions. Here's a list of Python features -you're not permitted to use: - -* Function calls. -* Inline if statements (``3 if foo else 5``). -* Automatic sequence unpacking (``*[1, 2, 3]``). -* List/set/dict comprehensions and generator expressions. -* Tuple/list/set/dict literals. - - - -Using a return converter ------------------------- - -By default the impl function Argument Clinic generates for you returns ``PyObject *``. -But your C function often computes some C type, then converts it into the ``PyObject *`` -at the last moment. Argument Clinic handles converting your inputs from Python types -into native C types--why not have it convert your return value from a native C type -into a Python type too? - -That's what a "return converter" does. It changes your impl function to return -some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate ``PyObject *``. - -The syntax for return converters is similar to that of parameter converters. -You specify the return converter like it was a return annotation on the -function itself. Return converters behave much the same as parameter converters; -they take arguments, the arguments are all keyword-only, and if you're not changing -any of the default arguments you can omit the parentheses. - -(If you use both ``"as"`` *and* a return converter for your function, -the ``"as"`` should come before the return converter.) - -There's one additional complication when using return converters: how do you -indicate an error has occurred? Normally, a function returns a valid (non-``NULL``) -pointer for success, and ``NULL`` for failure. But if you use an integer return converter, -all integers are valid. How can Argument Clinic detect an error? Its solution: each return -converter implicitly looks for a special value that indicates an error. If you return -that value, and an error has been set (``PyErr_Occurred()`` returns a true -value), then the generated code will propagate the error. Otherwise it will -encode the value you return like normal. - -Currently Argument Clinic supports only a few return converters:: - - bool - int - unsigned int - long - unsigned int - size_t - Py_ssize_t - float - double - DecodeFSDefault - -None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``char *``; return a NULL -pointer to indicate an error. - -(There's also an experimental ``NoneType`` converter, which lets you -return ``Py_None`` on success or ``NULL`` on failure, without having -to increment the reference count on ``Py_None``. I'm not sure it adds -enough clarity to be worth using.) - -To see all the return converters Argument Clinic supports, along with -their parameters (if any), -just run ``Tools/clinic/clinic.py --converters`` for the full list. - - -Cloning existing functions --------------------------- - -If you have a number of functions that look similar, you may be able to -use Clinic's "clone" feature. When you clone an existing function, -you reuse: - -* its parameters, including - - * their names, - - * their converters, with all parameters, - - * their default values, - - * their per-parameter docstrings, - - * their *kind* (whether they're positional only, - positional or keyword, or keyword only), and - -* its return converter. - -The only thing not copied from the original function is its docstring; -the syntax allows you to specify a new docstring. - -Here's the syntax for cloning a function:: - - /*[clinic input] - module.class.new_function [as c_basename] = module.class.existing_function - - Docstring for new_function goes here. - [clinic start generated code]*/ - -(The functions can be in different modules or classes. I wrote -``module.class`` in the sample just to illustrate that you must -use the full path to *both* functions.) - -Sorry, there's no syntax for partially-cloning a function, or cloning a function -then modifying it. Cloning is an all-or nothing proposition. - -Also, the function you are cloning from must have been previously defined -in the current file. - -Calling Python code -------------------- - -The rest of the advanced topics require you to write Python code -which lives inside your C file and modifies Argument Clinic's -runtime state. This is simple: you simply define a Python block. - -A Python block uses different delimiter lines than an Argument -Clinic function block. It looks like this:: - - /*[python input] - # python code goes here - [python start generated code]*/ - -All the code inside the Python block is executed at the -time it's parsed. All text written to stdout inside the block -is redirected into the "output" after the block. - -As an example, here's a Python block that adds a static integer -variable to the C code:: - - /*[python input] - print('static int __ignored_unused_variable__ = 0;') - [python start generated code]*/ - static int __ignored_unused_variable__ = 0; - /*[python checksum:...]*/ - - -Using a "self converter" ------------------------- - -Argument Clinic automatically adds a "self" parameter for you -using a default converter. It automatically sets the ``type`` -of this parameter to the "pointer to an instance" you specified -when you declared the type. However, you can override -Argument Clinic's converter and specify one yourself. -Just add your own ``self`` parameter as the first parameter in a -block, and ensure that its converter is an instance of -``self_converter`` or a subclass thereof. - -What's the point? This lets you override the type of ``self``, -or give it a different default name. - -How do you specify the custom type you want to cast ``self`` to? -If you only have one or two functions with the same type for ``self``, -you can directly use Argument Clinic's existing ``self`` converter, -passing in the type you want to use as the ``type`` parameter:: - - /*[clinic input] - - _pickle.Pickler.dump - - self: self(type="PicklerObject *") - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - -On the other hand, if you have a lot of functions that will use the same -type for ``self``, it's best to create your own converter, subclassing -``self_converter`` but overwriting the ``type`` member:: - - /*[python input] - class PicklerObject_converter(self_converter): - type = "PicklerObject *" - [python start generated code]*/ - - /*[clinic input] - - _pickle.Pickler.dump - - self: PicklerObject - obj: object - / - - Write a pickled representation of the given object to the open file. - [clinic start generated code]*/ - - - -Writing a custom converter --------------------------- - -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from ``CConverter``. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit--parsing this parameter means calling -a :c:func:`PyArg_ParseTuple` "converter function". - -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass ``CConverter.__init__``. Instead, you should -write a ``converter_init()`` function. ``converter_init()`` -always accepts a ``self`` parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -``converter_init()``. - -There are some additional members of ``CConverter`` you may wish -to specify in your subclass. Here's the current list: - -``type`` - The C type to use for this variable. - ``type`` should be a Python string specifying the type, e.g. ``int``. - If this is a pointer type, the type string should end with ``' *'``. - -``default`` - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - -``py_default`` - ``default`` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - -``c_default`` - ``default`` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - -``c_ignored_default`` - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups--although - properly-written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - -``converter`` - The name of the C converter function, as a string. - -``impl_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - -``parse_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - -Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``:: - - /*[python input] - - class capped_uint_converter(CConverter): - type = 'unsigned int' - converter = 'capped_uint_converter' - - [python start generated code]*/ - /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ - -This block adds a converter to Argument Clinic named ``capped_uint``. Parameters -declared as ``capped_uint`` will be declared as type ``unsigned int``, and will -be parsed by the ``'O&'`` format unit, which will call the -``capped_uint_converter`` converter function. ``capped_uint`` variables -automatically support default values. - -More sophisticated custom converters can insert custom C code to -handle initialization and cleanup. -You can see more examples of custom converters in the CPython -source tree; grep the C files for the string ``CConverter``. - -Writing a custom return converter ---------------------------------- - -Writing a custom return converter is much like writing -a custom converter. Except it's somewhat simpler, because return -converters are themselves much simpler. - -Return converters must subclass ``CReturnConverter``. -There are no examples yet of custom return converters, -because they are not widely used yet. If you wish to -write your own return converter, please read ``Tools/clinic/clinic.py``, -specifically the implementation of ``CReturnConverter`` and -all its subclasses. - -METH_O and METH_NOARGS ----------------------------------------------- - -To convert a function using ``METH_O``, make sure the function's -single argument is using the ``object`` converter, and mark the -arguments as positional-only:: - - /*[clinic input] - meth_o_sample - - argument: object - / - [clinic start generated code]*/ - - -To convert a function using ``METH_NOARGS``, just don't specify -any arguments. - -You can still use a self converter, a return converter, and specify -a ``type`` argument to the object converter for ``METH_O``. - -tp_new and tp_init functions ----------------------------------------------- - -You can convert ``tp_new`` and ``tp_init`` functions. Just name -them ``__new__`` or ``__init__`` as appropriate. Notes: - -* The function name generated for ``__new__`` doesn't end in ``__new__`` - like it would by default. It's just the name of the class, converted - into a valid C identifier. - -* No ``PyMethodDef`` ``#define`` is generated for these functions. - -* ``__init__`` functions return ``int``, not ``PyObject *``. - -* Use the docstring as the class docstring. - -* Although ``__new__`` and ``__init__`` functions must always - accept both the ``args`` and ``kwargs`` objects, when converting - you may specify any signature for these functions that you like. - (If your function doesn't support keywords, the parsing function - generated will throw an exception if it receives any.) - -Changing and redirecting Clinic's output ----------------------------------------- - -It can be inconvenient to have Clinic's output interspersed with -your conventional hand-edited C code. Luckily, Clinic is configurable: -you can buffer up its output for printing later (or earlier!), or write -its output to a separate file. You can also add a prefix or suffix to -every line of Clinic's generated output. - -While changing Clinic's output in this manner can be a boon to readability, -it may result in Clinic code using types before they are defined, or -your code attempting to use Clinic-generated code befire it is defined. -These problems can be easily solved by rearranging the declarations in your file, -or moving where Clinic's generated code goes. (This is why the default behavior -of Clinic is to output everything into the current block; while many people -consider this hampers readability, it will never require rearranging your -code to fix definition-before-use problems.) - -Let's start with defining some terminology: - -*field* - A field, in this context, is a subsection of Clinic's output. - For example, the ``#define`` for the ``PyMethodDef`` structure - is a field, called ``methoddef_define``. Clinic has seven - different fields it can output per function definition:: - - docstring_prototype - docstring_definition - methoddef_define - impl_prototype - parser_prototype - parser_definition - impl_definition - - All the names are of the form ``"<a>_<b>"``, - where ``"<a>"`` is the semantic object represented (the parsing function, - the impl function, the docstring, or the methoddef structure) and ``"<b>"`` - represents what kind of statement the field is. Field names that end in - ``"_prototype"`` - represent forward declarations of that thing, without the actual body/data - of the thing; field names that end in ``"_definition"`` represent the actual - definition of the thing, with the body/data of the thing. (``"methoddef"`` - is special, it's the only one that ends with ``"_define"``, representing that - it's a preprocessor #define.) - -*destination* - A destination is a place Clinic can write output to. There are - five built-in destinations: - - ``block`` - The default destination: printed in the output section of - the current Clinic block. - - ``buffer`` - A text buffer where you can save text for later. Text sent - here is appended to the end of any exsiting text. It's an - error to have any text left in the buffer when Clinic finishes - processing a file. - - ``file`` - A separate "clinic file" that will be created automatically by Clinic. - The filename chosen for the file is ``{basename}.clinic{extension}``, - where ``basename`` and ``extension`` were assigned the output - from ``os.path.splitext()`` run on the current file. (Example: - the ``file`` destination for ``_pickle.c`` would be written to - ``_pickle.clinic.c``.) - - **Important: When using a** ``file`` **destination, you** - *must check in* **the generated file!** - - ``two-pass`` - A buffer like ``buffer``. However, a two-pass buffer can only - be written once, and it prints out all text sent to it during - all of processing, even from Clinic blocks *after* the - - ``suppress`` - The text is suppressed--thrown away. - - -Clinic defines five new directives that let you reconfigure its output. - -The first new directive is ``dump``:: - - dump <destination> - -This dumps the current contents of the named destination into the output of -the current block, and empties it. This only works with ``buffer`` and -``two-pass`` destinations. - -The second new directive is ``output``. The most basic form of ``output`` -is like this:: - - output <field> <destination> - -This tells Clinic to output *field* to *destination*. ``output`` also -supports a special meta-destination, called ``everything``, which tells -Clinic to output *all* fields to that *destination*. - -``output`` has a number of other functions:: - - output push - output pop - output preset <preset> - - -``output push`` and ``output pop`` allow you to push and pop -configurations on an internal configuration stack, so that you -can temporarily modify the output configuration, then easily restore -the previous configuration. Simply push before your change to save -the current configuration, then pop when you wish to restore the -previous configuration. - -``output preset`` sets Clinic's output to one of several built-in -preset configurations, as follows: - - ``block`` - Clinic's original starting configuration. Writes everything - immediately after the input block. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write everything else to ``block``. - - ``file`` - Designed to write everything to the "clinic file" that it can. - You then ``#include`` this file near the top of your file. - You may need to rearrange your file to make this work, though - usually this just means creating forward declarations for various - ``typedef`` and ``PyTypeObject`` definitions. - - Suppress the ``parser_prototype`` - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - The default filename is ``"{dirname}/clinic/{basename}.h"``. - - ``buffer`` - Save up all most of the output from Clinic, to be written into - your file near the end. For Python files implementing modules - or builtin types, it's recommended that you dump the buffer - just above the static structures for your module or - builtin type; these are normally very near the end. Using - ``buffer`` may require even more editing than ``file``, if - your file has static ``PyMethodDef`` arrays defined in the - middle of the file. - - Suppress the ``parser_prototype``, ``impl_prototype``, - and ``docstring_prototype``, write the ``impl_definition`` to - ``block``, and write everything else to ``file``. - - ``two-pass`` - Similar to the ``buffer`` preset, but writes forward declarations to - the ``two-pass`` buffer, and definitions to the ``buffer``. - This is similar to the ``buffer`` preset, but may require - less editing than ``buffer``. Dump the ``two-pass`` buffer - near the top of your file, and dump the ``buffer`` near - the end just like you would when using the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``impl_definition`` - to ``block``, write ``docstring_prototype``, ``methoddef_define``, - and ``parser_prototype`` to ``two-pass``, write everything else - to ``buffer``. - - ``partial-buffer`` - Similar to the ``buffer`` preset, but writes more things to ``block``, - only writing the really big chunks of generated code to ``buffer``. - This avoids the definition-before-use problem of ``buffer`` completely, - at the small cost of having slightly more stuff in the block's output. - Dump the ``buffer`` near the end, just like you would when using - the ``buffer`` preset. - - Suppresses the ``impl_prototype``, write the ``docstring_definition`` - and ``parser_definition`` to ``buffer``, write everything else to ``block``. - -The third new directive is ``destination``:: - - destination <name> <command> [...] - -This performs an operation on the destination named ``name``. - -There are two defined subcommands: ``new`` and ``clear``. - -The ``new`` subcommand works like this:: - - destination <name> new <type> - -This creates a new destination with name ``<name>`` and type ``<type>``. - -There are five destination types: - - ``suppress`` - Throws the text away. - - ``block`` - Writes the text to the current block. This is what Clinic - originally did. - - ``buffer`` - A simple text buffer, like the "buffer" builtin destination above. - - ``file`` - A text file. The file destination takes an extra argument, - a template to use for building the filename, like so: - - destination <name> new <type> <file_template> - - The template can use three strings internally that will be replaced - by bits of the filename: - - {path} - The full path to the file, including directory and full filename. - {dirname} - The name of the directory the file is in. - {basename} - Just the name of the file, not including the directory. - {basename_root} - Basename with the extension clipped off - (everything up to but not including the last '.'). - {basename_extension} - The last '.' and everything after it. If the basename - does not contain a period, this will be the empty string. - - If there are no periods in the filename, {basename} and {filename} - are the same, and {extension} is empty. "{basename}{extension}" - is always exactly the same as "{filename}"." - - ``two-pass`` - A two-pass buffer, like the "two-pass" builtin destination above. - - -The ``clear`` subcommand works like this:: - - destination <name> clear - -It removes all the accumulated text up to this point in the destination. -(I don't know what you'd need this for, but I thought maybe it'd be -useful while someone's experimenting.) - -The fourth new directive is ``set``:: - - set line_prefix "string" - set line_suffix "string" - -``set`` lets you set two internal variables in Clinic. -``line_prefix`` is a string that will be prepended to every line of Clinic's output; -``line_suffix`` is a string that will be appended to every line of Clinic's output. - -Both of these support two format strings: - - ``{block comment start}`` - Turns into the string ``/*``, the start-comment text sequence for C files. - - ``{block comment end}`` - Turns into the string ``*/``, the end-comment text sequence for C files. - -The final new directive is one you shouldn't need to use directly, -called ``preserve``:: - - preserve - -This tells Clinic that the current contents of the output should be kept, unmodifed. -This is used internally by Clinic when dumping output into ``file`` files; wrapping -it in a Clinic block lets Clinic use its existing checksum functionality to ensure -the file was not modified by hand before it gets overwritten. - - -The #ifdef trick ----------------------------------------------- - -If you're converting a function that isn't available on all platforms, -there's a trick you can use to make life a little easier. The existing -code probably looks like this:: - - #ifdef HAVE_FUNCTIONNAME - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -And then in the ``PyMethodDef`` structure at the bottom the existing code -will have:: - - #ifdef HAVE_FUNCTIONNAME - {'functionname', ... }, - #endif /* HAVE_FUNCTIONNAME */ - -In this scenario, you should enclose the body of your impl function inside the ``#ifdef``, -like so:: - - #ifdef HAVE_FUNCTIONNAME - /*[clinic input] - module.functionname - ... - [clinic start generated code]*/ - static module_functionname(...) - { - ... - } - #endif /* HAVE_FUNCTIONNAME */ - -Then, remove those three lines from the ``PyMethodDef`` structure, -replacing them with the macro Argument Clinic generated:: - - MODULE_FUNCTIONNAME_METHODDEF - -(You can find the real name for this macro inside the generated code. -Or you can calculate it yourself: it's the name of your function as defined -on the first line of your block, but with periods changed to underscores, -uppercased, and ``"_METHODDEF"`` added to the end.) - -Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined? -The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either! - -Here's where Argument Clinic gets very clever. It actually detects that the -Argument Clinic block might be deactivated by the ``#ifdef``. When that -happens, it generates a little extra code that looks like this:: - - #ifndef MODULE_FUNCTIONNAME_METHODDEF - #define MODULE_FUNCTIONNAME_METHODDEF - #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */ - -That means the macro always works. If the function is defined, this turns -into the correct structure, including the trailing comma. If the function is -undefined, this turns into nothing. - -However, this causes one ticklish problem: where should Argument Clinic put this -extra code when using the "block" output preset? It can't go in the output block, -because that could be decativated by the ``#ifdef``. (That's the whole point!) - -In this situation, Argument Clinic writes the extra code to the "buffer" destination. -This may mean that you get a complaint from Argument Clinic:: - - Warning in file "Modules/posixmodule.c" on line 12357: - Destination buffer 'buffer' not empty at end of file, emptying. - -When this happens, just open your file, find the ``dump buffer`` block that -Argument Clinic added to your file (it'll be at the very bottom), then -move it above the ``PyMethodDef`` structure where that macro is used. - - - -Using Argument Clinic in Python files -------------------------------------- - -It's actually possible to use Argument Clinic to preprocess Python files. -There's no point to using Argument Clinic blocks, of course, as the output -wouldn't make any sense to the Python interpreter. But using Argument Clinic -to run Python blocks lets you use Python as a Python preprocessor! - -Since Python comments are different from C comments, Argument Clinic -blocks embedded in Python files look slightly different. They look like this:: - - #/*[python input] - #print("def foo(): pass") - #[python start generated code]*/ - def foo(): pass - #/*[python checksum:...]*/ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/cporting.rst --- a/Doc/howto/cporting.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/cporting.rst Mon Jan 25 17:05:13 2016 +0100 @@ -43,9 +43,10 @@ str/unicode Unification ----------------------- -Python 3's :func:`str` type is equivalent to Python 2's :func:`unicode`; the C -functions are called ``PyUnicode_*`` for both. The old 8-bit string type has become -:func:`bytes`, with C functions called ``PyBytes_*``. Python 2.6 and later provide a compatibility header, + +Python 3's :func:`str` (``PyString_*`` functions in C) type is equivalent to +Python 2's :func:`unicode` (``PyUnicode_*``). The old 8-bit string type has +become :func:`bytes`. Python 2.6 and later provide a compatibility header, :file:`bytesobject.h`, mapping ``PyBytes`` names to ``PyString`` ones. For best compatibility with Python 3, :c:type:`PyUnicode` should be used for textual data and :c:type:`PyBytes` for binary data. It's also important to remember that @@ -99,6 +100,25 @@ used in Python 2 was removed. In the C-API, ``PyInt_*`` functions are replaced by their ``PyLong_*`` equivalents. +The best course of action here is using the ``PyInt_*`` functions aliased to +``PyLong_*`` found in :file:`intobject.h`. The abstract ``PyNumber_*`` APIs +can also be used in some cases. :: + + #include "Python.h" + #include "intobject.h" + + static PyObject * + add_ints(PyObject *self, PyObject *args) { + int one, two; + PyObject *result; + + if (!PyArg_ParseTuple(args, "ii:add_ints", &one, &two)) + return NULL; + + return PyInt_FromLong(one + two); + } + + Module initialization and state =============================== @@ -233,7 +253,7 @@ * :c:func:`PyCapsule_GetName` always returns NULL. - * :c:func:`PyCapsule_SetName` always raises an exception and + * :c:func:`PyCapsule_SetName` always throws an exception and returns failure. (Since there's no way to store a name in a CObject, noisy failure of :c:func:`PyCapsule_SetName` was deemed preferable to silent failure here. If this is @@ -252,6 +272,6 @@ ============= If you are writing a new extension module, you might consider `Cython -<http://cython.org/>`_. It translates a Python-like language to C. The +<http://www.cython.org>`_. It translates a Python-like language to C. The extension modules it creates are compatible with Python 3 and Python 2. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/curses.rst --- a/Doc/howto/curses.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/curses.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,43 +5,39 @@ ********************************** :Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 +:Release: 2.03 .. topic:: Abstract - This document describes how to use the :mod:`curses` extension - module to control text-mode displays. + This document describes how to write text-mode programs with Python 2.x, using + the :mod:`curses` extension module to control the display. What is curses? =============== The curses library supplies a terminal-independent screen-painting and -keyboard-handling facility for text-based terminals; such terminals -include VT100s, the Linux console, and the simulated terminal provided -by various programs. Display terminals support various control codes -to perform common operations such as moving the cursor, scrolling the -screen, and erasing areas. Different terminals use widely differing -codes, and often have their own minor quirks. +keyboard-handling facility for text-based terminals; such terminals include +VT100s, the Linux console, and the simulated terminal provided by X11 programs +such as xterm and rxvt. Display terminals support various control codes to +perform common operations such as moving the cursor, scrolling the screen, and +erasing areas. Different terminals use widely differing codes, and often have +their own minor quirks. -In a world of graphical displays, one might ask "why bother"? It's -true that character-cell display terminals are an obsolete technology, -but there are niches in which being able to do fancy things with them -are still valuable. One niche is on small-footprint or embedded -Unixes that don't run an X server. Another is tools such as OS -installers and kernel configurators that may have to run before any -graphical support is available. +In a world of X displays, one might ask "why bother"? It's true that +character-cell display terminals are an obsolete technology, but there are +niches in which being able to do fancy things with them are still valuable. One +is on small-footprint or embedded Unixes that don't carry an X server. Another +is for tools like OS installers and kernel configurators that may have to run +before X is available. -The curses library provides fairly basic functionality, providing the -programmer with an abstraction of a display containing multiple -non-overlapping windows of text. The contents of a window can be -changed in various ways---adding text, erasing it, changing its -appearance---and the curses library will figure out what control codes -need to be sent to the terminal to produce the right output. curses -doesn't provide many user-interface concepts such as buttons, checkboxes, -or dialogs; if you need such features, consider a user interface library such as -`Urwid <https://pypi.python.org/pypi/urwid/>`_. +The curses library hides all the details of different terminals, and provides +the programmer with an abstraction of a display, containing multiple +non-overlapping windows. The contents of a window can be changed in various +ways-- adding text, erasing it, changing its appearance--and the curses library +will automagically figure out what control codes need to be sent to the terminal +to produce the right output. The curses library was originally written for BSD Unix; the later System V versions of Unix from AT&T added many enhancements and new functions. BSD curses @@ -53,13 +49,10 @@ versions of curses carried by some proprietary Unixes may not support everything, though. -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -<https://pypi.python.org/pypi/UniCurses>`_ is available. You could -also try `the Console module <http://effbot.org/zone/console-index.htm>`_ -written by Fredrik Lundh, which doesn't -use the same API as curses but provides cursor-addressable text output -and full support for mouse and keyboard input. +No one has made a Windows port of the curses module. On a Windows platform, try +the Console module written by Fredrik Lundh. The Console module provides +cursor-addressable text output, plus full support for mouse and keyboard input, +and is available from http://effbot.org/zone/console-index.htm. The Python curses module @@ -68,12 +61,11 @@ Thy Python module is a fairly simple wrapper over the C functions provided by curses; if you're already familiar with curses programming in C, it's really easy to transfer that knowledge to Python. The biggest difference is that the -Python interface makes things simpler by merging different C functions such as -:c:func:`addstr`, :c:func:`mvaddstr`, and :c:func:`mvwaddstr` into a single -:meth:`~curses.window.addstr` method. You'll see this covered in more -detail later. +Python interface makes things simpler, by merging different C functions such as +:func:`addstr`, :func:`mvaddstr`, :func:`mvwaddstr`, into a single +:meth:`addstr` method. You'll see this covered in more detail later. -This HOWTO is an introduction to writing text-mode programs with curses +This HOWTO is simply an introduction to writing text-mode programs with curses and Python. It doesn't attempt to be a complete guide to the curses API; for that, see the Python library guide's section on ncurses, and the C manual pages for ncurses. It will, however, give you the basic ideas. @@ -82,27 +74,25 @@ Starting and ending a curses application ======================================== -Before doing anything, curses must be initialized. This is done by -calling the :func:`~curses.initscr` function, which will determine the -terminal type, send any required setup codes to the terminal, and -create various internal data structures. If successful, -:func:`initscr` returns a window object representing the entire -screen; this is usually called ``stdscr`` after the name of the +Before doing anything, curses must be initialized. This is done by calling the +:func:`initscr` function, which will determine the terminal type, send any +required setup codes to the terminal, and create various internal data +structures. If successful, :func:`initscr` returns a window object representing +the entire screen; this is usually called ``stdscr``, after the name of the corresponding C variable. :: import curses stdscr = curses.initscr() -Usually curses applications turn off automatic echoing of keys to the -screen, in order to be able to read keys and only display them under -certain circumstances. This requires calling the -:func:`~curses.noecho` function. :: +Usually curses applications turn off automatic echoing of keys to the screen, in +order to be able to read keys and only display them under certain circumstances. +This requires calling the :func:`noecho` function. :: curses.noecho() -Applications will also commonly need to react to keys instantly, -without requiring the Enter key to be pressed; this is called cbreak -mode, as opposed to the usual buffered input mode. :: +Applications will also commonly need to react to keys instantly, without +requiring the Enter key to be pressed; this is called cbreak mode, as opposed to +the usual buffered input mode. :: curses.cbreak() @@ -113,18 +103,15 @@ :const:`curses.KEY_LEFT`. To get curses to do the job, you'll have to enable keypad mode. :: - stdscr.keypad(True) + stdscr.keypad(1) Terminating a curses application is much easier than starting one. You'll need -to call:: +to call :: - curses.nocbreak() - stdscr.keypad(False) - curses.echo() + curses.nocbreak(); stdscr.keypad(0); curses.echo() -to reverse the curses-friendly terminal settings. Then call the -:func:`~curses.endwin` function to restore the terminal to its original -operating mode. :: +to reverse the curses-friendly terminal settings. Then call the :func:`endwin` +function to restore the terminal to its original operating mode. :: curses.endwin() @@ -135,147 +122,102 @@ you type them, for example, which makes using the shell difficult. In Python you can avoid these complications and make debugging much easier by -importing the :func:`curses.wrapper` function and using it like this:: - - from curses import wrapper - - def main(stdscr): - # Clear screen - stdscr.clear() - - # This raises ZeroDivisionError when i == 10. - for i in range(0, 11): - v = i-10 - stdscr.addstr(i, 0, '10 divided by {} is {}'.format(v, 10/v)) - - stdscr.refresh() - stdscr.getkey() - - wrapper(main) - -The :func:`~curses.wrapper` function takes a callable object and does the -initializations described above, also initializing colors if color -support is present. :func:`wrapper` then runs your provided callable. -Once the callable returns, :func:`wrapper` will restore the original -state of the terminal. The callable is called inside a -:keyword:`try`...\ :keyword:`except` that catches exceptions, restores -the state of the terminal, and then re-raises the exception. Therefore -your terminal won't be left in a funny state on exception and you'll be -able to read the exception's message and traceback. +importing the module :mod:`curses.wrapper`. It supplies a :func:`wrapper` +function that takes a callable. It does the initializations described above, +and also initializes colors if color support is present. It then runs your +provided callable and finally deinitializes appropriately. The callable is +called inside a try-catch clause which catches exceptions, performs curses +deinitialization, and then passes the exception upwards. Thus, your terminal +won't be left in a funny state on exception. Windows and Pads ================ Windows are the basic abstraction in curses. A window object represents a -rectangular area of the screen, and supports methods to display text, +rectangular area of the screen, and supports various methods to display text, erase it, allow the user to input strings, and so forth. -The ``stdscr`` object returned by the :func:`~curses.initscr` function is a -window object that covers the entire screen. Many programs may need -only this single window, but you might wish to divide the screen into -smaller windows, in order to redraw or clear them separately. The -:func:`~curses.newwin` function creates a new window of a given size, -returning the new window object. :: +The ``stdscr`` object returned by the :func:`initscr` function is a window +object that covers the entire screen. Many programs may need only this single +window, but you might wish to divide the screen into smaller windows, in order +to redraw or clear them separately. The :func:`newwin` function creates a new +window of a given size, returning the new window object. :: - begin_x = 20; begin_y = 7 - height = 5; width = 40 + begin_x = 20 ; begin_y = 7 + height = 5 ; width = 40 win = curses.newwin(height, width, begin_y, begin_x) -Note that the coordinate system used in curses is unusual. -Coordinates are always passed in the order *y,x*, and the top-left -corner of a window is coordinate (0,0). This breaks the normal -convention for handling coordinates where the *x* coordinate comes -first. This is an unfortunate difference from most other computer -applications, but it's been part of curses since it was first written, -and it's too late to change things now. +A word about the coordinate system used in curses: coordinates are always passed +in the order *y,x*, and the top-left corner of a window is coordinate (0,0). +This breaks a common convention for handling coordinates, where the *x* +coordinate usually comes first. This is an unfortunate difference from most +other computer applications, but it's been part of curses since it was first +written, and it's too late to change things now. -Your application can determine the size of the screen by using the -:data:`curses.LINES` and :data:`curses.COLS` variables to obtain the *y* and -*x* sizes. Legal coordinates will then extend from ``(0,0)`` to -``(curses.LINES - 1, curses.COLS - 1)``. +When you call a method to display or erase text, the effect doesn't immediately +show up on the display. This is because curses was originally written with slow +300-baud terminal connections in mind; with these terminals, minimizing the time +required to redraw the screen is very important. This lets curses accumulate +changes to the screen, and display them in the most efficient manner. For +example, if your program displays some characters in a window, and then clears +the window, there's no need to send the original characters because they'd never +be visible. -When you call a method to display or erase text, the effect doesn't -immediately show up on the display. Instead you must call the -:meth:`~curses.window.refresh` method of window objects to update the -screen. - -This is because curses was originally written with slow 300-baud -terminal connections in mind; with these terminals, minimizing the -time required to redraw the screen was very important. Instead curses -accumulates changes to the screen and displays them in the most -efficient manner when you call :meth:`refresh`. For example, if your -program displays some text in a window and then clears the window, -there's no need to send the original text because they're never -visible. - -In practice, explicitly telling curses to redraw a window doesn't +Accordingly, curses requires that you explicitly tell it to redraw windows, +using the :func:`refresh` method of window objects. In practice, this doesn't really complicate programming with curses much. Most programs go into a flurry of activity, and then pause waiting for a keypress or some other action on the part of the user. All you have to do is to be sure that the screen has been -redrawn before pausing to wait for user input, by first calling -``stdscr.refresh()`` or the :meth:`refresh` method of some other relevant +redrawn before pausing to wait for user input, by simply calling +``stdscr.refresh()`` or the :func:`refresh` method of some other relevant window. A pad is a special case of a window; it can be larger than the actual display -screen, and only a portion of the pad displayed at a time. Creating a pad +screen, and only a portion of it displayed at a time. Creating a pad simply requires the pad's height and width, while refreshing a pad requires giving the coordinates of the on-screen area where a subsection of the pad will be -displayed. :: +displayed. :: pad = curses.newpad(100, 100) - # These loops fill the pad with letters; addch() is + # These loops fill the pad with letters; this is # explained in the next section - for y in range(0, 99): - for x in range(0, 99): - pad.addch(y,x, ord('a') + (x*x+y*y) % 26) + for y in range(0, 100): + for x in range(0, 100): + try: pad.addch(y,x, ord('a') + (x*x+y*y) % 26 ) + except curses.error: pass - # Displays a section of the pad in the middle of the screen. - # (0,0) : coordinate of upper-left corner of pad area to display. - # (5,5) : coordinate of upper-left corner of window area to be filled - # with pad content. - # (20, 75) : coordinate of lower-right corner of window area to be - # : filled with pad content. + # Displays a section of the pad in the middle of the screen pad.refresh( 0,0, 5,5, 20,75) -The :meth:`refresh` call displays a section of the pad in the rectangle +The :func:`refresh` call displays a section of the pad in the rectangle extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper left corner of the displayed section is coordinate (0,0) on the pad. Beyond that difference, pads are exactly like ordinary windows and support the same methods. -If you have multiple windows and pads on screen there is a more -efficient way to update the screen and prevent annoying screen flicker -as each part of the screen gets updated. :meth:`refresh` actually -does two things: - -1) Calls the :meth:`~curses.window.noutrefresh` method of each window - to update an underlying data structure representing the desired - state of the screen. -2) Calls the function :func:`~curses.doupdate` function to change the - physical screen to match the desired state recorded in the data structure. - -Instead you can call :meth:`noutrefresh` on a number of windows to -update the data structure, and then call :func:`doupdate` to update -the screen. +If you have multiple windows and pads on screen there is a more efficient way to +go, which will prevent annoying screen flicker at refresh time. Use the +:meth:`noutrefresh` method of each window to update the data structure +representing the desired state of the screen; then change the physical screen to +match the desired state in one go with the function :func:`doupdate`. The +normal :meth:`refresh` method calls :func:`doupdate` as its last act. Displaying Text =============== -From a C programmer's point of view, curses may sometimes look like a -twisty maze of functions, all subtly different. For example, -:c:func:`addstr` displays a string at the current cursor location in -the ``stdscr`` window, while :c:func:`mvaddstr` moves to a given y,x -coordinate first before displaying the string. :c:func:`waddstr` is just -like :c:func:`addstr`, but allows specifying a window to use instead of -using ``stdscr`` by default. :c:func:`mvwaddstr` allows specifying both -a window and a coordinate. +From a C programmer's point of view, curses may sometimes look like a twisty +maze of functions, all subtly different. For example, :func:`addstr` displays a +string at the current cursor location in the ``stdscr`` window, while +:func:`mvaddstr` moves to a given y,x coordinate first before displaying the +string. :func:`waddstr` is just like :func:`addstr`, but allows specifying a +window to use, instead of using ``stdscr`` by default. :func:`mvwaddstr` follows +similarly. -Fortunately the Python interface hides all these details. ``stdscr`` -is a window object like any other, and methods such as -:meth:`~curses.window.addstr` accept multiple argument forms. Usually there -are four different forms. +Fortunately the Python interface hides all these details; ``stdscr`` is a window +object like any other, and methods like :func:`addstr` accept multiple argument +forms. Usually there are four different forms. +---------------------------------+-----------------------------------------------+ | Form | Description | @@ -294,26 +236,17 @@ | | display *str* or *ch*, using attribute *attr* | +---------------------------------+-----------------------------------------------+ -Attributes allow displaying text in highlighted forms such as boldface, +Attributes allow displaying text in highlighted forms, such as in boldface, underline, reverse code, or in color. They'll be explained in more detail in the next subsection. - -The :meth:`~curses.window.addstr` method takes a Python string or -bytestring as the value to be displayed. The contents of bytestrings -are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`encoding` attribute; this defaults to -the default system encoding as returned by -:func:`locale.getpreferredencoding`. - -The :meth:`~curses.window.addch` methods take a character, which can be -either a string of length 1, a bytestring of length 1, or an integer. - -Constants are provided for extension characters; these constants are -integers greater than 255. For example, :const:`ACS_PLMINUS` is a +/- -symbol, and :const:`ACS_ULCORNER` is the upper left corner of a box -(handy for drawing borders). You can also use the appropriate Unicode -character. +The :func:`addstr` function takes a Python string as the value to be displayed, +while the :func:`addch` functions take a character, which can be either a Python +string of length 1 or an integer. If it's a string, you're limited to +displaying characters between 0 and 255. SVr4 curses provides constants for +extension characters; these constants are integers greater than 255. For +example, :const:`ACS_PLMINUS` is a +/- symbol, and :const:`ACS_ULCORNER` is the +upper left corner of a box (handy for drawing borders). Windows remember where the cursor was left after the last operation, so if you leave out the *y,x* coordinates, the string or character will be displayed @@ -323,11 +256,10 @@ won't be distracting; it can be confusing to have the cursor blinking at some apparently random location. -If your application doesn't need a blinking cursor at all, you can -call ``curs_set(False)`` to make it invisible. For compatibility -with older curses versions, there's a ``leaveok(bool)`` function -that's a synonym for :func:`~curses.curs_set`. When *bool* is true, the -curses library will attempt to suppress the flashing cursor, and you +If your application doesn't need a blinking cursor at all, you can call +``curs_set(0)`` to make it invisible. Equivalently, and for compatibility with +older curses versions, there's a ``leaveok(bool)`` function. When *bool* is +true, the curses library will attempt to suppress the flashing cursor, and you won't need to worry about leaving it in odd locations. @@ -335,16 +267,15 @@ -------------------- Characters can be displayed in different ways. Status lines in a text-based -application are commonly shown in reverse video, or a text viewer may need to +application are commonly shown in reverse video; a text viewer may need to highlight certain words. curses supports this by allowing you to specify an attribute for each cell on the screen. -An attribute is an integer, each bit representing a different -attribute. You can try to display text with multiple attribute bits -set, but curses doesn't guarantee that all the possible combinations -are available, or that they're all visually distinct. That depends on -the ability of the terminal being used, so it's safest to stick to the -most commonly available attributes, listed here. +An attribute is an integer, each bit representing a different attribute. You can +try to display text with multiple attribute bits set, but curses doesn't +guarantee that all the possible combinations are available, or that they're all +visually distinct. That depends on the ability of the terminal being used, so +it's safest to stick to the most commonly available attributes, listed here. +----------------------+--------------------------------------+ | Attribute | Description | @@ -373,11 +304,10 @@ most common such terminal is probably the Linux console, followed by color xterms. -To use color, you must call the :func:`~curses.start_color` function soon -after calling :func:`~curses.initscr`, to initialize the default color set -(the :func:`curses.wrapper` function does this automatically). Once that's -done, the :func:`~curses.has_colors` function returns TRUE if the terminal -in use can +To use color, you must call the :func:`start_color` function soon after calling +:func:`initscr`, to initialize the default color set (the +:func:`curses.wrapper.wrapper` function does this automatically). Once that's +done, the :func:`has_colors` function returns TRUE if the terminal in use can actually display color. (Note: curses uses the American spelling 'color', instead of the Canadian/British spelling 'colour'. If you're used to the British spelling, you'll have to resign yourself to misspelling it for the sake @@ -385,27 +315,25 @@ The curses library maintains a finite number of color pairs, containing a foreground (or text) color and a background color. You can get the attribute -value corresponding to a color pair with the :func:`~curses.color_pair` -function; this can be bitwise-OR'ed with other attributes such as -:const:`A_REVERSE`, but again, such combinations are not guaranteed to work -on all terminals. +value corresponding to a color pair with the :func:`color_pair` function; this +can be bitwise-OR'ed with other attributes such as :const:`A_REVERSE`, but +again, such combinations are not guaranteed to work on all terminals. An example, which displays a line of text using color pair 1:: - stdscr.addstr("Pretty text", curses.color_pair(1)) + stdscr.addstr( "Pretty text", curses.color_pair(1) ) stdscr.refresh() As I said before, a color pair consists of a foreground and background color. +:func:`start_color` initializes 8 basic colors when it activates color mode. +They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and +7:white. The curses module defines named constants for each of these colors: +:const:`curses.COLOR_BLACK`, :const:`curses.COLOR_RED`, and so forth. + The ``init_pair(n, f, b)`` function changes the definition of color pair *n*, to foreground color f and background color b. Color pair 0 is hard-wired to white on black, and cannot be changed. -Colors are numbered, and :func:`start_color` initializes 8 basic -colors when it activates color mode. They are: 0:black, 1:red, -2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The :mod:`curses` -module defines named constants for each of these colors: -:const:`curses.COLOR_BLACK`, :const:`curses.COLOR_RED`, and so forth. - Let's put all this together. To change color 1 to red text on a white background, you would call:: @@ -415,138 +343,94 @@ will change to the new colors. You can also display new text in this color with:: - stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1)) + stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1) ) Very fancy terminals can change the definitions of the actual colors to a given RGB value. This lets you change color 1, which is usually red, to purple or blue or any other color you like. Unfortunately, the Linux console doesn't support this, so I'm unable to try it out, and can't provide any examples. You -can check if your terminal can do this by calling -:func:`~curses.can_change_color`, which returns ``True`` if the capability is -there. If you're lucky enough to have such a talented terminal, consult your -system's man pages for more information. +can check if your terminal can do this by calling :func:`can_change_color`, +which returns TRUE if the capability is there. If you're lucky enough to have +such a talented terminal, consult your system's man pages for more information. User Input ========== -The C curses library offers only very simple input mechanisms. Python's -:mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid <https://pypi.python.org/pypi/urwid/>`_ have more extensive -collections of widgets.) +The curses library itself offers only very simple input mechanisms. Python's +support adds a text-input widget that makes up some of the lack. -There are two methods for getting input from a window: +The most common way to get input to a window is to use its :meth:`getch` method. +:meth:`getch` pauses and waits for the user to hit a key, displaying it if +:func:`echo` has been called earlier. You can optionally specify a coordinate +to which the cursor should be moved before pausing. -* :meth:`~curses.window.getch` refreshes the screen and then waits for - the user to hit a key, displaying the key if :func:`~curses.echo` has been - called earlier. You can optionally specify a coordinate to which - the cursor should be moved before pausing. - -* :meth:`~curses.window.getkey` does the same thing but converts the - integer to a string. Individual characters are returned as - 1-character strings, and special keys such as function keys return - longer strings containing a key name such as ``KEY_UP`` or ``^G``. - -It's possible to not wait for the user using the -:meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`getch` and :meth:`getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`getch` returns -``curses.ERR`` (a value of -1) and :meth:`getkey` raises an exception. -There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`getch`; if no input becomes -available within a specified delay (measured in tenths of a second), -curses raises an exception. +It's possible to change this behavior with the method :meth:`nodelay`. After +``nodelay(1)``, :meth:`getch` for the window becomes non-blocking and returns +``curses.ERR`` (a value of -1) when no input is ready. There's also a +:func:`halfdelay` function, which can be used to (in effect) set a timer on each +:meth:`getch`; if no input becomes available within a specified +delay (measured in tenths of a second), curses raises an exception. The :meth:`getch` method returns an integer; if it's between 0 and 255, it represents the ASCII code of the key pressed. Values greater than 255 are special keys such as Page Up, Home, or the cursor keys. You can compare the value returned to constants such as :const:`curses.KEY_PPAGE`, -:const:`curses.KEY_HOME`, or :const:`curses.KEY_LEFT`. The main loop of -your program may look something like this:: +:const:`curses.KEY_HOME`, or :const:`curses.KEY_LEFT`. Usually the main loop of +your program will look something like this:: while True: c = stdscr.getch() - if c == ord('p'): - PrintDocument() - elif c == ord('q'): - break # Exit the while loop - elif c == curses.KEY_HOME: - x = y = 0 + if c == ord('p'): PrintDocument() + elif c == ord('q'): break # Exit the while() + elif c == curses.KEY_HOME: x = y = 0 The :mod:`curses.ascii` module supplies ASCII class membership functions that -take either integer or 1-character string arguments; these may be useful in -writing more readable tests for such loops. It also supplies +take either integer or 1-character-string arguments; these may be useful in +writing more readable tests for your command interpreters. It also supplies conversion functions that take either integer or 1-character-string arguments and return the same type. For example, :func:`curses.ascii.ctrl` returns the control character corresponding to its argument. -There's also a method to retrieve an entire string, -:meth:`~curses.window.getstr`. It isn't used very often, because its -functionality is quite limited; the only editing keys available are -the backspace key and the Enter key, which terminates the string. It -can optionally be limited to a fixed number of characters. :: +There's also a method to retrieve an entire string, :const:`getstr()`. It isn't +used very often, because its functionality is quite limited; the only editing +keys available are the backspace key and the Enter key, which terminates the +string. It can optionally be limited to a fixed number of characters. :: curses.echo() # Enable echoing of characters # Get a 15-character string, with the cursor on the top line s = stdscr.getstr(0,0, 15) -The :mod:`curses.textpad` module supplies a text box that supports an -Emacs-like set of keybindings. Various methods of the -:class:`~curses.textpad.Textbox` class support editing with input -validation and gathering the edit results either with or without -trailing spaces. Here's an example:: - - import curses - from curses.textpad import Textbox, rectangle - - def main(stdscr): - stdscr.addstr(0, 0, "Enter IM message: (hit Ctrl-G to send)") - - editwin = curses.newwin(5,30, 2,1) - rectangle(stdscr, 1,0, 1+5+1, 1+30+1) - stdscr.refresh() - - box = Textbox(editwin) - - # Let the user edit until Ctrl-G is struck. - box.edit() - - # Get resulting contents - message = box.gather() - -See the library documentation on :mod:`curses.textpad` for more details. +The Python :mod:`curses.textpad` module supplies something better. With it, you +can turn a window into a text box that supports an Emacs-like set of +keybindings. Various methods of :class:`Textbox` class support editing with +input validation and gathering the edit results either with or without trailing +spaces. See the library documentation on :mod:`curses.textpad` for the +details. For More Information ==================== -This HOWTO doesn't cover some advanced topics, such as reading the -contents of the screen or capturing mouse events from an xterm -instance, but the Python library page for the :mod:`curses` module is now -reasonably complete. You should browse it next. +This HOWTO didn't cover some advanced topics, such as screen-scraping or +capturing mouse events from an xterm instance. But the Python library page for +the curses modules is now pretty complete. You should browse it next. -If you're in doubt about the detailed behavior of the curses -functions, consult the manual pages for your curses implementation, -whether it's ncurses or a proprietary Unix vendor's. The manual pages -will document any quirks, and provide complete lists of all the -functions, attributes, and :const:`ACS_\*` characters available to -you. +If you're in doubt about the detailed behavior of any of the ncurses entry +points, consult the manual pages for your curses implementation, whether it's +ncurses or a proprietary Unix vendor's. The manual pages will document any +quirks, and provide complete lists of all the functions, attributes, and +:const:`ACS_\*` characters available to you. -Because the curses API is so large, some functions aren't supported in -the Python interface. Often this isn't because they're difficult to -implement, but because no one has needed them yet. Also, Python -doesn't yet support the menu library associated with ncurses. -Patches adding support for these would be welcome; see -`the Python Developer's Guide <https://docs.python.org/devguide/>`_ to -learn more about submitting patches to Python. +Because the curses API is so large, some functions aren't supported in the +Python interface, not because they're difficult to implement, but because no one +has needed them yet. Feel free to add them and then submit a patch. Also, we +don't yet have support for the menu library associated with +ncurses; feel free to add that. -* `Writing Programs with NCURSES <http://invisible-island.net/ncurses/ncurses-intro.html>`_: - a lengthy tutorial for C programmers. -* `The ncurses man page <http://linux.die.net/man/3/ncurses>`_ -* `The ncurses FAQ <http://invisible-island.net/ncurses/ncurses.faq.html>`_ -* `"Use curses... don't swear" <http://www.youtube.com/watch?v=eN1eZtjLEnU>`_: - video of a PyCon 2013 talk on controlling terminals using curses or Urwid. -* `"Console Applications with Urwid" <http://www.pyvideo.org/video/1568/console-applications-with-urwid>`_: - video of a PyCon CA 2012 talk demonstrating some applications written using - Urwid. +If you write an interesting little program, feel free to contribute it as +another demo. We can always use more of them! + +The ncurses FAQ: http://invisible-island.net/ncurses/ncurses.faq.html + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/descriptor.rst --- a/Doc/howto/descriptor.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/descriptor.rst Mon Jan 25 17:05:13 2016 +0100 @@ -92,9 +92,9 @@ transforms ``b.x`` into ``type(b).__dict__['x'].__get__(b, type(b))``. The implementation works through a precedence chain that gives data descriptors priority over instance variables, instance variables priority over non-data -descriptors, and assigns lowest priority to :meth:`__getattr__` if provided. -The full C implementation can be found in :c:func:`PyObject_GenericGetAttr()` in -:source:`Objects/object.c`. +descriptors, and assigns lowest priority to :meth:`__getattr__` if provided. The +full C implementation can be found in :c:func:`PyObject_GenericGetAttr()` in +`Objects/object.c <http://svn.python.org/view/python/trunk/Objects/object.c?view=markup>`_\. For classes, the machinery is in :meth:`type.__getattribute__` which transforms ``B.x`` into ``B.__dict__['x'].__get__(None, B)``. In pure Python, it looks @@ -119,15 +119,15 @@ The object returned by ``super()`` also has a custom :meth:`__getattribute__` method for invoking descriptors. The call ``super(B, obj).m()`` searches ``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` -and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, +and then returns ``A.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a search using :meth:`object.__getattribute__`. The implementation details are in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. and a pure Python equivalent can be found in -`Guido's Tutorial`_. +`Objects/typeobject.c <http://svn.python.org/view/python/trunk/Objects/typeobject.c?view=markup>`_ +and a pure Python equivalent can be found in `Guido's Tutorial`_. -.. _`Guido's Tutorial`: https://www.python.org/download/releases/2.2.3/descrintro/#cooperation +.. _`Guido's Tutorial`: http://www.python.org/2.2.3/descrintro.html#cooperation The details above show that the mechanism for descriptors is embedded in the :meth:`__getattribute__()` methods for :class:`object`, :class:`type`, and @@ -210,36 +210,25 @@ self.fget = fget self.fset = fset self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ self.__doc__ = doc def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: - raise AttributeError("unreadable attribute") + raise AttributeError, "unreadable attribute" return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError("can't set attribute") + raise AttributeError, "can't set attribute" self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError("can't delete attribute") + raise AttributeError, "can't delete attribute" self.fdel(obj) - def getter(self, fget): - return type(self)(fget, self.fset, self.fdel, self.__doc__) - - def setter(self, fset): - return type(self)(self.fget, fset, self.fdel, self.__doc__) - - def deleter(self, fdel): - return type(self)(self.fget, self.fset, fdel, self.__doc__) - The :func:`property` builtin helps whenever a user interface has granted attribute access and then subsequent changes require the intervention of a method. @@ -300,9 +289,10 @@ The output suggests that bound and unbound methods are two different types. While they could have been implemented that way, the actual C implementation of -:c:type:`PyMethod_Type` in :source:`Objects/classobject.c` is a single object -with two different representations depending on whether the :attr:`im_self` -field is set or is *NULL* (the C equivalent of *None*). +:c:type:`PyMethod_Type` in +`Objects/classobject.c <http://svn.python.org/view/python/trunk/Objects/classobject.c?view=markup>`_ +is a single object with two different representations depending on whether the +:attr:`im_self` field is set or is *NULL* (the C equivalent of *None*). Likewise, the effects of calling a method object depend on the :attr:`im_self` field. If set (meaning bound), the original function (stored in the @@ -319,7 +309,7 @@ patterns of binding functions into methods. To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an +to a method when accessed as attributes. The non-data descriptor transforms a ``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``klass.f(*args)`` becomes ``f(*args)``. @@ -400,7 +390,7 @@ :func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure Python equivalent is:: - class Dict(object): + class Dict: . . . def fromkeys(klass, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/functional.rst --- a/Doc/howto/functional.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/functional.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,7 @@ ******************************** :Author: A. M. Kuchling -:Release: 0.32 +:Release: 0.31 In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the @@ -15,9 +15,9 @@ Introduction ============ -This section explains the basic concept of functional programming; if -you're just interested in learning about Python language features, -skip to the next section on :ref:`functional-howto-iterators`. +This section explains the basic concept of functional programming; if you're +just interested in learning about Python language features, skip to the next +section. Programming languages support decomposing problems in several different ways: @@ -173,8 +173,6 @@ a few functions specialized for the current task. -.. _functional-howto-iterators: - Iterators ========= @@ -183,26 +181,26 @@ An iterator is an object representing a stream of data; this object returns the data one element at a time. A Python iterator must support a method called -:meth:`~iterator.__next__` that takes no arguments and always returns the next -element of the stream. If there are no more elements in the stream, -:meth:`~iterator.__next__` must raise the :exc:`StopIteration` exception. -Iterators don't have to be finite, though; it's perfectly reasonable to write -an iterator that produces an infinite stream of data. +``__next__()`` that takes no arguments and always returns the next element of +the stream. If there are no more elements in the stream, ``__next__()`` must +raise the ``StopIteration`` exception. Iterators don't have to be finite, +though; it's perfectly reasonable to write an iterator that produces an infinite +stream of data. The built-in :func:`iter` function takes an arbitrary object and tries to return an iterator that will return the object's contents or elements, raising :exc:`TypeError` if the object doesn't support iteration. Several of Python's built-in data types support iteration, the most common being lists and -dictionaries. An object is called :term:`iterable` if you can get an iterator -for it. +dictionaries. An object is called an **iterable** object if you can get an +iterator for it. You can experiment with the iteration interface manually: >>> L = [1,2,3] >>> it = iter(L) - >>> it #doctest: +ELLIPSIS + >>> it <...iterator object at ...> - >>> it.__next__() # same as next(it) + >>> it.__next__() 1 >>> next(it) 2 @@ -215,9 +213,9 @@ >>> Python expects iterable objects in several different contexts, the most -important being the :keyword:`for` statement. In the statement ``for X in Y``, -Y must be an iterator or some object for which :func:`iter` can create an -iterator. These two statements are equivalent:: +important being the ``for`` statement. In the statement ``for X in Y``, Y must +be an iterator or some object for which ``iter()`` can create an iterator. +These two statements are equivalent:: for i in iter(obj): @@ -248,16 +246,16 @@ iterator argument and will return the largest or smallest element. The ``"in"`` and ``"not in"`` operators also support iterators: ``X in iterator`` is true if X is found in the stream returned by the iterator. You'll run into obvious -problems if the iterator is infinite; :func:`max`, :func:`min` +problems if the iterator is infinite; ``max()``, ``min()``, and ``"not in"`` will never return, and if the element X never appears in the stream, the -``"in"`` and ``"not in"`` operators won't return either. +``"in"`` operator won't return either. Note that you can only go forward in an iterator; there's no way to get the previous element, reset the iterator, or make a copy of it. Iterator objects can optionally provide these additional capabilities, but the iterator protocol -only specifies the :meth:`~iterator.__next__` method. Functions may therefore -consume all of the iterator's output, and if you need to do something different -with the same stream, you'll have to create a new iterator. +only specifies the ``next()`` method. Functions may therefore consume all of +the iterator's output, and if you need to do something different with the same +stream, you'll have to create a new iterator. @@ -269,11 +267,15 @@ iterator. Calling :func:`iter` on a dictionary returns an iterator that will loop over the -dictionary's keys:: +dictionary's keys: + +.. not a doctest since dict ordering varies across Pythons + +:: >>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, ... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} - >>> for key in m: #doctest: +SKIP + >>> for key in m: ... print(key, m[key]) Mar 3 Feb 2 @@ -294,19 +296,18 @@ Applying :func:`iter` to a dictionary always loops over the keys, but dictionaries have methods that return other iterators. If you want to iterate over values or key/value pairs, you can explicitly call the -:meth:`~dict.values` or :meth:`~dict.items` methods to get an appropriate -iterator. +:meth:`values` or :meth:`items` methods to get an appropriate iterator. The :func:`dict` constructor can accept an iterator that returns a finite stream of ``(key, value)`` tuples: >>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')] - >>> dict(iter(L)) #doctest: +SKIP + >>> dict(iter(L)) {'Italy': 'Rome', 'US': 'Washington DC', 'France': 'Paris'} -Files also support iteration by calling the :meth:`~io.TextIOBase.readline` -method until there are no more lines in the file. This means you can read each -line of a file like this:: +Files also support iteration by calling the ``readline()`` method until there +are no more lines in the file. This means you can read each line of a file like +this:: for line in file: # do something for each line @@ -409,9 +410,12 @@ lengths of all the sequences. If you have two lists of length 3, the output list is 9 elements long: +.. doctest:: + :options: +NORMALIZE_WHITESPACE + >>> seq1 = 'abc' >>> seq2 = (1,2,3) - >>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE + >>> [(x,y) for x in seq1 for y in seq2] [('a', 1), ('a', 2), ('a', 3), ('b', 1), ('b', 2), ('b', 3), ('c', 1), ('c', 2), ('c', 3)] @@ -421,9 +425,9 @@ comprehension below is a syntax error, while the second one is correct:: # Syntax error - [x, y for x in seq1 for y in seq2] + [ x,y for x in seq1 for y in seq2] # Correct - [(x, y) for x in seq1 for y in seq2] + [ (x,y) for x in seq1 for y in seq2] Generators @@ -444,13 +448,15 @@ Here's the simplest example of a generator function: - >>> def generate_ints(N): - ... for i in range(N): - ... yield i +.. testcode:: -Any function containing a :keyword:`yield` keyword is a generator function; -this is detected by Python's :term:`bytecode` compiler which compiles the -function specially as a result. + def generate_ints(N): + for i in range(N): + yield i + +Any function containing a ``yield`` keyword is a generator function; this is +detected by Python's :term:`bytecode` compiler which compiles the function +specially as a result. When you call a generator function, it doesn't return a single value; instead it returns a generator object that supports the iterator protocol. On executing @@ -458,13 +464,12 @@ ``return`` statement. The big difference between ``yield`` and a ``return`` statement is that on reaching a ``yield`` the generator's state of execution is suspended and local variables are preserved. On the next call to the -generator's :meth:`~generator.__next__` method, the function will resume -executing. +generator's ``.__next__()`` method, the function will resume executing. Here's a sample usage of the ``generate_ints()`` generator: >>> gen = generate_ints(3) - >>> gen #doctest: +ELLIPSIS + >>> gen <generator object generate_ints at ...> >>> next(gen) 0 @@ -481,21 +486,22 @@ You could equally write ``for i in generate_ints(5)``, or ``a,b,c = generate_ints(3)``. -Inside a generator function, ``return value`` causes ``StopIteration(value)`` -to be raised from the :meth:`~generator.__next__` method. Once this happens, or -the bottom of the function is reached, the procession of values ends and the -generator cannot yield any further values. +Inside a generator function, the ``return`` statement can only be used without a +value, and signals the end of the procession of values; after executing a +``return`` the generator cannot return any further values. ``return`` with a +value, such as ``return 5``, is a syntax error inside a generator function. The +end of the generator's results can also be indicated by raising +``StopIteration`` manually, or by just letting the flow of execution fall off +the bottom of the function. You could achieve the effect of generators manually by writing your own class and storing all the local variables of the generator as instance variables. For example, returning a list of integers could be done by setting ``self.count`` to -0, and having the :meth:`~iterator.__next__` method increment ``self.count`` and -return it. +0, and having the ``__next__()`` method increment ``self.count`` and return it. However, for a moderately complicated generator, writing a corresponding class can be much messier. -The test suite included with Python's library, -:source:`Lib/test/test_generators.py`, contains +The test suite included with Python's library, ``test_generators.py``, contains a number of more interesting examples. Here's one generator that implements an in-order traversal of a tree using generators recursively. :: @@ -538,23 +544,23 @@ The parentheses aren't always necessary, but it's easier to always add them instead of having to remember when they're needed. -(:pep:`342` explains the exact rules, which are that a ``yield``-expression must +(PEP 342 explains the exact rules, which are that a ``yield``-expression must always be parenthesized except when it occurs at the top-level expression on the right-hand side of an assignment. This means you can write ``val = yield i`` but have to use parentheses when there's an operation, as in ``val = (yield i) + 12``.) -Values are sent into a generator by calling its :meth:`send(value) -<generator.send>` method. This method resumes the generator's code and the -``yield`` expression returns the specified value. If the regular -:meth:`~generator.__next__` method is called, the ``yield`` returns ``None``. +Values are sent into a generator by calling its ``send(value)`` method. This +method resumes the generator's code and the ``yield`` expression returns the +specified value. If the regular ``__next__()`` method is called, the ``yield`` +returns ``None``. Here's a simple counter that increments by 1 and allows changing the value of the internal counter. .. testcode:: - def counter(maximum): + def counter (maximum): i = 0 while i < maximum: val = (yield i) @@ -566,16 +572,16 @@ And here's an example of changing the counter: - >>> it = counter(10) #doctest: +SKIP - >>> next(it) #doctest: +SKIP + >>> it = counter(10) + >>> next(it) 0 - >>> next(it) #doctest: +SKIP + >>> next(it) 1 - >>> it.send(8) #doctest: +SKIP + >>> it.send(8) 8 - >>> next(it) #doctest: +SKIP + >>> next(it) 9 - >>> next(it) #doctest: +SKIP + >>> next(it) Traceback (most recent call last): File "t.py", line 15, in ? it.next() @@ -583,23 +589,20 @@ Because ``yield`` will often be returning ``None``, you should always check for this case. Don't just use its value in expressions unless you're sure that the -:meth:`~generator.send` method will be the only method used to resume your -generator function. +``send()`` method will be the only method used resume your generator function. -In addition to :meth:`~generator.send`, there are two other methods on -generators: +In addition to ``send()``, there are two other new methods on generators: -* :meth:`throw(type, value=None, traceback=None) <generator.throw>` is used to - raise an exception inside the generator; the exception is raised by the - ``yield`` expression where the generator's execution is paused. +* ``throw(type, value=None, traceback=None)`` is used to raise an exception + inside the generator; the exception is raised by the ``yield`` expression + where the generator's execution is paused. -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the - generator to terminate the iteration. On receiving this exception, the - generator's code must either raise :exc:`GeneratorExit` or - :exc:`StopIteration`; catching the exception and doing anything else is - illegal and will trigger a :exc:`RuntimeError`. :meth:`~generator.close` - will also be called by Python's garbage collector when the generator is - garbage-collected. +* ``close()`` raises a :exc:`GeneratorExit` exception inside the generator to + terminate the iteration. On receiving this exception, the generator's code + must either raise :exc:`GeneratorExit` or :exc:`StopIteration`; catching the + exception and doing anything else is illegal and will trigger a + :exc:`RuntimeError`. ``close()`` will also be called by Python's garbage + collector when the generator is garbage-collected. If you need to run cleanup code when a :exc:`GeneratorExit` occurs, I suggest using a ``try: ... finally:`` suite instead of catching :exc:`GeneratorExit`. @@ -621,12 +624,13 @@ Two of Python's built-in functions, :func:`map` and :func:`filter` duplicate the features of generator expressions: -:func:`map(f, iterA, iterB, ...) <map>` returns an iterator over the sequence +``map(f, iterA, iterB, ...)`` returns an iterator over the sequence ``f(iterA[0], iterB[0]), f(iterA[1], iterB[1]), f(iterA[2], iterB[2]), ...``. >>> def upper(s): ... return s.upper() + >>> list(map(upper, ['sentence', 'fragment'])) ['SENTENCE', 'FRAGMENT'] >>> [upper(s) for s in ['sentence', 'fragment']] @@ -634,11 +638,11 @@ You can of course achieve the same effect with a list comprehension. -:func:`filter(predicate, iter) <filter>` returns an iterator over all the -sequence elements that meet a certain condition, and is similarly duplicated by -list comprehensions. A **predicate** is a function that returns the truth -value of some condition; for use with :func:`filter`, the predicate must take a -single value. +``filter(predicate, iter)`` returns an iterator over all the sequence elements +that meet a certain condition, and is similarly duplicated by list +comprehensions. A **predicate** is a function that returns the truth value of +some condition; for use with :func:`filter`, the predicate must take a single +value. >>> def is_even(x): ... return (x % 2) == 0 @@ -653,8 +657,8 @@ [0, 2, 4, 6, 8] -:func:`enumerate(iter) <enumerate>` counts off the elements in the iterable, -returning 2-tuples containing the count and each element. :: +``enumerate(iter)`` counts off the elements in the iterable, returning 2-tuples +containing the count and each element. :: >>> for item in enumerate(['subject', 'verb', 'object']): ... print(item) @@ -670,28 +674,29 @@ if line.strip() == '': print('Blank line at line #%i' % i) -:func:`sorted(iterable, key=None, reverse=False) <sorted>` collects all the -elements of the iterable into a list, sorts the list, and returns the sorted -result. The *key* and *reverse* arguments are passed through to the -constructed list's :meth:`~list.sort` method. :: +``sorted(iterable, [key=None], [reverse=False])`` collects all the elements of +the iterable into a list, sorts the list, and returns the sorted result. The +``key``, and ``reverse`` arguments are passed through to the constructed list's +``.sort()`` method. :: >>> import random >>> # Generate 8 random numbers between [0, 10000) >>> rand_list = random.sample(range(10000), 8) - >>> rand_list #doctest: +SKIP + >>> rand_list [769, 7953, 9828, 6431, 8442, 9878, 6213, 2207] - >>> sorted(rand_list) #doctest: +SKIP + >>> sorted(rand_list) [769, 2207, 6213, 6431, 7953, 8442, 9828, 9878] - >>> sorted(rand_list, reverse=True) #doctest: +SKIP + >>> sorted(rand_list, reverse=True) [9878, 9828, 8442, 7953, 6431, 6213, 2207, 769] -(For a more detailed discussion of sorting, see the :ref:`sortinghowto`.) +(For a more detailed discussion of sorting, see the Sorting mini-HOWTO in the +Python wiki at http://wiki.python.org/moin/HowTo/Sorting.) -The :func:`any(iter) <any>` and :func:`all(iter) <all>` built-ins look at the -truth values of an iterable's contents. :func:`any` returns ``True`` if any element -in the iterable is a true value, and :func:`all` returns ``True`` if all of the -elements are true values: +The ``any(iter)`` and ``all(iter)`` built-ins look at the truth values of an +iterable's contents. :func:`any` returns True if any element in the iterable is +a true value, and :func:`all` returns True if all of the elements are true +values: >>> any([0,1,0]) True @@ -707,7 +712,7 @@ True -:func:`zip(iterA, iterB, ...) <zip>` takes one element from each iterable and +``zip(iterA, iterB, ...)`` takes one element from each iterable and returns them in a tuple:: zip(['a', 'b', 'c'], (1, 2, 3)) => @@ -747,44 +752,42 @@ Creating new iterators ---------------------- -:func:`itertools.count(n) <itertools.count>` returns an infinite stream of -integers, increasing by 1 each time. You can optionally supply the starting -number, which defaults to 0:: +``itertools.count(n)`` returns an infinite stream of integers, increasing by 1 +each time. You can optionally supply the starting number, which defaults to 0:: itertools.count() => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... itertools.count(10) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... -:func:`itertools.cycle(iter) <itertools.cycle>` saves a copy of the contents of -a provided iterable and returns a new iterator that returns its elements from -first to last. The new iterator will repeat these elements infinitely. :: +``itertools.cycle(iter)`` saves a copy of the contents of a provided iterable +and returns a new iterator that returns its elements from first to last. The +new iterator will repeat these elements infinitely. :: itertools.cycle([1,2,3,4,5]) => 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ... -:func:`itertools.repeat(elem, [n]) <itertools.repeat>` returns the provided -element *n* times, or returns the element endlessly if *n* is not provided. :: +``itertools.repeat(elem, [n])`` returns the provided element ``n`` times, or +returns the element endlessly if ``n`` is not provided. :: itertools.repeat('abc') => abc, abc, abc, abc, abc, abc, abc, abc, abc, abc, ... itertools.repeat('abc', 5) => abc, abc, abc, abc, abc -:func:`itertools.chain(iterA, iterB, ...) <itertools.chain>` takes an arbitrary -number of iterables as input, and returns all the elements of the first -iterator, then all the elements of the second, and so on, until all of the -iterables have been exhausted. :: +``itertools.chain(iterA, iterB, ...)`` takes an arbitrary number of iterables as +input, and returns all the elements of the first iterator, then all the elements +of the second, and so on, until all of the iterables have been exhausted. :: itertools.chain(['a', 'b', 'c'], (1, 2, 3)) => a, b, c, 1, 2, 3 -:func:`itertools.islice(iter, [start], stop, [step]) <itertools.islice>` returns -a stream that's a slice of the iterator. With a single *stop* argument, it -will return the first *stop* elements. If you supply a starting index, you'll -get *stop-start* elements, and if you supply a value for *step*, elements -will be skipped accordingly. Unlike Python's string and list slicing, you can't -use negative values for *start*, *stop*, or *step*. :: +``itertools.islice(iter, [start], stop, [step])`` returns a stream that's a +slice of the iterator. With a single ``stop`` argument, it will return the +first ``stop`` elements. If you supply a starting index, you'll get +``stop-start`` elements, and if you supply a value for ``step``, elements will +be skipped accordingly. Unlike Python's string and list slicing, you can't use +negative values for ``start``, ``stop``, or ``step``. :: itertools.islice(range(10), 8) => 0, 1, 2, 3, 4, 5, 6, 7 @@ -793,10 +796,9 @@ itertools.islice(range(10), 2, 8, 2) => 2, 4, 6 -:func:`itertools.tee(iter, [n]) <itertools.tee>` replicates an iterator; it -returns *n* independent iterators that will all return the contents of the -source iterator. -If you don't supply a value for *n*, the default is 2. Replicating iterators +``itertools.tee(iter, [n])`` replicates an iterator; it returns ``n`` +independent iterators that will all return the contents of the source iterator. +If you don't supply a value for ``n``, the default is 2. Replicating iterators requires saving some of the contents of the source iterator, so this can consume significant memory if the iterator is large and one of the new iterators is consumed more than the others. :: @@ -814,21 +816,19 @@ Calling functions on elements ----------------------------- -The :mod:`operator` module contains a set of functions corresponding to Python's -operators. Some examples are :func:`operator.add(a, b) <operator.add>` (adds -two values), :func:`operator.ne(a, b) <operator.ne>` (same as ``a != b``), and -:func:`operator.attrgetter('id') <operator.attrgetter>` -(returns a callable that fetches the ``.id`` attribute). +The ``operator`` module contains a set of functions corresponding to Python's +operators. Some examples are ``operator.add(a, b)`` (adds two values), +``operator.ne(a, b)`` (same as ``a!=b``), and ``operator.attrgetter('id')`` +(returns a callable that fetches the ``"id"`` attribute). -:func:`itertools.starmap(func, iter) <itertools.starmap>` assumes that the -iterable will return a stream of tuples, and calls *func* using these tuples as -the arguments:: +``itertools.starmap(func, iter)`` assumes that the iterable will return a stream +of tuples, and calls ``f()`` using these tuples as the arguments:: itertools.starmap(os.path.join, - [('/bin', 'python'), ('/usr', 'bin', 'java'), - ('/usr', 'bin', 'perl'), ('/usr', 'bin', 'ruby')]) + [('/usr', 'bin', 'java'), ('/bin', 'python'), + ('/usr', 'bin', 'perl'),('/usr', 'bin', 'ruby')]) => - /bin/python, /usr/bin/java, /usr/bin/perl, /usr/bin/ruby + /usr/bin/java, /bin/python, /usr/bin/perl, /usr/bin/ruby Selecting elements @@ -837,19 +837,20 @@ Another group of functions chooses a subset of an iterator's elements based on a predicate. -:func:`itertools.filterfalse(predicate, iter) <itertools.filterfalse>` is the -opposite of :func:`filter`, returning all elements for which the predicate -returns false:: +``itertools.filterfalse(predicate, iter)`` is the opposite, returning all +elements for which the predicate returns false:: itertools.filterfalse(is_even, itertools.count()) => 1, 3, 5, 7, 9, 11, 13, 15, ... -:func:`itertools.takewhile(predicate, iter) <itertools.takewhile>` returns -elements for as long as the predicate returns true. Once the predicate returns -false, the iterator will signal the end of its results. :: +``itertools.takewhile(predicate, iter)`` returns elements for as long as the +predicate returns true. Once the predicate returns false, the iterator will +signal the end of its results. + +:: def less_than_10(x): - return x < 10 + return (x < 10) itertools.takewhile(less_than_10, itertools.count()) => 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 @@ -857,9 +858,10 @@ itertools.takewhile(is_even, itertools.count()) => 0 -:func:`itertools.dropwhile(predicate, iter) <itertools.dropwhile>` discards -elements while the predicate returns true, and then returns the rest of the -iterable's results. :: +``itertools.dropwhile(predicate, iter)`` discards elements while the predicate +returns true, and then returns the rest of the iterable's results. + +:: itertools.dropwhile(less_than_10, itertools.count()) => 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... @@ -867,89 +869,18 @@ itertools.dropwhile(is_even, itertools.count()) => 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ... -:func:`itertools.compress(data, selectors) <itertools.compress>` takes two -iterators and returns only those elements of *data* for which the corresponding -element of *selectors* is true, stopping whenever either one is exhausted:: - - itertools.compress([1,2,3,4,5], [True, True, False, False, True]) => - 1, 2, 5 - - -Combinatoric functions ----------------------- - -The :func:`itertools.combinations(iterable, r) <itertools.combinations>` -returns an iterator giving all possible *r*-tuple combinations of the -elements contained in *iterable*. :: - - itertools.combinations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 3), (2, 4), (2, 5), - (3, 4), (3, 5), - (4, 5) - - itertools.combinations([1, 2, 3, 4, 5], 3) => - (1, 2, 3), (1, 2, 4), (1, 2, 5), (1, 3, 4), (1, 3, 5), (1, 4, 5), - (2, 3, 4), (2, 3, 5), (2, 4, 5), - (3, 4, 5) - -The elements within each tuple remain in the same order as -*iterable* returned them. For example, the number 1 is always before -2, 3, 4, or 5 in the examples above. A similar function, -:func:`itertools.permutations(iterable, r=None) <itertools.permutations>`, -removes this constraint on the order, returning all possible -arrangements of length *r*:: - - itertools.permutations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 1), (2, 3), (2, 4), (2, 5), - (3, 1), (3, 2), (3, 4), (3, 5), - (4, 1), (4, 2), (4, 3), (4, 5), - (5, 1), (5, 2), (5, 3), (5, 4) - - itertools.permutations([1, 2, 3, 4, 5]) => - (1, 2, 3, 4, 5), (1, 2, 3, 5, 4), (1, 2, 4, 3, 5), - ... - (5, 4, 3, 2, 1) - -If you don't supply a value for *r* the length of the iterable is used, -meaning that all the elements are permuted. - -Note that these functions produce all of the possible combinations by -position and don't require that the contents of *iterable* are unique:: - - itertools.permutations('aba', 3) => - ('a', 'b', 'a'), ('a', 'a', 'b'), ('b', 'a', 'a'), - ('b', 'a', 'a'), ('a', 'a', 'b'), ('a', 'b', 'a') - -The identical tuple ``('a', 'a', 'b')`` occurs twice, but the two 'a' -strings came from different positions. - -The :func:`itertools.combinations_with_replacement(iterable, r) <itertools.combinations_with_replacement>` -function relaxes a different constraint: elements can be repeated -within a single tuple. Conceptually an element is selected for the -first position of each tuple and then is replaced before the second -element is selected. :: - - itertools.combinations_with_replacement([1, 2, 3, 4, 5], 2) => - (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), - (2, 2), (2, 3), (2, 4), (2, 5), - (3, 3), (3, 4), (3, 5), - (4, 4), (4, 5), - (5, 5) - Grouping elements ----------------- -The last function I'll discuss, :func:`itertools.groupby(iter, key_func=None) -<itertools.groupby>`, is the most complicated. ``key_func(elem)`` is a function -that can compute a key value for each element returned by the iterable. If you -don't supply a key function, the key is simply each element itself. +The last function I'll discuss, ``itertools.groupby(iter, key_func=None)``, is +the most complicated. ``key_func(elem)`` is a function that can compute a key +value for each element returned by the iterable. If you don't supply a key +function, the key is simply each element itself. -:func:`~itertools.groupby` collects all the consecutive elements from the -underlying iterable that have the same key value, and returns a stream of -2-tuples containing a key value and an iterator for the elements with that key. +``groupby()`` collects all the consecutive elements from the underlying iterable +that have the same key value, and returns a stream of 2-tuples containing a key +value and an iterator for the elements with that key. :: @@ -959,7 +890,7 @@ ... ] - def get_state(city_state): + def get_state (city_state): return city_state[1] itertools.groupby(city_list, get_state) => @@ -975,9 +906,9 @@ iterator-3 => ('Flagstaff', 'AZ'), ('Phoenix', 'AZ'), ('Tucson', 'AZ') -:func:`~itertools.groupby` assumes that the underlying iterable's contents will -already be sorted based on the key. Note that the returned iterators also use -the underlying iterable, so you have to consume the results of iterator-1 before +``groupby()`` assumes that the underlying iterable's contents will already be +sorted based on the key. Note that the returned iterators also use the +underlying iterable, so you have to consume the results of iterator-1 before requesting iterator-2 and its corresponding key. @@ -995,34 +926,33 @@ ``g(b, c)`` that's equivalent to ``f(1, b, c)``; you're filling in a value for one of ``f()``'s parameters. This is called "partial function application". -The constructor for :func:`~functools.partial` takes the arguments -``(function, arg1, arg2, ..., kwarg1=value1, kwarg2=value2)``. The resulting -object is callable, so you can just call it to invoke ``function`` with the -filled-in arguments. +The constructor for ``partial`` takes the arguments ``(function, arg1, arg2, +... kwarg1=value1, kwarg2=value2)``. The resulting object is callable, so you +can just call it to invoke ``function`` with the filled-in arguments. Here's a small but realistic example:: import functools - def log(message, subsystem): - """Write the contents of 'message' to the specified subsystem.""" + def log (message, subsystem): + "Write the contents of 'message' to the specified subsystem." print('%s: %s' % (subsystem, message)) ... server_log = functools.partial(log, subsystem='server') server_log('Unable to open socket') -:func:`functools.reduce(func, iter, [initial_value]) <functools.reduce>` -cumulatively performs an operation on all the iterable's elements and, -therefore, can't be applied to infinite iterables. *func* must be a function -that takes two elements and returns a single value. :func:`functools.reduce` -takes the first two elements A and B returned by the iterator and calculates -``func(A, B)``. It then requests the third element, C, calculates -``func(func(A, B), C)``, combines this result with the fourth element returned, -and continues until the iterable is exhausted. If the iterable returns no -values at all, a :exc:`TypeError` exception is raised. If the initial value is -supplied, it's used as a starting point and ``func(initial_value, A)`` is the -first calculation. :: +``functools.reduce(func, iter, [initial_value])`` cumulatively performs an +operation on all the iterable's elements and, therefore, can't be applied to +infinite iterables. (Note it is not in :mod:`builtins`, but in the +:mod:`functools` module.) ``func`` must be a function that takes two elements +and returns a single value. :func:`functools.reduce` takes the first two +elements A and B returned by the iterator and calculates ``func(A, B)``. It +then requests the third element, C, calculates ``func(func(A, B), C)``, combines +this result with the fourth element returned, and continues until the iterable +is exhausted. If the iterable returns no values at all, a :exc:`TypeError` +exception is raised. If the initial value is supplied, it's used as a starting +point and ``func(initial_value, A)`` is the first calculation. :: >>> import operator, functools >>> functools.reduce(operator.concat, ['A', 'BB', 'C']) @@ -1048,8 +978,8 @@ >>> sum([]) 0 -For many uses of :func:`functools.reduce`, though, it can be clearer to just -write the obvious :keyword:`for` loop:: +For many uses of :func:`functools.reduce`, though, it can be clearer to just write the +obvious :keyword:`for` loop:: import functools # Instead of: @@ -1060,17 +990,6 @@ for i in [1,2,3]: product *= i -A related function is `itertools.accumulate(iterable, func=operator.add) <itertools.accumulate`. -It performs the same calculation, but instead of returning only the -final result, :func:`accumulate` returns an iterator that also yields -each partial result:: - - itertools.accumulate([1,2,3,4,5]) => - 1, 3, 6, 10, 15 - - itertools.accumulate([1,2,3,4,5], operator.mul) => - 1, 2, 6, 24, 120 - The operator module ------------------- @@ -1104,23 +1023,28 @@ existing_files = filter(os.path.exists, file_list) If the function you need doesn't exist, you need to write it. One way to write -small functions is to use the :keyword:`lambda` statement. ``lambda`` takes a -number of parameters and an expression combining these parameters, and creates -an anonymous function that returns the value of the expression:: +small functions is to use the ``lambda`` statement. ``lambda`` takes a number +of parameters and an expression combining these parameters, and creates a small +function that returns the value of the expression:: + + lowercase = lambda x: x.lower() + + print_assign = lambda name, value: name + '=' + str(value) adder = lambda x, y: x+y - print_assign = lambda name, value: name + '=' + str(value) - An alternative is to just use the ``def`` statement and define a function in the usual way:: - def adder(x, y): - return x + y + def lowercase(x): + return x.lower() def print_assign(name, value): return name + '=' + str(value) + def adder(x,y): + return x + y + Which alternative is preferable? That's a style question; my usual course is to avoid using ``lambda``. @@ -1129,7 +1053,9 @@ expression, which means you can't have multiway ``if... elif... else`` comparisons or ``try... except`` statements. If you try to do too much in a ``lambda`` statement, you'll end up with an overly complicated expression that's -hard to read. Quick, what's the following code doing? :: +hard to read. Quick, what's the following code doing? + +:: import functools total = functools.reduce(lambda a, b: (0, a[1] + b[1]), items)[1] @@ -1139,7 +1065,7 @@ little bit better:: import functools - def combine(a, b): + def combine (a, b): return 0, a[1] + b[1] total = functools.reduce(combine, items)[1] @@ -1159,12 +1085,12 @@ Fredrik Lundh once suggested the following set of rules for refactoring uses of ``lambda``: -1. Write a lambda function. -2. Write a comment explaining what the heck that lambda does. -3. Study the comment for a while, and think of a name that captures the essence +1) Write a lambda function. +2) Write a comment explaining what the heck that lambda does. +3) Study the comment for a while, and think of a name that captures the essence of the comment. -4. Convert the lambda to a def statement, using that name. -5. Remove the comment. +4) Convert the lambda to a def statement, using that name. +5) Remove the comment. I really like these rules, but you're free to disagree about whether this lambda-free style is better. @@ -1244,6 +1170,51 @@ .. comment + Topics to place + ----------------------------- + + XXX os.walk() + + XXX Need a large example. + + But will an example add much? I'll post a first draft and see + what the comments say. + +.. comment + + Original outline: + Introduction + Idea of FP + Programs built out of functions + Functions are strictly input-output, no internal state + Opposed to OO programming, where objects have state + + Why FP? + Formal provability + Assignment is difficult to reason about + Not very relevant to Python + Modularity + Small functions that do one thing + Debuggability: + Easy to test due to lack of state + Easy to verify output from intermediate steps + Composability + You assemble a toolbox of functions that can be mixed + + Tackling a problem + Need a significant example + + Iterators + Generators + The itertools module + List comprehensions + Small functions and the lambda statement + Built-in functions + map + filter + +.. comment + Handy little function for printing part of an iterator -- used while writing this document. @@ -1254,3 +1225,5 @@ sys.stdout.write(str(elem)) sys.stdout.write(', ') print(elem[-1]) + + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/index.rst --- a/Doc/howto/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,6 +13,7 @@ .. toctree:: :maxdepth: 1 + advocacy.rst pyporting.rst cporting.rst curses.rst @@ -26,7 +27,4 @@ unicode.rst urllib2.rst webservers.rst - argparse.rst - ipaddress.rst - clinic.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/ipaddress.rst --- a/Doc/howto/ipaddress.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,341 +0,0 @@ -.. _ipaddress-howto: - -*************************************** -An introduction to the ipaddress module -*************************************** - -:author: Peter Moody -:author: Nick Coghlan - -.. topic:: Overview - - This document aims to provide a gentle introduction to the - :mod:`ipaddress` module. It is aimed primarily at users that aren't - already familiar with IP networking terminology, but may also be useful - to network engineers wanting an overview of how :mod:`ipaddress` - represents IP network addressing concepts. - - -Creating Address/Network/Interface objects -========================================== - -Since :mod:`ipaddress` is a module for inspecting and manipulating IP addresses, -the first thing you'll want to do is create some objects. You can use -:mod:`ipaddress` to create objects from strings and integers. - - -A Note on IP Versions ---------------------- - -For readers that aren't particularly familiar with IP addressing, it's -important to know that the Internet Protocol is currently in the process -of moving from version 4 of the protocol to version 6. This transition is -occurring largely because version 4 of the protocol doesn't provide enough -addresses to handle the needs of the whole world, especially given the -increasing number of devices with direct connections to the internet. - -Explaining the details of the differences between the two versions of the -protocol is beyond the scope of this introduction, but readers need to at -least be aware that these two versions exist, and it will sometimes be -necessary to force the use of one version or the other. - - -IP Host Addresses ------------------ - -Addresses, often referred to as "host addresses" are the most basic unit -when working with IP addressing. The simplest way to create addresses is -to use the :func:`ipaddress.ip_address` factory function, which automatically -determines whether to create an IPv4 or IPv6 address based on the passed in -value: - -.. testsetup:: - >>> import ipaddress - -:: - - >>> ipaddress.ip_address('192.0.2.1') - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address('2001:DB8::1') - IPv6Address('2001:db8::1') - -Addresses can also be created directly from integers. Values that will -fit within 32 bits are assumed to be IPv4 addresses:: - - >>> ipaddress.ip_address(3221225985) - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address(42540766411282592856903984951653826561) - IPv6Address('2001:db8::1') - -To force the use of IPv4 or IPv6 addresses, the relevant classes can be -invoked directly. This is particularly useful to force creation of IPv6 -addresses for small integers:: - - >>> ipaddress.ip_address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv4Address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv6Address(1) - IPv6Address('::1') - - -Defining Networks ------------------ - -Host addresses are usually grouped together into IP networks, so -:mod:`ipaddress` provides a way to create, inspect and manipulate network -definitions. IP network objects are constructed from strings that define the -range of host addresses that are part of that network. The simplest form -for that information is a "network address/network prefix" pair, where the -prefix defines the number of leading bits that are compared to determine -whether or not an address is part of the network and the network address -defines the expected value of those bits. - -As for addresses, a factory function is provided that determines the correct -IP version automatically:: - - >>> ipaddress.ip_network('192.0.2.0/24') - IPv4Network('192.0.2.0/24') - >>> ipaddress.ip_network('2001:db8::0/96') - IPv6Network('2001:db8::/96') - -Network objects cannot have any host bits set. The practical effect of this -is that ``192.0.2.1/24`` does not describe a network. Such definitions are -referred to as interface objects since the ip-on-a-network notation is -commonly used to describe network interfaces of a computer on a given network -and are described further in the next section. - -By default, attempting to create a network object with host bits set will -result in :exc:`ValueError` being raised. To request that the -additional bits instead be coerced to zero, the flag ``strict=False`` can -be passed to the constructor:: - - >>> ipaddress.ip_network('192.0.2.1/24') - Traceback (most recent call last): - ... - ValueError: 192.0.2.1/24 has host bits set - >>> ipaddress.ip_network('192.0.2.1/24', strict=False) - IPv4Network('192.0.2.0/24') - -While the string form offers significantly more flexibility, networks can -also be defined with integers, just like host addresses. In this case, the -network is considered to contain only the single address identified by the -integer, so the network prefix includes the entire network address:: - - >>> ipaddress.ip_network(3221225984) - IPv4Network('192.0.2.0/32') - >>> ipaddress.ip_network(42540766411282592856903984951653826560) - IPv6Network('2001:db8::/128') - -As with addresses, creation of a particular kind of network can be forced -by calling the class constructor directly instead of using the factory -function. - - -Host Interfaces ---------------- - -As mentioned just above, if you need to describe an address on a particular -network, neither the address nor the network classes are sufficient. -Notation like ``192.0.2.1/24`` is commonly used by network engineers and the -people who write tools for firewalls and routers as shorthand for "the host -``192.0.2.1`` on the network ``192.0.2.0/24``", Accordingly, :mod:`ipaddress` -provides a set of hybrid classes that associate an address with a particular -network. The interface for creation is identical to that for defining network -objects, except that the address portion isn't constrained to being a network -address. - - >>> ipaddress.ip_interface('192.0.2.1/24') - IPv4Interface('192.0.2.1/24') - >>> ipaddress.ip_interface('2001:db8::1/96') - IPv6Interface('2001:db8::1/96') - -Integer inputs are accepted (as with networks), and use of a particular IP -version can be forced by calling the relevant constructor directly. - - -Inspecting Address/Network/Interface Objects -============================================ - -You've gone to the trouble of creating an IPv(4|6)(Address|Network|Interface) -object, so you probably want to get information about it. :mod:`ipaddress` -tries to make doing this easy and intuitive. - -Extracting the IP version:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr6 = ipaddress.ip_address('2001:db8::1') - >>> addr6.version - 6 - >>> addr4.version - 4 - -Obtaining the network from an interface:: - - >>> host4 = ipaddress.ip_interface('192.0.2.1/24') - >>> host4.network - IPv4Network('192.0.2.0/24') - >>> host6 = ipaddress.ip_interface('2001:db8::1/96') - >>> host6.network - IPv6Network('2001:db8::/96') - -Finding out how many individual addresses are in a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.num_addresses - 256 - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.num_addresses - 4294967296 - -Iterating through the "usable" addresses on a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> for x in net4.hosts(): - ... print(x) # doctest: +ELLIPSIS - 192.0.2.1 - 192.0.2.2 - 192.0.2.3 - 192.0.2.4 - ... - 192.0.2.252 - 192.0.2.253 - 192.0.2.254 - - -Obtaining the netmask (i.e. set bits corresponding to the network prefix) or -the hostmask (any bits that are not part of the netmask): - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.netmask - IPv4Address('255.255.255.0') - >>> net4.hostmask - IPv4Address('0.0.0.255') - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.netmask - IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') - >>> net6.hostmask - IPv6Address('::ffff:ffff') - - -Exploding or compressing the address:: - - >>> addr6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0001' - >>> addr6.compressed - '2001:db8::1' - >>> net6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0000/96' - >>> net6.compressed - '2001:db8::/96' - -While IPv4 doesn't support explosion or compression, the associated objects -still provide the relevant properties so that version neutral code can -easily ensure the most concise or most verbose form is used for IPv6 -addresses while still correctly handling IPv4 addresses. - - -Networks as lists of Addresses -============================== - -It's sometimes useful to treat networks as lists. This means it is possible -to index them like this:: - - >>> net4[1] - IPv4Address('192.0.2.1') - >>> net4[-1] - IPv4Address('192.0.2.255') - >>> net6[1] - IPv6Address('2001:db8::1') - >>> net6[-1] - IPv6Address('2001:db8::ffff:ffff') - - -It also means that network objects lend themselves to using the list -membership test syntax like this:: - - if address in network: - # do something - -Containment testing is done efficiently based on the network prefix:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr4 in ipaddress.ip_network('192.0.2.0/24') - True - >>> addr4 in ipaddress.ip_network('192.0.3.0/24') - False - - -Comparisons -=========== - -:mod:`ipaddress` provides some simple, hopefully intuitive ways to compare -objects, where it makes sense:: - - >>> ipaddress.ip_address('192.0.2.1') < ipaddress.ip_address('192.0.2.2') - True - -A :exc:`TypeError` exception is raised if you try to compare objects of -different versions or different types. - - -Using IP Addresses with other modules -===================================== - -Other modules that use IP addresses (such as :mod:`socket`) usually won't -accept objects from this module directly. Instead, they must be coerced to -an integer or string that the other module will accept:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> str(addr4) - '192.0.2.1' - >>> int(addr4) - 3221225985 - - -Getting more detail when instance creation fails -================================================ - -When creating address/network/interface objects using the version-agnostic -factory functions, any errors will be reported as :exc:`ValueError` with -a generic error message that simply says the passed in value was not -recognized as an object of that type. The lack of a specific error is -because it's necessary to know whether the value is *supposed* to be IPv4 -or IPv6 in order to provide more detail on why it has been rejected. - -To support use cases where it is useful to have access to this additional -detail, the individual class constructors actually raise the -:exc:`ValueError` subclasses :exc:`ipaddress.AddressValueError` and -:exc:`ipaddress.NetmaskValueError` to indicate exactly which part of -the definition failed to parse correctly. - -The error messages are significantly more detailed when using the -class constructors directly. For example:: - - >>> ipaddress.ip_address("192.168.0.256") - Traceback (most recent call last): - ... - ValueError: '192.168.0.256' does not appear to be an IPv4 or IPv6 address - >>> ipaddress.IPv4Address("192.168.0.256") - Traceback (most recent call last): - ... - ipaddress.AddressValueError: Octet 256 (> 255) not permitted in '192.168.0.256' - - >>> ipaddress.ip_network("192.168.0.1/64") - Traceback (most recent call last): - ... - ValueError: '192.168.0.1/64' does not appear to be an IPv4 or IPv6 network - >>> ipaddress.IPv4Network("192.168.0.1/64") - Traceback (most recent call last): - ... - ipaddress.NetmaskValueError: '64' is not a valid netmask - -However, both of the module specific exceptions have :exc:`ValueError` as their -parent class, so if you're not concerned with the particular type of error, -you can still write code like the following:: - - try: - network = ipaddress.IPv4Network(address) - except ValueError: - print('address/netmask is invalid for IPv4:', address) - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/logging-cookbook.rst --- a/Doc/howto/logging-cookbook.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/logging-cookbook.rst Mon Jan 25 17:05:13 2016 +0100 @@ -97,11 +97,11 @@ Multiple handlers and formatters -------------------------------- -Loggers are plain Python objects. The :meth:`~Logger.addHandler` method has no -minimum or maximum quota for the number of handlers you may add. Sometimes it -will be beneficial for an application to log all messages of all severities to a -text file while simultaneously logging errors or above to the console. To set -this up, simply configure the appropriate handlers. The logging calls in the +Loggers are plain Python objects. The :func:`addHandler` method has no minimum +or maximum quota for the number of handlers you may add. Sometimes it will be +beneficial for an application to log all messages of all severities to a text +file while simultaneously logging errors or above to the console. To set this +up, simply configure the appropriate handlers. The logging calls in the application code will remain unchanged. Here is a slight modification to the previous simple module-based configuration example:: @@ -325,15 +325,6 @@ MainThread: Look out! -.. versionchanged:: 3.5 - Prior to Python 3.5, the :class:`QueueListener` always passed every message - received from the queue to every handler it was initialized with. (This was - because it was assumed that level filtering was all done on the other side, - where the queue is filled.) From 3.5 onwards, this behaviour can be changed - by passing a keyword argument ``respect_handler_level=True`` to the - listener's constructor. When this is done, the listener compares the level - of each message with the handler's level, and only passes a message to a - handler if it's appropriate to do so. .. _network-logging: @@ -425,7 +416,7 @@ Simple TCP socket-based logging receiver suitable for testing. """ - allow_reuse_address = True + allow_reuse_address = 1 def __init__(self, host='localhost', port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, @@ -468,9 +459,8 @@ Note that there are some security issues with pickle in some scenarios. If these affect you, you can use an alternative serialization scheme by overriding -the :meth:`~handlers.SocketHandler.makePickle` method and implementing your -alternative there, as well as adapting the above script to use your alternative -serialization. +the :meth:`makePickle` method and implementing your alternative there, as +well as adapting the above script to use your alternative serialization. .. _context-info: @@ -519,9 +509,9 @@ msg, kwargs = self.process(msg, kwargs) self.logger.debug(msg, *args, **kwargs) -The :meth:`~LoggerAdapter.process` method of :class:`LoggerAdapter` is where the -contextual information is added to the logging output. It's passed the message -and keyword arguments of the logging call, and it passes back (potentially) +The :meth:`process` method of :class:`LoggerAdapter` is where the contextual +information is added to the logging output. It's passed the message and +keyword arguments of the logging call, and it passes back (potentially) modified versions of these to use in the call to the underlying logger. The default implementation of this method leaves the message alone, but inserts an 'extra' key in the keyword argument whose value is the dict-like object @@ -533,32 +523,70 @@ customized strings with your :class:`Formatter` instances which know about the keys of the dict-like object. If you need a different method, e.g. if you want to prepend or append the contextual information to the message string, -you just need to subclass :class:`LoggerAdapter` and override -:meth:`~LoggerAdapter.process` to do what you need. Here is a simple example:: +you just need to subclass :class:`LoggerAdapter` and override :meth:`process` +to do what you need. Here's an example script which uses this class, which +also illustrates what dict-like behaviour is needed from an arbitrary +'dict-like' object for use in the constructor:: - class CustomAdapter(logging.LoggerAdapter): - """ - This example adapter expects the passed in dict-like object to have a - 'connid' key, whose value in brackets is prepended to the log message. - """ - def process(self, msg, kwargs): - return '[%s] %s' % (self.extra['connid'], msg), kwargs + import logging -which you can use like this:: + class ConnInfo: + """ + An example class which shows how an arbitrary class can be used as + the 'extra' context information repository passed to a LoggerAdapter. + """ - logger = logging.getLogger(__name__) - adapter = CustomAdapter(logger, {'connid': some_conn_id}) + def __getitem__(self, name): + """ + To allow this instance to look like a dict. + """ + from random import choice + if name == 'ip': + result = choice(['127.0.0.1', '192.168.0.1']) + elif name == 'user': + result = choice(['jim', 'fred', 'sheila']) + else: + result = self.__dict__.get(name, '?') + return result -Then any events that you log to the adapter will have the value of -``some_conn_id`` prepended to the log messages. + def __iter__(self): + """ + To allow iteration over keys, which will be merged into + the LogRecord dict before formatting and output. + """ + keys = ['ip', 'user'] + keys.extend(self.__dict__.keys()) + return keys.__iter__() -Using objects other than dicts to pass contextual information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + if __name__ == '__main__': + from random import choice + levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) + a1 = logging.LoggerAdapter(logging.getLogger('a.b.c'), + { 'ip' : '123.231.231.123', 'user' : 'sheila' }) + logging.basicConfig(level=logging.DEBUG, + format='%(asctime)-15s %(name)-5s %(levelname)-8s IP: %(ip)-15s User: %(user)-8s %(message)s') + a1.debug('A debug message') + a1.info('An info message with %s', 'some parameters') + a2 = logging.LoggerAdapter(logging.getLogger('d.e.f'), ConnInfo()) + for x in range(10): + lvl = choice(levels) + lvlname = logging.getLevelName(lvl) + a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') -You don't need to pass an actual dict to a :class:`LoggerAdapter` - you could -pass an instance of a class which implements ``__getitem__`` and ``__iter__`` so -that it looks like a dict to logging. This would be useful if you want to -generate values dynamically (whereas the values in a dict would be constant). +When this script is run, the output should look something like this:: + + 2008-01-18 14:49:54,023 a.b.c DEBUG IP: 123.231.231.123 User: sheila A debug message + 2008-01-18 14:49:54,023 a.b.c INFO IP: 123.231.231.123 User: sheila An info message with some parameters + 2008-01-18 14:49:54,023 d.e.f CRITICAL IP: 192.168.0.1 User: jim A message at CRITICAL level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: jim A message at INFO level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: fred A message at ERROR level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: sheila A message at ERROR level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: jim A message at WARNING level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: fred A message at INFO level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters + 2008-01-18 14:49:54,033 d.e.f WARNING IP: 127.0.0.1 User: jim A message at WARNING level with 2 parameters .. _filters-contextual: @@ -643,22 +671,22 @@ *multiple processes* is *not* supported, because there is no standard way to serialize access to a single file across multiple processes in Python. If you need to log to a single file from multiple processes, one way of doing this is -to have all the processes log to a :class:`~handlers.SocketHandler`, and have a -separate process which implements a socket server which reads from the socket -and logs to file. (If you prefer, you can dedicate one thread in one of the -existing processes to perform this function.) -:ref:`This section <network-logging>` documents this approach in more detail and -includes a working socket receiver which can be used as a starting point for you -to adapt in your own applications. +to have all the processes log to a :class:`SocketHandler`, and have a separate +process which implements a socket server which reads from the socket and logs +to file. (If you prefer, you can dedicate one thread in one of the existing +processes to perform this function.) :ref:`This section <network-logging>` +documents this approach in more detail and includes a working socket receiver +which can be used as a starting point for you to adapt in your own +applications. If you are using a recent version of Python which includes the :mod:`multiprocessing` module, you could write your own handler which uses the -:class:`~multiprocessing.Lock` class from this module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide +:class:`Lock` class from this module to serialize access to the file from +your processes. The existing :class:`FileHandler` and subclasses do not make +use of :mod:`multiprocessing` at present, though they may do so in the future. +Note that at present, the :mod:`multiprocessing` module does not provide working lock functionality on all platforms (see -https://bugs.python.org/issue3770). +http://bugs.python.org/issue3770). .. currentmodule:: logging.handlers @@ -713,7 +741,9 @@ break logger = logging.getLogger(record.name) logger.handle(record) # No level or filter logic applied - just do it! - except Exception: + except (KeyboardInterrupt, SystemExit): + raise + except: import sys, traceback print('Whoops! Problem:', file=sys.stderr) traceback.print_exc(file=sys.stderr) @@ -848,7 +878,7 @@ }, 'loggers': { 'foo': { - 'handlers': ['foofile'] + 'handlers' : ['foofile'] } }, 'root': { @@ -888,7 +918,7 @@ file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`~handlers.RotatingFileHandler`:: +logging package provides a :class:`RotatingFileHandler`:: import glob import logging @@ -1006,7 +1036,7 @@ call ``str()`` on that object to get the actual format string. Consider the following two classes:: - class BraceMessage: + class BraceMessage(object): def __init__(self, fmt, *args, **kwargs): self.fmt = fmt self.args = args @@ -1015,7 +1045,7 @@ def __str__(self): return self.fmt.format(*self.args, **self.kwargs) - class DollarMessage: + class DollarMessage(object): def __init__(self, fmt, **kwargs): self.fmt = fmt self.kwargs = kwargs @@ -1066,46 +1096,12 @@ string. That's because the __ notation is just syntax sugar for a constructor call to one of the XXXMessage classes. -If you prefer, you can use a :class:`LoggerAdapter` to achieve a similar effect -to the above, as in the following example:: - - import logging - - class Message(object): - def __init__(self, fmt, args): - self.fmt = fmt - self.args = args - - def __str__(self): - return self.fmt.format(*self.args) - - class StyleAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super(StyleAdapter, self).__init__(logger, extra or {}) - - def log(self, level, msg, *args, **kwargs): - if self.isEnabledFor(level): - msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, Message(msg, args), (), **kwargs) - - logger = StyleAdapter(logging.getLogger(__name__)) - - def main(): - logger.debug('Hello, {}', 'world!') - - if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - main() - -The above script should log the message ``Hello, world!`` when run with -Python 3.2 or later. - .. currentmodule:: logging .. _custom-logrecord: -Customizing ``LogRecord`` +Customising ``LogRecord`` ------------------------- Every logging event is represented by a :class:`LogRecord` instance. @@ -1262,7 +1258,7 @@ Below is an example of a logging configuration dictionary - it's taken from the `documentation on the Django project <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_. -This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: +This dictionary is passed to :func:`~logging.config.dictConfig` to put the configuration into effect:: LOGGING = { 'version': 1, @@ -1317,12 +1313,12 @@ } For more information about this configuration, you can see the `relevant -section <https://docs.djangoproject.com/en/1.6/topics/logging/#configuring-logging>`_ +section <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_ of the Django documentation. .. _cookbook-rotator-namer: -Using a rotator and namer to customize log rotation processing +Using a rotator and namer to customise log rotation processing -------------------------------------------------------------- An example of how you can define a namer and rotator is given in the following @@ -1347,1010 +1343,3 @@ "container" such as you’d find in an actual gzip file. This snippet is just for illustration purposes. -A more elaborate multiprocessing example ----------------------------------------- - -The following working example shows how logging can be used with multiprocessing -using configuration files. The configurations are fairly simple, but serve to -illustrate how more complex ones could be implemented in a real multiprocessing -scenario. - -In the example, the main process spawns a listener process and some worker -processes. Each of the main process, the listener and the workers have three -separate configurations (the workers all share the same configuration). We can -see logging in the main process, how the workers log to a QueueHandler and how -the listener implements a QueueListener and a more complex logging -configuration, and arranges to dispatch events received via the queue to the -handlers specified in the configuration. Note that these configurations are -purely illustrative, but you should be able to adapt this example to your own -scenario. - -Here's the script - the docstrings and the comments hopefully explain how it -works:: - - import logging - import logging.config - import logging.handlers - from multiprocessing import Process, Queue, Event, current_process - import os - import random - import time - - class MyHandler: - """ - A simple handler for logging events. It runs in the listener process and - dispatches events to loggers based on the name in the received record, - which then get dispatched, by the logging system, to the handlers - configured for those loggers. - """ - def handle(self, record): - logger = logging.getLogger(record.name) - # The process name is transformed just to show that it's the listener - # doing the logging to files and console - record.processName = '%s (for %s)' % (current_process().name, record.processName) - logger.handle(record) - - def listener_process(q, stop_event, config): - """ - This could be done in the main process, but is just done in a separate - process for illustrative purposes. - - This initialises logging according to the specified configuration, - starts the listener and waits for the main process to signal completion - via the event. The listener is then stopped, and the process exits. - """ - logging.config.dictConfig(config) - listener = logging.handlers.QueueListener(q, MyHandler()) - listener.start() - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - stop_event.wait() - listener.stop() - - def worker_process(config): - """ - A number of these are spawned for the purpose of illustration. In - practice, they could be a heterogeneous bunch of processes rather than - ones which are identical to each other. - - This initialises logging according to the specified configuration, - and logs a hundred messages with random levels to randomly selected - loggers. - - A small sleep is added to allow other processes a chance to run. This - is not strictly needed, but it mixes the output from the different - processes a bit more than if it's left out. - """ - logging.config.dictConfig(config) - levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL] - loggers = ['foo', 'foo.bar', 'foo.bar.baz', - 'spam', 'spam.ham', 'spam.ham.eggs'] - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - for i in range(100): - lvl = random.choice(levels) - logger = logging.getLogger(random.choice(loggers)) - logger.log(lvl, 'Message no. %d', i) - time.sleep(0.01) - - def main(): - q = Queue() - # The main process gets a simple configuration which prints to the console. - config_initial = { - 'version': 1, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'] - }, - } - # The worker process configuration is just a QueueHandler attached to the - # root logger, which allows all messages to be sent to the queue. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_worker = { - 'version': 1, - 'disable_existing_loggers': True, - 'handlers': { - 'queue': { - 'class': 'logging.handlers.QueueHandler', - 'queue': q, - }, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['queue'] - }, - } - # The listener process configuration shows that the full flexibility of - # logging configuration is available to dispatch events to handlers however - # you want. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_listener = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - }, - 'simple': { - 'class': 'logging.Formatter', - 'format': '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - 'formatter': 'simple', - }, - 'file': { - 'class': 'logging.FileHandler', - 'filename': 'mplog.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'foofile': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-foo.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'errors': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-errors.log', - 'mode': 'w', - 'level': 'ERROR', - 'formatter': 'detailed', - }, - }, - 'loggers': { - 'foo': { - 'handlers': ['foofile'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console', 'file', 'errors'] - }, - } - # Log some initial events, just to show that logging in the parent works - # normally. - logging.config.dictConfig(config_initial) - logger = logging.getLogger('setup') - logger.info('About to create workers ...') - workers = [] - for i in range(5): - wp = Process(target=worker_process, name='worker %d' % (i + 1), - args=(config_worker,)) - workers.append(wp) - wp.start() - logger.info('Started worker: %s', wp.name) - logger.info('About to create listener ...') - stop_event = Event() - lp = Process(target=listener_process, name='listener', - args=(q, stop_event, config_listener)) - lp.start() - logger.info('Started listener') - # We now hang around for the workers to finish their work. - for wp in workers: - wp.join() - # Workers all done, listening can now stop. - # Logging in the parent still works normally. - logger.info('Telling listener to stop ...') - stop_event.set() - lp.join() - logger.info('All done.') - - if __name__ == '__main__': - main() - - -Inserting a BOM into messages sent to a SysLogHandler ------------------------------------------------------ - -`RFC 5424 <http://tools.ietf.org/html/rfc5424>`_ requires that a -Unicode message be sent to a syslog daemon as a set of bytes which have the -following structure: an optional pure-ASCII component, followed by a UTF-8 Byte -Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the `relevant -section of the specification <http://tools.ietf.org/html/rfc5424#section-6>`_.) - -In Python 3.1, code was added to -:class:`~logging.handlers.SysLogHandler` to insert a BOM into the message, but -unfortunately, it was implemented incorrectly, with the BOM appearing at the -beginning of the message and hence not allowing any pure-ASCII component to -appear before it. - -As this behaviour is broken, the incorrect BOM insertion code is being removed -from Python 3.2.4 and later. However, it is not being replaced, and if you -want to produce RFC 5424-compliant messages which include a BOM, an optional -pure-ASCII sequence before it and arbitrary Unicode after it, encoded using -UTF-8, then you need to do the following: - -#. Attach a :class:`~logging.Formatter` instance to your - :class:`~logging.handlers.SysLogHandler` instance, with a format string - such as:: - - 'ASCII section\ufeffUnicode section' - - The Unicode code point U+FEFF, when encoded using UTF-8, will be - encoded as a UTF-8 BOM -- the byte-string ``b'\xef\xbb\xbf'``. - -#. Replace the ASCII section with whatever placeholders you like, but make sure - that the data that appears in there after substitution is always ASCII (that - way, it will remain unchanged after UTF-8 encoding). - -#. Replace the Unicode section with whatever placeholders you like; if the data - which appears there after substitution contains characters outside the ASCII - range, that's fine -- it will be encoded using UTF-8. - -The formatted message *will* be encoded using UTF-8 encoding by -``SysLogHandler``. If you follow the above rules, you should be able to produce -RFC 5424-compliant messages. If you don't, logging may not complain, but your -messages will not be RFC 5424-compliant, and your syslog daemon may complain. - - -Implementing structured logging -------------------------------- - -Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be cirumstances where you want to output -messages in a structured format which *is* capable of being parsed by a program -(without needing complex regular expressions to parse the log message). This is -straightforward to achieve using the logging package. There are a number of -ways in which this could be achieved, but the following is a simple approach -which uses JSON to serialise the event in a machine-parseable manner:: - - import json - import logging - - class StructuredMessage(object): - def __init__(self, message, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - return '%s >>> %s' % (self.message, json.dumps(self.kwargs)) - - _ = StructuredMessage # optional, to improve readability - - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456)) - -If the above script is run, it prints:: - - message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"} - -Note that the order of items might be different according to the version of -Python used. - -If you need more specialised processing, you can use a custom JSON encoder, -as in the following complete example:: - - from __future__ import unicode_literals - - import json - import logging - - # This next bit is to ensure the script runs unchanged on 2.x and 3.x - try: - unicode - except NameError: - unicode = str - - class Encoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, set): - return tuple(o) - elif isinstance(o, unicode): - return o.encode('unicode_escape').decode('ascii') - return super(Encoder, self).default(o) - - class StructuredMessage(object): - def __init__(self, message, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - s = Encoder().encode(self.kwargs) - return '%s >>> %s' % (self.message, s) - - _ = StructuredMessage # optional, to improve readability - - def main(): - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', set_value={1, 2, 3}, snowman='\u2603')) - - if __name__ == '__main__': - main() - -When the above script is run, it prints:: - - message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]} - -Note that the order of items might be different according to the version of -Python used. - - -.. _custom-handlers: - -.. currentmodule:: logging.config - -Customizing handlers with :func:`dictConfig` --------------------------------------------- - -There are times when you want to customize logging handlers in particular ways, -and if you use :func:`dictConfig` you may be able to do this without -subclassing. As an example, consider that you may want to set the ownership of a -log file. On POSIX, this is easily done using :func:`shutil.chown`, but the file -handlers in the stdlib don't offer built-in support. You can customize handler -creation using a plain function such as:: - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - -You can then specify, in a logging configuration passed to :func:`dictConfig`, -that a logging handler be created by calling this function:: - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - -In this example I am setting the ownership using the ``pulse`` user and group, -just for the purposes of illustration. Putting it together into a working -script, ``chowntest.py``:: - - import logging, logging.config, os, shutil - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(LOGGING) - logger = logging.getLogger('mylogger') - logger.debug('A debug message') - -To run this, you will probably need to run as ``root``:: - - $ sudo python3.3 chowntest.py - $ cat chowntest.log - 2013-11-05 09:34:51,128 DEBUG mylogger A debug message - $ ls -l chowntest.log - -rw-r--r-- 1 pulse pulse 55 2013-11-05 09:34 chowntest.log - -Note that this example uses Python 3.3 because that's where :func:`shutil.chown` -makes an appearance. This approach should work with any Python version that -supports :func:`dictConfig` - namely, Python 2.7, 3.2 or later. With pre-3.3 -versions, you would need to implement the actual ownership change using e.g. -:func:`os.chown`. - -In practice, the handler-creating function may be in a utility module somewhere -in your project. Instead of the line in the configuration:: - - '()': owned_file_handler, - -you could use e.g.:: - - '()': 'ext://project.util.owned_file_handler', - -where ``project.util`` can be replaced with the actual name of the package -where the function resides. In the above working script, using -``'ext://__main__.owned_file_handler'`` should work. Here, the actual callable -is resolved by :func:`dictConfig` from the ``ext://`` specification. - -This example hopefully also points the way to how you could implement other -types of file change - e.g. setting specific POSIX permission bits - in the -same way, using :func:`os.chmod`. - -Of course, the approach could also be extended to types of handler other than a -:class:`~logging.FileHandler` - for example, one of the rotating file handlers, -or a different type of handler altogether. - - -.. currentmodule:: logging - -.. _formatting-styles: - -Using particular formatting styles throughout your application --------------------------------------------------------------- - -In Python 3.2, the :class:`~logging.Formatter` gained a ``style`` keyword -parameter which, while defaulting to ``%`` for backward compatibility, allowed -the specification of ``{`` or ``$`` to support the formatting approaches -supported by :meth:`str.format` and :class:`string.Template`. Note that this -governs the formatting of logging messages for final output to logs, and is -completely orthogonal to how an individual logging message is constructed. - -Logging calls (:meth:`~Logger.debug`, :meth:`~Logger.info` etc.) only take -positional parameters for the actual logging message itself, with keyword -parameters used only for determining options for how to handle the logging call -(e.g. the ``exc_info`` keyword parameter to indicate that traceback information -should be logged, or the ``extra`` keyword parameter to indicate additional -contextual information to be added to the log). So you cannot directly make -logging calls using :meth:`str.format` or :class:`string.Template` syntax, -because internally the logging package uses %-formatting to merge the format -string and the variable arguments. There would no changing this while preserving -backward compatibility, since all logging calls which are out there in existing -code will be using %-format strings. - -There have been suggestions to associate format styles with specific loggers, -but that approach also runs into backward compatibility problems because any -existing code could be using a given logger name and using %-formatting. - -For logging to work interoperably between any third-party libraries and your -code, decisions about formatting need to be made at the level of the -individual logging call. This opens up a couple of ways in which alternative -formatting styles can be accommodated. - - -Using LogRecord factories -^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Python 3.2, along with the :class:`~logging.Formatter` changes mentioned -above, the logging package gained the ability to allow users to set their own -:class:`LogRecord` subclasses, using the :func:`setLogRecordFactory` function. -You can use this to set your own subclass of :class:`LogRecord`, which does the -Right Thing by overriding the :meth:`~LogRecord.getMessage` method. The base -class implementation of this method is where the ``msg % args`` formatting -happens, and where you can substitute your alternate formatting; however, you -should be careful to support all formatting styles and allow %-formatting as -the default, to ensure interoperability with other code. Care should also be -taken to call ``str(self.msg)``, just as the base implementation does. - -Refer to the reference documentation on :func:`setLogRecordFactory` and -:class:`LogRecord` for more information. - - -Using custom message objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There is another, perhaps simpler way that you can use {}- and $- formatting to -construct your individual log messages. You may recall (from -:ref:`arbitrary-object-messages`) that when logging you can use an arbitrary -object as a message format string, and that the logging package will call -:func:`str` on that object to get the actual format string. Consider the -following two classes:: - - class BraceMessage(object): - def __init__(self, fmt, *args, **kwargs): - self.fmt = fmt - self.args = args - self.kwargs = kwargs - - def __str__(self): - return self.fmt.format(*self.args, **self.kwargs) - - class DollarMessage(object): - def __init__(self, fmt, **kwargs): - self.fmt = fmt - self.kwargs = kwargs - - def __str__(self): - from string import Template - return Template(self.fmt).substitute(**self.kwargs) - -Either of these can be used in place of a format string, to allow {}- or -$-formatting to be used to build the actual "message" part which appears in the -formatted log output in place of “%(message)s” or “{message}” or “$message”. -If you find it a little unwieldy to use the class names whenever you want to log -something, you can make it more palatable if you use an alias such as ``M`` or -``_`` for the message (or perhaps ``__``, if you are using ``_`` for -localization). - -Examples of this approach are given below. Firstly, formatting with -:meth:`str.format`:: - - >>> __ = BraceMessage - >>> print(__('Message with {0} {1}', 2, 'placeholders')) - Message with 2 placeholders - >>> class Point: pass - ... - >>> p = Point() - >>> p.x = 0.5 - >>> p.y = 0.5 - >>> print(__('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', point=p)) - Message with coordinates: (0.50, 0.50) - -Secondly, formatting with :class:`string.Template`:: - - >>> __ = DollarMessage - >>> print(__('Message with $num $what', num=2, what='placeholders')) - Message with 2 placeholders - >>> - -One thing to note is that you pay no significant performance penalty with this -approach: the actual formatting happens not when you make the logging call, but -when (and if) the logged message is actually about to be output to a log by a -handler. So the only slightly unusual thing which might trip you up is that the -parentheses go around the format string and the arguments, not just the format -string. That’s because the __ notation is just syntax sugar for a constructor -call to one of the ``XXXMessage`` classes shown above. - - -.. _filters-dictconfig: - -.. currentmodule:: logging.config - -Configuring filters with :func:`dictConfig` -------------------------------------------- - -You *can* configure filters using :func:`~logging.config.dictConfig`, though it -might not be obvious at first glance how to do it (hence this recipe). Since -:class:`~logging.Filter` is the only filter class included in the standard -library, and it is unlikely to cater to many requirements (it's only there as a -base class), you will typically need to define your own :class:`~logging.Filter` -subclass with an overridden :meth:`~logging.Filter.filter` method. To do this, -specify the ``()`` key in the configuration dictionary for the filter, -specifying a callable which will be used to create the filter (a class is the -most obvious, but you can provide any callable which returns a -:class:`~logging.Filter` instance). Here is a complete example:: - - import logging - import logging.config - import sys - - class MyFilter(logging.Filter): - def __init__(self, param=None): - self.param = param - - def filter(self, record): - if self.param is None: - allow = True - else: - allow = self.param not in record.msg - if allow: - record.msg = 'changed: ' + record.msg - return allow - - LOGGING = { - 'version': 1, - 'filters': { - 'myfilter': { - '()': MyFilter, - 'param': 'noshow', - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'filters': ['myfilter'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'] - }, - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.debug('hello') - logging.debug('hello - noshow') - -This example shows how you can pass configuration data to the callable which -constructs the instance, in the form of keyword parameters. When run, the above -script will print:: - - changed: hello - -which shows that the filter is working as configured. - -A couple of extra points to note: - -* If you can't refer to the callable directly in the configuration (e.g. if it - lives in a different module, and you can't import it directly where the - configuration dictionary is), you can use the form ``ext://...`` as described - in :ref:`logging-config-dict-externalobj`. For example, you could have used - the text ``'ext://__main__.MyFilter'`` instead of ``MyFilter`` in the above - example. - -* As well as for filters, this technique can also be used to configure custom - handlers and formatters. See :ref:`logging-config-dict-userdef` for more - information on how logging supports using user-defined objects in its - configuration, and see the other cookbook recipe :ref:`custom-handlers` above. - - -.. _custom-format-exception: - -Customized exception formatting -------------------------------- - -There might be times when you want to do customized exception formatting - for -argument's sake, let's say you want exactly one line per logged event, even -when exception information is present. You can do this with a custom formatter -class, as shown in the following example:: - - import logging - - class OneLineExceptionFormatter(logging.Formatter): - def formatException(self, exc_info): - """ - Format an exception so that it prints on a single line. - """ - result = super(OneLineExceptionFormatter, self).formatException(exc_info) - return repr(result) # or format into one line however you want to - - def format(self, record): - s = super(OneLineExceptionFormatter, self).format(record) - if record.exc_text: - s = s.replace('\n', '') + '|' - return s - - def configure_logging(): - fh = logging.FileHandler('output.txt', 'w') - f = OneLineExceptionFormatter('%(asctime)s|%(levelname)s|%(message)s|', - '%d/%m/%Y %H:%M:%S') - fh.setFormatter(f) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(fh) - - def main(): - configure_logging() - logging.info('Sample message') - try: - x = 1 / 0 - except ZeroDivisionError as e: - logging.exception('ZeroDivisionError: %s', e) - - if __name__ == '__main__': - main() - -When run, this produces a file with exactly two lines:: - - 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| - -While the above treatment is simplistic, it points the way to how exception -information can be formatted to your liking. The :mod:`traceback` module may be -helpful for more specialized needs. - -.. _spoken-messages: - -Speaking logging messages -------------------------- - -There might be situations when it is desirable to have logging messages rendered -in an audible rather than a visible format. This is easy to do if you have text- -to-speech (TTS) functionality available in your system, even if it doesn't have -a Python binding. Most TTS systems have a command line program you can run, and -this can be invoked from a handler using :mod:`subprocess`. It's assumed here -that TTS command line programs won't expect to interact with users or take a -long time to complete, and that the frequency of logged messages will be not so -high as to swamp the user with messages, and that it's acceptable to have the -messages spoken one at a time rather than concurrently, The example implementation -below waits for one message to be spoken before the next is processed, and this -might cause other handlers to be kept waiting. Here is a short example showing -the approach, which assumes that the ``espeak`` TTS package is available:: - - import logging - import subprocess - import sys - - class TTSHandler(logging.Handler): - def emit(self, record): - msg = self.format(record) - # Speak slowly in a female English voice - cmd = ['espeak', '-s150', '-ven+f3', msg] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # wait for the program to finish - p.communicate() - - def configure_logging(): - h = TTSHandler() - root = logging.getLogger() - root.addHandler(h) - # the default formatter just returns the message - root.setLevel(logging.DEBUG) - - def main(): - logging.info('Hello') - logging.debug('Goodbye') - - if __name__ == '__main__': - configure_logging() - sys.exit(main()) - -When run, this script should say "Hello" and then "Goodbye" in a female voice. - -The above approach can, of course, be adapted to other TTS systems and even -other systems altogether which can process messages via external programs run -from a command line. - - -.. _buffered-logging: - -Buffering logging messages and outputting them conditionally ------------------------------------------------------------- - -There might be situations where you want to log messages in a temporary area -and only output them if a certain condition occurs. For example, you may want to -start logging debug events in a function, and if the function completes without -errors, you don't want to clutter the log with the collected debug information, -but if there is an error, you want all the debug information to be output as well -as the error. - -Here is an example which shows how you could do this using a decorator for your -functions where you want logging to behave this way. It makes use of the -:class:`logging.handlers.MemoryHandler`, which allows buffering of logged events -until some condition occurs, at which point the buffered events are ``flushed`` -- passed to another handler (the ``target`` handler) for processing. By default, -the ``MemoryHandler`` flushed when its buffer gets filled up or an event whose -level is greater than or equal to a specified threshold is seen. You can use this -recipe with a more specialised subclass of ``MemoryHandler`` if you want custom -flushing behavior. - -The example script has a simple function, ``foo``, which just cycles through -all the logging levels, writing to ``sys.stderr`` to say what level it's about -to log at, and then actually logging a message that that level. You can pass a -parameter to ``foo`` which, if true, will log at ERROR and CRITICAL levels - -otherwise, it only logs at DEBUG, INFO and WARNING levels. - -The script just arranges to decorate ``foo`` with a decorator which will do the -conditional logging that's required. The decorator takes a logger as a parameter -and attaches a memory handler for the duration of the call to the decorated -function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer. These -default to a :class:`~logging.StreamHandler` which writes to ``sys.stderr``, -``logging.ERROR`` and ``100`` respectively. - -Here's the script:: - - import logging - from logging.handlers import MemoryHandler - import sys - - logger = logging.getLogger(__name__) - logger.addHandler(logging.NullHandler()) - - def log_if_errors(logger, target_handler=None, flush_level=None, capacity=None): - if target_handler is None: - target_handler = logging.StreamHandler() - if flush_level is None: - flush_level = logging.ERROR - if capacity is None: - capacity = 100 - handler = MemoryHandler(capacity, flushLevel=flush_level, target=target_handler) - - def decorator(fn): - def wrapper(*args, **kwargs): - logger.addHandler(handler) - try: - return fn(*args, **kwargs) - except Exception: - logger.exception('call failed') - raise - finally: - super(MemoryHandler, handler).flush() - logger.removeHandler(handler) - return wrapper - - return decorator - - def write_line(s): - sys.stderr.write('%s\n' % s) - - def foo(fail=False): - write_line('about to log at DEBUG ...') - logger.debug('Actually logged at DEBUG') - write_line('about to log at INFO ...') - logger.info('Actually logged at INFO') - write_line('about to log at WARNING ...') - logger.warning('Actually logged at WARNING') - if fail: - write_line('about to log at ERROR ...') - logger.error('Actually logged at ERROR') - write_line('about to log at CRITICAL ...') - logger.critical('Actually logged at CRITICAL') - return fail - - decorated_foo = log_if_errors(logger)(foo) - - if __name__ == '__main__': - logger.setLevel(logging.DEBUG) - write_line('Calling undecorated foo with False') - assert not foo(False) - write_line('Calling undecorated foo with True') - assert foo(True) - write_line('Calling decorated foo with False') - assert not decorated_foo(False) - write_line('Calling decorated foo with True') - assert decorated_foo(True) - -When this script is run, the following output should be observed:: - - Calling undecorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling undecorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - about to log at CRITICAL ... - Calling decorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling decorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - Actually logged at DEBUG - Actually logged at INFO - Actually logged at WARNING - Actually logged at ERROR - about to log at CRITICAL ... - Actually logged at CRITICAL - -As you can see, actual logging output only occurs when an event is logged whose -severity is ERROR or greater, but in that case, any previous events at lower -severities are also logged. - -You can of course use the conventional means of decoration:: - - @log_if_errors(logger) - def foo(fail=False): - ... - - -.. _utc-formatting: - -Formatting times using UTC (GMT) via configuration --------------------------------------------------- - -Sometimes you want to format times using UTC, which can be done using a class -such as `UTCFormatter`, shown below:: - - import logging - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - -and you can then use the ``UTCFormatter`` in your code instead of -:class:`~logging.Formatter`. If you want to do that via configuration, you can -use the :func:`~logging.config.dictConfig` API with an approach illustrated by -the following complete example:: - - import logging - import logging.config - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'utc': { - '()': UTCFormatter, - 'format': '%(asctime)s %(message)s', - }, - 'local': { - 'format': '%(asctime)s %(message)s', - } - }, - 'handlers': { - 'console1': { - 'class': 'logging.StreamHandler', - 'formatter': 'utc', - }, - 'console2': { - 'class': 'logging.StreamHandler', - 'formatter': 'local', - }, - }, - 'root': { - 'handlers': ['console1', 'console2'], - } - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.warning('The local time is %s', time.asctime()) - -When this script is run, it should print something like:: - - 2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - 2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - -showing how the time is formatted both as local time and UTC, one for each -handler. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/logging.rst --- a/Doc/howto/logging.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/logging.rst Mon Jan 25 17:05:13 2016 +0100 @@ -63,8 +63,6 @@ they are used to track. The standard levels and their applicability are described below (in increasing order of severity): -.. tabularcolumns:: |l|L| - +--------------+---------------------------------------------+ | Level | When it's used | +==============+=============================================+ @@ -122,8 +120,7 @@ ^^^^^^^^^^^^^^^^^ A very common situation is that of recording logging events in a file, so let's -look at that next. Be sure to try the following in a newly-started Python -interpreter, and don't just continue from the session described above:: +look at that next:: import logging logging.basicConfig(filename='example.log',level=logging.DEBUG) @@ -239,7 +236,7 @@ compatibility: the logging package pre-dates newer formatting options such as :meth:`str.format` and :class:`string.Template`. These newer formatting options *are* supported, but exploring them is outside the scope of this -tutorial: see :ref:`formatting-styles` for more information. +tutorial. Changing the format of displayed messages @@ -333,9 +330,6 @@ to output. * Formatters specify the layout of log records in the final output. -Log event information is passed between loggers, handlers, filters and -formatters in a :class:`LogRecord` instance. - Logging is performed by calling methods on instances of the :class:`Logger` class (hereafter called :dfn:`loggers`). Each instance has a name, and they are conceptually arranged in a namespace hierarchy using dots (periods) as @@ -380,13 +374,6 @@ *format* keyword argument. For all options regarding how a format string is constructed, see :ref:`formatter-objects`. -Logging Flow -^^^^^^^^^^^^ - -The flow of log event information in loggers and handlers is illustrated in the -following diagram. - -.. image:: logging_flow.png Loggers ^^^^^^^ @@ -470,13 +457,12 @@ :class:`~logging.Handler` objects are responsible for dispatching the appropriate log messages (based on the log messages' severity) to the handler's -specified destination. :class:`Logger` objects can add zero or more handler -objects to themselves with an :meth:`~Logger.addHandler` method. As an example -scenario, an application may want to send all log messages to a log file, all -log messages of error or higher to stdout, and all messages of critical to an -email address. This scenario requires three individual handlers where each -handler is responsible for sending messages of a specific severity to a specific -location. +specified destination. Logger objects can add zero or more handler objects to +themselves with an :func:`addHandler` method. As an example scenario, an +application may want to send all log messages to a log file, all log messages +of error or higher to stdout, and all messages of critical to an email address. +This scenario requires three individual handlers where each handler is +responsible for sending messages of a specific severity to a specific location. The standard library includes quite a few handler types (see :ref:`useful-handlers`); the tutorials use mainly :class:`StreamHandler` and @@ -487,17 +473,16 @@ developers who are using the built-in handler objects (that is, not creating custom handlers) are the following configuration methods: -* The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the +* The :meth:`Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why are there two :func:`setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. -* :meth:`~Handler.setFormatter` selects a Formatter object for this handler to - use. +* :func:`setFormatter` selects a Formatter object for this handler to use. -* :meth:`~Handler.addFilter` and :meth:`~Handler.removeFilter` respectively - configure and deconfigure filter objects on handlers. +* :func:`addFilter` and :func:`removeFilter` respectively configure and + deconfigure filter objects on handlers. Application code should not directly instantiate and use instances of :class:`Handler`. Instead, the :class:`Handler` class is a base class that @@ -666,22 +651,6 @@ code approach, mainly separation of configuration and code and the ability of noncoders to easily modify the logging properties. -.. warning:: The :func:`fileConfig` function takes a default parameter, - ``disable_existing_loggers``, which defaults to ``True`` for reasons of - backward compatibility. This may or may not be what you want, since it - will cause any loggers existing before the :func:`fileConfig` call to - be disabled unless they (or an ancestor) are explicitly named in the - configuration. Please refer to the reference documentation for more - information, and specify ``False`` for this parameter if you wish. - - The dictionary passed to :func:`dictConfig` can also specify a Boolean - value with key ``disable_existing_loggers``, which if not specified - explicitly in the dictionary also defaults to being interpreted as - ``True``. This leads to the logger-disabling behaviour described above, - which may not be what you want - in which case, provide the key - explicitly with a value of ``False``. - - .. currentmodule:: logging Note that the class names referenced in config files need to be either relative @@ -795,14 +764,13 @@ libraries, then the logger name specified can be 'orgname.foo' rather than just 'foo'. -.. note:: It is strongly advised that you *do not add any handlers other - than* :class:`~logging.NullHandler` *to your library's loggers*. This is - because the configuration of handlers is the prerogative of the application - developer who uses your library. The application developer knows their - target audience and what handlers are most appropriate for their - application: if you add handlers 'under the hood', you might well interfere - with their ability to carry out unit tests and deliver logs which suit their - requirements. +**PLEASE NOTE:** It is strongly advised that you *do not add any handlers other +than* :class:`~logging.NullHandler` *to your library's loggers*. This is +because the configuration of handlers is the prerogative of the application +developer who uses your library. The application developer knows their target +audience and what handlers are most appropriate for their application: if you +add handlers 'under the hood', you might well interfere with their ability to +carry out unit tests and deliver logs which suit their requirements. Logging Levels @@ -901,10 +869,10 @@ disk files, rotating the log file at certain timed intervals. #. :class:`~handlers.SocketHandler` instances send messages to TCP/IP - sockets. Since 3.4, Unix domain sockets are also supported. + sockets. #. :class:`~handlers.DatagramHandler` instances send messages to UDP - sockets. Since 3.4, Unix domain sockets are also supported. + sockets. #. :class:`~handlers.SMTPHandler` instances send messages to a designated email address. @@ -951,16 +919,16 @@ use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format -string (which is applied to each message in the batch), there is provision for -header and trailer format strings. +:class:`BufferingFormatter` can be used. In addition to the format string (which +is applied to each message in the batch), there is provision for header and +trailer format strings. When filtering based on logger level and/or handler level is not enough, instances of :class:`Filter` can be added to both :class:`Logger` and -:class:`Handler` instances (through their :meth:`~Handler.addFilter` method). -Before deciding to process a message further, both loggers and handlers consult -all their filters for permission. If any filter returns a false value, the -message is not processed further. +:class:`Handler` instances (through their :meth:`addFilter` method). Before +deciding to process a message further, both loggers and handlers consult all +their filters for permission. If any filter returns a false value, the message +is not processed further. The basic :class:`Filter` functionality allows filtering by specific logger name. If this feature is used, messages sent to the named logger and its @@ -978,19 +946,17 @@ cause the application using logging to terminate prematurely. :class:`SystemExit` and :class:`KeyboardInterrupt` exceptions are never -swallowed. Other exceptions which occur during the :meth:`~Handler.emit` method -of a :class:`Handler` subclass are passed to its :meth:`~Handler.handleError` -method. +swallowed. Other exceptions which occur during the :meth:`emit` method of a +:class:`Handler` subclass are passed to its :meth:`handleError` method. -The default implementation of :meth:`~Handler.handleError` in :class:`Handler` -checks to see if a module-level variable, :data:`raiseExceptions`, is set. If -set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is -swallowed. +The default implementation of :meth:`handleError` in :class:`Handler` checks +to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a +traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is - because during development, you typically want to be notified of any - exceptions that occur. It's advised that you set :data:`raiseExceptions` to - ``False`` for production usage. +**Note:** The default value of :data:`raiseExceptions` is ``True``. This is because +during development, you typically want to be notified of any exceptions that +occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production +usage. .. currentmodule:: logging @@ -1002,11 +968,11 @@ In the preceding sections and examples, it has been assumed that the message passed when logging the event is a string. However, this is not the only possibility. You can pass an arbitrary object as a message, and its -:meth:`~object.__str__` method will be called when the logging system needs to -convert it to a string representation. In fact, if you want to, you can avoid +:meth:`__str__` method will be called when the logging system needs to convert +it to a string representation. In fact, if you want to, you can avoid computing a string representation altogether - for example, the -:class:`~handlers.SocketHandler` emits an event by pickling it and sending it -over the wire. +:class:`SocketHandler` emits an event by pickling it and sending it over the +wire. Optimization @@ -1015,10 +981,9 @@ Formatting of message arguments is deferred until it cannot be avoided. However, computing the arguments passed to the logging method can also be expensive, and you may want to avoid doing it if the logger will just throw -away your event. To decide what to do, you can call the -:meth:`~Logger.isEnabledFor` method which takes a level argument and returns -true if the event would be created by the Logger for that level of call. -You can write code like this:: +away your event. To decide what to do, you can call the :meth:`isEnabledFor` +method which takes a level argument and returns true if the event would be +created by the Logger for that level of call. You can write code like this:: if logger.isEnabledFor(logging.DEBUG): logger.debug('Message with %s, %s', expensive_func1(), @@ -1027,15 +992,6 @@ so that if the logger's threshold is set above ``DEBUG``, the calls to :func:`expensive_func1` and :func:`expensive_func2` are never made. -.. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more - expensive than you'd like (e.g. for deeply nested loggers where an explicit - level is only set high up in the logger hierarchy). In such cases (or if you - want to avoid calling a method in tight loops), you can cache the result of a - call to :meth:`~Logger.isEnabledFor` in a local or instance variable, and use - that instead of calling the method each time. Such a cached value would only - need to be recomputed when the logging configuration changes dynamically - while the application is running (which is not all that common). - There are other optimizations which can be made for specific applications which need more precise control over what logging information is collected. Here's a list of things you can do to avoid processing during logging which you don't @@ -1045,12 +1001,6 @@ | What you don't want to collect | How to avoid collecting it | +===============================================+========================================+ | Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | -| | This avoids calling | -| | :func:`sys._getframe`, which may help | -| | to speed up your code in environments | -| | like PyPy (which can't speed up code | -| | that uses :func:`sys._getframe`), if | -| | and when PyPy supports Python 3.x. | +-----------------------------------------------+----------------------------------------+ | Threading information. | Set ``logging.logThreads`` to ``0``. | +-----------------------------------------------+----------------------------------------+ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/logging_flow.png Binary file Doc/howto/logging_flow.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/pyporting.rst --- a/Doc/howto/pyporting.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/pyporting.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,385 +10,706 @@ With Python 3 being the future of Python while Python 2 is still in active use, it is good to have your project available for both major releases of - Python. This guide is meant to help you figure out how best to support both - Python 2 & 3 simultaneously. + Python. This guide is meant to help you choose which strategy works best + for your project to support both Python 2 & 3 along with how to execute + that strategy. If you are looking to port an extension module instead of pure Python code, please see :ref:`cporting-howto`. - If you would like to read one core Python developer's take on why Python 3 - came into existence, you can read Nick Coghlan's `Python 3 Q & A`_. - For help with porting, you can email the python-porting_ mailing list with - questions. +Choosing a Strategy +=================== -The Short Explanation -===================== +When a project makes the decision that it's time to support both Python 2 & 3, +a decision needs to be made as to how to go about accomplishing that goal. +The chosen strategy will depend on how large the project's existing +codebase is and how much divergence you want from your Python 2 codebase from +your Python 3 one (e.g., starting a new version with Python 3). -To make your project be single-source Python 2/3 compatible, the basic steps -are: +If your project is brand-new or does not have a large codebase, then you may +want to consider writing/porting :ref:`all of your code for Python 3 +and use 3to2 <use_3to2>` to port your code for Python 2. -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``pip install coverage``) -#. Learn the differences between Python 2 & 3 -#. Use Modernize_ or Futurize_ to update your code (``pip install modernize`` or - ``pip install future``, respectively) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 & 3 (tox_ can help test - against multiple versions of Python; ``pip install tox``) +If you would prefer to maintain a codebase which is semantically **and** +syntactically compatible with Python 2 & 3 simultaneously, you can write +:ref:`use_same_source`. While this tends to lead to somewhat non-idiomatic +code, it does mean you keep a rapid development process for you, the developer. -If you are dropping support for Python 2 entirely, then after you learn the -differences between Python 2 & 3 you can run 2to3_ over your code and skip the -rest of the steps outlined above. +Finally, you do have the option of :ref:`using 2to3 <use_2to3>` to translate +Python 2 code into Python 3 code (with some manual help). This can take the +form of branching your code and using 2to3 to start a Python 3 branch. You can +also have users perform the translation as installation time automatically so +that you only have to maintain a Python 2 codebase. +Regardless of which approach you choose, porting is not as hard or +time-consuming as you might initially think. You can also tackle the problem +piece-meal as a good portion of porting is simply updating your code to follow +current best practices in a Python 2/3 compatible way. -Details -======= -A key point about supporting Python 2 & 3 simultaneously is that you can start -**today**! Even if your dependencies are not supporting Python 3 yet that does -not mean you can't modernize your code **now** to support Python 3. Most changes -required to support Python 3 lead to cleaner code using newer practices even in -Python 2. +Universal Bits of Advice +------------------------ -Another key point is that modernizing your Python 2 code to also support -Python 3 is largely automated for you. While you might have to make some API -decisions thanks to Python 3 clarifying text data versus binary data, the -lower-level work is now mostly done for you and thus can at least benefit from -the automated changes immediately. +Regardless of what strategy you pick, there are a few things you should +consider. -Keep those key points in mind while you read on about the details of porting -your code to support Python 2 & 3 simultaneously. +One is make sure you have a robust test suite. You need to make sure everything +continues to work, just like when you support a new minor version of Python. +This means making sure your test suite is thorough and is ported properly +between Python 2 & 3. You will also most likely want to use something like tox_ +to automate testing between both a Python 2 and Python 3 VM. +Two, once your project has Python 3 support, make sure to add the proper +classifier on the Cheeseshop_ (PyPI_). To have your project listed as Python 3 +compatible it must have the +`Python 3 classifier <http://pypi.python.org/pypi?:action=browse&c=533>`_ +(from +http://techspot.zzzeek.org/2011/01/24/zzzeek-s-guide-to-python-3-porting/):: -Drop support for Python 2.6 and older -------------------------------------- + setup( + name='Your Library', + version='1.0', + classifiers=[ + # make sure to use :: Python *and* :: Python :: 3 so + # that pypi can list the package on the python 3 page + 'Programming Language :: Python', + 'Programming Language :: Python :: 3' + ], + packages=['yourlibrary'], + # make sure to add custom_fixers to the MANIFEST.in + include_package_data=True, + # ... + ) -While you can make Python 2.5 work with Python 3, it is **much** easier if you -only have to work with Python 2.7. If dropping Python 2.5 is not an -option then the six_ project can help you support Python 2.5 & 3 simultaneously -(``pip install six``). Do realize, though, that nearly all the projects listed -in this HOWTO will not be available to you. -If you are able to skip Python 2.5 and older, then the required changes -to your code should continue to look and feel like idiomatic Python code. At -worst you will have to use a function instead of a method in some instances or -have to import a function instead of using a built-in one, but otherwise the -overall transformation should not feel foreign to you. +Doing so will cause your project to show up in the +`Python 3 packages list +<http://pypi.python.org/pypi?:action=browse&c=533&show=all>`_. You will know +you set the classifier properly as visiting your project page on the Cheeseshop +will show a Python 3 logo in the upper-left corner of the page. -But you should aim for only supporting Python 2.7. Python 2.6 is no longer -supported and thus is not receiving bugfixes. This means **you** will have to -work around any issues you come across with Python 2.6. There are also some -tools mentioned in this HOWTO which do not support Python 2.6 (e.g., Pylint_), -and this will become more commonplace as time goes on. It will simply be easier -for you if you only support the versions of Python that you have to support. +Three, the six_ project provides a library which helps iron out differences +between Python 2 & 3. If you find there is a sticky point that is a continual +point of contention in your translation or maintenance of code, consider using +a source-compatible solution relying on six. If you have to create your own +Python 2/3 compatible solution, you can use ``sys.version_info[0] >= 3`` as a +guard. -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- +Four, read all the approaches. Just because some bit of advice applies to one +approach more than another doesn't mean that some advice doesn't apply to other +strategies. -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. +Five, drop support for older Python versions if possible. `Python 2.5`_ +introduced a lot of useful syntax and libraries which have become idiomatic +in Python 3. `Python 2.6`_ introduced future statements which makes +compatibility much easier if you are going from Python 2 to 3. +`Python 2.7`_ continues the trend in the stdlib. So choose the newest version +of Python which you believe can be your minimum support version +and work from there. -Have good test coverage + +.. _tox: http://codespeak.net/tox/ +.. _Cheeseshop: +.. _PyPI: http://pypi.python.org/ +.. _six: http://packages.python.org/six +.. _Python 2.7: http://www.python.org/2.7.x +.. _Python 2.6: http://www.python.org/2.6.x +.. _Python 2.5: http://www.python.org/2.5.x +.. _Python 2.4: http://www.python.org/2.4.x +.. _Python 2.3: http://www.python.org/2.3.x +.. _Python 2.2: http://www.python.org/2.2.x + + +.. _use_3to2: + +Python 3 and 3to2 +================= + +If you are starting a new project or your codebase is small enough, you may +want to consider writing your code for Python 3 and backporting to Python 2 +using 3to2_. Thanks to Python 3 being more strict about things than Python 2 +(e.g., bytes vs. strings), the source translation can be easier and more +straightforward than from Python 2 to 3. Plus it gives you more direct +experience developing in Python 3 which, since it is the future of Python, is a +good thing long-term. + +A drawback of this approach is that 3to2 is a third-party project. This means +that the Python core developers (and thus this guide) can make no promises +about how well 3to2 works at any time. There is nothing to suggest, though, +that 3to2 is not a high-quality project. + + +.. _3to2: https://bitbucket.org/amentajo/lib3to2/overview + + +.. _use_2to3: + +Python 2 and 2to3 +================= + +Included with Python since 2.6, the 2to3_ tool (and :mod:`lib2to3` module) +helps with porting Python 2 to Python 3 by performing various source +translations. This is a perfect solution for projects which wish to branch +their Python 3 code from their Python 2 codebase and maintain them as +independent codebases. You can even begin preparing to use this approach +today by writing future-compatible Python code which works cleanly in +Python 2 in conjunction with 2to3; all steps outlined below will work +with Python 2 code up to the point when the actual use of 2to3 occurs. + +Use of 2to3 as an on-demand translation step at install time is also possible, +preventing the need to maintain a separate Python 3 codebase, but this approach +does come with some drawbacks. While users will only have to pay the +translation cost once at installation, you as a developer will need to pay the +cost regularly during development. If your codebase is sufficiently large +enough then the translation step ends up acting like a compilation step, +robbing you of the rapid development process you are used to with Python. +Obviously the time required to translate a project will vary, so do an +experimental translation just to see how long it takes to evaluate whether you +prefer this approach compared to using :ref:`use_same_source` or simply keeping +a separate Python 3 codebase. + +Below are the typical steps taken by a project which uses a 2to3-based approach +to supporting Python 2 & 3. + + +Support Python 2.7 +------------------ + +As a first step, make sure that your project is compatible with `Python 2.7`_. +This is just good to do as Python 2.7 is the last release of Python 2 and thus +will be used for a rather long time. It also allows for use of the ``-3`` flag +to Python to help discover places in your code which 2to3 cannot handle but are +known to cause issues. + +Try to Support `Python 2.6`_ and Newer Only +------------------------------------------- + +While not possible for all projects, if you can support `Python 2.6`_ and newer +**only**, your life will be much easier. Various future statements, stdlib +additions, etc. exist only in Python 2.6 and later which greatly assist in +porting to Python 3. But if you project must keep support for `Python 2.5`_ (or +even `Python 2.4`_) then it is still possible to port to Python 3. + +Below are the benefits you gain if you only have to support Python 2.6 and +newer. Some of these options are personal choice while others are +**strongly** recommended (the ones that are more for personal choice are +labeled as such). If you continue to support older versions of Python then you +at least need to watch out for situations that these solutions fix. + + +``from __future__ import print_function`` +''''''''''''''''''''''''''''''''''''''''' + +This is a personal choice. 2to3 handles the translation from the print +statement to the print function rather well so this is an optional step. This +future statement does help, though, with getting used to typing +``print('Hello, World')`` instead of ``print 'Hello, World'``. + + +``from __future__ import unicode_literals`` +''''''''''''''''''''''''''''''''''''''''''' + +Another personal choice. You can always mark what you want to be a (unicode) +string with a ``u`` prefix to get the same effect. But regardless of whether +you use this future statement or not, you **must** make sure you know exactly +which Python 2 strings you want to be bytes, and which are to be strings. This +means you should, **at minimum** mark all strings that are meant to be text +strings with a ``u`` prefix if you do not use this future statement. + + +Bytes literals +'''''''''''''' + +This is a **very** important one. The ability to prefix Python 2 strings that +are meant to contain bytes with a ``b`` prefix help to very clearly delineate +what is and is not a Python 3 string. When you run 2to3 on code, all Python 2 +strings become Python 3 strings **unless** they are prefixed with ``b``. + +There are some differences between byte literals in Python 2 and those in +Python 3 thanks to the bytes type just being an alias to ``str`` in Python 2. +Probably the biggest "gotcha" is that indexing results in different values. In +Python 2, the value of ``b'py'[1]`` is ``'y'``, while in Python 3 it's ``121``. +You can avoid this disparity by always slicing at the size of a single element: +``b'py'[1:2]`` is ``'y'`` in Python 2 and ``b'y'`` in Python 3 (i.e., close +enough). + +You cannot concatenate bytes and strings in Python 3. But since in Python +2 has bytes aliased to ``str``, it will succeed: ``b'a' + u'b'`` works in +Python 2, but ``b'a' + 'b'`` in Python 3 is a :exc:`TypeError`. A similar issue +also comes about when doing comparisons between bytes and strings. + + +Supporting `Python 2.5`_ and Newer Only +--------------------------------------- + +If you are supporting `Python 2.5`_ and newer there are still some features of +Python that you can utilize. + + +``from __future__ import absolute_import`` +'''''''''''''''''''''''''''''''''''''''''' + +Implicit relative imports (e.g., importing ``spam.bacon`` from within +``spam.eggs`` with the statement ``import bacon``) does not work in Python 3. +This future statement moves away from that and allows the use of explicit +relative imports (e.g., ``from . import bacon``). + +In `Python 2.5`_ you must use +the __future__ statement to get to use explicit relative imports and prevent +implicit ones. In `Python 2.6`_ explicit relative imports are available without +the statement, but you still want the __future__ statement to prevent implicit +relative imports. In `Python 2.7`_ the __future__ statement is not needed. In +other words, unless you are only supporting Python 2.7 or a version earlier +than Python 2.5, use the __future__ statement. + + + +Handle Common "Gotchas" ----------------------- -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you can't easily get past 90%). If you -don't already have a tool to measure test coverage then coverage.py_ is -recommended. +There are a few things that just consistently come up as sticking points for +people which 2to3 cannot handle automatically or can easily be done in Python 2 +to help modernize your code. -Learn the differences between Python 2 & 3 -------------------------------------------- -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. Typically the two best ways of doing that -is reading the `"What's New"`_ doc for each release of Python 3 and the -`Porting to Python 3`_ book (which is free online). There is also a handy -`cheat sheet`_ from the Python-Future project. +``from __future__ import division`` +''''''''''''''''''''''''''''''''''' +While the exact same outcome can be had by using the ``-Qnew`` argument to +Python, using this future statement lifts the requirement that your users use +the flag to get the expected behavior of division in Python 3 +(e.g., ``1/2 == 0.5; 1//2 == 0``). -Update your code ----------------- -Once you feel like you know what is different in Python 3 compared to Python 2, -it's time to update your code! You have a choice between two tools in porting -your code automatically: Modernize_ and Futurize_. Which tool you choose will -depend on how much like Python 3 you want your code to be. Futurize_ does its -best to make Python 3 idioms and practices exist in Python 2, e.g. backporting -the ``bytes`` type from Python 3 so that you have semantic parity between the -major versions of Python. Modernize_, -on the other hand, is more conservative and targets a Python 2/3 subset of -Python, relying on six_ to help provide compatibility. -Regardless of which tool you choose, they will update your code to run under -Python 3 while staying compatible with the version of Python 2 you started with. -Depending on how conservative you want to be, you may want to run the tool over -your test suite first and visually inspect the diff to make sure the -transformation is accurate. After you have transformed your test suite and -verified that all the tests still pass as expected, then you can transform your -application code knowing that any tests which fail is a translation failure. +Specify when opening a file as binary +''''''''''''''''''''''''''''''''''''' -Unfortunately the tools can't automate everything to make your code work under -Python 3 and so there are a handful of things you will need to update manually -to get full Python 3 support (which of these steps are necessary vary between -the tools). Read the documentation for the tool you choose to use to see what it -fixes by default and what it can do optionally to know what will (not) be fixed -for you and what you may have to fix on your own (e.g. using ``io.open()`` over -the built-in ``open()`` function is off by default in Modernize). Luckily, -though, there are only a couple of things to watch out for which can be -considered large issues that may be hard to debug if not watched for. - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2``; all division between ``int`` values -result in a ``float``. This change has actually been planned since Python 2.2 -which was released in 2002. Since then users have been encouraged to add -``from __future__ import division`` to any and all files which use the ``/`` and -``//`` operators or to be running the interpreter with the ``-Q`` flag. If you -have not been doing this then you will need to go through your code and do two -things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). - -Text versus binary data -+++++++++++++++++++++++ - -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. - -To make the distinction between text and binary data clearer and more -pronounced, Python 3 did what most languages created in the age of the internet -have done and made text and binary data distinct types that cannot blindly be -mixed together (Python predates widespread access to the internet). For any code -that only deals with text or only binary data, this separation doesn't pose an -issue. But for code that has to deal with both, it does mean you might have to -now care about when you are using text compared to binary data, which is why -this cannot be entirely automated. - -To start, you will need to decide which APIs take text and which take binary -(it is **highly** recommended you don't design APIs that can take both due to -the difficulty of keeping the code working; as stated earlier it is difficult to -do well). In Python 2 this means making sure the APIs that take text can work -with ``unicode`` in Python 2 and those that work with binary data work with the -``bytes`` type from Python 3 and thus a subset of ``str`` in Python 2 (which the -``bytes`` type in Python 2 is an alias for). Usually the biggest issue is -realizing which methods exist for which types in Python 2 & 3 simultaneously -(for text that's ``unicode`` in Python 2 and ``str`` in Python 3, for binary -that's ``str``/``bytes`` in Python 2 and ``bytes`` in Python 3). The following -table lists the **unique** methods of each data type across Python 2 & 3 -(e.g., the ``decode()`` method is usable on the equivalent binary data type in -either Python 2 or 3, but it can't be used by the text data type consistently -between Python 2 and 3 because ``str`` in Python 3 doesn't have the method). Do -note that as of Python 3.5 the ``__mod__`` method was added to the bytes type. - -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== - -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. - -The next issue is making sure you know whether the string literals in your code -represent text or binary data. At minimum you should add a ``b`` prefix to any -literal that presents binary data. For text you should either use the -``from __future__ import unicode_literals`` statement or add a ``u`` prefix to -the text literal. - -As part of this dichotomy you also need to be careful about opening files. Unless you have been working on Windows, there is a chance you have not always bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for binary reading). Under Python 3, binary files and text files are clearly distinct and mutually incompatible; see the :mod:`io` module for details. Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing to read and/or write binary data) or text access -(allowing to read and/or write text data). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). +binary access (allowing to read and/or write bytes data) or text access +(allowing to read and/or write unicode data). -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 & 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. +Text files +'''''''''' -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. +Text files created using ``open()`` under Python 2 return byte strings, +while under Python 3 they return unicode strings. Depending on your porting +strategy, this can be an issue. -To summarize: +If you want text files to return unicode strings in Python 2, you have two +possibilities: -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, use a ``u`` prefix or - :mod:`__future__` import statement for text literals -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing binary data +* Under Python 2.6 and higher, use :func:`io.open`. Since :func:`io.open` + is essentially the same function in both Python 2 and Python 3, it will + help iron out any issues that might arise. -Prevent compatibility regressions ---------------------------------- +* If pre-2.6 compatibility is needed, then you should use :func:`codecs.open` + instead. This will make sure that you get back unicode strings in Python 2. -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. +Subclass ``object`` +''''''''''''''''''' -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: +New-style classes have been around since `Python 2.2`_. You need to make sure +you are subclassing from ``object`` to avoid odd edge cases involving method +resolution order, etc. This continues to be totally valid in Python 3 (although +unneeded as all classes implicitly inherit from ``object``). - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - from __future__ import unicode_literals -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. +Deal With the Bytes/String Dichotomy +'''''''''''''''''''''''''''''''''''' +One of the biggest issues people have when porting code to Python 3 is handling +the bytes/string dichotomy. Because Python 2 allowed the ``str`` type to hold +textual data, people have over the years been rather loose in their delineation +of what ``str`` instances held text compared to bytes. In Python 3 you cannot +be so care-free anymore and need to properly handle the difference. The key +handling this issue to make sure that **every** string literal in your +Python 2 code is either syntactically of functionally marked as either bytes or +text data. After this is done you then need to make sure your APIs are designed +to either handle a specific type or made to be properly polymorphic. -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. +Mark Up Python 2 String Literals +******************************** -Check which dependencies block your transition ----------------------------------------------- +First thing you must do is designate every single string literal in Python 2 +as either textual or bytes data. If you are only supporting Python 2.6 or +newer, this can be accomplished by marking bytes literals with a ``b`` prefix +and then designating textual data with a ``u`` prefix or using the +``unicode_literals`` future statement. -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com . +If your project supports versions of Python pre-dating 2.6, then you should use +the six_ project and its ``b()`` function to denote bytes literals. For text +literals you can either use six's ``u()`` function or use a ``u`` prefix. -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- +Decide what APIs Will Accept +**************************** -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell -anyone using your code that you support Python 2 **and** 3. Ideally you will -also want to add classifiers for each major/minor version of Python you now -support. +In Python 2 it was very easy to accidentally create an API that accepted both +bytes and textual data. But in Python 3, thanks to the more strict handling of +disparate types, this loose usage of bytes and text together tends to fail. -Use continuous integration to stay compatible ---------------------------------------------- +Take the dict ``{b'a': 'bytes', u'a': 'text'}`` in Python 2.6. It creates the +dict ``{u'a': 'text'}`` since ``b'a' == u'a'``. But in Python 3 the equivalent +dict creates ``{b'a': 'bytes', 'a': 'text'}``, i.e., no lost data. Similar +issues can crop up when transitioning Python 2 code to Python 3. -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 & 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. +This means you need to choose what an API is going to accept and create and +consistently stick to that API in both Python 2 and 3. -You may also want to use use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. -And that's mostly it! At this point your code base is compatible with both -Python 2 and 3 simultaneously. Your testing will also be set up so that you -don't accidentally break Python 2 or 3 compatibility regardless of which version -you typically run your tests under while developing. +Bytes / Unicode Comparison +************************** +In Python 3, mixing bytes and unicode is forbidden in most situations; it +will raise a :class:`TypeError` where Python 2 would have attempted an implicit +coercion between types. However, there is one case where it doesn't and +it can be very misleading:: -Dropping Python 2 support completely -==================================== + >>> b"" == "" + False -If you are able to fully drop support for Python 2, then the steps required -to transition to Python 3 simplify greatly. +This is because an equality comparison is required by the language to always +succeed (and return ``False`` for incompatible types). However, this also +means that code incorrectly ported to Python 3 can display buggy behaviour +if such comparisons are silently executed. To detect such situations, +Python 3 has a ``-b`` flag that will display a warning:: -#. Update your code to only support Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help) -#. Learn the differences between Python 2 & 3 -#. Use 2to3_ to rewrite your code to run only under Python 3 + $ python3 -b + >>> b"" == "" + __main__:1: BytesWarning: Comparison between bytes and string + False -After this your code will be fully Python 3 compliant but in a way that is not -supported by Python 2. You should also update the classifiers in your -``setup.py`` to contain ``Programming Language :: Python :: 3 :: Only``. +To turn the warning into an exception, use the ``-bb`` flag instead:: + $ python3 -bb + >>> b"" == "" + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + BytesWarning: Comparison between bytes and string -.. _2to3: https://docs.python.org/3/library/2to3.html -.. _caniusepython3: https://pypi.python.org/pypi/caniusepython3 -.. _cheat sheet: http://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.python.org/pypi/coverage -.. _Futurize: http://python-future.org/automatic_conversion.html -.. _Modernize: http://python-modernize.readthedocs.org/en/latest/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.python.org/pypi/pylint -.. _Python 3 Q & A: http://ncoghlan-devs-python-notes.readthedocs.org/en/latest/python3/questions_and_answers.html -.. _python-future: http://python-future.org/ -.. _python-porting: https://mail.python.org/mailman/listinfo/python-porting -.. _six: https://pypi.python.org/pypi/six -.. _tox: https://pypi.python.org/pypi/tox -.. _trove classifier: https://pypi.python.org/pypi?%3Aaction=list_classifiers -.. _"What's New": https://docs.python.org/3/whatsnew/index.html +Indexing bytes objects +'''''''''''''''''''''' + +Another potentially surprising change is the indexing behaviour of bytes +objects in Python 3:: + + >>> b"xyz"[0] + 120 + +Indeed, Python 3 bytes objects (as well as :class:`bytearray` objects) +are sequences of integers. But code converted from Python 2 will often +assume that indexing a bytestring produces another bytestring, not an +integer. To reconcile both behaviours, use slicing:: + + >>> b"xyz"[0:1] + b'x' + >>> n = 1 + >>> b"xyz"[n:n+1] + b'y' + +The only remaining gotcha is that an out-of-bounds slice returns an empty +bytes object instead of raising ``IndexError``: + + >>> b"xyz"[3] + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + IndexError: index out of range + >>> b"xyz"[3:4] + b'' + + +``__str__()``/``__unicode__()`` +''''''''''''''''''''''''''''''' + +In Python 2, objects can specify both a string and unicode representation of +themselves. In Python 3, though, there is only a string representation. This +becomes an issue as people can inadvertently do things in their ``__str__()`` +methods which have unpredictable results (e.g., infinite recursion if you +happen to use the ``unicode(self).encode('utf8')`` idiom as the body of your +``__str__()`` method). + +There are two ways to solve this issue. One is to use a custom 2to3 fixer. The +blog post at http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/ +specifies how to do this. That will allow 2to3 to change all instances of ``def +__unicode(self): ...`` to ``def __str__(self): ...``. This does require you +define your ``__str__()`` method in Python 2 before your ``__unicode__()`` +method. + +The other option is to use a mixin class. This allows you to only define a +``__unicode__()`` method for your class and let the mixin derive +``__str__()`` for you (code from +http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/):: + + import sys + + class UnicodeMixin(object): + + """Mixin class to handle defining the proper __str__/__unicode__ + methods in Python 2 or 3.""" + + if sys.version_info[0] >= 3: # Python 3 + def __str__(self): + return self.__unicode__() + else: # Python 2 + def __str__(self): + return self.__unicode__().encode('utf8') + + + class Spam(UnicodeMixin): + + def __unicode__(self): + return u'spam-spam-bacon-spam' # 2to3 will remove the 'u' prefix + + +Don't Index on Exceptions +''''''''''''''''''''''''' + +In Python 2, the following worked:: + + >>> exc = Exception(1, 2, 3) + >>> exc.args[1] + 2 + >>> exc[1] # Python 2 only! + 2 + +But in Python 3, indexing directly on an exception is an error. You need to +make sure to only index on the :attr:`BaseException.args` attribute which is a +sequence containing all arguments passed to the :meth:`__init__` method. + +Even better is to use the documented attributes the exception provides. + +Don't use ``__getslice__`` & Friends +'''''''''''''''''''''''''''''''''''' + +Been deprecated for a while, but Python 3 finally drops support for +``__getslice__()``, etc. Move completely over to :meth:`__getitem__` and +friends. + + +Updating doctests +''''''''''''''''' + +2to3_ will attempt to generate fixes for doctests that it comes across. It's +not perfect, though. If you wrote a monolithic set of doctests (e.g., a single +docstring containing all of your doctests), you should at least consider +breaking the doctests up into smaller pieces to make it more manageable to fix. +Otherwise it might very well be worth your time and effort to port your tests +to :mod:`unittest`. + + +Update `map` for imbalanced input sequences +''''''''''''''''''''''''''''''''''''''''''' + +With Python 2, `map` would pad input sequences of unequal length with +`None` values, returning a sequence as long as the longest input sequence. + +With Python 3, if the input sequences to `map` are of unequal length, `map` +will stop at the termination of the shortest of the sequences. For full +compatibility with `map` from Python 2.x, also wrap the sequences in +:func:`itertools.zip_longest`, e.g. ``map(func, *sequences)`` becomes +``list(map(func, itertools.zip_longest(*sequences)))``. + +Eliminate ``-3`` Warnings +------------------------- + +When you run your application's test suite, run it using the ``-3`` flag passed +to Python. This will cause various warnings to be raised during execution about +things that 2to3 cannot handle automatically (e.g., modules that have been +removed). Try to eliminate those warnings to make your code even more portable +to Python 3. + + +Run 2to3 +-------- + +Once you have made your Python 2 code future-compatible with Python 3, it's +time to use 2to3_ to actually port your code. + + +Manually +'''''''' + +To manually convert source code using 2to3_, you use the ``2to3`` script that +is installed with Python 2.6 and later.:: + + 2to3 <directory or file to convert> + +This will cause 2to3 to write out a diff with all of the fixers applied for the +converted source code. If you would like 2to3 to go ahead and apply the changes +you can pass it the ``-w`` flag:: + + 2to3 -w <stuff to convert> + +There are other flags available to control exactly which fixers are applied, +etc. + + +During Installation +''''''''''''''''''' + +When a user installs your project for Python 3, you can have either +:mod:`distutils` or Distribute_ run 2to3_ on your behalf. +For distutils, use the following idiom:: + + try: # Python 3 + from distutils.command.build_py import build_py_2to3 as build_py + except ImportError: # Python 2 + from distutils.command.build_py import build_py + + setup(cmdclass = {'build_py': build_py}, + # ... + ) + +For Distribute:: + + setup(use_2to3=True, + # ... + ) + +This will allow you to not have to distribute a separate Python 3 version of +your project. It does require, though, that when you perform development that +you at least build your project and use the built Python 3 source for testing. + + +Verify & Test +------------- + +At this point you should (hopefully) have your project converted in such a way +that it works in Python 3. Verify it by running your unit tests and making sure +nothing has gone awry. If you miss something then figure out how to fix it in +Python 3, backport to your Python 2 code, and run your code through 2to3 again +to verify the fix transforms properly. + + +.. _2to3: http://docs.python.org/py3k/library/2to3.html +.. _Distribute: http://packages.python.org/distribute/ + + +.. _use_same_source: + +Python 2/3 Compatible Source +============================ + +While it may seem counter-intuitive, you can write Python code which is +source-compatible between Python 2 & 3. It does lead to code that is not +entirely idiomatic Python (e.g., having to extract the currently raised +exception from ``sys.exc_info()[1]``), but it can be run under Python 2 +**and** Python 3 without using 2to3_ as a translation step (although the tool +should be used to help find potential portability problems). This allows you to +continue to have a rapid development process regardless of whether you are +developing under Python 2 or Python 3. Whether this approach or using +:ref:`use_2to3` works best for you will be a per-project decision. + +To get a complete idea of what issues you will need to deal with, see the +`What's New in Python 3.0`_. Others have reorganized the data in other formats +such as http://docs.pythonsprints.com/python3_porting/py-porting.html . + +The following are some steps to take to try to support both Python 2 & 3 from +the same source code. + + +.. _What's New in Python 3.0: http://docs.python.org/release/3.0/whatsnew/3.0.html + + +Follow The Steps for Using 2to3_ +-------------------------------- + +All of the steps outlined in how to +:ref:`port Python 2 code with 2to3 <use_2to3>` apply +to creating a Python 2/3 codebase. This includes trying only support Python 2.6 +or newer (the :mod:`__future__` statements work in Python 3 without issue), +eliminating warnings that are triggered by ``-3``, etc. + +You should even consider running 2to3_ over your code (without committing the +changes). This will let you know where potential pain points are within your +code so that you can fix them properly before they become an issue. + + +Use six_ +-------- + +The six_ project contains many things to help you write portable Python code. +You should make sure to read its documentation from beginning to end and use +any and all features it provides. That way you will minimize any mistakes you +might make in writing cross-version code. + + +Capturing the Currently Raised Exception +---------------------------------------- + +One change between Python 2 and 3 that will require changing how you code (if +you support `Python 2.5`_ and earlier) is +accessing the currently raised exception. In Python 2.5 and earlier the syntax +to access the current exception is:: + + try: + raise Exception() + except Exception, exc: + # Current exception is 'exc' + pass + +This syntax changed in Python 3 (and backported to `Python 2.6`_ and later) +to:: + + try: + raise Exception() + except Exception as exc: + # Current exception is 'exc' + # In Python 3, 'exc' is restricted to the block; Python 2.6 will "leak" + pass + +Because of this syntax change you must change to capturing the current +exception to:: + + try: + raise Exception() + except Exception: + import sys + exc = sys.exc_info()[1] + # Current exception is 'exc' + pass + +You can get more information about the raised exception from +:func:`sys.exc_info` than simply the current exception instance, but you most +likely don't need it. + +.. note:: + In Python 3, the traceback is attached to the exception instance + through the ``__traceback__`` attribute. If the instance is saved in + a local variable that persists outside of the ``except`` block, the + traceback will create a reference cycle with the current frame and its + dictionary of local variables. This will delay reclaiming dead + resources until the next cyclic :term:`garbage collection` pass. + + In Python 2, this problem only occurs if you save the traceback itself + (e.g. the third element of the tuple returned by :func:`sys.exc_info`) + in a variable. + + +Other Resources +=============== + +The authors of the following blog posts, wiki pages, and books deserve special +thanks for making public their tips for porting Python 2 code to Python 3 (and +thus helping provide information for this document): + +* http://python3porting.com/ +* http://docs.pythonsprints.com/python3_porting/py-porting.html +* http://techspot.zzzeek.org/2011/01/24/zzzeek-s-guide-to-python-3-porting/ +* http://dabeaz.blogspot.com/2011/01/porting-py65-and-my-superboard-to.html +* http://lucumr.pocoo.org/2011/1/22/forwards-compatible-python/ +* http://lucumr.pocoo.org/2010/2/11/porting-to-python-3-a-guide/ +* http://wiki.python.org/moin/PortingPythonToPy3k + +If you feel there is something missing from this document that should be added, +please email the python-porting_ mailing list. + +.. _python-porting: http://mail.python.org/mailman/listinfo/python-porting diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/regex.rst --- a/Doc/howto/regex.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/regex.rst Mon Jan 25 17:05:13 2016 +0100 @@ -104,25 +104,13 @@ or ``\``, you can precede them with a backslash to remove their special meaning: ``\[`` or ``\\``. -Some of the special sequences beginning with ``'\'`` represent -predefined sets of characters that are often useful, such as the set -of digits, the set of letters, or the set of anything that isn't -whitespace. - -Let's take an example: ``\w`` matches any alphanumeric character. If -the regex pattern is expressed in bytes, this is equivalent to the -class ``[a-zA-Z0-9_]``. If the regex pattern is a string, ``\w`` will -match all the characters marked as letters in the Unicode database -provided by the :mod:`unicodedata` module. You can use the more -restricted definition of ``\w`` in a string pattern by supplying the -:const:`re.ASCII` flag when compiling the regular expression. - -The following list of special sequences isn't complete. For a complete -list of sequences and expanded class definitions for Unicode string -patterns, see the last part of :ref:`Regular Expression Syntax -<re-syntax>` in the Standard Library reference. In general, the -Unicode versions match any character that's in the appropriate -category in the Unicode database. +Some of the special sequences beginning with ``'\'`` represent predefined sets +of characters that are often useful, such as the set of digits, the set of +letters, or the set of anything that isn't whitespace. The following predefined +special sequences are a subset of those available. The equivalent classes are +for bytes patterns. For a complete list of sequences and expanded class +definitions for Unicode string patterns, see the last part of +:ref:`Regular Expression Syntax <re-syntax>`. ``\d`` Matches any decimal digit; this is equivalent to the class ``[0-9]``. @@ -172,8 +160,9 @@ For example, ``ca*t`` will match ``ct`` (0 ``a`` characters), ``cat`` (1 ``a``), ``caaat`` (3 ``a`` characters), and so forth. The RE engine has various internal limitations stemming from the size of C's ``int`` type that will -prevent it from matching over 2 billion ``a`` characters; patterns -are usually not written to match that much data. +prevent it from matching over 2 billion ``a`` characters; you probably don't +have enough memory to construct a string that large, so you shouldn't run into +that limit. Repetitions such as ``*`` are :dfn:`greedy`; when repeating a RE, the matching engine will try to repeat it as many times as possible. If later portions of the @@ -272,7 +261,7 @@ >>> import re >>> p = re.compile('ab*') >>> p - re.compile('ab*') + <_sre.SRE_Pattern object at 0x...> :func:`re.compile` also accepts an optional *flags* argument, used to enable various special features and syntax variations. We'll go over the available @@ -364,10 +353,10 @@ | | returns them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.regex.match` and :meth:`~re.regex.search` return ``None`` if no match can be found. If -they're successful, a :ref:`match object <match-objects>` instance is returned, -containing information about the match: where it starts and ends, the substring -it matched, and more. +:meth:`match` and :meth:`search` return ``None`` if no match can be found. If +they're successful, a ``MatchObject`` instance is returned, containing +information about the match: where it starts and ends, the substring it matched, +and more. You can learn about this by interactively experimenting with the :mod:`re` module. If you have :mod:`tkinter` available, you may also want to look at @@ -384,7 +373,7 @@ >>> import re >>> p = re.compile('[a-z]+') >>> p - re.compile('[a-z]+') + <_sre.SRE_Pattern object at 0x...> Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. @@ -397,16 +386,16 @@ None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`match` will return a :ref:`match object <match-objects>`, so you -should store the result in a variable for later use. :: +case, :meth:`match` will return a :class:`MatchObject`, so you should store the +result in a variable for later use. :: >>> m = p.match('tempo') - >>> m #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 5), match='tempo'> + >>> m + <_sre.SRE_Match object at 0x...> -Now you can query the :ref:`match object <match-objects>` for information -about the matching string. :ref:`match object <match-objects>` instances -also have several methods and attributes; the most important ones are: +Now you can query the :class:`MatchObject` for information about the matching +string. :class:`MatchObject` instances also have several methods and +attributes; the most important ones are: +------------------+--------------------------------------------+ | Method/Attribute | Purpose | @@ -430,8 +419,8 @@ >>> m.span() (0, 5) -:meth:`~re.match.group` returns the substring that was matched by the RE. :meth:`~re.match.start` -and :meth:`~re.match.end` return the starting and ending index of the match. :meth:`~re.match.span` +:meth:`group` returns the substring that was matched by the RE. :meth:`start` +and :meth:`end` return the starting and ending index of the match. :meth:`span` returns both start and end indexes in a single tuple. Since the :meth:`match` method only checks if the RE matches at the start of a string, :meth:`start` will always be zero. However, the :meth:`search` method of patterns @@ -440,16 +429,15 @@ >>> print(p.match('::: message')) None - >>> m = p.search('::: message'); print(m) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(4, 11), match='message'> + >>> m = p.search('::: message') ; print(m) + <_sre.SRE_Match object at 0x...> >>> m.group() 'message' >>> m.span() (4, 11) -In actual programs, the most common style is to store the -:ref:`match object <match-objects>` in a variable, and then check if it was -``None``. This usually looks like:: +In actual programs, the most common style is to store the :class:`MatchObject` +in a variable, and then check if it was ``None``. This usually looks like:: p = re.compile( ... ) m = p.match( 'string goes here' ) @@ -459,18 +447,18 @@ print('No match') Two pattern methods return all of the matches for a pattern. -:meth:`~re.regex.findall` returns a list of matching strings:: +:meth:`findall` returns a list of matching strings:: >>> p = re.compile('\d+') >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') ['12', '11', '10'] :meth:`findall` has to create the entire list before it can be returned as the -result. The :meth:`~re.regex.finditer` method returns a sequence of -:ref:`match object <match-objects>` instances as an :term:`iterator`:: +result. The :meth:`finditer` method returns a sequence of :class:`MatchObject` +instances as an :term:`iterator`:: >>> iterator = p.finditer('12 drummers drumming, 11 ... 10 ...') - >>> iterator #doctest: +ELLIPSIS + >>> iterator <callable_iterator object at 0x...> >>> for match in iterator: ... print(match.span()) @@ -484,27 +472,38 @@ ---------------------- You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions -take the same arguments as the corresponding pattern method with +:mod:`re` module also provides top-level functions called :func:`match`, +:func:`search`, :func:`findall`, :func:`sub`, and so forth. These functions +take the same arguments as the corresponding pattern method, with the RE string added as the first argument, and still return either ``None`` or a -:ref:`match object <match-objects>` instance. :: +:class:`MatchObject` instance. :: >>> print(re.match(r'From\s+', 'Fromage amk')) None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 5), match='From '> + >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') + <_sre.SRE_Match object at 0x...> Under the hood, these functions simply create a pattern object for you -and call the appropriate method on it. They also store the compiled -object in a cache, so future calls using the same RE won't need to -parse the pattern again and again. +and call the appropriate method on it. They also store the compiled object in a +cache, so future calls using the same RE are faster. Should you use these module-level functions, or should you get the -pattern and call its methods yourself? If you're accessing a regex -within a loop, pre-compiling it will save a few function calls. -Outside of loops, there's not much difference thanks to the internal -cache. +pattern and call its methods yourself? That choice depends on how +frequently the RE will be used, and on your personal coding style. If the RE is +being used at only one point in the code, then the module functions are probably +more convenient. If a program contains a lot of regular expressions, or re-uses +the same ones in several locations, then it might be worthwhile to collect all +the definitions in one place, in a section of code that compiles all the REs +ahead of time. To take an example from the standard library, here's an extract +from the now-defunct Python 2 standard :mod:`xmllib` module:: + + ref = re.compile( ... ) + entityref = re.compile( ... ) + charref = re.compile( ... ) + starttagopen = re.compile( ... ) + +I generally prefer to work with the compiled object, even for one-time uses, but +few people will be as much of a purist about this as I am. Compilation Flags @@ -524,10 +523,6 @@ +---------------------------------+--------------------------------------------+ | Flag | Meaning | +=================================+============================================+ -| :const:`ASCII`, :const:`A` | Makes several escapes like ``\w``, ``\b``, | -| | ``\s`` and ``\d`` match only on ASCII | -| | characters with the respective property. | -+---------------------------------+--------------------------------------------+ | :const:`DOTALL`, :const:`S` | Make ``.`` match any character, including | | | newlines | +---------------------------------+--------------------------------------------+ @@ -539,7 +534,11 @@ | | ``$`` | +---------------------------------+--------------------------------------------+ | :const:`VERBOSE`, :const:`X` | Enable verbose REs, which can be organized | -| (for 'extended') | more cleanly and understandably. | +| | more cleanly and understandably. | ++---------------------------------+--------------------------------------------+ +| :const:`ASCII`, :const:`A` | Makes several escapes like ``\w``, ``\b``, | +| | ``\s`` and ``\d`` match only on ASCII | +| | characters with the respective property. | +---------------------------------+--------------------------------------------+ @@ -558,8 +557,7 @@ LOCALE :noindex: - Make ``\w``, ``\W``, ``\b``, and ``\B``, dependent on the current locale - instead of the Unicode database. + Make ``\w``, ``\W``, ``\b``, and ``\B``, dependent on the current locale. Locales are a feature of the C library intended to help in writing programs that take account of language differences. For example, if you're processing French @@ -684,8 +682,8 @@ For example, if you wish to match the word ``From`` only at the beginning of a line, the RE to use is ``^From``. :: - >>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 4), match='From'> + >>> print(re.search('^From', 'From Here to Eternity')) + <_sre.SRE_Match object at 0x...> >>> print(re.search('^From', 'Reciting From Memory')) None @@ -696,12 +694,12 @@ Matches at the end of a line, which is defined as either the end of the string, or any location followed by a newline character. :: - >>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(6, 7), match='}'> + >>> print(re.search('}$', '{block}')) + <_sre.SRE_Match object at 0x...> >>> print(re.search('}$', '{block} ')) None - >>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(6, 7), match='}'> + >>> print(re.search('}$', '{block}\n')) + <_sre.SRE_Match object at 0x...> To match a literal ``'$'``, use ``\$`` or enclose it inside a character class, as in ``[$]``. @@ -725,8 +723,8 @@ match when it's contained inside another word. :: >>> p = re.compile(r'\bclass\b') - >>> print(p.search('no class at all')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(3, 8), match='class'> + >>> print(p.search('no class at all')) + <_sre.SRE_Match object at 0x...> >>> print(p.search('the declassified algorithm')) None >>> print(p.search('one subclass is')) @@ -743,8 +741,8 @@ >>> p = re.compile('\bclass\b') >>> print(p.search('no class at all')) None - >>> print(p.search('\b' + 'class' + '\b')) #doctest: +ELLIPSIS - <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'> + >>> print(p.search('\b' + 'class' + '\b') ) + <_sre.SRE_Match object at 0x...> Second, inside a character class, where there's no use for this assertion, ``\b`` represents the backspace character, for compatibility with Python's @@ -788,9 +786,9 @@ index of the text that they match; this can be retrieved by passing an argument to :meth:`group`, :meth:`start`, :meth:`end`, and :meth:`span`. Groups are numbered starting with 0. Group 0 is always present; it's the whole RE, so -:ref:`match object <match-objects>` methods all have group 0 as their default -argument. Later we'll see how to express groups that don't capture the span -of text that they match. :: +:class:`MatchObject` methods all have group 0 as their default argument. Later +we'll see how to express groups that don't capture the span of text that they +match. :: >>> p = re.compile('(a)b') >>> m = p.match('ab') @@ -852,10 +850,11 @@ problem. Both of them use a common syntax for regular expression extensions, so we'll look at that first. -Perl 5 is well known for its powerful additions to standard regular expressions. -For these new features the Perl developers couldn't choose new single-keystroke metacharacters -or new special sequences beginning with ``\`` without making Perl's regular -expressions confusingly different from standard REs. If they chose ``&`` as a +Perl 5 added several additional features to standard regular expressions, and +the Python :mod:`re` module supports most of them. It would have been +difficult to choose new single-keystroke metacharacters or new special sequences +beginning with ``\`` to represent the new features without making Perl's regular +expressions confusingly different from standard REs. If you chose ``&`` as a new metacharacter, for example, old expressions would be assuming that ``&`` was a regular character and wouldn't have escaped it by writing ``\&`` or ``[&]``. @@ -867,15 +866,22 @@ assertion) and ``(?:foo)`` is something else (a non-capturing group containing the subexpression ``foo``). -Python supports several of Perl's extensions and adds an extension -syntax to Perl's extension syntax. If the first character after the -question mark is a ``P``, you know that it's an extension that's -specific to Python. +Python adds an extension syntax to Perl's extension syntax. If the first +character after the question mark is a ``P``, you know that it's an extension +that's specific to Python. Currently there are two such extensions: +``(?P<name>...)`` defines a named group, and ``(?P=name)`` is a backreference to +a named group. If future versions of Perl 5 add similar features using a +different syntax, the :mod:`re` module will be changed to support the new +syntax, while preserving the Python-specific syntax for compatibility's sake. -Now that we've looked at the general extension syntax, we can return -to the features that simplify working with groups in complex REs. +Now that we've looked at the general extension syntax, we can return to the +features that simplify working with groups in complex REs. Since groups are +numbered from left to right and a complex expression may use many groups, it can +become difficult to keep track of the correct numbering. Modifying such a +complex RE is annoying, too: insert a new group near the beginning and you +change the numbers of everything that follows it. -Sometimes you'll want to use a group to denote a part of a regular expression, +Sometimes you'll want to use a group to collect a part of a regular expression, but aren't interested in retrieving the group's contents. You can make this fact explicit by using a non-capturing group: ``(?:...)``, where you can replace the ``...`` with any other regular expression. :: @@ -901,11 +907,11 @@ The syntax for a named group is one of the Python-specific extensions: ``(?P<name>...)``. *name* is, obviously, the name of the group. Named groups -behave exactly like capturing groups, and additionally associate a name -with a group. The :ref:`match object <match-objects>` methods that deal with -capturing groups all accept either integers that refer to the group by number -or strings that contain the desired group's name. Named groups are still -given numbers, so you can retrieve information about a group in two ways:: +also behave exactly like capturing groups, and additionally associate a name +with a group. The :class:`MatchObject` methods that deal with capturing groups +all accept either integers that refer to the group by number or strings that +contain the desired group's name. Named groups are still given numbers, so you +can retrieve information about a group in two ways:: >>> p = re.compile(r'(?P<word>\b\w+\b)') >>> m = p.search( '(((( Lots of punctuation )))' ) @@ -968,10 +974,9 @@ ``.*[.].*$`` Notice that the ``.`` needs to be treated specially because it's a -metacharacter, so it's inside a character class to only match that -specific character. Also notice the trailing ``$``; this is added to -ensure that all the rest of the string must be included in the -extension. This regular expression matches ``foo.bar`` and +metacharacter; I've put it inside a character class. Also notice the trailing +``$``; this is added to ensure that all the rest of the string must be included +in the extension. This regular expression matches ``foo.bar`` and ``autoexec.bat`` and ``sendmail.cf`` and ``printers.conf``. Now, consider complicating the problem a bit; what if you want to match @@ -1004,18 +1009,17 @@ A negative lookahead cuts through all this confusion: -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` +``.*[.](?!bat$).*$`` The negative lookahead means: if the expression ``bat`` doesn't match at this point, try the rest of the pattern; if ``bat$`` does match, the whole pattern will fail. The trailing ``$`` is required to ensure that something like ``sample.batch``, where the extension only starts with -``bat``, will be allowed. The ``[^.]*`` makes sure that the pattern works -when there are multiple dots in the filename. +``bat``, will be allowed. Excluding another filename extension is now easy; simply add it as an alternative inside the assertion. The following pattern excludes filenames that end in either ``bat`` or ``exe``: -``.*[.](?!bat$|exe$)[^.]*$`` +``.*[.](?!bat$|exe$).*$`` Modifying Strings @@ -1046,7 +1050,7 @@ The :meth:`split` method of a pattern splits a string apart wherever the RE matches, returning a list of the pieces. It's similar to the :meth:`split` method of strings but provides much more generality in the -delimiters that you can split by; string :meth:`split` only supports splitting by +delimiters that you can split by; :meth:`split` only supports splitting by whitespace or by a fixed string. As you'd expect, there's a module-level :func:`re.split` function, too. @@ -1101,6 +1105,7 @@ with a different string. The :meth:`sub` method takes a replacement value, which can be either a string or a function, and the string to be processed. + .. method:: .sub(replacement, string[, count=0]) :noindex: @@ -1139,7 +1144,7 @@ If *replacement* is a string, any backslash escapes in it are processed. That is, ``\n`` is converted to a single newline character, ``\r`` is converted to a -carriage return, and so forth. Unknown escapes such as ``\&`` are left alone. +carriage return, and so forth. Unknown escapes such as ``\j`` are left alone. Backreferences, such as ``\6``, are replaced with the substring matched by the corresponding group in the RE. This lets you incorporate portions of the original text in the resulting replacement string. @@ -1170,16 +1175,16 @@ *replacement* can also be a function, which gives you even more control. If *replacement* is a function, the function is called for every non-overlapping -occurrence of *pattern*. On each call, the function is passed a -:ref:`match object <match-objects>` argument for the match and can use this -information to compute the desired replacement string and return it. +occurrence of *pattern*. On each call, the function is passed a +:class:`MatchObject` argument for the match and can use this information to +compute the desired replacement string and return it. -In the following example, the replacement function translates decimals into +In the following example, the replacement function translates decimals into hexadecimal:: - >>> def hexrepl(match): + >>> def hexrepl( match ): ... "Return the hex string for a decimal number" - ... value = int(match.group()) + ... value = int( match.group() ) ... return hex(value) ... >>> p = re.compile(r'\d+') @@ -1356,3 +1361,4 @@ reference for programming in Python. (The first edition covered Python's now-removed :mod:`regex` module, which won't help you much.) Consider checking it out from your library. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/sockets.rst --- a/Doc/howto/sockets.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/sockets.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,6 +19,12 @@ Sockets ======= +Sockets are used nearly everywhere, but are one of the most severely +misunderstood technologies around. This is a 10,000 foot overview of sockets. +It's not really a tutorial - you'll still have work to do in getting things +working. It doesn't cover the fine points (and there are a lot of them), but I +hope it will give you enough background to begin using them decently. + I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really know what you're doing (in which case this HOWTO isn't for you!), you'll get @@ -78,11 +84,9 @@ serversocket.listen(5) A couple things to notice: we used ``socket.gethostname()`` so that the socket -would be visible to the outside world. If we had used ``s.bind(('localhost', -80))`` or ``s.bind(('127.0.0.1', 80))`` we would still have a "server" socket, -but one that was only visible within the same machine. ``s.bind(('', 80))`` -specifies that the socket is reachable by any address the machine happens to -have. +would be visible to the outside world. If we had used ``s.bind(('', 80))`` or +``s.bind(('localhost', 80))`` or ``s.bind(('127.0.0.1', 80))`` we would still +have a "server" socket, but one that was only visible within the same machine. A second thing to note: low number ports are usually reserved for "well known" services (HTTP, SNMP etc). If you're playing around, use a nice high number (4 @@ -149,7 +153,7 @@ there, you may wait forever for the reply, because the request may still be in your output buffer. -Now we come to the major stumbling block of sockets - ``send`` and ``recv`` operate +Now we come the major stumbling block of sockets - ``send`` and ``recv`` operate on the network buffers. They do not necessarily handle all the bytes you hand them (or expect from them), because their major focus is handling the network buffers. In general, they return when the associated network buffers have been @@ -160,7 +164,7 @@ When a ``recv`` returns 0 bytes, it means the other side has closed (or is in the process of closing) the connection. You will not receive any more data on this connection. Ever. You may be able to send data successfully; I'll talk -more about this later. +about that some on the next page. A protocol like HTTP uses a socket for only one transfer. The client sends a request, then reads a reply. That's it. The socket is discarded. This means that @@ -180,7 +184,7 @@ Assuming you don't want to end the connection, the simplest solution is a fixed length message:: - class MySocket: + class mysocket: """demonstration class only - coded for clarity, not efficiency """ @@ -189,8 +193,8 @@ if sock is None: self.sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM) - else: - self.sock = sock + else: + self.sock = sock def connect(self, host, port): self.sock.connect((host, port)) @@ -204,15 +208,13 @@ totalsent = totalsent + sent def myreceive(self): - chunks = [] - bytes_recd = 0 - while bytes_recd < MSGLEN: - chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048)) + msg = b'' + while len(msg) < MSGLEN: + chunk = self.sock.recv(MSGLEN-len(msg)) if chunk == b'': raise RuntimeError("socket connection broken") - chunks.append(chunk) - bytes_recd = bytes_recd + len(chunk) - return b''.join(chunks) + msg = msg + chunk + return msg The sending code here is usable for almost any messaging scheme - in Python you send strings, and you can use ``len()`` to determine its length (even if it has @@ -234,7 +236,7 @@ following message. You'll need to put that aside and hold onto it, until it's needed. -Prefixing the message with its length (say, as 5 numeric characters) gets more +Prefixing the message with it's length (say, as 5 numeric characters) gets more complex, because (believe it or not), you may not get all 5 characters in one ``recv``. In playing around, you'll get away with it; but in high network loads, your code will very quickly break unless you use two ``recv`` loops - the first diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/sorting.rst --- a/Doc/howto/sorting.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/sorting.rst Mon Jan 25 17:05:13 2016 +0100 @@ -42,7 +42,7 @@ Key Functions ============= -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a +Both :meth:`list.sort` and :func:`sorted` have *key* parameter to specify a function to be called on each list element prior to making comparisons. For example, here's a case-insensitive string comparison: @@ -89,7 +89,7 @@ The key-function patterns shown above are very common, so Python provides convenience functions to make accessor functions easier and faster. The :mod:`operator` module has :func:`~operator.itemgetter`, -:func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. +:func:`~operator.attrgetter`, and an :func:`~operator.methodcaller` function. Using those functions, the above examples become simpler and faster: @@ -114,7 +114,7 @@ ======================== Both :meth:`list.sort` and :func:`sorted` accept a *reverse* parameter with a -boolean value. This is used to flag descending sorts. For example, to get the +boolean value. This is using to flag descending sorts. For example, to get the student data in reverse *age* order: >>> sorted(student_tuples, key=itemgetter(2), reverse=True) @@ -225,7 +225,7 @@ def cmp_to_key(mycmp): 'Convert a cmp= function into a key= function' - class K: + class K(object): def __init__(self, obj, *args): self.obj = obj def __lt__(self, other): diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/unicode.rst --- a/Doc/howto/unicode.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/unicode.rst Mon Jan 25 17:05:13 2016 +0100 @@ -28,15 +28,15 @@ as 'naïve' and 'café', and some publications have house styles which require spellings such as 'coöperate'.) -For a while people just wrote programs that didn't display accents. -In the mid-1980s an Apple II BASIC program written by a French speaker -might have lines like these:: +For a while people just wrote programs that didn't display accents. I remember +looking at Apple ][ BASIC programs, published in French-language publications in +the mid-1980s, that had lines like these:: - PRINT "MISE A JOUR TERMINEE" - PRINT "PARAMETRES ENREGISTRES" + PRINT "FICHIER EST COMPLETE." + PRINT "CARACTERE NON ACCEPTE." -Those messages should contain accents (terminée, paramètre, enregistrés) and -they just look wrong to someone who can read French. +Those messages should contain accents, and they just look wrong to someone who +can read French. In the 1980s, almost all personal computers were 8-bit, meaning that bytes could hold values ranging from 0 to 255. ASCII codes only went up to 127, so some @@ -44,12 +44,12 @@ machines had different codes, however, which led to problems exchanging files. Eventually various commonly used sets of values for the 128--255 range emerged. Some were true standards, defined by the International Standards Organization, -and some were *de facto* conventions that were invented by one company or +and some were **de facto** conventions that were invented by one company or another and managed to catch on. 255 characters aren't very many. For example, you can't fit both the accented characters used in Western Europe and the Cyrillic alphabet used for Russian -into the 128--255 range because there are more than 128 such characters. +into the 128--255 range because there are more than 127 such characters. You could write files using different codes (all your Russian files in a coding system called KOI8, all your French files in a different coding system called @@ -62,19 +62,16 @@ to represent many different characters from many different alphabets; an initial goal was to have Unicode contain the alphabets for every single human language. It turns out that even 16 bits isn't enough to meet that goal, and the modern -Unicode specification uses a wider range of codes, 0 through 1,114,111 ( -``0x10FFFF`` in base 16). +Unicode specification uses a wider range of codes, 0 through 1,114,111 (0x10ffff +in base 16). There's a related ISO standard, ISO 10646. Unicode and ISO 10646 were originally separate efforts, but the specifications were merged with the 1.1 revision of Unicode. -(This discussion of Unicode's history is highly simplified. The -precise historical details aren't necessary for understanding how to -use Unicode effectively, but if you're curious, consult the Unicode -consortium site listed in the References or -the `Wikipedia entry for Unicode <http://en.wikipedia.org/wiki/Unicode#History>`_ -for more information.) +(This discussion of Unicode's history is highly simplified. I don't think the +average Python programmer needs to worry about the historical details; consult +the Unicode consortium site listed in the References for more information.) Definitions @@ -90,11 +87,9 @@ The Unicode standard describes how characters are represented by **code points**. A code point is an integer value, usually denoted in base 16. In the -standard, a code point is written using the notation ``U+12CA`` to mean the -character with value ``0x12ca`` (4,810 decimal). The Unicode standard contains -a lot of tables listing characters and their corresponding code points: - -.. code-block:: none +standard, a code point is written using the notation U+12ca to mean the +character with value 0x12ca (4,810 decimal). The Unicode standard contains a lot +of tables listing characters and their corresponding code points:: 0061 'a'; LATIN SMALL LETTER A 0062 'b'; LATIN SMALL LETTER B @@ -103,7 +98,7 @@ 007B '{'; LEFT CURLY BRACKET Strictly, these definitions imply that it's meaningless to say 'this is -character ``U+12CA``'. ``U+12CA`` is a code point, which represents some particular +character U+12ca'. U+12ca is a code point, which represents some particular character; in this case, it represents the character 'ETHIOPIC SYLLABLE WI'. In informal contexts, this distinction between code points and characters will sometimes be forgotten. @@ -120,15 +115,13 @@ --------- To summarize the previous section: a Unicode string is a sequence of code -points, which are numbers from 0 through ``0x10FFFF`` (1,114,111 decimal). This +points, which are numbers from 0 through 0x10ffff (1,114,111 decimal). This sequence needs to be represented as a set of bytes (meaning, values from 0 through 255) in memory. The rules for translating a Unicode string into a sequence of bytes are called an **encoding**. The first encoding you might think of is an array of 32-bit integers. In this -representation, the string "Python" would look like this: - -.. code-block:: none +representation, the string "Python" would look like this:: P y t h o n 0x50 00 00 00 79 00 00 00 74 00 00 00 68 00 00 00 6f 00 00 00 6e 00 00 00 @@ -140,10 +133,10 @@ 1. It's not portable; different processors order the bytes differently. 2. It's very wasteful of space. In most texts, the majority of the code points - are less than 127, or less than 255, so a lot of space is occupied by ``0x00`` + are less than 127, or less than 255, so a lot of space is occupied by zero bytes. The above string takes 24 bytes compared to the 6 bytes needed for an ASCII representation. Increased RAM usage doesn't matter too much (desktop - computers have gigabytes of RAM, and strings aren't usually that large), but + computers have megabytes of RAM, and strings aren't usually that large), but expanding our usage of disk and network bandwidth by a factor of 4 is intolerable. @@ -182,12 +175,14 @@ UTF-8 is one of the most commonly used encodings. UTF stands for "Unicode Transformation Format", and the '8' means that 8-bit numbers are used in the -encoding. (There are also a UTF-16 and UTF-32 encodings, but they are less -frequently used than UTF-8.) UTF-8 uses the following rules: +encoding. (There's also a UTF-16 encoding, but it's less frequently used than +UTF-8.) UTF-8 uses the following rules: -1. If the code point is < 128, it's represented by the corresponding byte value. -2. If the code point is >= 128, it's turned into a sequence of two, three, or - four bytes, where each byte of the sequence is between 128 and 255. +1. If the code point is <128, it's represented by the corresponding byte value. +2. If the code point is between 128 and 0x7ff, it's turned into two byte values + between 128 and 255. +3. Code points >0x7ff are turned into three- or four-byte sequences, where each + byte of the sequence is between 128 and 255. UTF-8 has several convenient properties: @@ -197,8 +192,8 @@ processed by C functions such as ``strcpy()`` and sent through protocols that can't handle zero bytes. 3. A string of ASCII text is also valid UTF-8 text. -4. UTF-8 is fairly compact; the majority of commonly used characters can be - represented with one or two bytes. +4. UTF-8 is fairly compact; the majority of code points are turned into two + bytes, and values less than 128 occupy only a single byte. 5. If bytes are corrupted or lost, it's possible to determine the start of the next UTF-8-encoded code point and resynchronize. It's also unlikely that random 8-bit data will look like valid UTF-8. @@ -208,23 +203,25 @@ References ---------- -The `Unicode Consortium site <http://www.unicode.org>`_ has character charts, a +The Unicode Consortium site at <http://www.unicode.org> has character charts, a glossary, and PDF versions of the Unicode specification. Be prepared for some -difficult reading. `A chronology <http://www.unicode.org/history/>`_ of the -origin and development of Unicode is also available on the site. +difficult reading. <http://www.unicode.org/history/> is a chronology of the +origin and development of Unicode. -To help understand the standard, Jukka Korpela has written `an introductory -guide <http://www.cs.tut.fi/~jkorpela/unicode/guide.html>`_ to reading the -Unicode character tables. +To help understand the standard, Jukka Korpela has written an introductory guide +to reading the Unicode character tables, available at +<http://www.cs.tut.fi/~jkorpela/unicode/guide.html>. -Another `good introductory article <http://www.joelonsoftware.com/articles/Unicode.html>`_ -was written by Joel Spolsky. -If this introduction didn't make things clear to you, you should try -reading this alternate article before continuing. +Another good introductory article was written by Joel Spolsky +<http://www.joelonsoftware.com/articles/Unicode.html>. +If this introduction didn't make things clear to you, you should try reading this +alternate article before continuing. -Wikipedia entries are often helpful; see the entries for "`character encoding -<http://en.wikipedia.org/wiki/Character_encoding>`_" and `UTF-8 -<http://en.wikipedia.org/wiki/UTF-8>`_, for example. +.. Jason Orendorff XXX http://www.jorendorff.com/articles/unicode/ is broken + +Wikipedia entries are often helpful; see the entries for "character encoding" +<http://en.wikipedia.org/wiki/Character_encoding> and UTF-8 +<http://en.wikipedia.org/wiki/UTF-8>, for example. Python's Unicode Support @@ -236,35 +233,12 @@ The String Type --------------- -Since Python 3.0, the language features a :class:`str` type that contain Unicode +Since Python 3.0, the language features a ``str`` type that contain Unicode characters, meaning any string created using ``"unicode rocks!"``, ``'unicode rocks!'``, or the triple-quoted string syntax is stored as Unicode. -The default encoding for Python source code is UTF-8, so you can simply -include a Unicode character in a string literal:: - - try: - with open('/tmp/input.txt', 'r') as f: - ... - except OSError: - # 'File not found' error message. - print("Fichier non trouvé") - -You can use a different encoding from UTF-8 by putting a specially-formatted -comment as the first or second line of the source code:: - - # -*- coding: <encoding name> -*- - -Side note: Python 3 also supports using Unicode characters in identifiers:: - - répertoire = "/tmp/records.log" - with open(répertoire, "w") as f: - f.write("test\n") - -If you can't enter a particular character in your editor or want to -keep the source code ASCII-only for some reason, you can also use -escape sequences in string literals. (Depending on your system, -you may see the actual capital-delta glyph instead of a \u escape.) :: +To insert a Unicode character that is not part ASCII, e.g., any letters with +accents, one can use escape sequences in their string literals as such:: >>> "\N{GREEK CAPITAL LETTER DELTA}" # Using the character name '\u0394' @@ -273,27 +247,23 @@ >>> "\U00000394" # Using a 32-bit hex value '\u0394' -In addition, one can create a string using the :func:`~bytes.decode` method of -:class:`bytes`. This method takes an *encoding* argument, such as ``UTF-8``, -and optionally an *errors* argument. +In addition, one can create a string using the :func:`decode` method of +:class:`bytes`. This method takes an encoding, such as UTF-8, and, optionally, +an *errors* argument. The *errors* argument specifies the response when the input string can't be converted according to the encoding's rules. Legal values for this argument are -``'strict'`` (raise a :exc:`UnicodeDecodeError` exception), ``'replace'`` (use -``U+FFFD``, ``REPLACEMENT CHARACTER``), ``'ignore'`` (just leave the -character out of the Unicode result), or ``'backslashreplace'`` (inserts a -``\xNN`` escape sequence). -The following examples show the differences:: +'strict' (raise a :exc:`UnicodeDecodeError` exception), 'replace' (use U+FFFD, +'REPLACEMENT CHARACTER'), or 'ignore' (just leave the character out of the +Unicode result). The following examples show the differences:: - >>> b'\x80abc'.decode("utf-8", "strict") #doctest: +NORMALIZE_WHITESPACE + >>> b'\x80abc'.decode("utf-8", "strict") Traceback (most recent call last): - ... - UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: - invalid start byte + File "<stdin>", line 1, in ? + UnicodeDecodeError: 'utf8' codec can't decode byte 0x80 in position 0: + unexpected code byte >>> b'\x80abc'.decode("utf-8", "replace") - '\ufffdabc' - >>> b'\x80abc'.decode("utf-8", "backslashreplace") - '\\x80abc' + '?abc' >>> b'\x80abc'.decode("utf-8", "ignore") 'abc' @@ -303,8 +273,8 @@ Encodings are specified as strings containing the encoding's name. Python 3.2 comes with roughly 100 different encodings; see the Python Library Reference at :ref:`standard-encodings` for a list. Some encodings have multiple names; for -example, ``'latin-1'``, ``'iso_8859_1'`` and ``'8859``' are all synonyms for -the same encoding. +example, 'latin-1', 'iso_8859_1' and '8859' are all synonyms for the same +encoding. One-character Unicode strings can also be created with the :func:`chr` built-in function, which takes integers and returns a Unicode string of length 1 @@ -320,45 +290,36 @@ Converting to Bytes ------------------- -The opposite method of :meth:`bytes.decode` is :meth:`str.encode`, -which returns a :class:`bytes` representation of the Unicode string, encoded in the -requested *encoding*. - -The *errors* parameter is the same as the parameter of the -:meth:`~bytes.decode` method but supports a few more possible handlers. As well as -``'strict'``, ``'ignore'``, and ``'replace'`` (which in this case -inserts a question mark instead of the unencodable character), there is -also ``'xmlcharrefreplace'`` (inserts an XML character reference), -``backslashreplace`` (inserts a ``\uNNNN`` escape sequence) and -``namereplace`` (inserts a ``\N{...}`` escape sequence). - -The following example shows the different results:: +Another important str method is ``.encode([encoding], [errors='strict'])``, +which returns a ``bytes`` representation of the Unicode string, encoded in the +requested encoding. The ``errors`` parameter is the same as the parameter of +the :meth:`decode` method, with one additional possibility; as well as 'strict', +'ignore', and 'replace' (which in this case inserts a question mark instead of +the unencodable character), you can also pass 'xmlcharrefreplace' which uses +XML's character references. The following example shows the different results:: >>> u = chr(40960) + 'abcd' + chr(1972) >>> u.encode('utf-8') b'\xea\x80\x80abcd\xde\xb4' - >>> u.encode('ascii') #doctest: +NORMALIZE_WHITESPACE + >>> u.encode('ascii') Traceback (most recent call last): - ... + File "<stdin>", line 1, in ? UnicodeEncodeError: 'ascii' codec can't encode character '\ua000' in - position 0: ordinal not in range(128) + position 0: ordinal not in range(128) >>> u.encode('ascii', 'ignore') b'abcd' >>> u.encode('ascii', 'replace') b'?abcd?' >>> u.encode('ascii', 'xmlcharrefreplace') b'ꀀabcd޴' - >>> u.encode('ascii', 'backslashreplace') - b'\\ua000abcd\\u07b4' - >>> u.encode('ascii', 'namereplace') - b'\\N{YI SYLLABLE IT}abcd\\u07b4' -The low-level routines for registering and accessing the available -encodings are found in the :mod:`codecs` module. Implementing new -encodings also requires understanding the :mod:`codecs` module. -However, the encoding and decoding functions returned by this module -are usually more low-level than is comfortable, and writing new encodings -is a specialized task, so the module won't be covered in this HOWTO. +The low-level routines for registering and accessing the available encodings are +found in the :mod:`codecs` module. However, the encoding and decoding functions +returned by this module are usually more low-level than is comfortable, so I'm +not going to describe the :mod:`codecs` module here. If you need to implement a +completely new encoding, you'll need to learn about the :mod:`codecs` module +interfaces, but implementing encodings is a specialized task that also won't be +covered here. Consult the Python documentation to learn more about this module. Unicode Literals in Python Source Code @@ -370,11 +331,12 @@ not four:: >>> s = "a\xac\u1234\u20ac\U00008000" - ... # ^^^^ two-digit hex escape - ... # ^^^^^^ four-digit Unicode escape - ... # ^^^^^^^^^^ eight-digit Unicode escape - >>> [ord(c) for c in s] - [97, 172, 4660, 8364, 32768] + ^^^^ two-digit hex escape + ^^^^^ four-digit Unicode escape + ^^^^^^^^^^ eight-digit Unicode escape + >>> for c in s: print(ord(c), end=" ") + ... + 97 172 4660 8364 32768 Using escape sequences for code points greater than 127 is fine in small doses, but becomes an annoyance if you're using many accented characters, as you would @@ -404,14 +366,14 @@ ``coding: name`` or ``coding=name`` in the comment. If you don't include such a comment, the default encoding used will be UTF-8 as -already mentioned. See also :pep:`263` for more information. +already mentioned. Unicode Properties ------------------ The Unicode specification includes a database of information about code points. -For each defined code point, the information includes the character's +For each code point that's defined, the information includes the character's name, its category, the numeric value if applicable (Unicode has characters representing the Roman numerals and fractions such as one-third and four-fifths). There are also properties related to the code point's use in @@ -431,9 +393,7 @@ # Get numeric value of second character print(unicodedata.numeric(u[1])) -When run, this prints: - -.. code-block:: none +When run, this prints:: 0 00e9 Ll LATIN SMALL LETTER E WITH ACUTE 1 0bf2 No TAMIL NUMBER ONE THOUSAND @@ -448,62 +408,25 @@ from the above output, ``'Ll'`` means 'Letter, lowercase', ``'No'`` means "Number, other", ``'Mn'`` is "Mark, nonspacing", and ``'So'`` is "Symbol, other". See -`the General Category Values section of the Unicode Character Database documentation <http://www.unicode.org/reports/tr44/#General_Category_Values>`_ for a +<http://www.unicode.org/reports/tr44/#General_Category_Values> for a list of category codes. - -Unicode Regular Expressions ---------------------------- - -The regular expressions supported by the :mod:`re` module can be provided -either as bytes or strings. Some of the special character sequences such as -``\d`` and ``\w`` have different meanings depending on whether -the pattern is supplied as bytes or a string. For example, -``\d`` will match the characters ``[0-9]`` in bytes but -in strings will match any character that's in the ``'Nd'`` category. - -The string in this example has the number 57 written in both Thai and -Arabic numerals:: - - import re - p = re.compile('\d+') - - s = "Over \u0e55\u0e57 57 flavours" - m = p.search(s) - print(repr(m.group())) - -When executed, ``\d+`` will match the Thai numerals and print them -out. If you supply the :const:`re.ASCII` flag to -:func:`~re.compile`, ``\d+`` will match the substring "57" instead. - -Similarly, ``\w`` matches a wide variety of Unicode characters but -only ``[a-zA-Z0-9_]`` in bytes or if :const:`re.ASCII` is supplied, -and ``\s`` will match either Unicode whitespace characters or -``[ \t\n\r\f\v]``. - - References ---------- -.. comment should these be mentioned earlier, e.g. at the start of the "introduction to Unicode" first section? - -Some good alternative discussions of Python's Unicode support are: - -* `Processing Text Files in Python 3 <http://python-notes.curiousefficiency.org/en/latest/python3/text_file_processing.html>`_, by Nick Coghlan. -* `Pragmatic Unicode <http://nedbatchelder.com/text/unipain.html>`_, a PyCon 2012 presentation by Ned Batchelder. - -The :class:`str` type is described in the Python library reference at -:ref:`textseq`. +The ``str`` type is described in the Python library reference at +:ref:`typesseq`. The documentation for the :mod:`unicodedata` module. The documentation for the :mod:`codecs` module. -Marc-André Lemburg gave `a presentation titled "Python and Unicode" (PDF slides) -<https://downloads.egenix.com/python/Unicode-EPC2002-Talk.pdf>`_ at -EuroPython 2002. The slides are an excellent overview of the design of Python -2's Unicode features (where the Unicode string type is called ``unicode`` and -literals start with ``u``). +Marc-André Lemburg gave a presentation at EuroPython 2002 titled "Python and +Unicode". A PDF version of his slides is available at +<http://downloads.egenix.com/python/Unicode-EPC2002-Talk.pdf>, and is an +excellent overview of the design of Python's Unicode features (based on Python +2, where the Unicode string type is called ``unicode`` and literals start with +``u``). Reading and Writing Unicode Data @@ -521,16 +444,16 @@ Unicode data is usually converted to a particular encoding before it gets written to disk or sent over a socket. It's possible to do all the work -yourself: open a file, read an 8-bit bytes object from it, and convert the bytes -with ``bytes.decode(encoding)``. However, the manual approach is not recommended. +yourself: open a file, read an 8-bit byte string from it, and convert the string +with ``str(bytes, encoding)``. However, the manual approach is not recommended. One problem is the multi-byte nature of encodings; one Unicode character can be represented by several bytes. If you want to read the file in arbitrary-sized -chunks (say, 1024 or 4096 bytes), you need to write error-handling code to catch the case +chunks (say, 1K or 4K), you need to write error-handling code to catch the case where only part of the bytes encoding a single Unicode character are read at the end of a chunk. One solution would be to read the entire file into memory and then perform the decoding, but that prevents you from working with files that -are extremely large; if you need to read a 2 GiB file, you need 2 GiB of RAM. +are extremely large; if you need to read a 2Gb file, you need 2Gb of RAM. (More, really, since for at least a moment you'd need to have both the encoded string and its Unicode version in memory.) @@ -538,14 +461,13 @@ of partial coding sequences. The work of implementing this has already been done for you: the built-in :func:`open` function can return a file-like object that assumes the file's contents are in a specified encoding and accepts Unicode -parameters for methods such as :meth:`~io.TextIOBase.read` and -:meth:`~io.TextIOBase.write`. This works through :func:`open`\'s *encoding* and -*errors* parameters which are interpreted just like those in :meth:`str.encode` -and :meth:`bytes.decode`. +parameters for methods such as ``.read()`` and ``.write()``. This works through +:func:`open`\'s *encoding* and *errors* parameters which are interpreted just +like those in string objects' :meth:`encode` and :meth:`decode` methods. Reading Unicode from a file is therefore simple:: - with open('unicode.txt', encoding='utf-8') as f: + with open('unicode.rst', encoding='utf-8') as f: for line in f: print(repr(line)) @@ -557,7 +479,7 @@ f.seek(0) print(repr(f.readline()[:1])) -The Unicode character ``U+FEFF`` is used as a byte-order mark (BOM), and is often +The Unicode character U+FEFF is used as a byte-order mark (BOM), and is often written as the first character of a file in order to assist with autodetection of the file's byte ordering. Some encodings, such as UTF-16, expect a BOM to be present at the start of a file; when such an encoding is used, the BOM will be @@ -583,7 +505,7 @@ Windows, Python uses the name "mbcs" to refer to whatever the currently configured encoding is. On Unix systems, there will only be a filesystem encoding if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if -you haven't, the default encoding is UTF-8. +you haven't, the default encoding is ASCII. The :func:`sys.getfilesystemencoding` function returns the encoding to use on your current system, in case you want to do the encoding manually, but there's @@ -598,13 +520,13 @@ Functions in the :mod:`os` module such as :func:`os.stat` will also accept Unicode filenames. -The :func:`os.listdir` function returns filenames and raises an issue: should it return -the Unicode version of filenames, or should it return bytes containing +Function :func:`os.listdir`, which returns filenames, raises an issue: should it return +the Unicode version of filenames, or should it return byte strings containing the encoded versions? :func:`os.listdir` will do both, depending on whether you -provided the directory path as bytes or a Unicode string. If you pass a +provided the directory path as a byte string or a Unicode string. If you pass a Unicode string as the path, filenames will be decoded using the filesystem's encoding and a list of Unicode strings will be returned, while passing a byte -path will return the filenames as bytes. For example, +path will return the byte string versions of the filenames. For example, assuming the default filesystem encoding is UTF-8, running the following program:: @@ -619,13 +541,13 @@ will produce the following output:: amk:~$ python t.py - [b'filename\xe4\x94\x80abc', ...] - ['filename\u4500abc', ...] + [b'.svn', b'filename\xe4\x94\x80abc', ...] + ['.svn', 'filename\u4500abc', ...] The first list contains UTF-8-encoded filenames, and the second list contains the Unicode versions. -Note that on most occasions, the Unicode APIs should be used. The bytes APIs +Note that in most occasions, the Unicode APIs should be used. The bytes APIs should only be used on systems where undecodable file names can be present, i.e. Unix systems. @@ -638,13 +560,13 @@ The most important tip is: - Software should only work with Unicode strings internally, decoding the input - data as soon as possible and encoding the output only at the end. + Software should only work with Unicode strings internally, converting to a + particular encoding on output. If you attempt to write processing functions that accept both Unicode and byte strings, you will find your program vulnerable to bugs wherever you combine the -two different kinds of strings. There is no automatic encoding or decoding: if -you do e.g. ``str + bytes``, a :exc:`TypeError` will be raised. +two different kinds of strings. There is no automatic encoding or decoding if +you do e.g. ``str + bytes``, a :exc:`TypeError` is raised for this expression. When using data coming from a web browser or some other untrusted source, a common technique is to check for illegal characters in a string before using the @@ -656,77 +578,68 @@ clever way to hide malicious text in the encoded bytestream. -Converting Between File Encodings -''''''''''''''''''''''''''''''''' - -The :class:`~codecs.StreamRecoder` class can transparently convert between -encodings, taking a stream that returns data in encoding #1 -and behaving like a stream returning data in encoding #2. - -For example, if you have an input file *f* that's in Latin-1, you -can wrap it with a :class:`~codecs.StreamRecoder` to return bytes encoded in -UTF-8:: - - new_f = codecs.StreamRecoder(f, - # en/decoder: used by read() to encode its results and - # by write() to decode its input. - codecs.getencoder('utf-8'), codecs.getdecoder('utf-8'), - - # reader/writer: used to read and write to the stream. - codecs.getreader('latin-1'), codecs.getwriter('latin-1') ) - - -Files in an Unknown Encoding -'''''''''''''''''''''''''''' - -What can you do if you need to make a change to a file, but don't know -the file's encoding? If you know the encoding is ASCII-compatible and -only want to examine or modify the ASCII parts, you can open the file -with the ``surrogateescape`` error handler:: - - with open(fname, 'r', encoding="ascii", errors="surrogateescape") as f: - data = f.read() - - # make changes to the string 'data' - - with open(fname + '.new', 'w', - encoding="ascii", errors="surrogateescape") as f: - f.write(data) - -The ``surrogateescape`` error handler will decode any non-ASCII bytes -as code points in the Unicode Private Use Area ranging from U+DC80 to -U+DCFF. These private code points will then be turned back into the -same bytes when the ``surrogateescape`` error handler is used when -encoding the data and writing it back out. - - References ---------- -One section of `Mastering Python 3 Input/Output -<http://pyvideo.org/video/289/pycon-2010--mastering-python-3-i-o>`_, -a PyCon 2010 talk by David Beazley, discusses text processing and binary data handling. - -The `PDF slides for Marc-André Lemburg's presentation "Writing Unicode-aware -Applications in Python" -<https://downloads.egenix.com/python/LSM2005-Developing-Unicode-aware-applications-in-Python.pdf>`_ -discuss questions of character encodings as well as how to internationalize +The PDF slides for Marc-André Lemburg's presentation "Writing Unicode-aware +Applications in Python" are available at +<http://downloads.egenix.com/python/LSM2005-Developing-Unicode-aware-applications-in-Python.pdf> +and discuss questions of character encodings as well as how to internationalize and localize an application. These slides cover Python 2.x only. -`The Guts of Unicode in Python -<http://pyvideo.org/video/1768/the-guts-of-unicode-in-python>`_ -is a PyCon 2013 talk by Benjamin Peterson that discusses the internal Unicode -representation in Python 3.3. - Acknowledgements ================ -The initial draft of this document was written by Andrew Kuchling. -It has since been revised further by Alexander Belopolsky, Georg Brandl, -Andrew Kuchling, and Ezio Melotti. +Thanks to the following people who have noted errors or offered suggestions on +this article: Nicholas Bastin, Marius Gedminas, Kent Johnson, Ken Krugler, +Marc-André Lemburg, Martin von Löwis, Chad Whitacre. -Thanks to the following people who have noted errors or offered -suggestions on this article: Éric Araujo, Nicholas Bastin, Nick -Coghlan, Marius Gedminas, Kent Johnson, Ken Krugler, Marc-André -Lemburg, Martin von Löwis, Terry J. Reedy, Chad Whitacre. +.. comment + Revision History + + Version 1.0: posted August 5 2005. + + Version 1.01: posted August 7 2005. Corrects factual and markup errors; adds + several links. + + Version 1.02: posted August 16 2005. Corrects factual errors. + + Version 1.1: Feb-Nov 2008. Updates the document with respect to Python 3 changes. + + Version 1.11: posted June 20 2010. Notes that Python 3.x is not covered, + and that the HOWTO only covers 2.x. + +.. comment Describe Python 3.x support (new section? new document?) +.. comment Additional topic: building Python w/ UCS2 or UCS4 support +.. comment Describe use of codecs.StreamRecoder and StreamReaderWriter + +.. comment + Original outline: + + - [ ] Unicode introduction + - [ ] ASCII + - [ ] Terms + - [ ] Character + - [ ] Code point + - [ ] Encodings + - [ ] Common encodings: ASCII, Latin-1, UTF-8 + - [ ] Unicode Python type + - [ ] Writing unicode literals + - [ ] Obscurity: -U switch + - [ ] Built-ins + - [ ] unichr() + - [ ] ord() + - [ ] unicode() constructor + - [ ] Unicode type + - [ ] encode(), decode() methods + - [ ] Unicodedata module for character properties + - [ ] I/O + - [ ] Reading/writing Unicode data into files + - [ ] Byte-order marks + - [ ] Unicode filenames + - [ ] Writing Unicode programs + - [ ] Do everything in Unicode + - [ ] Declaring source code encodings (PEP 263) + - [ ] Other issues + - [ ] Building Python (UCS2, UCS4) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/urllib2.rst --- a/Doc/howto/urllib2.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/urllib2.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,7 +26,7 @@ A tutorial on *Basic Authentication*, with examples in Python. -**urllib.request** is a Python module for fetching URLs +**urllib.request** is a `Python <http://www.python.org>`_ module for fetching URLs (Uniform Resource Locators). It offers a very simple interface, in the form of the *urlopen* function. This is capable of fetching URLs using a variety of different protocols. It also offers a slightly more complex interface for @@ -53,11 +53,11 @@ The simplest way to use urllib.request is as follows:: import urllib.request - with urllib.request.urlopen('http://python.org/') as response: - html = response.read() + response = urllib.request.urlopen('http://python.org/') + html = response.read() If you wish to retrieve a resource via URL and store it in a temporary location, -you can do so via the :func:`~urllib.request.urlretrieve` function:: +you can do so via the :func:`urlretrieve` function:: import urllib.request local_filename, headers = urllib.request.urlretrieve('http://python.org/') @@ -79,8 +79,8 @@ import urllib.request req = urllib.request.Request('http://www.voidspace.org.uk') - with urllib.request.urlopen(req) as response: - the_page = response.read() + response = urllib.request.urlopen(req) + the_page = response.read() Note that urllib.request makes use of the same Request interface to handle all URL schemes. For example, you can make an FTP request like so:: @@ -97,7 +97,7 @@ ---- Sometimes you want to send data to a URL (often the URL will refer to a CGI -(Common Gateway Interface) script or other web application). With HTTP, +(Common Gateway Interface) script [#]_ or other web application). With HTTP, this is often done using what's known as a **POST** request. This is often what your browser does when you submit a HTML form that you filled in on the web. Not all POSTs have to come from forms: you can use a POST to transmit arbitrary data @@ -115,10 +115,10 @@ 'language' : 'Python' } data = urllib.parse.urlencode(values) - data = data.encode('ascii') # data should be bytes + data = data.encode('utf-8') # data should be bytes req = urllib.request.Request(url, data) - with urllib.request.urlopen(req) as response: - the_page = response.read() + response = urllib.request.urlopen(req) + the_page = response.read() Note that other encodings are sometimes required (e.g. for file upload from HTML forms - see `HTML Specification, Form Submission @@ -144,7 +144,7 @@ >>> data['location'] = 'Northampton' >>> data['language'] = 'Python' >>> url_values = urllib.parse.urlencode(data) - >>> print(url_values) # The order may differ from below. #doctest: +SKIP + >>> print(url_values) name=Somebody+Here&language=Python&location=Northampton >>> url = 'http://www.example.com/example.cgi' >>> full_url = url + '?' + url_values @@ -160,7 +160,7 @@ to your HTTP request. Some websites [#]_ dislike being browsed by programs, or send different versions -to different browsers [#]_. By default urllib identifies itself as +to different browsers [#]_ . By default urllib identifies itself as ``Python-urllib/x.y`` (where ``x`` and ``y`` are the major and minor version numbers of the Python release, e.g. ``Python-urllib/2.5``), which may confuse the site, or just plain @@ -174,17 +174,17 @@ import urllib.request url = 'http://www.someserver.com/cgi-bin/register.cgi' - user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)' + user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' values = {'name' : 'Michael Foord', 'location' : 'Northampton', 'language' : 'Python' } headers = { 'User-Agent' : user_agent } - data = urllib.parse.urlencode(values) - data = data.encode('ascii') + data = urllib.parse.urlencode(values) + data = data.encode('utf-8') req = urllib.request.Request(url, data, headers) - with urllib.request.urlopen(req) as response: - the_page = response.read() + response = urllib.request.urlopen(req) + the_page = response.read() The response also has two useful methods. See the section on `info and geturl`_ which comes after we have a look at what happens when things go wrong. @@ -214,9 +214,9 @@ >>> req = urllib.request.Request('http://www.pretend_server.org') >>> try: urllib.request.urlopen(req) - ... except urllib.error.URLError as e: - ... print(e.reason) #doctest: +SKIP - ... + >>> except urllib.error.URLError as e: + >>> print(e.reason) + >>> (4, 'getaddrinfo failed') @@ -322,17 +322,18 @@ >>> req = urllib.request.Request('http://www.python.org/fish.html') >>> try: - ... urllib.request.urlopen(req) - ... except urllib.error.HTTPError as e: - ... print(e.code) - ... print(e.read()) #doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... + >>> urllib.request.urlopen(req) + >>> except urllib.error.HTTPError as e: + >>> print(e.code) + >>> print(e.read()) + >>> 404 - b'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n\n\n<html - ... - <title>Page Not Found\n - ... + + + Error 404: File Not Found + ...... etc... Wrapping it Up -------------- @@ -454,12 +455,12 @@ When authentication is required, the server sends a header (as well as the 401 error code) requesting authentication. This specifies the authentication scheme -and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME +and a 'realm'. The header looks like : ``Www-authenticate: SCHEME realm="REALM"``. e.g. :: - WWW-Authenticate: Basic realm="cPanel Users" + Www-authenticate: Basic realm="cPanel Users" The client should then retry the request with the appropriate name and password @@ -504,10 +505,9 @@ In the above example we only supplied our ``HTTPBasicAuthHandler`` to ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` - environment variable is set), ``UnknownHandler``, ``HTTPHandler``, + -- ``ProxyHandler``, ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. + ``FileHandler``, ``HTTPErrorProcessor``. ``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme component and the hostname and optionally the port number) @@ -522,11 +522,10 @@ ======= **urllib** will auto-detect your proxy settings and use those. This is through -the ``ProxyHandler``, which is part of the normal handler chain when a proxy -setting is detected. Normally that's a good thing, but there are occasions -when it may not be helpful [#]_. One way to do this is to setup our own -``ProxyHandler``, with no proxies defined. This is done using similar steps to -setting up a `Basic Authentication`_ handler: :: +the ``ProxyHandler`` which is part of the normal handler chain. Normally that's +a good thing, but there are occasions when it may not be helpful [#]_. One way +to do this is to setup our own ``ProxyHandler``, with no proxies defined. This +is done using similar steps to setting up a `Basic Authentication`_ handler : :: >>> proxy_support = urllib.request.ProxyHandler({}) >>> opener = urllib.request.build_opener(proxy_support) @@ -572,7 +571,12 @@ This document was reviewed and revised by John Lee. -.. [#] Google for example. +.. [#] For an introduction to the CGI protocol see + `Writing Web Applications in Python `_. +.. [#] Like Google for example. The *proper* way to use google from a program + is to use `PyGoogle `_ of course. See + `Voidspace Google `_ + for some examples of using the Google API. .. [#] Browser sniffing is a very bad practise for website design - building sites using web standards is much more sensible. Unfortunately a lot of sites still send different versions to different browsers. @@ -586,5 +590,5 @@ scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/howto/webservers.rst --- a/Doc/howto/webservers.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/howto/webservers.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,7 +26,7 @@ While this HOWTO tries to give an overview of Python in the web, it cannot always be as up to date as desired. Web development in Python is rapidly moving forward, so the wiki page on `Web Programming - `_ may be more in sync with + `_ may be more in sync with recent development. @@ -86,7 +86,7 @@ applications, instead of presenting a "500 Internal Server Error" message The Python wiki features a page on `CGI scripts - `_ with some additional information + `_ with some additional information about CGI in Python. @@ -146,7 +146,7 @@ tutorial also describes the most common gotchas that might arise. * On lighttpd you need to use the `CGI module - `_\ , which can be configured + `_\ , which can be configured in a straightforward way. It boils down to setting ``cgi.assign`` properly. @@ -210,7 +210,7 @@ ---------- People coming from PHP often find it hard to grasp how to use Python in the web. -Their first thought is mostly `mod_python `_\ , +Their first thought is mostly `mod_python `_\ , because they think that this is the equivalent to ``mod_php``. Actually, there are many differences. What ``mod_python`` does is embed the interpreter into the Apache process, thus speeding up requests by not having to start a Python @@ -260,6 +260,13 @@ These days, FastCGI is never used directly. Just like ``mod_python``, it is only used for the deployment of WSGI applications. +.. seealso:: + + * `FastCGI, SCGI, and Apache: Background and Future + `_ + is a discussion on why the concept of FastCGI and SCGI is better than that + of mod_python. + Setting up FastCGI ^^^^^^^^^^^^^^^^^^ @@ -273,8 +280,8 @@ to be loaded by Apache. * lighttpd ships its own `FastCGI module - `_ as well as an - `SCGI module `_. + `_ as well as an + `SCGI module `_. * `nginx `_ also supports `FastCGI `_. @@ -302,13 +309,13 @@ WSGIServer(app).run() This is a simple WSGI application, but you need to install `flup -`_ first, as flup handles the low level +`_ first, as flup handles the low level FastCGI access. .. seealso:: There is some documentation on `setting up Django with FastCGI - `_, most of + `_, most of which can be reused for other WSGI-compliant frameworks and libraries. Only the ``manage.py`` part has to be changed, the example used here can be used instead. Django does more or less the exact same thing. @@ -479,7 +486,7 @@ There are far more components than can be presented here. The Python wiki has a page about these components, called - `Web Components `_. + `Web Components `_. Templates @@ -515,13 +522,13 @@ * `Mako `_ * `Genshi `_ - * `Jinja `_ + * `Jinja `_ .. seealso:: There are many template engines competing for attention, because it is pretty easy to create them in Python. The page `Templating - `_ in the wiki lists a big, + `_ in the wiki lists a big, ever-growing number of these. The three listed above are considered "second generation" template engines and are a good place to start. @@ -571,11 +578,11 @@ .. seealso:: - * `Persistence Tools `_ lists + * `Persistence Tools `_ lists possibilities on how to save data in the file system. Some of these modules are part of the standard library - * `Database Programming `_ + * `Database Programming `_ helps with choosing a method for saving data * `SQLAlchemy `_, the most powerful OR-Mapper @@ -637,7 +644,7 @@ Django ^^^^^^ -`Django `_ is a framework consisting of several +`Django `_ is a framework consisting of several tightly coupled elements which were written from scratch and work together very well. It includes an ORM which is quite powerful while being simple to use, and has a great online administration interface which makes it possible to edit @@ -650,7 +657,7 @@ It has a big, international community, the members of which have created many web sites. There are also a lot of add-on projects which extend Django's normal functionality. This is partly due to Django's well written `online -documentation `_ and the `Django book +documentation `_ and the `Django book `_. @@ -658,7 +665,7 @@ Although Django is an MVC-style framework, it names the elements differently, which is described in the `Django FAQ - `_. + `_. TurboGears @@ -680,7 +687,7 @@ The newest version of TurboGears, version 2.0, moves even further in direction of WSGI support and a component-based architecture. TurboGears 2 is based on the WSGI stack of another popular component-based web framework, `Pylons -`_. +`_. Zope @@ -701,7 +708,7 @@ separate framework based on the Zope components: `Grok `_. -Zope is also the infrastructure used by the `Plone `_ content +Zope is also the infrastructure used by the `Plone `_ content management system, one of the most powerful and popular content management systems available. @@ -725,7 +732,9 @@ .. seealso:: The Python wiki contains an extensive list of `web frameworks - `_. + `_. Most frameworks also have their own mailing lists and IRC channels, look out - for these on the projects' web sites. + for these on the projects' web sites. There is also a general "Python in the + Web" IRC channel on freenode called `#python.web + `_. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-alternative-new-api.py --- a/Doc/includes/email-alternative-new-api.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -import smtplib - -from email.message import EmailMessage -from email.headerregistry import Address -from email.utils import make_msgid - -# Create the base text message. -msg = EmailMessage() -msg['Subject'] = "Ayons asperges pour le déjeuner" -msg['From'] = Address("Pepé Le Pew", "pepe@example.com") -msg['To'] = (Address("Penelope Pussycat", "penelope@example.com"), - Address("Fabrette Pussycat", "fabrette@example.com")) -msg.set_content("""\ -Salut! - -Cela ressemble à un excellent recipie[1] déjeuner. - -[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 - ---Pepé -""") - -# Add the html version. This converts the message into a multipart/alternative -# container, with the original text message as the first part and the new html -# message as the second part. -asparagus_cid = make_msgid() -msg.add_alternative("""\ - - - -

Salut!<\p> -

Cela ressemble à un excellent - - - -""".format(asparagus_cid=asparagus_cid[1:-1]), subtype='html') -# note that we needed to peel the <> off the msgid for use in the html. - -# Now add the related image to the html part. -with open("roasted-asparagus.jpg", 'rb') as img: - msg.get_payload()[1].add_related(img.read(), 'image', 'jpeg', - cid=asparagus_cid) - -# Make a local copy of what we are going to send. -with open('outgoing.msg', 'wb') as f: - f.write(bytes(msg)) - -# Send the message via local SMTP server. -with smtplib.SMTP('localhost') as s: - s.send_message(msg) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-alternative.py --- a/Doc/includes/email-alternative.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-alternative.py Mon Jan 25 17:05:13 2016 +0100 @@ -17,14 +17,14 @@ msg['To'] = you # Create the body of the message (a plain-text and an HTML version). -text = "Hi!\nHow are you?\nHere is the link you wanted:\nhttps://www.python.org" +text = "Hi!\nHow are you?\nHere is the link you wanted:\nhttp://www.python.org" html = """\

Hi!
How are you?
- Here is the link you wanted. + Here is the link you wanted.

diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-dir.py --- a/Doc/includes/email-dir.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-dir.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,7 +8,7 @@ # For guessing MIME type based on file name extension import mimetypes -from argparse import ArgumentParser +from optparse import OptionParser from email import encoders from email.message import Message @@ -22,36 +22,44 @@ def main(): - parser = ArgumentParser(description="""\ + parser = OptionParser(usage="""\ Send the contents of a directory as a MIME message. + +Usage: %prog [options] + Unless the -o option is given, the email is sent by forwarding to your local SMTP server, which then does the normal delivery process. Your local machine must be running an SMTP server. """) - parser.add_argument('-d', '--directory', - help="""Mail the contents of the specified directory, - otherwise use the current directory. Only the regular - files in the directory are sent, and we don't recurse to - subdirectories.""") - parser.add_argument('-o', '--output', - metavar='FILE', - help="""Print the composed message to FILE instead of - sending the message to the SMTP server.""") - parser.add_argument('-s', '--sender', required=True, - help='The value of the From: header (required)') - parser.add_argument('-r', '--recipient', required=True, - action='append', metavar='RECIPIENT', - default=[], dest='recipients', - help='A To: header value (at least one required)') - args = parser.parse_args() - directory = args.directory + parser.add_option('-d', '--directory', + type='string', action='store', + help="""Mail the contents of the specified directory, + otherwise use the current directory. Only the regular + files in the directory are sent, and we don't recurse to + subdirectories.""") + parser.add_option('-o', '--output', + type='string', action='store', metavar='FILE', + help="""Print the composed message to FILE instead of + sending the message to the SMTP server.""") + parser.add_option('-s', '--sender', + type='string', action='store', metavar='SENDER', + help='The value of the From: header (required)') + parser.add_option('-r', '--recipient', + type='string', action='append', metavar='RECIPIENT', + default=[], dest='recipients', + help='A To: header value (at least one required)') + opts, args = parser.parse_args() + if not opts.sender or not opts.recipients: + parser.print_help() + sys.exit(1) + directory = opts.directory if not directory: directory = '.' # Create the enclosing (outer) message outer = MIMEMultipart() outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory) - outer['To'] = COMMASPACE.join(args.recipients) - outer['From'] = args.sender + outer['To'] = COMMASPACE.join(opts.recipients) + outer['From'] = opts.sender outer.preamble = 'You will not see this in a MIME-aware mail reader.\n' for filename in os.listdir(directory): @@ -68,19 +76,23 @@ ctype = 'application/octet-stream' maintype, subtype = ctype.split('/', 1) if maintype == 'text': - with open(path) as fp: - # Note: we should handle calculating the charset - msg = MIMEText(fp.read(), _subtype=subtype) + fp = open(path) + # Note: we should handle calculating the charset + msg = MIMEText(fp.read(), _subtype=subtype) + fp.close() elif maintype == 'image': - with open(path, 'rb') as fp: - msg = MIMEImage(fp.read(), _subtype=subtype) + fp = open(path, 'rb') + msg = MIMEImage(fp.read(), _subtype=subtype) + fp.close() elif maintype == 'audio': - with open(path, 'rb') as fp: - msg = MIMEAudio(fp.read(), _subtype=subtype) + fp = open(path, 'rb') + msg = MIMEAudio(fp.read(), _subtype=subtype) + fp.close() else: - with open(path, 'rb') as fp: - msg = MIMEBase(maintype, subtype) - msg.set_payload(fp.read()) + fp = open(path, 'rb') + msg = MIMEBase(maintype, subtype) + msg.set_payload(fp.read()) + fp.close() # Encode the payload using Base64 encoders.encode_base64(msg) # Set the filename parameter @@ -88,12 +100,14 @@ outer.attach(msg) # Now send or store the message composed = outer.as_string() - if args.output: - with open(args.output, 'w') as fp: - fp.write(composed) + if opts.output: + fp = open(opts.output, 'w') + fp.write(composed) + fp.close() else: - with smtplib.SMTP('localhost') as s: - s.sendmail(args.sender, args.recipients, composed) + s = smtplib.SMTP('localhost') + s.sendmail(opts.sender, opts.recipients, composed) + s.quit() if __name__ == '__main__': diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-headers.py --- a/Doc/includes/email-headers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-headers.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,8 @@ # Import the email modules we'll need from email.parser import Parser -# If the e-mail headers are in a file, uncomment these two lines: -# with open(messagefile) as fp: -# headers = Parser().parse(fp) +# If the e-mail headers are in a file, uncomment this line: +#headers = Parser().parse(open(messagefile, 'r')) # Or for parsing headers in a string, use: headers = Parser().parsestr('From: \n' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-mime.py --- a/Doc/includes/email-mime.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-mime.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,8 +20,9 @@ for file in pngfiles: # Open the files in binary mode. Let the MIMEImage class automatically # guess the specific image type. - with open(file, 'rb') as fp: - img = MIMEImage(fp.read()) + fp = open(file, 'rb') + img = MIMEImage(fp.read()) + fp.close() msg.attach(img) # Send the email via our own SMTP server. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-read-alternative-new-api.py --- a/Doc/includes/email-read-alternative-new-api.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,75 +0,0 @@ -import os -import sys -import tempfile -import mimetypes -import webbrowser - -# Import the email modules we'll need -from email import policy -from email.parser import BytesParser - -# An imaginary module that would make this work and be safe. -from imaginary import magic_html_parser - -# In a real program you'd get the filename from the arguments. -with open('outgoing.msg', 'rb') as fp: - msg = BytesParser(policy=policy.default).parse(fp) - -# Now the header items can be accessed as a dictionary, and any non-ASCII will -# be converted to unicode: -print('To:', msg['to']) -print('From:', msg['from']) -print('Subject:', msg['subject']) - -# If we want to print a priview of the message content, we can extract whatever -# the least formatted payload is and print the first three lines. Of course, -# if the message has no plain text part printing the first three lines of html -# is probably useless, but this is just a conceptual example. -simplest = msg.get_body(preferencelist=('plain', 'html')) -print() -print(''.join(simplest.get_content().splitlines(keepends=True)[:3])) - -ans = input("View full message?") -if ans.lower()[0] == 'n': - sys.exit() - -# We can extract the richest alternative in order to display it: -richest = msg.get_body() -partfiles = {} -if richest['content-type'].maintype == 'text': - if richest['content-type'].subtype == 'plain': - for line in richest.get_content().splitlines(): - print(line) - sys.exit() - elif richest['content-type'].subtype == 'html': - body = richest - else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -elif richest['content-type'].content_type == 'multipart/related': - body = richest.get_body(preferencelist=('html')) - for part in richest.iter_attachments(): - fn = part.get_filename() - if fn: - extension = os.path.splitext(part.get_filename())[1] - else: - extension = mimetypes.guess_extension(part.get_content_type()) - with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as f: - f.write(part.get_content()) - # again strip the <> to go from email form of cid to html form. - partfiles[part['content-id'][1:-1]] = f.name -else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - # The magic_html_parser has to rewrite the href="cid:...." attributes to - # point to the filenames in partfiles. It also has to do a safety-sanitize - # of the html. It could be written using html.parser. - f.write(magic_html_parser(body.get_content(), partfiles)) -webbrowser.open(f.name) -os.remove(f.name) -for fn in partfiles.values(): - os.remove(fn) - -# Of course, there are lots of email messages that could break this simple -# minded program, but it will handle the most common ones. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-simple.py --- a/Doc/includes/email-simple.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-simple.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,9 +6,10 @@ # Open a plain text file for reading. For this example, assume that # the text file contains only ASCII characters. -with open(textfile) as fp: - # Create a text/plain message - msg = MIMEText(fp.read()) +fp = open(textfile, 'rb') +# Create a text/plain message +msg = MIMEText(fp.read()) +fp.close() # me == the sender's email address # you == the recipient's email address diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/email-unpack.py --- a/Doc/includes/email-unpack.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/email-unpack.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,27 +8,41 @@ import errno import mimetypes -from argparse import ArgumentParser +from optparse import OptionParser def main(): - parser = ArgumentParser(description="""\ + parser = OptionParser(usage="""\ Unpack a MIME message into a directory of files. + +Usage: %prog [options] msgfile """) - parser.add_argument('-d', '--directory', required=True, - help="""Unpack the MIME message into the named - directory, which will be created if it doesn't already - exist.""") - parser.add_argument('msgfile') - args = parser.parse_args() - - with open(args.msgfile) as fp: - msg = email.message_from_file(fp) + parser.add_option('-d', '--directory', + type='string', action='store', + help="""Unpack the MIME message into the named + directory, which will be created if it doesn't already + exist.""") + opts, args = parser.parse_args() + if not opts.directory: + parser.print_help() + sys.exit(1) try: - os.mkdir(args.directory) - except FileExistsError: - pass + msgfile = args[0] + except IndexError: + parser.print_help() + sys.exit(1) + + try: + os.mkdir(opts.directory) + except OSError as e: + # Ignore directory exists error + if e.errno != errno.EEXIST: + raise + + fp = open(msgfile) + msg = email.message_from_file(fp) + fp.close() counter = 1 for part in msg.walk(): @@ -45,8 +59,9 @@ ext = '.bin' filename = 'part-%03d%s' % (counter, ext) counter += 1 - with open(os.path.join(args.directory, filename), 'wb') as fp: - fp.write(part.get_payload(decode=True)) + fp = open(os.path.join(opts.directory, filename), 'wb') + fp.write(part.get_payload(decode=True)) + fp.close() if __name__ == '__main__': diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_benchmarks.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/includes/mp_benchmarks.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,243 @@ +# +# Simple benchmarks for the multiprocessing package +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + +import time +import sys +import multiprocessing +import threading +import queue +import gc + +if sys.platform == 'win32': + _timer = time.clock +else: + _timer = time.time + +delta = 1 + + +#### TEST_QUEUESPEED + +def queuespeed_func(q, c, iterations): + a = '0' * 256 + c.acquire() + c.notify() + c.release() + + for i in range(iterations): + q.put(a) + + q.put('STOP') + +def test_queuespeed(Process, q, c): + elapsed = 0 + iterations = 1 + + while elapsed < delta: + iterations *= 2 + + p = Process(target=queuespeed_func, args=(q, c, iterations)) + c.acquire() + p.start() + c.wait() + c.release() + + result = None + t = _timer() + + while result != 'STOP': + result = q.get() + + elapsed = _timer() - t + + p.join() + + print(iterations, 'objects passed through the queue in', elapsed, 'seconds') + print('average number/sec:', iterations/elapsed) + + +#### TEST_PIPESPEED + +def pipe_func(c, cond, iterations): + a = '0' * 256 + cond.acquire() + cond.notify() + cond.release() + + for i in range(iterations): + c.send(a) + + c.send('STOP') + +def test_pipespeed(): + c, d = multiprocessing.Pipe() + cond = multiprocessing.Condition() + elapsed = 0 + iterations = 1 + + while elapsed < delta: + iterations *= 2 + + p = multiprocessing.Process(target=pipe_func, + args=(d, cond, iterations)) + cond.acquire() + p.start() + cond.wait() + cond.release() + + result = None + t = _timer() + + while result != 'STOP': + result = c.recv() + + elapsed = _timer() - t + p.join() + + print(iterations, 'objects passed through connection in',elapsed,'seconds') + print('average number/sec:', iterations/elapsed) + + +#### TEST_SEQSPEED + +def test_seqspeed(seq): + elapsed = 0 + iterations = 1 + + while elapsed < delta: + iterations *= 2 + + t = _timer() + + for i in range(iterations): + a = seq[5] + + elapsed = _timer() - t + + print(iterations, 'iterations in', elapsed, 'seconds') + print('average number/sec:', iterations/elapsed) + + +#### TEST_LOCK + +def test_lockspeed(l): + elapsed = 0 + iterations = 1 + + while elapsed < delta: + iterations *= 2 + + t = _timer() + + for i in range(iterations): + l.acquire() + l.release() + + elapsed = _timer() - t + + print(iterations, 'iterations in', elapsed, 'seconds') + print('average number/sec:', iterations/elapsed) + + +#### TEST_CONDITION + +def conditionspeed_func(c, N): + c.acquire() + c.notify() + + for i in range(N): + c.wait() + c.notify() + + c.release() + +def test_conditionspeed(Process, c): + elapsed = 0 + iterations = 1 + + while elapsed < delta: + iterations *= 2 + + c.acquire() + p = Process(target=conditionspeed_func, args=(c, iterations)) + p.start() + + c.wait() + + t = _timer() + + for i in range(iterations): + c.notify() + c.wait() + + elapsed = _timer() - t + + c.release() + p.join() + + print(iterations * 2, 'waits in', elapsed, 'seconds') + print('average number/sec:', iterations * 2 / elapsed) + +#### + +def test(): + manager = multiprocessing.Manager() + + gc.disable() + + print('\n\t######## testing Queue.Queue\n') + test_queuespeed(threading.Thread, queue.Queue(), + threading.Condition()) + print('\n\t######## testing multiprocessing.Queue\n') + test_queuespeed(multiprocessing.Process, multiprocessing.Queue(), + multiprocessing.Condition()) + print('\n\t######## testing Queue managed by server process\n') + test_queuespeed(multiprocessing.Process, manager.Queue(), + manager.Condition()) + print('\n\t######## testing multiprocessing.Pipe\n') + test_pipespeed() + + print() + + print('\n\t######## testing list\n') + test_seqspeed(list(range(10))) + print('\n\t######## testing list managed by server process\n') + test_seqspeed(manager.list(list(range(10)))) + print('\n\t######## testing Array("i", ..., lock=False)\n') + test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=False)) + print('\n\t######## testing Array("i", ..., lock=True)\n') + test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=True)) + + print() + + print('\n\t######## testing threading.Lock\n') + test_lockspeed(threading.Lock()) + print('\n\t######## testing threading.RLock\n') + test_lockspeed(threading.RLock()) + print('\n\t######## testing multiprocessing.Lock\n') + test_lockspeed(multiprocessing.Lock()) + print('\n\t######## testing multiprocessing.RLock\n') + test_lockspeed(multiprocessing.RLock()) + print('\n\t######## testing lock managed by server process\n') + test_lockspeed(manager.Lock()) + print('\n\t######## testing rlock managed by server process\n') + test_lockspeed(manager.RLock()) + + print() + + print('\n\t######## testing threading.Condition\n') + test_conditionspeed(threading.Thread, threading.Condition()) + print('\n\t######## testing multiprocessing.Condition\n') + test_conditionspeed(multiprocessing.Process, multiprocessing.Condition()) + print('\n\t######## testing condition managed by a server process\n') + test_conditionspeed(multiprocessing.Process, manager.Condition()) + + gc.enable() + +if __name__ == '__main__': + multiprocessing.freeze_support() + test() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_newtype.py --- a/Doc/includes/mp_newtype.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/mp_newtype.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,11 @@ +# +# This module shows how to use arbitrary callables with a subclass of +# `BaseManager`. +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + from multiprocessing import freeze_support from multiprocessing.managers import BaseManager, BaseProxy import operator @@ -19,10 +27,12 @@ # Proxy type for generator objects class GeneratorProxy(BaseProxy): - _exposed_ = ['__next__'] + _exposed_ = ('next', '__next__') def __iter__(self): return self def __next__(self): + return self._callmethod('next') + def __next__(self): return self._callmethod('__next__') # Function to return the operator module @@ -80,6 +90,8 @@ op = manager.operator() print('op.add(23, 45) =', op.add(23, 45)) print('op.pow(2, 94) =', op.pow(2, 94)) + print('op.getslice(range(10), 2, 6) =', op.getslice(list(range(10)), 2, 6)) + print('op.repeat(range(5), 3) =', op.repeat(list(range(5)), 3)) print('op._exposed_ =', op._exposed_) ## diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_pool.py --- a/Doc/includes/mp_pool.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/mp_pool.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,10 @@ +# +# A test of `multiprocessing.Pool` class +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + import multiprocessing import time import random @@ -39,115 +46,269 @@ # def test(): + print('cpu_count() = %d\n' % multiprocessing.cpu_count()) + + # + # Create pool + # + PROCESSES = 4 print('Creating pool with %d processes\n' % PROCESSES) + pool = multiprocessing.Pool(PROCESSES) + print('pool = %s' % pool) + print() - with multiprocessing.Pool(PROCESSES) as pool: - # - # Tests - # + # + # Tests + # - TASKS = [(mul, (i, 7)) for i in range(10)] + \ - [(plus, (i, 8)) for i in range(10)] + TASKS = [(mul, (i, 7)) for i in range(10)] + \ + [(plus, (i, 8)) for i in range(10)] - results = [pool.apply_async(calculate, t) for t in TASKS] - imap_it = pool.imap(calculatestar, TASKS) - imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) + results = [pool.apply_async(calculate, t) for t in TASKS] + imap_it = pool.imap(calculatestar, TASKS) + imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) - print('Ordered results using pool.apply_async():') - for r in results: - print('\t', r.get()) - print() + print('Ordered results using pool.apply_async():') + for r in results: + print('\t', r.get()) + print() - print('Ordered results using pool.imap():') - for x in imap_it: - print('\t', x) - print() + print('Ordered results using pool.imap():') + for x in imap_it: + print('\t', x) + print() - print('Unordered results using pool.imap_unordered():') - for x in imap_unordered_it: - print('\t', x) - print() + print('Unordered results using pool.imap_unordered():') + for x in imap_unordered_it: + print('\t', x) + print() - print('Ordered results using pool.map() --- will block till complete:') - for x in pool.map(calculatestar, TASKS): - print('\t', x) - print() + print('Ordered results using pool.map() --- will block till complete:') + for x in pool.map(calculatestar, TASKS): + print('\t', x) + print() - # - # Test error handling - # + # + # Simple benchmarks + # - print('Testing error handling:') + N = 100000 + print('def pow3(x): return x**3') + t = time.time() + A = list(map(pow3, range(N))) + print('\tmap(pow3, range(%d)):\n\t\t%s seconds' % \ + (N, time.time() - t)) + + t = time.time() + B = pool.map(pow3, range(N)) + print('\tpool.map(pow3, range(%d)):\n\t\t%s seconds' % \ + (N, time.time() - t)) + + t = time.time() + C = list(pool.imap(pow3, range(N), chunksize=N//8)) + print('\tlist(pool.imap(pow3, range(%d), chunksize=%d)):\n\t\t%s' \ + ' seconds' % (N, N//8, time.time() - t)) + + assert A == B == C, (len(A), len(B), len(C)) + print() + + L = [None] * 1000000 + print('def noop(x): pass') + print('L = [None] * 1000000') + + t = time.time() + A = list(map(noop, L)) + print('\tmap(noop, L):\n\t\t%s seconds' % \ + (time.time() - t)) + + t = time.time() + B = pool.map(noop, L) + print('\tpool.map(noop, L):\n\t\t%s seconds' % \ + (time.time() - t)) + + t = time.time() + C = list(pool.imap(noop, L, chunksize=len(L)//8)) + print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \ + (len(L)//8, time.time() - t)) + + assert A == B == C, (len(A), len(B), len(C)) + print() + + del A, B, C, L + + # + # Test error handling + # + + print('Testing error handling:') + + try: + print(pool.apply(f, (5,))) + except ZeroDivisionError: + print('\tGot ZeroDivisionError as expected from pool.apply()') + else: + raise AssertionError('expected ZeroDivisionError') + + try: + print(pool.map(f, list(range(10)))) + except ZeroDivisionError: + print('\tGot ZeroDivisionError as expected from pool.map()') + else: + raise AssertionError('expected ZeroDivisionError') + + try: + print(list(pool.imap(f, list(range(10))))) + except ZeroDivisionError: + print('\tGot ZeroDivisionError as expected from list(pool.imap())') + else: + raise AssertionError('expected ZeroDivisionError') + + it = pool.imap(f, list(range(10))) + for i in range(10): try: - print(pool.apply(f, (5,))) + x = next(it) except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.apply()') + if i == 5: + pass + except StopIteration: + break else: - raise AssertionError('expected ZeroDivisionError') + if i == 5: + raise AssertionError('expected ZeroDivisionError') + assert i == 9 + print('\tGot ZeroDivisionError as expected from IMapIterator.next()') + print() + + # + # Testing timeouts + # + + print('Testing ApplyResult.get() with timeout:', end=' ') + res = pool.apply_async(calculate, TASKS[0]) + while 1: + sys.stdout.flush() try: - print(pool.map(f, list(range(10)))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.map()') - else: - raise AssertionError('expected ZeroDivisionError') + sys.stdout.write('\n\t%s' % res.get(0.02)) + break + except multiprocessing.TimeoutError: + sys.stdout.write('.') + print() + print() + print('Testing IMapIterator.next() with timeout:', end=' ') + it = pool.imap(calculatestar, TASKS) + while 1: + sys.stdout.flush() try: - print(list(pool.imap(f, list(range(10))))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from list(pool.imap())') - else: - raise AssertionError('expected ZeroDivisionError') + sys.stdout.write('\n\t%s' % it.next(0.02)) + except StopIteration: + break + except multiprocessing.TimeoutError: + sys.stdout.write('.') + print() + print() - it = pool.imap(f, list(range(10))) - for i in range(10): - try: - x = next(it) - except ZeroDivisionError: - if i == 5: - pass - except StopIteration: - break - else: - if i == 5: - raise AssertionError('expected ZeroDivisionError') + # + # Testing callback + # - assert i == 9 - print('\tGot ZeroDivisionError as expected from IMapIterator.next()') - print() + print('Testing callback:') - # - # Testing timeouts - # + A = [] + B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729] - print('Testing ApplyResult.get() with timeout:', end=' ') - res = pool.apply_async(calculate, TASKS[0]) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % res.get(0.02)) - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() + r = pool.apply_async(mul, (7, 8), callback=A.append) + r.wait() - print('Testing IMapIterator.next() with timeout:', end=' ') - it = pool.imap(calculatestar, TASKS) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % it.next(0.02)) - except StopIteration: - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() + r = pool.map_async(pow3, list(range(10)), callback=A.extend) + r.wait() + + if A == B: + print('\tcallbacks succeeded\n') + else: + print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B)) + + # + # Check there are no outstanding tasks + # + + assert not pool._cache, 'cache = %r' % pool._cache + + # + # Check close() methods + # + + print('Testing close():') + + for worker in pool._pool: + assert worker.is_alive() + + result = pool.apply_async(time.sleep, [0.5]) + pool.close() + pool.join() + + assert result.get() is None + + for worker in pool._pool: + assert not worker.is_alive() + + print('\tclose() succeeded\n') + + # + # Check terminate() method + # + + print('Testing terminate():') + + pool = multiprocessing.Pool(2) + DELTA = 0.1 + ignore = pool.apply(pow3, [2]) + results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)] + pool.terminate() + pool.join() + + for worker in pool._pool: + assert not worker.is_alive() + + print('\tterminate() succeeded\n') + + # + # Check garbage collection + # + + print('Testing garbage collection:') + + pool = multiprocessing.Pool(2) + DELTA = 0.1 + processes = pool._pool + ignore = pool.apply(pow3, [2]) + results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)] + + results = pool = None + + time.sleep(DELTA * 2) + + for worker in processes: + assert not worker.is_alive() + + print('\tgarbage collection succeeded\n') if __name__ == '__main__': multiprocessing.freeze_support() + + assert len(sys.argv) in (1, 2) + + if len(sys.argv) == 1 or sys.argv[1] == 'processes': + print(' Using processes '.center(79, '-')) + elif sys.argv[1] == 'threads': + print(' Using threads '.center(79, '-')) + import multiprocessing.dummy as multiprocessing + else: + print('Usage:\n\t%s [processes | threads]' % sys.argv[0]) + raise SystemExit(2) + test() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_synchronize.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/includes/mp_synchronize.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,278 @@ +# +# A test file for the `multiprocessing` package +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + +import time +import sys +import random +from queue import Empty + +import multiprocessing # may get overwritten + + +#### TEST_VALUE + +def value_func(running, mutex): + random.seed() + time.sleep(random.random()*4) + + mutex.acquire() + print('\n\t\t\t' + str(multiprocessing.current_process()) + ' has finished') + running.value -= 1 + mutex.release() + +def test_value(): + TASKS = 10 + running = multiprocessing.Value('i', TASKS) + mutex = multiprocessing.Lock() + + for i in range(TASKS): + p = multiprocessing.Process(target=value_func, args=(running, mutex)) + p.start() + + while running.value > 0: + time.sleep(0.08) + mutex.acquire() + print(running.value, end=' ') + sys.stdout.flush() + mutex.release() + + print() + print('No more running processes') + + +#### TEST_QUEUE + +def queue_func(queue): + for i in range(30): + time.sleep(0.5 * random.random()) + queue.put(i*i) + queue.put('STOP') + +def test_queue(): + q = multiprocessing.Queue() + + p = multiprocessing.Process(target=queue_func, args=(q,)) + p.start() + + o = None + while o != 'STOP': + try: + o = q.get(timeout=0.3) + print(o, end=' ') + sys.stdout.flush() + except Empty: + print('TIMEOUT') + + print() + + +#### TEST_CONDITION + +def condition_func(cond): + cond.acquire() + print('\t' + str(cond)) + time.sleep(2) + print('\tchild is notifying') + print('\t' + str(cond)) + cond.notify() + cond.release() + +def test_condition(): + cond = multiprocessing.Condition() + + p = multiprocessing.Process(target=condition_func, args=(cond,)) + print(cond) + + cond.acquire() + print(cond) + cond.acquire() + print(cond) + + p.start() + + print('main is waiting') + cond.wait() + print('main has woken up') + + print(cond) + cond.release() + print(cond) + cond.release() + + p.join() + print(cond) + + +#### TEST_SEMAPHORE + +def semaphore_func(sema, mutex, running): + sema.acquire() + + mutex.acquire() + running.value += 1 + print(running.value, 'tasks are running') + mutex.release() + + random.seed() + time.sleep(random.random()*2) + + mutex.acquire() + running.value -= 1 + print('%s has finished' % multiprocessing.current_process()) + mutex.release() + + sema.release() + +def test_semaphore(): + sema = multiprocessing.Semaphore(3) + mutex = multiprocessing.RLock() + running = multiprocessing.Value('i', 0) + + processes = [ + multiprocessing.Process(target=semaphore_func, + args=(sema, mutex, running)) + for i in range(10) + ] + + for p in processes: + p.start() + + for p in processes: + p.join() + + +#### TEST_JOIN_TIMEOUT + +def join_timeout_func(): + print('\tchild sleeping') + time.sleep(5.5) + print('\n\tchild terminating') + +def test_join_timeout(): + p = multiprocessing.Process(target=join_timeout_func) + p.start() + + print('waiting for process to finish') + + while 1: + p.join(timeout=1) + if not p.is_alive(): + break + print('.', end=' ') + sys.stdout.flush() + + +#### TEST_EVENT + +def event_func(event): + print('\t%r is waiting' % multiprocessing.current_process()) + event.wait() + print('\t%r has woken up' % multiprocessing.current_process()) + +def test_event(): + event = multiprocessing.Event() + + processes = [multiprocessing.Process(target=event_func, args=(event,)) + for i in range(5)] + + for p in processes: + p.start() + + print('main is sleeping') + time.sleep(2) + + print('main is setting event') + event.set() + + for p in processes: + p.join() + + +#### TEST_SHAREDVALUES + +def sharedvalues_func(values, arrays, shared_values, shared_arrays): + for i in range(len(values)): + v = values[i][1] + sv = shared_values[i].value + assert v == sv + + for i in range(len(values)): + a = arrays[i][1] + sa = list(shared_arrays[i][:]) + assert a == sa + + print('Tests passed') + +def test_sharedvalues(): + values = [ + ('i', 10), + ('h', -2), + ('d', 1.25) + ] + arrays = [ + ('i', list(range(100))), + ('d', [0.25 * i for i in range(100)]), + ('H', list(range(1000))) + ] + + shared_values = [multiprocessing.Value(id, v) for id, v in values] + shared_arrays = [multiprocessing.Array(id, a) for id, a in arrays] + + p = multiprocessing.Process( + target=sharedvalues_func, + args=(values, arrays, shared_values, shared_arrays) + ) + p.start() + p.join() + + assert p.exitcode == 0 + + +#### + +def test(namespace=multiprocessing): + global multiprocessing + + multiprocessing = namespace + + for func in [test_value, test_queue, test_condition, + test_semaphore, test_join_timeout, test_event, + test_sharedvalues]: + + print('\n\t######## %s\n' % func.__name__) + func() + + ignore = multiprocessing.active_children() # cleanup any old processes + if hasattr(multiprocessing, '_debug_info'): + info = multiprocessing._debug_info() + if info: + print(info) + raise ValueError('there should be no positive refcounts left') + + +if __name__ == '__main__': + multiprocessing.freeze_support() + + assert len(sys.argv) in (1, 2) + + if len(sys.argv) == 1 or sys.argv[1] == 'processes': + print(' Using processes '.center(79, '-')) + namespace = multiprocessing + elif sys.argv[1] == 'manager': + print(' Using processes and a manager '.center(79, '-')) + namespace = multiprocessing.Manager() + namespace.Process = multiprocessing.Process + namespace.current_process = multiprocessing.current_process + namespace.active_children = multiprocessing.active_children + elif sys.argv[1] == 'threads': + print(' Using threads '.center(79, '-')) + import multiprocessing.dummy as namespace + else: + print('Usage:\n\t%s [processes | manager | threads]' % sys.argv[0]) + raise SystemExit(2) + + test(namespace) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_webserver.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/includes/mp_webserver.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,70 @@ +# +# Example where a pool of http servers share a single listening socket +# +# On Windows this module depends on the ability to pickle a socket +# object so that the worker processes can inherit a copy of the server +# object. (We import `multiprocessing.reduction` to enable this pickling.) +# +# Not sure if we should synchronize access to `socket.accept()` method by +# using a process-shared lock -- does not seem to be necessary. +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + +import os +import sys + +from multiprocessing import Process, current_process, freeze_support +from http.server import HTTPServer +from http.server import SimpleHTTPRequestHandler + +if sys.platform == 'win32': + import multiprocessing.reduction # make sockets pickable/inheritable + + +def note(format, *args): + sys.stderr.write('[%s]\t%s\n' % (current_process().name, format % args)) + + +class RequestHandler(SimpleHTTPRequestHandler): + # we override log_message() to show which process is handling the request + def log_message(self, format, *args): + note(format, *args) + +def serve_forever(server): + note('starting server') + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +def runpool(address, number_of_processes): + # create a single server object -- children will each inherit a copy + server = HTTPServer(address, RequestHandler) + + # create child processes to act as workers + for i in range(number_of_processes - 1): + Process(target=serve_forever, args=(server,)).start() + + # main process also acts as a worker + serve_forever(server) + + +def test(): + DIR = os.path.join(os.path.dirname(__file__), '..') + ADDRESS = ('localhost', 8000) + NUMBER_OF_PROCESSES = 4 + + print('Serving at http://%s:%d using %d worker processes' % \ + (ADDRESS[0], ADDRESS[1], NUMBER_OF_PROCESSES)) + print('To exit press Ctrl-' + ['C', 'Break'][sys.platform=='win32']) + + os.chdir(DIR) + runpool(ADDRESS, NUMBER_OF_PROCESSES) + + +if __name__ == '__main__': + freeze_support() + test() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/mp_workers.py --- a/Doc/includes/mp_workers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/mp_workers.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,16 @@ +# +# Simple example which uses a pool of workers to carry out some tasks. +# +# Notice that the results will probably not come out of the output +# queue in the same in the same order as the corresponding tasks were +# put on the input queue. If it is important to get the results back +# in the original order then consider using `Pool.map()` or +# `Pool.imap()` (which will save on the amount of code needed anyway). +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# + import time import random diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/noddy.c --- a/Doc/includes/noddy.c Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/noddy.c Mon Jan 25 17:05:13 2016 +0100 @@ -38,7 +38,7 @@ }; PyMODINIT_FUNC -PyInit_noddy(void) +PyInit_noddy(void) { PyObject* m; diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/run-func.c --- a/Doc/includes/run-func.c Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/run-func.c Mon Jan 25 17:05:13 2016 +0100 @@ -13,7 +13,7 @@ } Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); + pName = PyUnicode_FromString(argv[1]); /* Error checking of pName left out */ pModule = PyImport_Import(pName); @@ -63,8 +63,6 @@ fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); return 1; } - if (Py_FinalizeEx() < 0) { - return 120; - } + Py_Finalize(); return 0; } diff -r 6db40a9955dc -r 0d413f60cc23 Doc/includes/typestruct.h --- a/Doc/includes/typestruct.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/includes/typestruct.h Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,7 @@ typedef struct _typeobject { PyObject_VAR_HEAD - const char *tp_name; /* For printing, in format "." */ - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ + char *tp_name; /* For printing, in format "." */ + int tp_basicsize, tp_itemsize; /* For allocation */ /* Methods to implement standard operations */ @@ -9,8 +9,7 @@ printfunc tp_print; getattrfunc tp_getattr; setattrfunc tp_setattr; - PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) - or tp_reserved (Python 3) */ + void *tp_reserved; reprfunc tp_repr; /* Method suites for standard classes */ @@ -31,9 +30,9 @@ PyBufferProcs *tp_as_buffer; /* Flags to define presence of optional/expanded features */ - unsigned long tp_flags; + long tp_flags; - const char *tp_doc; /* Documentation string */ + char *tp_doc; /* Documentation string */ /* call function for all accessible objects */ traverseproc tp_traverse; @@ -45,7 +44,7 @@ richcmpfunc tp_richcompare; /* weak reference enabler */ - Py_ssize_t tp_weaklistoffset; + long tp_weaklistoffset; /* Iterators */ getiterfunc tp_iter; @@ -59,7 +58,7 @@ PyObject *tp_dict; descrgetfunc tp_descr_get; descrsetfunc tp_descr_set; - Py_ssize_t tp_dictoffset; + long tp_dictoffset; initproc tp_init; allocfunc tp_alloc; newfunc tp_new; @@ -70,11 +69,5 @@ PyObject *tp_cache; PyObject *tp_subclasses; PyObject *tp_weaklist; - destructor tp_del; - - /* Type attribute cache version tag. Added in version 2.6 */ - unsigned int tp_version_tag; - - destructor tp_finalize; } PyTypeObject; diff -r 6db40a9955dc -r 0d413f60cc23 Doc/install/index.rst --- a/Doc/install/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/install/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,12 +1,12 @@ -.. highlightlang:: none +.. _packaging-install-index: -.. _install-index: +****************************** + Installing Python Projects +****************************** -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward +:Author: The Fellowship of the Packaging +:Release: |version| +:Date: |today| .. TODO: Fill in XXX comments @@ -14,1087 +14,43 @@ about Python and aren't about to learn the language just in order to install and maintain it for their users, i.e. system administrators. Thus, I have to be sure to explain the basics at some point: - sys.path and PYTHONPATH at least. Should probably give pointers to + sys.path and PYTHONPATH at least. Should probably give pointers to other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. Finally, it might be useful to include all the material from my "Care - and Feeding of a Python Installation" talk in here somewhere. Yow! + and Feeding of a Python Installation" talk in here somewhere. Yow! -This document describes the Python Distribution Utilities ("Distutils") from the -end-user's point-of-view, describing how to extend the capabilities of a -standard Python installation by building and installing third-party Python -modules and extensions. +.. topic:: Abstract + This document describes Packaging from the end-user's point of view: it + explains how to extend the functionality of a standard Python installation by + building and installing third-party Python modules and applications. -.. note:: - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. +This guide is split into a simple overview followed by a longer presentation of +the :program:`pysetup` script, the Python package management tool used to +build, distribute, search for, install, remove and list Python distributions. +.. TODO integrate install and pysetup instead of duplicating -.. _inst-intro: +.. toctree:: + :maxdepth: 2 + :numbered: - -Introduction -============ - -Although Python's extensive standard library covers many programming needs, -there often comes a time when you need to add some new functionality to your -Python installation in the form of third-party modules. This might be necessary -to support your own programming, or to support an application that you want to -use and that happens to be written in Python. - -In the past, there has been little support for adding third-party modules to an -existing Python installation. With the introduction of the Python Distribution -Utilities (Distutils for short) in Python 2.0, this changed. - -This document is aimed primarily at the people who need to install third-party -Python modules: end-users and system administrators who just need to get some -Python application running, and existing Python programmers who want to add some -new goodies to their toolbox. You don't need to know Python to read this -document; there will be some brief forays into using Python's interactive mode -to explore your installation, but that's it. If you're looking for information -on how to distribute your own Python modules so that others may use them, see -the :ref:`distutils-index` manual. :ref:`debug-setup-script` may also be of -interest. - - -.. _inst-trivial-install: - -Best case: trivial installation -------------------------------- - -In the best case, someone will have prepared a special version of the module -distribution you want to install that is targeted specifically at your platform -and is installed just like any other software on your platform. For example, -the module developer might make an executable installer available for Windows -users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, -Mandrake, and many others), a Debian package for users of Debian-based Linux -systems, and so forth. - -In that case, you would download the installer appropriate to your platform and -do the obvious thing with it: run it if it's an executable installer, ``rpm ---install`` it if it's an RPM, etc. You don't need to run Python or a setup -script, you don't need to compile anything---you might not even need to read any -instructions (although it's always a good idea to do so anyway). - -Of course, things will not always be that easy. You might be interested in a -module distribution that doesn't have an easy-to-use installer for your -platform. In that case, you'll have to start with the source distribution -released by the module's author/maintainer. Installing from a source -distribution is not too hard, as long as the modules are packaged in the -standard way. The bulk of this document is about building and installing -modules from standard source distributions. - - -.. _inst-new-standard: - -The new standard: Distutils ---------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and Mac OS X, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and Mac OS X. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`:: - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :data:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :data:`site.USER_SITE`). -Here are the values for UNIX, including Mac OS X: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`--prefix` option defines the installation base, and -the :option:`--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to -:option:`--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`--prefix` or :option:`--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`--prefix` -and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`--prefix` option; the -:option:`--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the same--- -they only differ when you use the Unix "prefix scheme" and supply different -``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will -override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme," where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation directory ----you just have to remember that there are two types of modules to worry about, -Python and extension modules, which can conveniently be both controlled by one -option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`):: - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, :: - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it:: - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same major version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and Mac OS X, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the `--no-user-cfg` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default:: - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following:: - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or - :option:`-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`-o32` option, and the linker will be passed -:option:`-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState Web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) + install + pysetup + pysetup-config + pysetup-servers .. seealso:: - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. + :ref:`packaging-index` + The manual for developers of Python projects who want to package and + distribute them. This describes how to use :mod:`packaging` to make + projects easily found and added to an existing Python installation. - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -http://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check http://www.sourceware.org/cygwin/ and http://www.mingw.org/ for more - information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. + :mod:`packaging` + A library reference for developers of packaging tools wanting to use + standalone building blocks like :mod:`~packaging.version` or + :mod:`~packaging.metadata`, or extend Packaging itself. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/install/install.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/install/install.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1119 @@ +.. highlightlang:: none + +==================================== +Installing Python projects: overview +==================================== + +.. _packaging-install-intro: + +Introduction +============ + +Although Python's extensive standard library covers many programming needs, +there often comes a time when you need to add new functionality to your Python +installation in the form of third-party modules. This might be necessary to +support your own programming, or to support an application that you want to use +and that happens to be written in Python. + +In the past, there was little support for adding third-party modules to an +existing Python installation. With the introduction of the Python Distribution +Utilities (Distutils for short) in Python 2.0, this changed. However, not all +problems were solved; end-users had to rely on ``easy_install`` or +``pip`` to download third-party modules from PyPI, uninstall distributions or do +other maintenance operations. Packaging is a more complete replacement for +Distutils, in the standard library, with a backport named Distutils2 available +for older Python versions. + +This document is aimed primarily at people who need to install third-party +Python modules: end-users and system administrators who just need to get some +Python application running, and existing Python programmers who want to add +new goodies to their toolbox. You don't need to know Python to read this +document; there will be some brief forays into using Python's interactive mode +to explore your installation, but that's it. If you're looking for information +on how to distribute your own Python modules so that others may use them, see +the :ref:`packaging-index` manual. + + +.. _packaging-trivial-install: + +Best case: trivial installation +------------------------------- + +In the best case, someone will have prepared a special version of the module +distribution you want to install that is targeted specifically at your platform +and can be installed just like any other software on your platform. For example, +the module's developer might make an executable installer available for Windows +users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, +Mandrake, and many others), a Debian package for users of Debian and derivative +systems, and so forth. + +In that case, you would use the standard system tools to download and install +the specific installer for your platform and its dependencies. + +Of course, things will not always be that easy. You might be interested in a +module whose distribution doesn't have an easy-to-use installer for your +platform. In that case, you'll have to start with the source distribution +released by the module's author/maintainer. Installing from a source +distribution is not too hard, as long as the modules are packaged in the +standard way. The bulk of this document addresses the building and installing +of modules from standard source distributions. + + +.. _packaging-distutils: + +The Python standard: Distutils +------------------------------ + +If you download a source distribution of a module, it will be obvious whether +it was packaged and distributed using Distutils. First, the distribution's name +and version number will be featured prominently in the name of the downloaded +archive, e.g. :file:`foo-1.0.tar.gz` or :file:`widget-0.9.7.zip`. Next, the +archive will unpack into a similarly-named directory: :file:`foo-1.0` or +:file:`widget-0.9.7`. Additionally, the distribution may contain a +:file:`setup.cfg` file and a file named :file:`README.txt` ---or possibly just +:file:`README`--- explaining that building and installing the module +distribution is a simple matter of issuing the following command at your shell's +prompt:: + + python setup.py install + +Third-party projects have extended Distutils to work around its limitations or +add functionality. After some years of near-inactivity in Distutils, a new +maintainer has started to standardize good ideas in PEPs and implement them in a +new, improved version of Distutils, called Distutils2 or Packaging. + + +.. _packaging-new-standard: + +The new standard: Packaging +--------------------------- + +The rules described in the first paragraph above apply to Packaging-based +projects too: a source distribution will have a name like +:file:`widget-0.9.7.zip`. One of the main differences with Distutils is that +distributions no longer have a :file:`setup.py` script; it used to cause a +number of issues. Now there is a unique script installed with Python itself:: + + pysetup install widget-0.9.7.zip + +Running this command is enough to build and install projects (Python modules or +packages, scripts or whole applications), without even having to unpack the +archive. It is also compatible with Distutils-based distributions. + +Unless you have to perform non-standard installations or customize the build +process, you can stop reading this manual ---the above command is everything you +need to get out of it. + +With :program:`pysetup`, you won't even have to manually download a distribution +before installing it; see :ref:`packaging-pysetup`. + + +.. _packaging-standard-install: + +Standard build and install +========================== + +As described in section :ref:`packaging-new-standard`, building and installing +a module distribution using Packaging usually comes down to one simple +command:: + + pysetup run install_dist + +This is a command that should be run in a terminal. On Windows, it is called a +command prompt and found in :menuselection:`Start --> Accessories`; Powershell +is a popular alternative. + + +.. _packaging-platform-variations: + +Platform variations +------------------- + +The setup command is meant to be run from the root directory of the source +distribution, i.e. the top-level subdirectory that the module source +distribution unpacks into. For example, if you've just downloaded a module +source distribution :file:`foo-1.0.tar.gz` onto a Unix system, the normal +steps to follow are these:: + + gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 + cd foo-1.0 + pysetup run install_dist + +On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the +archive file to :file:`C:\\Temp`, then it would unpack into +:file:`C:\\Temp\\foo-1.0`. To actually unpack the archive, you can use either +an archive manipulator with a graphical user interface (such as WinZip or 7-Zip) +or a command-line tool (such as :program:`unzip`, :program:`pkunzip` or, again, +:program:`7z`). Then, open a command prompt window and run:: + + cd c:\Temp\foo-1.0 + pysetup run install_dist + + +.. _packaging-splitting-up: + +Splitting the job up +-------------------- + +Running ``pysetup run install_dist`` builds and installs all modules in one go. If you +prefer to work incrementally ---especially useful if you want to customize the +build process, or if things are going wrong--- you can use the setup script to +do one thing at a time. This is a valuable tool when different users will perform +separately the build and install steps. For example, you might want to build a +module distribution and hand it off to a system administrator for installation +(or do it yourself, but with super-user or admin privileges). + +For example, to build everything in one step and then install everything +in a second step, you aptly invoke two distinct Packaging commands:: + + pysetup run build + pysetup run install_dist + +If you do this, you will notice that invoking the :command:`install_dist` command +first runs the :command:`build` command, which ---in this case--- quickly +notices it can spare itself the work, since everything in the :file:`build` +directory is up-to-date. + +You may often ignore this ability to divide the process in steps if all you do +is installing modules downloaded from the Internet, but it's very handy for +more advanced tasks. If you find yourself in the need for distributing your own +Python modules and extensions, though, you'll most likely run many individual +Packaging commands. + + +.. _packaging-how-build-works: + +How building works +------------------ + +As implied above, the :command:`build` command is responsible for collecting +and placing the files to be installed into a *build directory*. By default, +this is :file:`build`, under the distribution root. If you're excessively +concerned with speed, or want to keep the source tree pristine, you can specify +a different build directory with the :option:`--build-base` option. For example:: + + pysetup run build --build-base /tmp/pybuild/foo-1.0 + +(Or you could do this permanently with a directive in your system or personal +Packaging configuration file; see section :ref:`packaging-config-files`.) +In the usual case, however, all this is unnecessary. + +The build tree's default layout looks like so:: + + --- build/ --- lib/ + or + --- build/ --- lib./ + temp./ + +where ```` expands to a brief description of the current OS/hardware +platform and Python version. The first form, with just a :file:`lib` directory, +is used for pure module distributions (module distributions that +include only pure Python modules). If a module distribution contains any +extensions (modules written in C/C++), then the second form, with two ```` +directories, is used. In that case, the :file:`temp.{plat}` directory holds +temporary files generated during the compile/link process which are not intended +to be installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory +contains all Python modules (pure Python and extensions) to be installed. + +In the future, more directories will be added to handle Python scripts, +documentation, binary executables, and whatever else is required to install +Python modules and applications. + + +.. _packaging-how-install-works: + +How installation works +---------------------- + +After the :command:`build` command is run (whether explicitly or by the +:command:`install_dist` command on your behalf), the work of the :command:`install_dist` +command is relatively simple: all it has to do is copy the contents of +:file:`build/lib` (or :file:`build/lib.{plat}`) to the installation directory +of your choice. + +If you don't choose an installation directory ---i.e., if you just run +``pysetup run install_dist``\ --- then the :command:`install_dist` command +installs to the standard location for third-party Python modules. This location +varies by platform and depending on how you built/installed Python itself. On +Unix (and Mac OS X, which is also Unix-based), it also depends on whether the +module distribution being installed is pure Python or contains extensions +("non-pure"): + ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Platform | Standard installation location | Default value | Notes | ++=================+=====================================================+==================================================+=======+ +| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ + +Notes: + +(1) + Most Linux distributions include Python as a standard part of the system, so + :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on + Linux. If you build Python yourself on Linux (or any Unix-like system), the + default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. + +(2) + The default installation directory on Windows was :file:`C:\\Program + Files\\Python` under Python 1.6a1, 1.5.2, and earlier. + +:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python +is installed to, and where it finds its libraries at run-time. They are always +the same under Windows, and very often the same under Unix and Mac OS X. You +can find out what your Python installation uses for :file:`{prefix}` and +:file:`{exec-prefix}` by running Python in interactive mode and typing a few +simple commands. + +.. TODO link to Doc/using instead of duplicating + +To start the interactive Python interpreter, you need to follow a slightly +different recipe for each platform. Under Unix, just type :command:`python` at +the shell prompt. Under Windows (assuming the Python executable is on your +:envvar:`PATH`, which is the usual case), you can choose :menuselection:`Start --> Run`, +type ``python`` and press ``enter``. Alternatively, you can simply execute +:command:`python` at a command prompt (:menuselection:`Start --> Accessories`) +or in Powershell. + +Once the interpreter is started, you type Python code at the prompt. For +example, on my Linux system, I type the three Python statements shown below, +and get the output as shown, to find out my :file:`{prefix}` and :file:`{exec-prefix}`:: + + Python 3.3 (r32:88445, Apr 2 2011, 10:43:54) + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.prefix + '/usr' + >>> sys.exec_prefix + '/usr' + +A few other placeholders are used in this document: :file:`{X.Y}` stands for the +version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by +the value of :data:`sys.abiflags` or the empty string for platforms which don't +define ABI flags; :file:`{distname}` will be replaced by the name of the module +distribution being installed. Dots and capitalization are important in the +paths; for example, a value that uses ``python3.2`` on UNIX will typically use +``Python32`` on Windows. + +If you don't want to install modules to the standard location, or if you don't +have permission to write there, then you need to read about alternate +installations in section :ref:`packaging-alt-install`. If you want to customize your +installation directories more heavily, see section :ref:`packaging-custom-install`. + + +.. _packaging-alt-install: + +Alternate installation +====================== + +Often, it is necessary or desirable to install modules to a location other than +the standard location for third-party Python modules. For example, on a Unix +system you might not have permission to write to the standard third-party module +directory. Or you might wish to try out a module before making it a standard +part of your local Python installation. This is especially true when upgrading +a distribution already present: you want to make sure your existing base of +scripts still works with the new version before actually upgrading. + +The Packaging :command:`install_dist` command is designed to make installing module +distributions to an alternate location simple and painless. The basic idea is +that you supply a base directory for the installation, and the +:command:`install_dist` command picks a set of directories (called an *installation +scheme*) under this base directory in which to install files. The details +differ across platforms, so read whichever of the following sections applies to +you. + +Note that the various alternate installation schemes are mutually exclusive: you +can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or +``--install-base`` and ``--install-platbase``, but you can't mix from these +groups. + + +.. _packaging-alt-install-user: + +Alternate installation: the user scheme +--------------------------------------- + +This scheme is designed to be the most convenient solution for users that don't +have write permission to the global site-packages directory or don't want to +install into it. It is enabled with a simple option:: + + pysetup run install_dist --user + +Files will be installed into subdirectories of :data:`site.USER_BASE` (written +as :file:`{userbase}` hereafter). This scheme installs pure Python modules and +extension modules in the same location (also known as :data:`site.USER_SITE`). +Here are the values for UNIX, including non-framework builds on Mac OS X: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{userbase}/lib/python{X.Y}/site-packages` +scripts :file:`{userbase}/bin` +data :file:`{userbase}` +C headers :file:`{userbase}/include/python{X.Y}` +=============== =========================================================== + +Framework builds on Mac OS X use these paths: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{userbase}/lib/python/site-packages` +scripts :file:`{userbase}/bin` +data :file:`{userbase}` +C headers :file:`{userbase}/include/python` +=============== =========================================================== + +And here are the values used on Windows: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{userbase}\\Python{XY}\\site-packages` +scripts :file:`{userbase}\\Scripts` +data :file:`{userbase}` +C headers :file:`{userbase}\\Python{XY}\\Include` +=============== =========================================================== + +The advantage of using this scheme compared to the other ones described below is +that the user site-packages directory is under normal conditions always included +in :data:`sys.path` (see :mod:`site` for more information), which means that +there is no additional step to perform after running ``pysetup`` to finalize the +installation. + +The :command:`build_ext` command also has a ``--user`` option to add +:file:`{userbase}/include` to the compiler search path for header files and +:file:`{userbase}/lib` to the compiler search path for libraries as well as to +the runtime search path for shared C libraries (rpath). + + +.. _packaging-alt-install-home: + +Alternate installation: the home scheme +--------------------------------------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the concept of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +In spite of its name's origin, this scheme can be used by anyone, regardless +of the operating system. + +Installing a new module distribution in this way is as simple as :: + + pysetup run install_dist --home + +where you can supply any directory you like for the :option:`--home` option. On +Unix, lazy typists can just type a tilde (``~``); the :command:`install_dist` command +will expand this to your home directory:: + + pysetup run install_dist --home ~ + +To make Python find the distributions installed with this scheme, you may have +to :ref:`modify Python's search path ` or edit +:mod:`sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit +:data:`sys.path`. + +The :option:`--home` option defines the base directory for the installation. +Under it, files are installed to the following directories: + +=============== =========================================================== +Type of file Installation directory +=============== =========================================================== +modules :file:`{home}/lib/python` +scripts :file:`{home}/bin` +data :file:`{home}` +C headers :file:`{home}/include/python` +=============== =========================================================== + +(Mentally replace slashes with backslashes if you're on Windows.) + + +.. _packaging-alt-install-prefix-unix: + +Alternate installation: Unix (the prefix scheme) +------------------------------------------------ + +The "prefix scheme" is useful when you wish to use one Python installation to +run the build command, but install modules into the third-party module directory +of a different Python installation (or something that looks like a different +Python installation). If this sounds a trifle unusual, it is ---that's why the +user and home schemes come before. However, there are at least two known cases +where the prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. This can be done with :: + + pysetup run install_dist --prefix /usr/local + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could +be done with :: + + pysetup run install_dist --prefix=/mnt/@server/export + +In either case, the :option:`--prefix` option defines the installation base, and +the :option:`--exec-prefix` option defines the platform-specific installation +base, which is used for platform-specific files. (Currently, this just means +non-pure module distributions, but could be expanded to C libraries, binary +executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to +:option:`--prefix`. Files are installed as follows: + +================= ========================================================== +Type of file Installation directory +================= ========================================================== +Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` +extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` +scripts :file:`{prefix}/bin` +data :file:`{prefix}` +C headers :file:`{prefix}/include/python{X.Y}{abiflags}` +================= ========================================================== + +.. XXX misses an entry for platinclude + +There is no requirement that :option:`--prefix` or :option:`--exec-prefix` +actually point to an alternate Python installation; if the directories listed +above do not already exist, they are created at installation time. + +Incidentally, the real reason the prefix scheme is important is simply that a +standard Unix installation uses the prefix scheme, but with :option:`--prefix` +and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and +``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, +but every time you run ``pysetup run install_dist`` without any other +options, you're using it. + +Note that installing extensions to an alternate Python installation doesn't have +anything to do with how those extensions are built: in particular, extensions +will be compiled using the Python header files (:file:`Python.h` and friends) +installed with the Python interpreter used to run the build command. It is +therefore your responsibility to ensure compatibility between the interpreter +intended to run extensions installed in this way and the interpreter used to +build these same extensions. To avoid problems, it is best to make sure that +the two interpreters are the same version of Python (possibly different builds, +or possibly copies of the same build). (Of course, if your :option:`--prefix` +and :option:`--exec-prefix` don't even point to an alternate Python installation, +this is immaterial.) + + +.. _packaging-alt-install-prefix-windows: + +Alternate installation: Windows (the prefix scheme) +--------------------------------------------------- + +Windows has a different and vaguer notion of home directories than Unix, and +since its standard Python installation is simpler, the :option:`--prefix` option +has traditionally been used to install additional packages to arbitrary +locations. :: + + pysetup run install_dist --prefix "\Temp\Python" + +to install modules to the :file:`\\Temp\\Python` directory on the current drive. + +The installation base is defined by the :option:`--prefix` option; the +:option:`--exec-prefix` option is not supported under Windows, which means that +pure Python modules and extension modules are installed into the same location. +Files are installed as follows: + +=============== ========================================================== +Type of file Installation directory +=============== ========================================================== +modules :file:`{prefix}\\Lib\\site-packages` +scripts :file:`{prefix}\\Scripts` +data :file:`{prefix}` +C headers :file:`{prefix}\\Include` +=============== ========================================================== + + +.. _packaging-custom-install: + +Custom installation +=================== + +Sometimes, the alternate installation schemes described in section +:ref:`packaging-alt-install` just don't do what you want. You might want to tweak +just one or two directories while keeping everything under the same base +directory, or you might want to completely redefine the installation scheme. +In either case, you're creating a *custom installation scheme*. + +To create a custom installation scheme, you start with one of the alternate +schemes and override some of the installation directories used for the various +types of files, using these options: + +====================== ======================= +Type of file Override option +====================== ======================= +Python modules ``--install-purelib`` +extension modules ``--install-platlib`` +all modules ``--install-lib`` +scripts ``--install-scripts`` +data ``--install-data`` +C headers ``--install-headers`` +====================== ======================= + +These override options can be relative, absolute, +or explicitly defined in terms of one of the installation base directories. +(There are two installation base directories, and they are normally the same +---they only differ when you use the Unix "prefix scheme" and supply different +``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will +override values computed or given for ``--install-purelib`` and +``--install-platlib``, and is recommended for schemes that don't make a +difference between Python and extension modules.) + +For example, say you're installing a module distribution to your home directory +under Unix, but you want scripts to go in :file:`~/scripts` rather than +:file:`~/bin`. As you might expect, you can override this directory with the +:option:`--install-scripts` option and, in this case, it makes most sense to supply +a relative path, which will be interpreted relative to the installation base +directory (in our example, your home directory):: + + pysetup run install_dist --home ~ --install-scripts scripts + +Another Unix example: suppose your Python installation was built and installed +with a prefix of :file:`/usr/local/python`. Thus, in a standard installation, +scripts will wind up in :file:`/usr/local/python/bin`. If you want them in +:file:`/usr/local/bin` instead, you would supply this absolute directory for +the :option:`--install-scripts` option:: + + pysetup run install_dist --install-scripts /usr/local/bin + +This command performs an installation using the "prefix scheme", where the +prefix is whatever your Python interpreter was installed with ---in this case, +:file:`/usr/local/python`. + +If you maintain Python on Windows, you might want third-party modules to live in +a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` +itself. This is almost as easy as customizing the script installation directory +---you just have to remember that there are two types of modules to worry about, +Python and extension modules, which can conveniently be both controlled by one +option:: + + pysetup run install_dist --install-lib Site + +.. XXX Nothing is installed right under prefix in windows, is it?? + +The specified installation directory is relative to :file:`{prefix}`. Of +course, you also have to ensure that this directory is in Python's module +search path, such as by putting a :file:`.pth` file in a site directory (see +:mod:`site`). See section :ref:`packaging-search-path` to find out how to modify +Python's search path. + +If you want to define an entire installation scheme, you just have to supply all +of the installation directory options. Using relative paths is recommended here. +For example, if you want to maintain all Python module-related files under +:file:`python` in your home directory, and you want a separate directory for +each platform that you use your home directory from, you might define the +following installation scheme:: + + pysetup run install_dist --home ~ \ + --install-purelib python/lib \ + --install-platlib python/'lib.$PLAT' \ + --install-scripts python/scripts \ + --install-data python/data + +or, equivalently, :: + + pysetup run install_dist --home ~/python \ + --install-purelib lib \ + --install-platlib 'lib.$PLAT' \ + --install-scripts scripts \ + --install-data data + +``$PLAT`` doesn't need to be defined as an environment variable ---it will also +be expanded by Packaging as it parses your command line options, just as it +does when parsing your configuration file(s). (More on that later.) + +Obviously, specifying the entire installation scheme every time you install a +new module distribution would be very tedious. To spare you all that work, you +can store it in a Packaging configuration file instead (see section +:ref:`packaging-config-files`), like so:: + + [install_dist] + install-base = $HOME + install-purelib = python/lib + install-platlib = python/lib.$PLAT + install-scripts = python/scripts + install-data = python/data + +or, equivalently, :: + + [install_dist] + install-base = $HOME/python + install-purelib = lib + install-platlib = lib.$PLAT + install-scripts = scripts + install-data = data + +Note that these two are *not* equivalent if you override their installation +base directory when running the setup script. For example, :: + + pysetup run install_dist --install-base /tmp + +would install pure modules to :file:`/tmp/python/lib` in the first case, and +to :file:`/tmp/lib` in the second case. (For the second case, you'd probably +want to supply an installation base of :file:`/tmp/python`.) + +You may have noticed the use of ``$HOME`` and ``$PLAT`` in the sample +configuration file. These are Packaging configuration variables, which +bear a strong resemblance to environment variables. In fact, you can use +environment variables in configuration files on platforms that have such a notion, but +Packaging additionally defines a few extra variables that may not be in your +environment, such as ``$PLAT``. Of course, on systems that don't have +environment variables, such as Mac OS 9, the configuration variables supplied by +the Packaging are the only ones you can use. See section :ref:`packaging-config-files` +for details. + +.. XXX which vars win out eventually in case of clash env or Packaging? + +.. XXX need some Windows examples---when would custom installation schemes be + needed on those platforms? + + +.. XXX Move this section to Doc/using + +.. _packaging-search-path: + +Modifying Python's search path +------------------------------ + +When the Python interpreter executes an :keyword:`import` statement, it searches +for both Python code and extension modules along a search path. A default value +for this path is configured into the Python binary when the interpreter is built. +You can obtain the search path by importing the :mod:`sys` module and printing +the value of ``sys.path``. :: + + $ python + Python 2.2 (#11, Oct 3 2002, 13:31:27) + [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.path + ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', + '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', + '/usr/local/lib/python2.3/site-packages'] + >>> + +The null string in ``sys.path`` represents the current working directory. + +The expected convention for locally installed packages is to put them in the +:file:`{...}/site-packages/` directory, but you may want to choose a different +location for some reason. For example, if your site kept by convention all web +server-related software under :file:`/www`. Add-on Python modules might then +belong in :file:`/www/python`, and in order to import them, this directory would +have to be added to ``sys.path``. There are several ways to solve this problem. + +The most convenient way is to add a path configuration file to a directory +that's already on Python's path, usually to the :file:`.../site-packages/` +directory. Path configuration files have an extension of :file:`.pth`, and each +line must contain a single path that will be appended to ``sys.path``. (Because +the new paths are appended to ``sys.path``, modules in the added directories +will not override standard modules. This means you can't use this mechanism for +installing fixed versions of standard modules.) + +Paths can be absolute or relative, in which case they're relative to the +directory containing the :file:`.pth` file. See the documentation of +the :mod:`site` module for more information. + +A slightly less convenient way is to edit the :file:`site.py` file in Python's +standard library, and modify ``sys.path``. :file:`site.py` is automatically +imported when the Python interpreter is executed, unless the :option:`-S` switch +is supplied to suppress this behaviour. So you could simply edit +:file:`site.py` and add two lines to it:: + + import sys + sys.path.append('/www/python/') + +However, if you reinstall the same major version of Python (perhaps when +upgrading from 3.3 to 3.3.1, for example) :file:`site.py` will be overwritten by +the stock version. You'd have to remember that it was modified and save a copy +before doing the installation. + +Alternatively, there are two environment variables that can modify ``sys.path``. +:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python +installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, +the search path will be set to ``['', '/www/python/lib/pythonX.Y/', +'/www/python/lib/pythonX.Y/plat-linux2', ...]``. + +The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be +added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is +set to ``/www/python:/opt/py``, the search path will begin with +``['/www/python', '/opt/py']``. (Note that directories must exist in order to +be added to ``sys.path``; the :mod:`site` module removes non-existent paths.) + +Finally, ``sys.path`` is just a regular Python list, so any Python application +can modify it by adding or removing entries. + + +.. _packaging-config-files: + +Configuration files for Packaging +================================= + +As mentioned above, you can use configuration files to store personal or site +preferences for any option supported by any Packaging command. Depending on your +platform, you can use one of two or three possible configuration files. These +files will be read before parsing the command-line, so they take precedence over +default values. In turn, the command-line will override configuration files. +Lastly, if there are multiple configuration files, values from files read +earlier will be overridden by values from files read later. + +.. XXX "one of two or three possible..." seems wrong info. Below always 3 files + are indicated in the tables. + + +.. _packaging-config-filenames: + +Location and names of configuration files +----------------------------------------- + +The name and location of the configuration files vary slightly across +platforms. On Unix and Mac OS X, these are the three configuration files listed +in the order they are processed: + ++--------------+----------------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+==========================================================+=======+ +| system | :file:`{prefix}/lib/python{ver}/packaging/packaging.cfg` | \(1) | ++--------------+----------------------------------------------------------+-------+ +| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | ++--------------+----------------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+----------------------------------------------------------+-------+ + +Similarly, the configuration files on Windows ---also listed in the order they +are processed--- are these: + ++--------------+-------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+=================================================+=======+ +| system | :file:`{prefix}\\Lib\\packaging\\packaging.cfg` | \(4) | ++--------------+-------------------------------------------------+-------+ +| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | ++--------------+-------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+-------------------------------------------------+-------+ + +On all platforms, the *personal* file can be temporarily disabled by +means of the `--no-user-cfg` option. + +Notes: + +(1) + Strictly speaking, the system-wide configuration file lives in the directory + where Packaging is installed. + +(2) + On Unix, if the :envvar:`HOME` environment variable is not defined, the + user's home directory will be determined with the :func:`getpwuid` function + from the standard :mod:`pwd` module. Packaging uses the + :func:`os.path.expanduser` function to do this. + +(3) + I.e., in the current directory (usually the location of the setup script). + +(4) + (See also note (1).) Python's default installation prefix is + :file:`C:\\Python`, so the system configuration file is normally + :file:`C:\\Python\\Lib\\packaging\\packaging.cfg`. + +(5) + On Windows, if the :envvar:`HOME` environment variable is not defined, + :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will + be tried. Packaging uses the :func:`os.path.expanduser` function to do this. + + +.. _packaging-config-syntax: + +Syntax of configuration files +----------------------------- + +All Packaging configuration files share the same syntax. Options defined in +them are grouped into sections, and each Packaging command gets its own section. +Additionally, there's a ``global`` section for options that affect every command. +Sections consist of one or more lines containing a single option specified as +``option = value``. + +.. XXX use dry-run in the next example or use a pysetup option as example + +For example, here's a complete configuration file that forces all commands to +run quietly by default:: + + [global] + verbose = 0 + +If this was the system configuration file, it would affect all processing +of any Python module distribution by any user on the current system. If it was +installed as your personal configuration file (on systems that support them), +it would affect only module distributions processed by you. Lastly, if it was +used as the :file:`setup.cfg` for a particular module distribution, it would +affect that distribution only. + +.. XXX "(on systems that support them)" seems wrong info + +If you wanted to, you could override the default "build base" directory and +make the :command:`build\*` commands always forcibly rebuild all files with +the following:: + + [build] + build-base = blib + force = 1 + +which corresponds to the command-line arguments:: + + pysetup run build --build-base blib --force + +except that including the :command:`build` command on the command-line means +that command will be run. Including a particular command in configuration files +has no such implication; it only means that if the command is run, the options +for it in the configuration file will apply. (This is also true if you run +other commands that derive values from it.) + +You can find out the complete list of options for any command using the +:option:`--help` option, e.g.:: + + pysetup run build --help + +and you can find out the complete list of global options by using +:option:`--help` without a command:: + + pysetup run --help + +See also the "Reference" section of the "Distributing Python Modules" manual. + +.. XXX no links to the relevant section exist. + + +.. _packaging-building-ext: + +Building extensions: tips and tricks +==================================== + +Whenever possible, Packaging tries to use the configuration information made +available by the Python interpreter used to run `pysetup`. +For example, the same compiler and linker flags used to compile Python will also +be used for compiling extensions. Usually this will work well, but in +complicated situations this might be inappropriate. This section discusses how +to override the usual Packaging behaviour. + + +.. _packaging-tweak-flags: + +Tweaking compiler/linker flags +------------------------------ + +Compiling a Python extension written in C or C++ will sometimes require +specifying custom flags for the compiler and linker in order to use a particular +library or produce a special kind of object code. This is especially true if the +extension hasn't been tested on your platform, or if you're trying to +cross-compile Python. + +.. TODO update to new setup.cfg + +In the most general case, the extension author might have foreseen that +compiling the extensions would be complicated, and provided a :file:`Setup` file +for you to edit. This will likely only be done if the module distribution +contains many separate extension modules, or if they often require elaborate +sets of compiler flags in order to work. + +A :file:`Setup` file, if present, is parsed in order to get a list of extensions +to build. Each line in a :file:`Setup` describes a single module. Lines have +the following structure:: + + module ... [sourcefile ...] [cpparg ...] [library ...] + + +Let's examine each of the fields in turn. + +* *module* is the name of the extension module to be built, and should be a + valid Python identifier. You can't just change this in order to rename a module + (edits to the source code would also be needed), so this should be left alone. + +* *sourcefile* is anything that's likely to be a source code file, at least + judging by the filename. Filenames ending in :file:`.c` are assumed to be + written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are + assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed + to be in Objective C. + +* *cpparg* is an argument for the C preprocessor, and is anything starting with + :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. + +* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or + :option:`-L`. + +If a particular platform requires a special library on your platform, you can +add it by editing the :file:`Setup` file and running ``pysetup run build``. +For example, if the module defined by the line :: + + foo foomodule.c + +must be linked with the math library :file:`libm.a` on your platform, simply add +:option:`-lm` to the line:: + + foo foomodule.c -lm + +Arbitrary switches intended for the compiler or the linker can be supplied with +the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: + + foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm + +The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be +appended to the proper command line, so in the above example the compiler will +be passed the :option:`-o32` option, and the linker will be passed +:option:`-shared`. If a compiler option requires an argument, you'll have to +supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` +the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. + +Compiler flags can also be supplied through setting the :envvar:`CFLAGS` +environment variable. If set, the contents of :envvar:`CFLAGS` will be added to +the compiler flags specified in the :file:`Setup` file. + + +.. _packaging-non-ms-compilers: + +Using non-Microsoft compilers on Windows +---------------------------------------- + +.. sectionauthor:: Rene Liebscher + + + +Borland/CodeGear C++ +^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the necessary steps to use Packaging with the Borland +C++ compiler version 5.5. First you have to know that Borland's object file +format (OMF) is different from the format used by the Python version you can +download from the Python or ActiveState Web site. (Python is built with +Microsoft Visual C++, which uses COFF as the object file format.) For this +reason, you have to convert Python's library :file:`python25.lib` into the +Borland format. You can do this as follows: + +.. Should we mention that users have to create cfg-files for the compiler? +.. see also http://community.borland.com/article/0,1410,21205,00.html + +:: + + coff2omf python25.lib python25_bcpp.lib + +The :file:`coff2omf` program comes with the Borland compiler. The file +:file:`python25.lib` is in the :file:`Libs` directory of your Python +installation. If your extension uses other libraries (zlib, ...) you have to +convert them too. + +The converted files have to reside in the same directories as the normal +libraries. + +How does Packaging manage to use these libraries with their changed names? If +the extension needs a library (eg. :file:`foo`) Packaging checks first if it +finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then +uses this library. In the case it doesn't find such a special library it uses +the default name (:file:`foo.lib`.) [#]_ + +To let Packaging compile your extension with Borland, C++ you now have to +type:: + + pysetup run build --compiler bcpp + +If you want to use the Borland C++ compiler as the default, you could specify +this in your personal or system-wide configuration file for Packaging (see +section :ref:`packaging-config-files`.) + + +.. seealso:: + + `C++Builder Compiler `_ + Information about the free C++ compiler from Borland, including links to the + download pages. + + `Creating Python Extensions Using Borland's Free Compiler `_ + Document describing how to use Borland's free command-line C++ compiler to build + Python. + + +GNU C / Cygwin / MinGW +^^^^^^^^^^^^^^^^^^^^^^ + +This section describes the necessary steps to use Packaging with the GNU C/C++ +compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter +that was built with Cygwin, everything should work without any of these +following steps. + +Not all extensions can be built with MinGW or Cygwin, but many can. Extensions +most likely to not work are those that use C++ or depend on Microsoft Visual C +extensions. + +To let Packaging compile your extension with Cygwin, you have to type:: + + pysetup run build --compiler=cygwin + +and for Cygwin in no-cygwin mode [#]_ or for MinGW, type:: + + pysetup run build --compiler=mingw32 + +If you want to use any of these options/compilers as default, you should +consider writing it in your personal or system-wide configuration file for +Packaging (see section :ref:`packaging-config-files`.) + +Older Versions of Python and MinGW +"""""""""""""""""""""""""""""""""" +The following instructions only apply if you're using a version of Python +inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with +:file:`binutils-2.13.90-20030111-1`). + +These compilers require some special libraries. This task is more complex than +for Borland's C++, because there is no program to convert the library. First +you have to create a list of symbols which the Python DLL exports. (You can find +a good program for this task at +http://www.emmestech.com/software/pexports-0.43/download_pexports.html). + +.. I don't understand what the next line means. --amk + (inclusive the references on data structures.) + +:: + + pexports python25.dll > python25.def + +The location of an installed :file:`python25.dll` will depend on the +installation options and the version and language of Windows. In a "just for +me" installation, it will appear in the root of the installation directory. In +a shared installation, it will be located in the system directory. + +Then you can create from these information an import library for gcc. :: + + /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a + +The resulting library has to be placed in the same directory as +:file:`python25.lib`. (Should be the :file:`libs` directory under your Python +installation directory.) + +If your extension uses other libraries (zlib,...) you might have to convert +them too. The converted files have to reside in the same directories as the +normal libraries do. + + +.. seealso:: + + `Building Python modules on MS Windows platform with MinGW `_ + Information about building the required libraries for the MinGW + environment. + + +.. rubric:: Footnotes + +.. [#] This also means you could replace all existing COFF-libraries with + OMF-libraries of the same name. + +.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for + more information. + +.. [#] Then you have no POSIX emulation available, but you also don't need + :file:`cygwin1.dll`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/install/pysetup-config.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/install/pysetup-config.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,44 @@ +.. _packaging-pysetup-config: + +===================== +Pysetup Configuration +===================== + +Pysetup supports two configuration files: :file:`.pypirc` and :file:`packaging.cfg`. + +.. FIXME integrate with configfile instead of duplicating + +Configuring indexes +------------------- + +You can configure additional indexes in :file:`.pypirc` to be used for index-related +operations. By default, all configured index-servers and package-servers will be used +in an additive fashion. To limit operations to specific indexes, use the :option:`--index` +and :option:`--package-server options`:: + + $ pysetup install --index pypi --package-server django some.project + +Adding indexes to :file:`.pypirc`:: + + [packaging] + index-servers = + pypi + other + + package-servers = + django + + [pypi] + repository: + username: + password: + + [other] + repository: + username: + password: + + [django] + repository: + username: + password: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/install/pysetup-servers.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/install/pysetup-servers.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,61 @@ +.. _packaging-pysetup-servers: + +=============== +Package Servers +=============== + +Pysetup supports installing Python packages from *Package Servers* in addition +to PyPI indexes and mirrors. + +Package Servers are simple directory listings of Python distributions. Directories +can be served via HTTP or a local file system. This is useful when you want to +dump source distributions in a directory and not worry about the full index structure. + +Serving distributions from Apache +--------------------------------- +:: + + $ mkdir -p /var/www/html/python/distributions + $ cp *.tar.gz /var/www/html/python/distributions/ + + + ServerAdmin webmaster@domain.com + DocumentRoot "/var/www/html/python" + ServerName python.example.org + ErrorLog logs/python.example.org-error.log + CustomLog logs/python.example.org-access.log common + Options Indexes FollowSymLinks MultiViews + DirectoryIndex index.html index.htm + + + Options Indexes FollowSymLinks MultiViews + Order allow,deny + Allow from all + + + +Add the Apache based distribution server to :file:`.pypirc`:: + + [packaging] + package-servers = + apache + + [apache] + repository: http://python.example.org/distributions/ + + +Serving distributions from a file system +---------------------------------------- +:: + + $ mkdir -p /data/python/distributions + $ cp *.tar.gz /data/python/distributions/ + +Add the directory to :file:`.pypirc`:: + + [packaging] + package-servers = + local + + [local] + repository: file:///data/python/distributions/ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/install/pysetup.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/install/pysetup.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,164 @@ +.. _packaging-pysetup: + +================ +Pysetup Tutorial +================ + +Getting started +--------------- + +Pysetup is a simple script that supports the following features: + +- install, remove, list, and verify Python packages; +- search for available packages on PyPI or any *Simple Index*; +- verify installed packages (md5sum, installed files, version). + + +Finding out what's installed +---------------------------- + +Pysetup makes it easy to find out what Python packages are installed:: + + $ pysetup list virtualenv + 'virtualenv' 1.6 at '/opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info' + + $ pysetup list + 'pyverify' 0.8.1 at '/opt/python3.3/lib/python3.3/site-packages/pyverify-0.8.1.dist-info' + 'virtualenv' 1.6 at '/opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info' + ... + + +Installing a distribution +------------------------- + +Pysetup can install a Python project from the following sources: + +- PyPI and Simple Indexes; +- source directories containing a valid :file:`setup.py` or :file:`setup.cfg`; +- distribution source archives (:file:`project-1.0.tar.gz`, :file:`project-1.0.zip`); +- HTTP (http://host/packages/project-1.0.tar.gz). + + +Installing from PyPI and Simple Indexes:: + + $ pysetup install project + $ pysetup install project==1.0 + +Installing from a distribution source archive:: + + $ pysetup install project-1.0.tar.gz + +Installing from a source directory containing a valid :file:`setup.py` or +:file:`setup.cfg`:: + + $ cd path/to/source/directory + $ pysetup install + + $ pysetup install path/to/source/directory + +Installing from HTTP:: + + $ pysetup install http://host/packages/project-1.0.tar.gz + + +Retrieving metadata +------------------- + +You can gather metadata from two sources, a project's source directory or an +installed distribution. The `pysetup metadata` command can retrieve one or +more metadata fields using the `-f` option and a metadata field as the +argument. :: + + $ pysetup metadata virtualenv -f version -f name + Version: + 1.6 + Name: + virtualenv + + $ pysetup metadata virtualenv + Metadata-Version: + 1.0 + Name: + virtualenv + Version: + 1.6 + Platform: + UNKNOWN + Summary: + Virtual Python Environment builder + ... + +.. seealso:: + + There are three metadata versions, 1.0, 1.1, and 1.2. The following PEPs + describe specifics of the field names, and their semantics and usage. 1.0 + :PEP:`241`, 1.1 :PEP:`314`, and 1.2 :PEP:`345` + + +Removing a distribution +----------------------- + +You can remove one or more installed distributions using the `pysetup remove` +command:: + + $ pysetup remove virtualenv + removing 'virtualenv': + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/dependency_links.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/entry_points.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/not-zip-safe + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/PKG-INFO + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/SOURCES.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/top_level.txt + Proceed (y/n)? y + success: removed 6 files and 1 dirs + +The optional '-y' argument auto confirms, skipping the conformation prompt:: + + $ pysetup remove virtualenv -y + + +Getting help +------------ + +All pysetup actions take the `-h` and `--help` options which prints the commands +help string to stdout. :: + + $ pysetup remove -h + Usage: pysetup remove dist [-y] + or: pysetup remove --help + + Uninstall a Python package. + + positional arguments: + dist installed distribution name + + optional arguments: + -y auto confirm package removal + +Getting a list of all pysetup actions and global options:: + + $ pysetup --help + Usage: pysetup [options] action [action_options] + + Actions: + run: Run one or several commands + metadata: Display the metadata of a project + install: Install a project + remove: Remove a project + search: Search for a project in the indexes + list: List installed projects + graph: Display a graph + create: Create a project + generate-setup: Generate a backward-compatible setup.py + + To get more help on an action, use: + + pysetup action --help + + Global options: + --verbose (-v) run verbosely (default) + --quiet (-q) run quietly (turns verbosity off) + --dry-run (-n) don't actually do anything + --help (-h) show detailed help message + --no-user-cfg ignore pydistutils.cfg in your home directory + --version Display the version diff -r 6db40a9955dc -r 0d413f60cc23 Doc/installing/index.rst --- a/Doc/installing/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,222 +0,0 @@ -.. highlightlang:: none - -.. _installing-index: - -***************************** - Installing Python Modules -***************************** - -:Email: distutils-sig@python.org - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the installation part of the process. For a guide to -creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. - -.. note:: - - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* ``pip`` is the preferred installer program. Starting with Python 3.4, it - is included by default with the Python binary installers. -* a virtual environment is a semi-isolated Python environment that allows - packages to be installed for use by a particular application, rather than - being installed system wide -* ``pyvenv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments -* ``virtualenv`` is a third party alternative (and predecessor) to - ``pyvenv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``pyvenv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* the `Python Packaging Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub `__ and - `BitBucket `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). - - -Basic usage -=========== - -The standard packaging tools are all designed to be used from the command -line. - -The following command will install the latest version of a module and its -dependencies from the Python Packaging Index:: - - python -m pip install SomePackage - -.. note:: - - For POSIX users (including Mac OS X and Linux users), the examples in - this guide assume the use of a :term:`virtual environment`. - - For Windows users, the examples in this guide assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -It's also possible to specify an exact or minimum version directly on the -command line. When using comparator operators such as ``>``, ``<`` or some other -special character which get interpreted by shell, the package name and the -version should be enclosed within double quotes:: - - python -m pip install SomePackage==1.0.4 # specific version - python -m pip install "SomePackage>=1.0.4" # minimum version - -Normally, if a suitable module is already installed, attempting to install -it again will have no effect. Upgrading existing modules must be requested -explicitly:: - - python -m pip install --upgrade SomePackage - -More information and resources regarding ``pip`` and its capabilities can be -found in the `Python Packaging User Guide `__. - -``pyvenv`` has its own documentation at :ref:`scripts-pyvenv`. Installing -into an active virtual environment uses the commands shown above. - -.. seealso:: - - `Python Packaging User Guide: Installing Python Distribution Packages - `__ - - -How do I ...? -============= - -These are quick answers or links for some common tasks. - -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Setup for Installing Distribution Packages - `__ - - -.. installing-per-user-installation: - -... install packages just for the current user? ------------------------------------------------ - -Passing the ``--user`` option to ``python -m pip install`` will install a -package just for the current user, rather than for all users of the system. - - -... install scientific Python packages? ---------------------------------------- - -A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by -`other means -`__ -rather than attempting to install them with ``pip``. - -.. seealso:: - - `Python Packaging User Guide: Installing Scientific Packages - `__ - - -... work with multiple versions of Python installed in parallel? ----------------------------------------------------------------- - -On Linux, Mac OS X and other POSIX systems, use the versioned Python commands -in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: - - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 - -(appropriately versioned ``pip`` commands may also be available) - -On Windows, use the ``py`` Python launcher in combination with the ``-m`` -switch:: - - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 - -.. other questions: - - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches) - - -Common installation issues -========================== - -Installing into the system Python on Linux ------------------------------------------- - -On Linux systems, a Python installation will typically be included as part -of the distribution. Installing into this Python installation requires -root access to the system, and may interfere with the operation of the -system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. - -On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. - - -Installing binary extensions ----------------------------- - -Python has typically relied heavily on source based distribution, with end -users being expected to compile extension modules from source as part of -the installation process. - -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and Mac OS X through the -Python Packaging Index, this problem is expected to diminish over time, -as users are more regularly able to install pre-built extensions rather -than needing to build them themselves. - -Some of the solutions for installing `scientific software -`__ -that is not yet available as pre-built ``wheel`` files may also help with -obtaining other binary extensions without needing to build them locally. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/2to3.rst --- a/Doc/library/2to3.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/2to3.rst Mon Jan 25 17:05:13 2016 +0100 @@ -23,7 +23,7 @@ also located in the :file:`Tools/scripts` directory of the Python root. 2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. +directories are to recursively traversed for Python sources. Here is a sample Python 2.x source file, :file:`example.py`:: @@ -142,39 +142,6 @@ Removes usage of :func:`apply`. For example ``apply(function, *args, **kwargs)`` is converted to ``function(*args, **kwargs)``. -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - .. 2to3fixer:: basestring Converts :class:`basestring` to :class:`str`. @@ -271,7 +238,7 @@ .. 2to3fixer:: input - Converts ``input(prompt)`` to ``eval(input(prompt))``. + Converts ``input(prompt)`` to ``eval(input(prompt))`` .. 2to3fixer:: intern @@ -297,7 +264,8 @@ .. 2to3fixer:: long - Renames :class:`long` to :class:`int`. + Strips the ``L`` prefix on long literals and renames :class:`long` to + :class:`int`. .. 2to3fixer:: map @@ -323,11 +291,11 @@ Converts the use of iterator's :meth:`~iterator.next` methods to the :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. + :meth:`~object.__next__`. .. 2to3fixer:: nonzero - Renames :meth:`__nonzero__` to :meth:`~object.__bool__`. + Renames :meth:`~object.__nonzero__` to :meth:`~object.__bool__`. .. 2to3fixer:: numliterals @@ -375,10 +343,6 @@ Handles the move of :func:`reduce` to :func:`functools.reduce`. -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`imp.reload`. - .. 2to3fixer:: renames Changes :data:`sys.maxint` to :data:`sys.maxsize`. @@ -392,7 +356,7 @@ Replaces use of the :class:`set` constructor with set literals. This fixer is optional. -.. 2to3fixer:: standarderror +.. 2to3fixer:: standard_error Renames :exc:`StandardError` to :exc:`Exception`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/__future__.rst --- a/Doc/library/__future__.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/__future__.rst Mon Jan 25 17:05:13 2016 +0100 @@ -75,7 +75,7 @@ | division | 2.2.0a2 | 3.0 | :pep:`238`: | | | | | *Changing the Division Operator* | +------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | +| absolute_import | 2.5.0a1 | 2.7 | :pep:`328`: | | | | | *Imports: Multi-Line and Absolute/Relative* | +------------------+-------------+--------------+---------------------------------------------+ | with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | @@ -87,9 +87,6 @@ | unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | | | | | *Bytes literals in Python 3000* | +------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/__main__.rst --- a/Doc/library/__main__.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/__main__.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,19 +5,13 @@ .. module:: __main__ :synopsis: The environment where the top-level script is run. -``'__main__'`` is the name of the scope in which top-level code executes. -A module's __name__ is set equal to ``'__main__'`` when read from -standard input, a script, or from an interactive prompt. -A module can discover whether or not it is running in the main scope by -checking its own ``__name__``, which allows a common idiom for conditionally -executing code in a module when it is run as a script or with ``python --m`` but not when it is imported:: +This module represents the (otherwise anonymous) scope in which the +interpreter's main program executes --- commands read either from standard +input, from a script file, or from an interactive prompt. It is this +environment in which the idiomatic "conditional script" stanza causes a script +to run:: if __name__ == "__main__": - # execute only if run as a script main() -For a package, the same effect can be achieved by including a -``__main__.py`` module, the contents of which will be executed when the -module is run with ``-m``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/_dummy_thread.rst --- a/Doc/library/_dummy_thread.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/_dummy_thread.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ try: import _thread except ImportError: - import _dummy_thread as _thread + import dummy_thread as _thread Be careful to not use this module where deadlock might occur from a thread being created that blocks waiting for another thread to be created. This often occurs diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/_thread.rst --- a/Doc/library/_thread.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/_thread.rst Mon Jan 25 17:05:13 2016 +0100 @@ -93,16 +93,15 @@ Return the thread stack size used when creating new threads. The optional *size* argument specifies the stack size to be used for subsequently created threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB + integer value of at least 32,768 (32kB). If changing the thread stack size is + unsupported, a :exc:`ThreadError` is raised. If the specified stack size is + invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32kB is currently the minimum supported stack size value to guarantee sufficient stack space for the interpreter itself. Note that some platforms may have particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system + minimum stack size > 32kB or requiring allocation in multiples of the system memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is + information (4kB pages are common; using multiples of 4096 for the stack size is the suggested approach in the absence of more specific information). Availability: Windows, systems with POSIX threads. @@ -177,6 +176,10 @@ * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. +* Not all built-in functions that may block waiting for I/O allow other threads + to run. (The most popular ones (:func:`time.sleep`, :meth:`file.read`, + :func:`select.select`) work as expected.) + * It is not possible to interrupt the :meth:`acquire` method on a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/abc.rst --- a/Doc/library/abc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/abc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,9 +12,9 @@ -------------- This module provides the infrastructure for defining :term:`abstract base -classes ` (ABCs) in Python, as outlined in :pep:`3119`; -see the PEP for why this was added to Python. (See also :pep:`3141` and the -:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.) +classes ` (ABCs) in Python, as outlined in :pep:`3119`; see the PEP for why this +was added to Python. (See also :pep:`3141` and the :mod:`numbers` module +regarding a type hierarchy for numbers based on ABCs.) The :mod:`collections` module has some concrete classes that derive from ABCs; these can, of course, be further derived. In addition the @@ -23,7 +23,7 @@ hashable or a mapping. -This module provides the following classes: +This module provides the following class: .. class:: ABCMeta @@ -58,10 +58,6 @@ .. versionchanged:: 3.3 Returns the registered subclass, to allow usage as a class decorator. - .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the - :func:`get_cache_token` function. - You can also override this method in an abstract base class: .. method:: __subclasshook__(subclass) @@ -114,39 +110,26 @@ MyIterable.register(Foo) The ABC ``MyIterable`` defines the standard iterable method, - :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method - is also part of the ``MyIterable`` abstract base class, but it does not have - to be overridden in non-abstract derived classes. + :meth:`__iter__`, as an abstract method. The implementation given here can + still be called from subclasses. The :meth:`get_iterator` method is also + part of the ``MyIterable`` abstract base class, but it does not have to be + overridden in non-abstract derived classes. The :meth:`__subclasshook__` class method defined here says that any class - that has an :meth:`~iterator.__iter__` method in its - :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. + that has an :meth:`__iter__` method in its :attr:`__dict__` (or in that of + one of its base classes, accessed via the :attr:`__mro__` list) is + considered a ``MyIterable`` too. Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, - even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and + even though it does not define an :meth:`__iter__` method (it uses the + old-style iterable protocol, defined in terms of :meth:`__len__` and :meth:`__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. -.. class:: ABC - - A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC`, - avoiding sometimes confusing metaclass usage. - - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding metaclass - usage, as multiple inheritance may lead to metaclass conflicts. - - .. versionadded:: 3.4 - - The :mod:`abc` module also provides the following decorators: -.. decorator:: abstractmethod +.. decorator:: abstractmethod(function) A decorator indicating abstract methods. @@ -220,52 +203,43 @@ multiple-inheritance. -.. decorator:: abstractclassmethod +.. decorator:: abstractclassmethod(function) A subclass of the built-in :func:`classmethod`, indicating an abstract classmethod. Otherwise it is similar to :func:`abstractmethod`. - This special case is deprecated, as the :func:`classmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: + Usage:: class C(metaclass=ABCMeta): - @classmethod - @abstractmethod + @abstractclassmethod def my_abstract_classmethod(cls, ...): ... .. versionadded:: 3.2 .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. + Use :class:`classmethod` with :func:`abstractmethod` instead -.. decorator:: abstractstaticmethod +.. decorator:: abstractstaticmethod(function) A subclass of the built-in :func:`staticmethod`, indicating an abstract staticmethod. Otherwise it is similar to :func:`abstractmethod`. - This special case is deprecated, as the :func:`staticmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: + Usage:: class C(metaclass=ABCMeta): - @staticmethod - @abstractmethod + @abstractstaticmethod def my_abstract_staticmethod(...): ... .. versionadded:: 3.2 .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. + Use :class:`staticmethod` with :func:`abstractmethod` instead .. decorator:: abstractproperty(fget=None, fset=None, fdel=None, doc=None) - A subclass of the built-in :func:`property`, indicating an abstract - property. + A subclass of the built-in :func:`property`, indicating an abstract property. Using this function requires that the class's metaclass is :class:`ABCMeta` or is derived from it. A class that has a metaclass derived from @@ -273,56 +247,23 @@ and properties are overridden. The abstract properties can be called using any of the normal 'super' call mechanisms. - This special case is deprecated, as the :func:`property` decorator - is now correctly identified as abstract when applied to an abstract - method:: + Usage:: class C(metaclass=ABCMeta): - @property - @abstractmethod + @abstractproperty def my_abstract_property(self): ... - The above example defines a read-only property; you can also define a - read-write abstract property by appropriately marking one or more of the - underlying methods as abstract:: + This defines a read-only property; you can also define a read-write abstract + property using the 'long' form of property declaration:: class C(metaclass=ABCMeta): - @property - def x(self): - ... - - @x.setter - @abstractmethod - def x(self, val): - ... - - If only some components are abstract, only those components need to be - updated to create a concrete property in a subclass:: - - class D(C): - @C.x.setter - def x(self, val): - ... - + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. - - -The :mod:`abc` module also provides the following functions: - -.. function:: get_cache_token() - - Returns the current abstract base class cache token. - - The token is an opaque object (that supports equality testing) identifying - the current version of the abstract base class cache for virtual subclasses. - The token changes with every call to :meth:`ABCMeta.register` on any ABC. - - .. versionadded:: 3.4 + Use :class:`property` with :func:`abstractmethod` instead .. rubric:: Footnotes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/aifc.rst --- a/Doc/library/aifc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/aifc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -30,8 +30,8 @@ sampled. The number of channels indicate if the audio is mono, stereo, or quadro. Each frame consists of one sample per channel. The sample size is the size in bytes of each sample. Thus a frame consists of -``nchannels * samplesize`` bytes, and a second's worth of audio consists of -``nchannels * samplesize * framerate`` bytes. +*nchannels*\**samplesize* bytes, and a second's worth of audio consists of +*nchannels*\**samplesize*\**framerate* bytes. For example, CD quality audio has a sample size of two bytes (16 bits), uses two channels (stereo) and has a frame rate of 44,100 frames/second. This gives a @@ -51,11 +51,6 @@ used for writing, the file object should be seekable, unless you know ahead of time how many samples you are going to write in total and use :meth:`writeframesraw` and :meth:`setnframes`. - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`with` block completes, the :meth:`~aifc.close` method is called. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. Objects returned by :func:`.open` when a file is opened for reading have the following methods: @@ -97,9 +92,7 @@ .. method:: aifc.getparams() - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. + Return a tuple consisting of all of the above values in the above order. .. method:: aifc.getmarkers() @@ -225,18 +218,12 @@ Write data to the output file. This method can only be called after the audio file parameters have been set. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - .. method:: aifc.writeframesraw(data) Like :meth:`writeframes`, except that the header of the audio file is not updated. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - .. method:: aifc.close() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/allos.rst --- a/Doc/library/allos.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/allos.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,6 +16,7 @@ io.rst time.rst argparse.rst + optparse.rst getopt.rst logging.rst logging.config.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/argparse.rst --- a/Doc/library/argparse.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/argparse.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,12 +12,6 @@ -------------- -.. sidebar:: Tutorial - - This page contains the API reference information. For a more gentle - introduction to Python command-line parsing, have a look at the - :ref:`argparse tutorial `. - The :mod:`argparse` module makes it easy to write user-friendly command-line interfaces. The program defines what arguments it requires, and :mod:`argparse` will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` @@ -46,7 +40,7 @@ Assuming the Python code above is saved into a file called ``prog.py``, it can be run at the command line and provides useful help messages:: - $ python prog.py -h + $ prog.py -h usage: prog.py [-h] [--sum] N [N ...] Process some integers. @@ -61,15 +55,15 @@ When run with the appropriate arguments, it prints either the sum or the max of the command-line integers:: - $ python prog.py 1 2 3 4 + $ prog.py 1 2 3 4 4 - $ python prog.py 1 2 3 4 --sum + $ prog.py 1 2 3 4 --sum 10 If invalid arguments are passed in, it will issue an error:: - $ python prog.py a b c + $ prog.py a b c usage: prog.py [-h] [--sum] N [N ...] prog.py: error: argument N: invalid int value: 'a' @@ -130,149 +124,44 @@ ArgumentParser objects ---------------------- -.. class:: ArgumentParser(prog=None, usage=None, description=None, \ - epilog=None, parents=[], \ - formatter_class=argparse.HelpFormatter, \ - prefix_chars='-', fromfile_prefix_chars=None, \ - argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True) +.. class:: ArgumentParser([description], [epilog], [prog], [usage], [add_help], \ + [argument_default], [parents], [prefix_chars], \ + [conflict_handler], [formatter_class]) - Create a new :class:`ArgumentParser` object. All parameters should be passed - as keyword arguments. Each parameter has its own more detailed description - below, but in short they are: + Create a new :class:`ArgumentParser` object. Each parameter has its own more + detailed description below, but in short they are: - * prog_ - The name of the program (default: ``sys.argv[0]``) + * description_ - Text to display before the argument help. - * usage_ - The string describing the program usage (default: generated from - arguments added to parser) + * epilog_ - Text to display after the argument help. - * description_ - Text to display before the argument help (default: none) + * add_help_ - Add a -h/--help option to the parser. (default: ``True``) - * epilog_ - Text to display after the argument help (default: none) + * argument_default_ - Set the global default value for arguments. + (default: ``None``) * parents_ - A list of :class:`ArgumentParser` objects whose arguments should - also be included + also be included. - * formatter_class_ - A class for customizing the help output - - * prefix_chars_ - The set of characters that prefix optional arguments + * prefix_chars_ - The set of characters that prefix optional arguments. (default: '-') * fromfile_prefix_chars_ - The set of characters that prefix files from - which additional arguments should be read (default: ``None``) + which additional arguments should be read. (default: ``None``) - * argument_default_ - The global default value for arguments - (default: ``None``) + * formatter_class_ - A class for customizing the help output. - * conflict_handler_ - The strategy for resolving conflicting optionals - (usually unnecessary) + * conflict_handler_ - Usually unnecessary, defines strategy for resolving + conflicting optionals. - * add_help_ - Add a -h/--help option to the parser (default: ``True``) + * prog_ - The name of the program (default: + ``sys.argv[0]``) - * allow_abbrev_ - Allows long options to be abbreviated if the - abbreviation is unambiguous. (default: ``True``) - - .. versionchanged:: 3.5 - *allow_abbrev* parameter was added. + * usage_ - The string describing the program usage (default: generated) The following sections describe how each of these are used. -prog -^^^^ - -By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from):: - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir\myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.print_help() - usage: myprogram [-h] - - optional arguments: - -h, --help show this help message and exit - -Note that the program name, whether determined from ``sys.argv[0]`` or from the -``prog=`` argument, is available to help messages using the ``%(prog)s`` format -specifier. - -:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.add_argument('--foo', help='foo of the %(prog)s program') - >>> parser.print_help() - usage: myprogram [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo of the myprogram program - - -usage -^^^^^ - -By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: - - >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [options] - - positional arguments: - bar bar help - - optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The ``%(prog)s`` format specifier is available to fill in the program name in -your usage messages. - - description ^^^^^^^^^^^ @@ -320,6 +209,122 @@ argument to :class:`ArgumentParser`. +add_help +^^^^^^^^ + +By default, ArgumentParser objects add an option which simply displays +the parser's help message. For example, consider a file named +``myprogram.py`` containing the following code:: + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--foo', help='foo help') + args = parser.parse_args() + +If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser +help will be printed:: + + $ python myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo help + +Occasionally, it may be useful to disable the addition of this help option. +This can be achieved by passing ``False`` as the ``add_help=`` argument to +:class:`ArgumentParser`:: + + >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) + >>> parser.add_argument('--foo', help='foo help') + >>> parser.print_help() + usage: PROG [--foo FOO] + + optional arguments: + --foo FOO foo help + +The help option is typically ``-h/--help``. The exception to this is +if the ``prefix_chars=`` is specified and does not include ``-``, in +which case ``-h`` and ``--help`` are not valid options. In +this case, the first character in ``prefix_chars`` is used to prefix +the help options:: + + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') + >>> parser.print_help() + usage: PROG [+h] + + optional arguments: + +h, ++help show this help message and exit + + +prefix_chars +^^^^^^^^^^^^ + +Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. +Parsers that need to support different or additional prefix +characters, e.g. for options +like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument +to the ArgumentParser constructor:: + + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') + >>> parser.add_argument('+f') + >>> parser.add_argument('++bar') + >>> parser.parse_args('+f X ++bar Y'.split()) + Namespace(bar='Y', f='X') + +The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of +characters that does not include ``-`` will cause ``-f/--foo`` options to be +disallowed. + + +fromfile_prefix_chars +^^^^^^^^^^^^^^^^^^^^^ + +Sometimes, for example when dealing with a particularly long argument lists, it +may make sense to keep the list of arguments in a file rather than typing it out +at the command line. If the ``fromfile_prefix_chars=`` argument is given to the +:class:`ArgumentParser` constructor, then arguments that start with any of the +specified characters will be treated as files, and will be replaced by the +arguments they contain. For example:: + + >>> with open('args.txt', 'w') as fp: + ... fp.write('-f\nbar') + >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + >>> parser.add_argument('-f') + >>> parser.parse_args(['-f', 'foo', '@args.txt']) + Namespace(f='bar') + +Arguments read from a file must by default be one per line (but see also +:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they +were in the same place as the original file referencing argument on the command +line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` +is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. + +The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that +arguments will never be treated as file references. + + +argument_default +^^^^^^^^^^^^^^^^ + +Generally, argument defaults are specified either by passing a default to +:meth:`~ArgumentParser.add_argument` or by calling the +:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value +pairs. Sometimes however, it may be useful to specify a single parser-wide +default for arguments. This can be accomplished by passing the +``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, +to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` +calls, we supply ``argument_default=SUPPRESS``:: + + >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) + >>> parser.add_argument('--foo') + >>> parser.add_argument('bar', nargs='?') + >>> parser.parse_args(['--foo', '1', 'BAR']) + Namespace(bar='BAR', foo='1') + >>> parser.parse_args([]) + Namespace() + + parents ^^^^^^^ @@ -457,94 +462,6 @@ --foo int -prefix_chars -^^^^^^^^^^^^ - -Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. -Parsers that need to support different or additional prefix -characters, e.g. for options -like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') - >>> parser.add_argument('+f') - >>> parser.add_argument('++bar') - >>> parser.parse_args('+f X ++bar Y'.split()) - Namespace(bar='Y', f='X') - -The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of -characters that does not include ``-`` will cause ``-f/--foo`` options to be -disallowed. - - -fromfile_prefix_chars -^^^^^^^^^^^^^^^^^^^^^ - -Sometimes, for example when dealing with a particularly long argument lists, it -may make sense to keep the list of arguments in a file rather than typing it out -at the command line. If the ``fromfile_prefix_chars=`` argument is given to the -:class:`ArgumentParser` constructor, then arguments that start with any of the -specified characters will be treated as files, and will be replaced by the -arguments they contain. For example:: - - >>> with open('args.txt', 'w') as fp: - ... fp.write('-f\nbar') - >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') - >>> parser.add_argument('-f') - >>> parser.parse_args(['-f', 'foo', '@args.txt']) - Namespace(f='bar') - -Arguments read from a file must by default be one per line (but see also -:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they -were in the same place as the original file referencing argument on the command -line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` -is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. - -The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that -arguments will never be treated as file references. - - -argument_default -^^^^^^^^^^^^^^^^ - -Generally, argument defaults are specified either by passing a default to -:meth:`~ArgumentParser.add_argument` or by calling the -:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value -pairs. Sometimes however, it may be useful to specify a single parser-wide -default for arguments. This can be accomplished by passing the -``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, -to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` -calls, we supply ``argument_default=SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar', nargs='?') - >>> parser.parse_args(['--foo', '1', 'BAR']) - Namespace(bar='BAR', foo='1') - >>> parser.parse_args([]) - Namespace() - -.. _allow_abbrev: - -allow_abbrev -^^^^^^^^^^^^ - -Normally, when you pass an argument list to the -:meth:`~ArgumentParser.parse_args` method of an :class:`ArgumentParser`, -it :ref:`recognizes abbreviations ` of long options. - -This feature can be disabled by setting ``allow_abbrev`` to ``False``:: - - >>> parser = argparse.ArgumentParser(prog='PROG', allow_abbrev=False) - >>> parser.add_argument('--foobar', action='store_true') - >>> parser.add_argument('--foonley', action='store_false') - >>> parser.parse_args(['--foon']) - usage: PROG [-h] [--foobar] [--foonley] - PROG: error: unrecognized arguments: --foon - -.. versionadded:: 3.5 - - conflict_handler ^^^^^^^^^^^^^^^^ @@ -582,20 +499,22 @@ string was overridden. -add_help -^^^^^^^^ +prog +^^^^ -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: +By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine +how to display the name of the program in help messages. This default is almost +always desirable because it will make the help messages match how the program was +invoked on the command line. For example, consider a file named +``myprogram.py`` with the following code:: import argparse parser = argparse.ArgumentParser() parser.add_argument('--foo', help='foo help') args = parser.parse_args() -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed:: +The help for this program will display ``myprogram.py`` as the program name +(regardless of where the program was invoked from):: $ python myprogram.py --help usage: myprogram.py [-h] [--foo FOO] @@ -603,31 +522,76 @@ optional arguments: -h, --help show this help message and exit --foo FOO foo help - -Occasionally, it may be useful to disable the addition of this help option. -This can be achieved by passing ``False`` as the ``add_help=`` argument to -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> parser.add_argument('--foo', help='foo help') - >>> parser.print_help() - usage: PROG [--foo FOO] + $ cd .. + $ python subdir\myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] optional arguments: - --foo FOO foo help + -h, --help show this help message and exit + --foo FOO foo help -The help option is typically ``-h/--help``. The exception to this is -if the ``prefix_chars=`` is specified and does not include ``-``, in -which case ``-h`` and ``--help`` are not valid options. In -this case, the first character in ``prefix_chars`` is used to prefix -the help options:: +To change this default behavior, another value can be supplied using the +``prog=`` argument to :class:`ArgumentParser`:: - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') + >>> parser = argparse.ArgumentParser(prog='myprogram') >>> parser.print_help() - usage: PROG [+h] + usage: myprogram [-h] optional arguments: - +h, ++help show this help message and exit + -h, --help show this help message and exit + +Note that the program name, whether determined from ``sys.argv[0]`` or from the +``prog=`` argument, is available to help messages using the ``%(prog)s`` format +specifier. + +:: + + >>> parser = argparse.ArgumentParser(prog='myprogram') + >>> parser.add_argument('--foo', help='foo of the %(prog)s program') + >>> parser.print_help() + usage: myprogram [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo of the myprogram program + + +usage +^^^^^ + +By default, :class:`ArgumentParser` calculates the usage message from the +arguments it contains:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [-h] [--foo [FOO]] bar [bar ...] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The default message can be overridden with the ``usage=`` keyword argument:: + + >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [options] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The ``%(prog)s`` format specifier is available to fill in the program name in +your usage messages. The add_argument() method @@ -709,7 +673,7 @@ actions can do just about anything with the command-line arguments associated with them, though most actions simply add an attribute to the object returned by :meth:`~ArgumentParser.parse_args`. The ``action`` keyword argument specifies -how the command-line arguments should be handled. The supplied actions are: +how the command-line arguments should be handled. The supported actions are: * ``'store'`` - This just stores the argument's value. This is the default action. For example:: @@ -783,18 +747,28 @@ >>> parser.parse_args(['--version']) PROG 2.0 -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The recommended way to do -this is to extend :class:`Action`, overriding the ``__call__`` method -and optionally the ``__init__`` method. +You can also specify an arbitrary action by passing an object that implements +the Action API. The easiest way to do this is to extend +:class:`argparse.Action`, supplying an appropriate ``__call__`` method. The +``__call__`` method should accept four parameters: + +* ``parser`` - The ArgumentParser object which contains this action. + +* ``namespace`` - The :class:`Namespace` object that will be returned by + :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this + object. + +* ``values`` - The associated command-line arguments, with any type conversions + applied. (Type conversions are specified with the type_ keyword argument to + :meth:`~ArgumentParser.add_argument`. + +* ``option_string`` - The option string that was used to invoke this action. + The ``option_string`` argument is optional, and will be absent if the action + is associated with a positional argument. An example of a custom action:: >>> class FooAction(argparse.Action): - ... def __init__(self, option_strings, dest, nargs=None, **kwargs): - ... if nargs is not None: - ... raise ValueError("nargs not allowed") - ... super(FooAction, self).__init__(option_strings, dest, **kwargs) ... def __call__(self, parser, namespace, values, option_string=None): ... print('%r %r %r' % (namespace, values, option_string)) ... setattr(namespace, self.dest, values) @@ -808,7 +782,6 @@ >>> args Namespace(bar='1', foo='2') -For more details, see :class:`Action`. nargs ^^^^^ @@ -941,17 +914,6 @@ >>> parser.parse_args(''.split()) Namespace(foo=42) -If the ``default`` value is a string, the parser parses the value as if it -were a command-line argument. In particular, the parser applies any type_ -conversion argument, if provided, before setting the attribute on the -:class:`Namespace` return value. Otherwise, the parser uses the value as is:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--length', default='10', type=int) - >>> parser.add_argument('--width', default=10.5, type=int) - >>> parser.parse_args() - Namespace(length=10, width=10.5) - For positional arguments with nargs_ equal to ``?`` or ``*``, the ``default`` value is used when no command-line argument was present:: @@ -990,13 +952,10 @@ >>> parser.parse_args('2 temp.txt'.split()) Namespace(bar=<_io.TextIOWrapper name='temp.txt' encoding='UTF-8'>, foo=2) -See the section on the default_ keyword argument for information on when the -``type`` argument is applied to default arguments. - To ease the use of various types of files, the argparse module provides the -factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and -``errors=`` arguments of the :func:`open` function. For example, -``FileType('w')`` can be used to create a writable file:: +factory FileType which takes the ``mode=`` and ``bufsize=`` arguments of the +:func:`open` function. For example, ``FileType('w')`` can be used to create a +writable file:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('bar', type=argparse.FileType('w')) @@ -1040,33 +999,32 @@ ^^^^^^^ Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a container object as the *choices* keyword +These can be handled by passing a container object as the ``choices`` keyword argument to :meth:`~ArgumentParser.add_argument`. When the command line is -parsed, argument values will be checked, and an error message will be displayed -if the argument was not one of the acceptable values:: +parsed, argument values will be checked, and an error message will be displayed if +the argument was not one of the acceptable values:: - >>> parser = argparse.ArgumentParser(prog='game.py') - >>> parser.add_argument('move', choices=['rock', 'paper', 'scissors']) - >>> parser.parse_args(['rock']) - Namespace(move='rock') - >>> parser.parse_args(['fire']) - usage: game.py [-h] {rock,paper,scissors} - game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', - 'paper', 'scissors') + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('foo', choices='abc') + >>> parser.parse_args('c'.split()) + Namespace(foo='c') + >>> parser.parse_args('X'.split()) + usage: PROG [-h] {a,b,c} + PROG: error: argument foo: invalid choice: 'X' (choose from 'a', 'b', 'c') -Note that inclusion in the *choices* container is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* +Note that inclusion in the ``choices`` container is checked after any type_ +conversions have been performed, so the type of the objects in the ``choices`` container should match the type_ specified:: - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('foo', type=complex, choices=[1, 1j]) + >>> parser.parse_args('1j'.split()) + Namespace(foo=1j) + >>> parser.parse_args('-- -4'.split()) + usage: PROG [-h] {1,1j} + PROG: error: argument foo: invalid choice: (-4+0j) (choose from 1, 1j) -Any object that supports the ``in`` operator can be passed as the *choices* +Any object that supports the ``in`` operator can be passed as the ``choices`` value, so :class:`dict` objects, :class:`set` objects, custom containers, etc. are all supported. @@ -1137,9 +1095,6 @@ optional arguments: -h, --help show this help message and exit -As the help string supports %-formatting, if you want a literal ``%`` to appear -in the help string, you must escape it as ``%%``. - :mod:`argparse` supports silencing the help entry for certain options, by setting the ``help`` value to ``argparse.SUPPRESS``:: @@ -1155,7 +1110,7 @@ metavar ^^^^^^^ -When :class:`ArgumentParser` generates help messages, it needs some way to refer +When :class:`ArgumentParser` generates help messages, it need some way to refer to each expected argument. By default, ArgumentParser objects use the dest_ value as the "name" of each object. By default, for positional argument actions, the dest_ value is used directly, and for optional argument actions, @@ -1255,49 +1210,6 @@ >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') -Action classes -^^^^^^^^^^^^^^ - -Action classes implement the Action API, a callable which returns a callable -which processes arguments from the command-line. Any object which follows -this API may be passed as the ``action`` parameter to -:meth:`add_argument`. - -.. class:: Action(option_strings, dest, nargs=None, const=None, default=None, \ - type=None, choices=None, required=False, help=None, \ - metavar=None) - -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. - -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. - -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: - -* ``parser`` - The ArgumentParser object which contains this action. - -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. - -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. - -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. - -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. - The parse_args() method ----------------------- @@ -1431,14 +1343,12 @@ >>> parser.parse_args(['--', '-f']) Namespace(foo='-f', one=None) -.. _prefix-matching: -Argument abbreviations (prefix matching) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Argument abbreviations +^^^^^^^^^^^^^^^^^^^^^^ -The :meth:`~ArgumentParser.parse_args` method :ref:`by default ` -allows long options to be abbreviated to a prefix, if the abbreviation is -unambiguous (the prefix matches a unique option):: +The :meth:`~ArgumentParser.parse_args` method allows long options to be +abbreviated if the abbreviation is unambiguous:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('-bacon') @@ -1452,7 +1362,6 @@ PROG: error: ambiguous option: -ba could match -badger, -bacon An error is produced for arguments that could produce more than one options. -This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. Beyond ``sys.argv`` @@ -1515,10 +1424,7 @@ Sub-commands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ - [parser_class], [action], \ - [option_string], [dest], [help], \ - [metavar]) +.. method:: ArgumentParser.add_subparsers() Many programs split up their functionality into a number of sub-commands, for example, the ``svn`` program can invoke sub-commands like ``svn @@ -1527,38 +1433,11 @@ different functions which require different kinds of command-line arguments. :class:`ArgumentParser` supports the creation of such sub-commands with the :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally - called with no arguments and returns a special action object. This object + called with no arguments and returns an special action object. This object has a single method, :meth:`~ArgumentParser.add_parser`, which takes a command name and any :class:`ArgumentParser` constructor arguments, and returns an :class:`ArgumentParser` object that can be modified as usual. - Description of parameters: - - * title - title for the sub-parser group in help output; by default - "subcommands" if description is provided, otherwise uses title for - positional arguments - - * description - description for the sub-parser group in help output, by - default None - - * prog - usage information that will be displayed with sub-command help, - by default the name of the program and any positional arguments before the - subparser argument - - * parser_class - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) - - * action_ - the basic type of action to be taken when this argument is - encountered at the command line - - * dest_ - name of the attribute under which sub-command name will be - stored; by default None and no value is stored - - * help_ - help for sub-parser group in help output, by default None - - * metavar_ - string presenting available sub-commands in help; by default it - is None and presents sub-commands in form {cmd1, cmd2, ..} - Some example usage:: >>> # create the top-level parser @@ -1600,8 +1479,8 @@ positional arguments: {a,b} sub-command help - a a help - b b help + a a help + b b help optional arguments: -h, --help show this help message and exit @@ -1712,19 +1591,17 @@ FileType objects ^^^^^^^^^^^^^^^^ -.. class:: FileType(mode='r', bufsize=-1, encoding=None, errors=None) +.. class:: FileType(mode='r', bufsize=None) The :class:`FileType` factory creates objects that can be passed to the type argument of :meth:`ArgumentParser.add_argument`. Arguments that have - :class:`FileType` objects as their type will open command-line arguments as - files with the requested modes, buffer sizes, encodings and error handling - (see the :func:`open` function for more details):: + :class:`FileType` objects as their type will open command-line arguments as files + with the requested modes and buffer sizes:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--raw', type=argparse.FileType('wb', 0)) - >>> parser.add_argument('out', type=argparse.FileType('w', encoding='UTF-8')) - >>> parser.parse_args(['--raw', 'raw.dat', 'file.txt']) - Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>) + >>> parser.add_argument('--output', type=argparse.FileType('wb', 0)) + >>> parser.parse_args(['--output', 'out']) + Namespace(output=<_io.BufferedWriter name='out'>) FileType objects understand the pseudo-argument ``'-'`` and automatically convert this into ``sys.stdin`` for readable :class:`FileType` objects and @@ -1735,9 +1612,6 @@ >>> parser.parse_args(['-']) Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) - .. versionadded:: 3.4 - The *encodings* and *errors* keyword arguments. - Argument groups ^^^^^^^^^^^^^^^ @@ -1787,14 +1661,14 @@ --bar BAR bar help - Note that any arguments not in your user-defined groups will end up back - in the usual "positional arguments" and "optional arguments" sections. + Note that any arguments not your user defined groups will end up back in the + usual "positional arguments" and "optional arguments" sections. Mutual exclusion ^^^^^^^^^^^^^^^^ -.. method:: ArgumentParser.add_mutually_exclusive_group(required=False) +.. method:: add_mutually_exclusive_group(required=False) Create a mutually exclusive group. :mod:`argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the @@ -1923,12 +1797,6 @@ >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) -.. warning:: - :ref:`Prefix matching ` rules apply to - :meth:`parse_known_args`. The parser may consume an option even if it's just - a prefix of one of its known options, instead of leaving it in the remaining - arguments list. - Customizing file parsing ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1937,7 +1805,7 @@ Arguments that are read from a file (see the *fromfile_prefix_chars* keyword argument to the :class:`ArgumentParser` constructor) are read one - argument per line. :meth:`convert_arg_line_to_args` can be overridden for + argument per line. :meth:`convert_arg_line_to_args` can be overriden for fancier reading. This method takes a single argument *arg_line* which is a string read from @@ -1948,7 +1816,10 @@ as an argument:: def convert_arg_line_to_args(self, arg_line): - return arg_line.split() + for arg in arg_line.split(): + if not arg.strip(): + continue + yield arg Exiting methods @@ -1976,25 +1847,14 @@ :mod:`optparse` had either been copy-pasted over or monkey-patched, it no longer seemed practical to try to maintain the backwards compatibility. -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: - -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. - A partial upgrade path from :mod:`optparse` to :mod:`argparse`: * Replace all :meth:`optparse.OptionParser.add_option` calls with :meth:`ArgumentParser.add_argument` calls. -* Replace ``(options, args) = parser.parse_args()`` with ``args = +* Replace ``options, args = parser.parse_args()`` with ``args = parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in :mod:`argparse` context is called ``args``. + calls for the positional arguments. * Replace callback actions and the ``callback_*`` keyword arguments with ``type`` or ``action`` arguments. @@ -2011,4 +1871,4 @@ ``%(default)s`` and ``%(prog)s``. * Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. + ``parser.add_argument('--version', action='version', version='')`` diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/array.rst --- a/Doc/library/array.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/array.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,7 +21,7 @@ +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | Py_UNICODE | Unicode character | 2 | \(1) | +| ``'u'`` | Py_UCS4 | Unicode character | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -35,9 +35,9 @@ +-----------+--------------------+-------------------+-----------------------+-------+ | ``'L'`` | unsigned long | int | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | \(2) | +| ``'q'`` | signed long long | int | 8 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | \(2) | +| ``'Q'`` | unsigned long long | int | 8 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -47,16 +47,6 @@ Notes: (1) - The ``'u'`` type code corresponds to Python's obsolete unicode character - (:c:type:`Py_UNICODE` which is :c:type:`wchar_t`). Depending on the - platform, it can be 16 bits or 32 bits. - - ``'u'`` will be removed together with the rest of the :c:type:`Py_UNICODE` - API. - - .. deprecated-removed:: 3.3 4.0 - -(2) The ``'q'`` and ``'Q'`` type codes are available only if the platform C compiler used to build Python supports C :c:type:`long long`, or, on Windows, :c:type:`__int64`. @@ -73,8 +63,8 @@ .. class:: array(typecode[, initializer]) A new array whose items are restricted by *typecode*, and initialized - from the optional *initializer* value, which must be a list, a - :term:`bytes-like object`, or iterable over elements of the + from the optional *initializer* value, which must be a list, object + supporting the buffer interface, or iterable over elements of the appropriate type. If given a list or string, the initializer is passed to the new array's @@ -91,7 +81,7 @@ concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, -and may be used wherever :term:`bytes-like object`\ s are supported. +and may be used wherever buffer objects are supported. The following data items and methods are also supported: @@ -271,7 +261,9 @@ Packing and unpacking of External Data Representation (XDR) data as used in some remote procedure call systems. - `The Numerical Python Documentation `_ + `The Numerical Python Manual `_ The Numeric Python extension (NumPy) defines another array type; see - http://www.numpy.org/ for further information about Numerical Python. + http://numpy.sourceforge.net/ for further information about Numerical Python. + (A PDF version of the NumPy manual is available at + http://numpy.sourceforge.net/numdoc/numdoc.pdf). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ast.rst --- a/Doc/library/ast.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/ast.rst Mon Jan 25 17:05:13 2016 +0100 @@ -104,7 +104,7 @@ :mod:`ast` Helpers ------------------ -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: .. function:: parse(source, filename='', mode='exec') @@ -115,15 +115,13 @@ .. function:: literal_eval(node_or_string) - Safely evaluate an expression node or a string containing a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, and ``None``. + Safely evaluate an expression node or a string containing a Python + expression. The string or node provided may only consist of the following + Python literal structures: strings, bytes, numbers, tuples, lists, dicts, + sets, booleans, and ``None``. - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. + This can be used for safely evaluating strings containing Python expressions + from untrusted sources without the need to parse the values oneself. .. versionchanged:: 3.2 Now allows bytes and set literals. @@ -246,11 +244,6 @@ Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. The returned string will show the names and the values for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to ``False``. Attributes such as line + wanted *annotate_fields* must be set to False. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to ``True``. - -.. seealso:: - - `Green Tree Snakes `_, an external documentation resource, has good - details on working with Python ASTs. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asynchat.rst --- a/Doc/library/asynchat.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/asynchat.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,11 +10,6 @@ -------------- -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - This module builds on the :mod:`asyncore` infrastructure, simplifying asynchronous clients and servers and making it easier to handle protocols whose elements are terminated by arbitrary strings, or are of variable length. @@ -61,8 +56,8 @@ have only one method, :meth:`more`, which should return data to be transmitted on the channel. The producer indicates exhaustion (*i.e.* that it contains no more data) by - having its :meth:`more` method return the empty bytes object. At this point - the :class:`async_chat` object removes the producer from the fifo and starts + having its :meth:`more` method return the empty string. At this point the + :class:`async_chat` object removes the producer from the fifo and starts using the next producer, if any. When the producer fifo is empty the :meth:`handle_write` method does nothing. You use the channel object's :meth:`set_terminator` method to describe how to recognize the end of, or @@ -147,6 +142,40 @@ by the channel after :meth:`found_terminator` is called. +asynchat - Auxiliary Classes +------------------------------------------ + +.. class:: fifo(list=None) + + A :class:`fifo` holding data which has been pushed by the application but + not yet popped for writing to the channel. A :class:`fifo` is a list used + to hold data and/or producers until they are required. If the *list* + argument is provided then it should contain producers or data items to be + written to the channel. + + + .. method:: is_empty() + + Returns ``True`` if and only if the fifo is empty. + + + .. method:: first() + + Returns the least-recently :meth:`push`\ ed item from the fifo. + + + .. method:: push(data) + + Adds the given data (which may be a string or a producer object) to the + producer fifo. + + + .. method:: pop() + + If the fifo is not empty, returns ``True, first()``, deleting the popped + item. Returns ``False, None`` for an empty fifo. + + .. _asynchat-example: asynchat Example @@ -168,9 +197,6 @@ marshalled, after setting the channel terminator to ``None`` to ensure that any extraneous data sent by the web client are ignored. :: - - import asynchat - class http_request_handler(asynchat.async_chat): def __init__(self, sock, addr, sessions, log): @@ -192,7 +218,7 @@ def found_terminator(self): if self.reading_headers: self.reading_headers = False - self.parse_headers(b"".join(self.ibuffer)) + self.parse_headers("".join(self.ibuffer)) self.ibuffer = [] if self.op.upper() == b"POST": clen = self.headers.getheader("content-length") diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-dev.rst --- a/Doc/library/asyncio-dev.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,402 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-dev: - -Develop with asyncio -==================== - -Asynchronous programming is different than classical "sequential" programming. -This page lists common traps and explains how to avoid them. - - -.. _asyncio-debug-mode: - -Debug mode of asyncio ---------------------- - -The implementation of :mod:`asyncio` has been written for performance. -In order to ease the development of asynchronous code, you may wish to -enable *debug mode*. - -To enable all debug checks for an application: - -* Enable the asyncio debug mode globally by setting the environment variable - :envvar:`PYTHONASYNCIODEBUG` to ``1``, or by calling :meth:`BaseEventLoop.set_debug`. -* Set the log level of the :ref:`asyncio logger ` to - :py:data:`logging.DEBUG`. For example, call - ``logging.basicConfig(level=logging.DEBUG)`` at startup. -* Configure the :mod:`warnings` module to display :exc:`ResourceWarning` - warnings. For example, use the ``-Wdefault`` command line option of Python to - display them. - -Examples debug checks: - -* Log :ref:`coroutines defined but never "yielded from" - ` -* :meth:`~BaseEventLoop.call_soon` and :meth:`~BaseEventLoop.call_at` methods - raise an exception if they are called from the wrong thread. -* Log the execution time of the selector -* Log callbacks taking more than 100 ms to be executed. The - :attr:`BaseEventLoop.slow_callback_duration` attribute is the minimum - duration in seconds of "slow" callbacks. -* :exc:`ResourceWarning` warnings are emitted when transports and event loops - are :ref:`not closed explicitly `. - -.. seealso:: - - The :meth:`BaseEventLoop.set_debug` method and the :ref:`asyncio logger - `. - - -Cancellation ------------- - -Cancellation of tasks is not common in classic programming. In asynchronous -programming, not only it is something common, but you have to prepare your -code to handle it. - -Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel` -method. The :func:`wait_for` function cancels the waited task when the timeout -occurs. There are many other cases where a task can be cancelled indirectly. - -Don't call :meth:`~Future.set_result` or :meth:`~Future.set_exception` method -of :class:`Future` if the future is cancelled: it would fail with an exception. -For example, write:: - - if not fut.cancelled(): - fut.set_result('done') - -Don't schedule directly a call to the :meth:`~Future.set_result` or the -:meth:`~Future.set_exception` method of a future with -:meth:`BaseEventLoop.call_soon`: the future can be cancelled before its method -is called. - -If you wait for a future, you should check early if the future was cancelled to -avoid useless operations. Example:: - - @coroutine - def slow_operation(fut): - if fut.cancelled(): - return - # ... slow computation ... - yield from fut - # ... - -The :func:`shield` function can also be used to ignore cancellation. - - -.. _asyncio-multithreading: - -Concurrency and multithreading ------------------------------- - -An event loop runs in a thread and executes all callbacks and tasks in the same -thread. While a task is running in the event loop, no other task is running in -the same thread. But when the task uses ``yield from``, the task is suspended -and the event loop executes the next task. - -To schedule a callback from a different thread, the -:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example:: - - loop.call_soon_threadsafe(callback, *args) - -Most asyncio objects are not thread safe. You should only worry if you access -objects outside the event loop. For example, to cancel a future, don't call -directly its :meth:`Future.cancel` method, but:: - - loop.call_soon_threadsafe(fut.cancel) - -To handle signals and to execute subprocesses, the event loop must be run in -the main thread. - -To schedule a coroutine object from a different thread, the -:func:`run_coroutine_threadsafe` function should be used. It returns a -:class:`concurrent.futures.Future` to access the result:: - - future = asyncio.run_coroutine_threadsafe(coro_func(), loop) - result = future.result(timeout) # Wait for the result with a timeout - -The :meth:`BaseEventLoop.run_in_executor` method can be used with a thread pool -executor to execute a callback in different thread to not block the thread of -the event loop. - -.. seealso:: - - The :ref:`Synchronization primitives ` section describes ways - to synchronize tasks. - - The :ref:`Subprocess and threads ` section lists - asyncio limitations to run subprocesses from different threads. - - - - -.. _asyncio-handle-blocking: - -Handle blocking functions correctly ------------------------------------ - -Blocking functions should not be called directly. For example, if a function -blocks for 1 second, other tasks are delayed by 1 second which can have an -important impact on reactivity. - -For networking and subprocesses, the :mod:`asyncio` module provides high-level -APIs like :ref:`protocols `. - -An executor can be used to run a task in a different thread or even in a -different process, to not block the thread of the event loop. See the -:meth:`BaseEventLoop.run_in_executor` method. - -.. seealso:: - - The :ref:`Delayed calls ` section details how the - event loop handles time. - - -.. _asyncio-logger: - -Logging -------- - -The :mod:`asyncio` module logs information with the :mod:`logging` module in -the logger ``'asyncio'``. - - -.. _asyncio-coroutine-not-scheduled: - -Detect coroutine objects never scheduled ----------------------------------------- - -When a coroutine function is called and its result is not passed to -:func:`ensure_future` or to the :meth:`BaseEventLoop.create_task` method, -the execution of the coroutine object will never be scheduled which is -probably a bug. :ref:`Enable the debug mode of asyncio ` -to :ref:`log a warning ` to detect it. - -Example with the bug:: - - import asyncio - - @asyncio.coroutine - def test(): - print("never scheduled") - - test() - -Output in debug mode:: - - Coroutine test() at test.py:3 was never yielded from - Coroutine object created at (most recent call last): - File "test.py", line 7, in - test() - -The fix is to call the :func:`ensure_future` function or the -:meth:`BaseEventLoop.create_task` method with the coroutine object. - -.. seealso:: - - :ref:`Pending task destroyed `. - - -Detect exceptions never consumed --------------------------------- - -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If -:meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted -` when the future is deleted by the garbage collector, with the -traceback where the exception was raised. - -Example of unhandled exception:: - - import asyncio - - @asyncio.coroutine - def bug(): - raise Exception("not consumed") - - loop = asyncio.get_event_loop() - asyncio.ensure_future(bug()) - loop.run_forever() - loop.close() - -Output:: - - Task exception was never retrieved - future: exception=Exception('not consumed',)> - Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug - raise Exception("not consumed") - Exception: not consumed - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Output in debug mode:: - - Task exception was never retrieved - future: exception=Exception('not consumed',) created at test.py:8> - source_traceback: Object created at (most recent call last): - File "test.py", line 8, in - asyncio.ensure_future(bug()) - Traceback (most recent call last): - File "asyncio/tasks.py", line 237, in _step - result = next(coro) - File "asyncio/coroutines.py", line 79, in __next__ - return next(self.gen) - File "asyncio/coroutines.py", line 141, in coro - res = func(*args, **kw) - File "test.py", line 5, in bug - raise Exception("not consumed") - Exception: not consumed - -There are different options to fix this issue. The first option is to chain the -coroutine in another coroutine and use classic try/except:: - - @asyncio.coroutine - def handle_exception(): - try: - yield from bug() - except Exception: - print("exception consumed") - - loop = asyncio.get_event_loop() - asyncio.ensure_future(handle_exception()) - loop.run_forever() - loop.close() - -Another option is to use the :meth:`BaseEventLoop.run_until_complete` -function:: - - task = asyncio.ensure_future(bug()) - try: - loop.run_until_complete(task) - except Exception: - print("exception consumed") - -.. seealso:: - - The :meth:`Future.exception` method. - - -Chain coroutines correctly --------------------------- - -When a coroutine function calls other coroutine functions and tasks, they -should be chained explicitly with ``yield from``. Otherwise, the execution is -not guaranteed to be sequential. - -Example with different bugs using :func:`asyncio.sleep` to simulate slow -operations:: - - import asyncio - - @asyncio.coroutine - def create(): - yield from asyncio.sleep(3.0) - print("(1) create file") - - @asyncio.coroutine - def write(): - yield from asyncio.sleep(1.0) - print("(2) write into file") - - @asyncio.coroutine - def close(): - print("(3) close file") - - @asyncio.coroutine - def test(): - asyncio.ensure_future(create()) - asyncio.ensure_future(write()) - asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) - loop.stop() - - loop = asyncio.get_event_loop() - asyncio.ensure_future(test()) - loop.run_forever() - print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop)) - loop.close() - -Expected output:: - - (1) create file - (2) write into file - (3) close file - Pending tasks at exit: set() - -Actual output:: - - (3) close file - (2) write into file - Pending tasks at exit: {>} - Task was destroyed but it is pending! - task: > - -The loop stopped before the ``create()`` finished, ``close()`` has been called -before ``write()``, whereas coroutine functions were called in this order: -``create()``, ``write()``, ``close()``. - -To fix the example, tasks must be marked with ``yield from``:: - - @asyncio.coroutine - def test(): - yield from asyncio.ensure_future(create()) - yield from asyncio.ensure_future(write()) - yield from asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) - loop.stop() - -Or without ``asyncio.ensure_future()``:: - - @asyncio.coroutine - def test(): - yield from create() - yield from write() - yield from close() - yield from asyncio.sleep(2.0) - loop.stop() - - -.. _asyncio-pending-task-destroyed: - -Pending task destroyed ----------------------- - -If a pending task is destroyed, the execution of its wrapped :ref:`coroutine -` did not complete. It is probably a bug and so a warning is logged. - -Example of log:: - - Task was destroyed but it is pending! - task: wait_for=> - -:ref:`Enable the debug mode of asyncio ` to get the -traceback where the task was created. Example of log in debug mode:: - - Task was destroyed but it is pending! - source_traceback: Object created at (most recent call last): - File "test.py", line 15, in - task = asyncio.ensure_future(coro, loop=loop) - task: wait_for= created at test.py:15> - - -.. seealso:: - - :ref:`Detect coroutine objects never scheduled `. - -.. _asyncio-close-transports: - -Close transports and event loops --------------------------------- - -When a transport is no more needed, call its ``close()`` method to release -resources. Event loops must also be closed explicitly. - -If a transport or an event loop is not closed explicitly, a -:exc:`ResourceWarning` warning will be emitted in its destructor. By default, -:exc:`ResourceWarning` warnings are ignored. The :ref:`Debug mode of asyncio -` section explains how to display them. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-eventloop.rst --- a/Doc/library/asyncio-eventloop.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,922 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-event-loop: - -Base Event Loop -=============== - -The event loop is the central execution device provided by :mod:`asyncio`. -It provides multiple facilities, including: - -* Registering, executing and cancelling delayed calls (timeouts). - -* Creating client and server :ref:`transports ` for various - kinds of communication. - -* Launching subprocesses and the associated :ref:`transports - ` for communication with an external program. - -* Delegating costly function calls to a pool of threads. - -.. class:: BaseEventLoop - - Base class of event loops. - - This class is :ref:`not thread safe `. - -Run an event loop ------------------ - -.. method:: BaseEventLoop.run_forever() - - Run until :meth:`stop` is called. If :meth:`stop` is called before - :meth:`run_forever()` is called, this polls the I/O selector once - with a timeout of zero, runs all callbacks scheduled in response to - I/O events (and those that were already scheduled), and then exits. - If :meth:`stop` is called while :meth:`run_forever` is running, - this will run the current batch of callbacks and then exit. Note - that callbacks scheduled by callbacks will not run in that case; - they will run the next time :meth:`run_forever` is called. - - .. versionchanged:: 3.5.1 - -.. method:: BaseEventLoop.run_until_complete(future) - - Run until the :class:`Future` is done. - - If the argument is a :ref:`coroutine object `, it is wrapped by - :func:`ensure_future`. - - Return the Future's result, or raise its exception. - -.. method:: BaseEventLoop.is_running() - - Returns running status of event loop. - -.. method:: BaseEventLoop.stop() - - Stop running the event loop. - - This causes :meth:`run_forever` to exit at the next suitable - opportunity (see there for more details). - - .. versionchanged:: 3.5.1 - -.. method:: BaseEventLoop.is_closed() - - Returns ``True`` if the event loop was closed. - - .. versionadded:: 3.4.2 - -.. method:: BaseEventLoop.close() - - Close the event loop. The loop must not be running. Pending - callbacks will be lost. - - This clears the queues and shuts down the executor, but does not wait for - the executor to finish. - - This is idempotent and irreversible. No other methods should be called after - this one. - -.. _asyncio-pass-keywords: - -Calls ------ - -Most :mod:`asyncio` functions don't accept keywords. If you want to pass -keywords to your callback, use :func:`functools.partial`. For example, -``loop.call_soon(functools.partial(print, "Hello", flush=True))`` will call -``print("Hello", flush=True)``. - -.. note:: - :func:`functools.partial` is better than ``lambda`` functions, because - :mod:`asyncio` can inspect :func:`functools.partial` object to display - parameters in debug mode, whereas ``lambda`` functions have a poor - representation. - -.. method:: BaseEventLoop.call_soon(callback, \*args) - - Arrange for a callback to be called as soon as possible. The callback is - called after :meth:`call_soon` returns, when control returns to the event - loop. - - This operates as a FIFO queue, callbacks are called in the order in - which they are registered. Each callback will be called exactly once. - - Any positional arguments after the callback will be passed to the - callback when it is called. - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.call_soon_threadsafe(callback, \*args) - - Like :meth:`call_soon`, but thread safe. - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - -.. _asyncio-delayed-calls: - -Delayed calls -------------- - -The event loop has its own internal clock for computing timeouts. -Which clock is used depends on the (platform-specific) event loop -implementation; ideally it is a monotonic clock. This will generally be -a different clock than :func:`time.time`. - -.. note:: - - Timeouts (relative *delay* or absolute *when*) should not exceed one day. - - -.. method:: BaseEventLoop.call_later(delay, callback, *args) - - Arrange for the *callback* to be called after the given *delay* - seconds (either an int or float). - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - *callback* will be called exactly once per call to :meth:`call_later`. - If two callbacks are scheduled for exactly the same time, it is - undefined which will be called first. - - The optional positional *args* will be passed to the callback when it - is called. If you want the callback to be called with some named - arguments, use a closure or :func:`functools.partial`. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.call_at(when, callback, *args) - - Arrange for the *callback* to be called at the given absolute timestamp - *when* (an int or float), using the same time reference as - :meth:`BaseEventLoop.time`. - - This method's behavior is the same as :meth:`call_later`. - - An instance of :class:`asyncio.Handle` is returned, which can be - used to cancel the callback. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.time() - - Return the current time, as a :class:`float` value, according to the - event loop's internal clock. - -.. seealso:: - - The :func:`asyncio.sleep` function. - - -Tasks ------ - -.. method:: BaseEventLoop.create_task(coro) - - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. - - Third-party event loops can use their own subclass of :class:`Task` for - interoperability. In this case, the result type is a subclass of - :class:`Task`. - - This method was added in Python 3.4.2. Use the :func:`async` function to - support also older Python versions. - - .. versionadded:: 3.4.2 - -.. method:: BaseEventLoop.set_task_factory(factory) - - Set a task factory that will be used by - :meth:`BaseEventLoop.create_task`. - - If *factory* is ``None`` the default task factory will be set. - - If *factory* is a *callable*, it should have a signature matching - ``(loop, coro)``, where *loop* will be a reference to the active - event loop, *coro* will be a coroutine object. The callable - must return an :class:`asyncio.Future` compatible object. - - .. versionadded:: 3.4.4 - -.. method:: BaseEventLoop.get_task_factory() - - Return a task factory, or ``None`` if the default one is in use. - - .. versionadded:: 3.4.4 - - -Creating connections --------------------- - -.. coroutinemethod:: BaseEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None) - - Create a streaming transport connection to a given Internet *host* and - *port*: socket family :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), - socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - The chronological synopsis of the underlying operation is as follows: - - #. The connection is established, and a :ref:`transport ` - is created to represent it. - - #. *protocol_factory* is called without arguments and must return a - :ref:`protocol ` instance. - - #. The protocol instance is tied to the transport, and its - :meth:`connection_made` method is called. - - #. The coroutine returns successfully with the ``(transport, protocol)`` - pair. - - The created transport is an implementation-dependent bidirectional stream. - - .. note:: - *protocol_factory* can be any kind of callable, not necessarily - a class. For example, if you want to use a pre-created - protocol instance, you can pass ``lambda: my_protocol``. - - Options allowing to change how the connection is created: - - * *ssl*: if given and not false, a SSL/TLS transport is created - (by default a plain TCP transport is created). If *ssl* is - a :class:`ssl.SSLContext` object, this context is used to create - the transport; if *ssl* is :const:`True`, a context with some - unspecified default settings is used. - - .. seealso:: :ref:`SSL/TLS security considerations ` - - * *server_hostname*, is only for use together with *ssl*, - and sets or overrides the hostname that the target server's certificate - will be matched against. By default the value of the *host* argument - is used. If *host* is empty, there is no default and you must pass a - value for *server_hostname*. If *server_hostname* is an empty - string, hostname matching is disabled (which is a serious security - risk, allowing for man-in-the-middle-attacks). - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to getaddrinfo() for *host* resolution. - If given, these should all be integers from the corresponding - :mod:`socket` module constants. - - * *sock*, if given, should be an existing, already connected - :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* - and *local_addr* should be specified. - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* - are looked up using getaddrinfo(), similarly to *host* and *port*. - - .. versionchanged:: 3.5 - - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. - - .. seealso:: - - The :func:`open_connection` function can be used to get a pair of - (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol. - - -.. coroutinemethod:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None) - - Create datagram connection: socket family :py:data:`~socket.AF_INET` or - :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified), - socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a - callable returning a :ref:`protocol ` instance. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - Options changing how the connection is created: - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* - are looked up using :meth:`getaddrinfo`. - - * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used - to connect the socket to a remote address. The *remote_host* and - *remote_port* are looked up using :meth:`getaddrinfo`. - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to :meth:`getaddrinfo` for *host* - resolution. If given, these should all be integers from the - corresponding :mod:`socket` module constants. - - * *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified will automatically be set to True on - UNIX. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on Windows - and some UNIX's. If the :py:data:`~socket.SO_REUSEPORT` constant is not - defined then this capability is unsupported. - - * *allow_broadcast* tells the kernel to allow this endpoint to send - messages to the broadcast address. - - * *sock* can optionally be specified in order to use a preexisting, - already connected, :class:`socket.socket` object to be used by the - transport. If specified, *local_addr* and *remote_addr* should be omitted - (must be :const:`None`). - - On Windows with :class:`ProactorEventLoop`, this method is not supported. - - See :ref:`UDP echo client protocol ` and - :ref:`UDP echo server protocol ` examples. - - -.. coroutinemethod:: BaseEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None) - - Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket - type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket - family is used to communicate between processes on the same machine - efficiently. - - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - See the :meth:`BaseEventLoop.create_connection` method for parameters. - - Availability: UNIX. - - -Creating listening connections ------------------------------- - -.. coroutinemethod:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None) - - Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to - *host* and *port*. - - Return a :class:`Server` object, its :attr:`~Server.sockets` attribute - contains created sockets. Use the :meth:`Server.close` method to stop the - server: close listening sockets. - - Parameters: - - * The *host* parameter can be a string, in that case the TCP server is - bound to *host* and *port*. The *host* parameter can also be a sequence - of strings and in that case the TCP server is bound to all hosts of the - sequence. If *host* is an empty string or ``None``, all interfaces are - assumed and a list of multiple sockets will be returned (most likely one - for IPv4 and another one for IPv6). - - * *family* can be set to either :data:`socket.AF_INET` or - :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set - it will be determined from host (defaults to :data:`socket.AF_UNSPEC`). - - * *flags* is a bitmask for :meth:`getaddrinfo`. - - * *sock* can optionally be specified in order to use a preexisting - socket object. If specified, *host* and *port* should be omitted (must be - :const:`None`). - - * *backlog* is the maximum number of queued connections passed to - :meth:`~socket.socket.listen` (defaults to 100). - - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. - - * *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified will automatically be set to True on - UNIX. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on - Windows. - - This method is a :ref:`coroutine `. - - .. versionchanged:: 3.5 - - On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. - - .. seealso:: - - The function :func:`start_server` creates a (:class:`StreamReader`, - :class:`StreamWriter`) pair and calls back a function with this pair. - - .. versionchanged:: 3.5.1 - - The *host* parameter can now be a sequence of strings. - - -.. coroutinemethod:: BaseEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None) - - Similar to :meth:`BaseEventLoop.create_server`, but specific to the - socket family :py:data:`~socket.AF_UNIX`. - - This method is a :ref:`coroutine `. - - Availability: UNIX. - - -Watch file descriptors ----------------------- - -On Windows with :class:`SelectorEventLoop`, only socket handles are supported -(ex: pipe file descriptors are not supported). - -On Windows with :class:`ProactorEventLoop`, these methods are not supported. - -.. method:: BaseEventLoop.add_reader(fd, callback, \*args) - - Start watching the file descriptor for read availability and then call the - *callback* with specified arguments. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.remove_reader(fd) - - Stop watching the file descriptor for read availability. - -.. method:: BaseEventLoop.add_writer(fd, callback, \*args) - - Start watching the file descriptor for write availability and then call the - *callback* with specified arguments. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.remove_writer(fd) - - Stop watching the file descriptor for write availability. - -The :ref:`watch a file descriptor for read events ` -example uses the low-level :meth:`BaseEventLoop.add_reader` method to register -the file descriptor of a socket. - - -Low-level socket operations ---------------------------- - -.. coroutinemethod:: BaseEventLoop.sock_recv(sock, nbytes) - - Receive data from the socket. The return value is a bytes object - representing the data received. The maximum amount of data to be received - at once is specified by *nbytes*. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`socket.socket.recv` method. - -.. coroutinemethod:: BaseEventLoop.sock_sendall(sock, data) - - Send data to the socket. The socket must be connected to a remote socket. - This method continues to send data from *data* until either all data has - been sent or an error occurs. ``None`` is returned on success. On error, - an exception is raised, and there is no way to determine how much data, if - any, was successfully processed by the receiving end of the connection. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`socket.socket.sendall` method. - -.. coroutinemethod:: BaseEventLoop.sock_connect(sock, address) - - Connect to a remote socket at *address*. - - The *address* must be already resolved to avoid the trap of hanging the - entire event loop when the address requires doing a DNS lookup. For - example, it must be an IP address, not an hostname, for - :py:data:`~socket.AF_INET` and :py:data:`~socket.AF_INET6` address families. - Use :meth:`getaddrinfo` to resolve the hostname asynchronously. - - With :class:`SelectorEventLoop` event loop, the socket *sock* must be - non-blocking. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`BaseEventLoop.create_connection` method, the - :func:`open_connection` function and the :meth:`socket.socket.connect` - method. - - -.. coroutinemethod:: BaseEventLoop.sock_accept(sock) - - Accept a connection. The socket must be bound to an address and listening - for connections. The return value is a pair ``(conn, address)`` where *conn* - is a *new* socket object usable to send and receive data on the connection, - and *address* is the address bound to the socket on the other end of the - connection. - - The socket *sock* must be non-blocking. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`BaseEventLoop.create_server` method, the :func:`start_server` - function and the :meth:`socket.socket.accept` method. - - -Resolve host name ------------------ - -.. coroutinemethod:: BaseEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0) - - This method is a :ref:`coroutine `, similar to - :meth:`socket.getaddrinfo` function but non-blocking. - -.. coroutinemethod:: BaseEventLoop.getnameinfo(sockaddr, flags=0) - - This method is a :ref:`coroutine `, similar to - :meth:`socket.getnameinfo` function but non-blocking. - - -Connect pipes -------------- - -On Windows with :class:`SelectorEventLoop`, these methods are not supported. -Use :class:`ProactorEventLoop` to support pipes on Windows. - -.. coroutinemethod:: BaseEventLoop.connect_read_pipe(protocol_factory, pipe) - - Register read pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`Protocol` - interface. *pipe* is a :term:`file-like object `. - Return pair ``(transport, protocol)``, where *transport* supports the - :class:`ReadTransport` interface. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - - This method is a :ref:`coroutine `. - -.. coroutinemethod:: BaseEventLoop.connect_write_pipe(protocol_factory, pipe) - - Register write pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`BaseProtocol` - interface. *pipe* is :term:`file-like object `. - Return pair ``(transport, protocol)``, where *transport* supports - :class:`WriteTransport` interface. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - - This method is a :ref:`coroutine `. - -.. seealso:: - - The :meth:`BaseEventLoop.subprocess_exec` and - :meth:`BaseEventLoop.subprocess_shell` methods. - - -UNIX signals ------------- - -Availability: UNIX only. - -.. method:: BaseEventLoop.add_signal_handler(signum, callback, \*args) - - Add a handler for a signal. - - Raise :exc:`ValueError` if the signal number is invalid or uncatchable. - Raise :exc:`RuntimeError` if there is a problem setting up the handler. - - :ref:`Use functools.partial to pass keywords to the callback - `. - -.. method:: BaseEventLoop.remove_signal_handler(sig) - - Remove a handler for a signal. - - Return ``True`` if a signal handler was removed, ``False`` if not. - -.. seealso:: - - The :mod:`signal` module. - - -Executor --------- - -Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or -pool of processes). By default, an event loop uses a thread pool executor -(:class:`~concurrent.futures.ThreadPoolExecutor`). - -.. coroutinemethod:: BaseEventLoop.run_in_executor(executor, func, \*args) - - Arrange for a *func* to be called in the specified executor. - - The *executor* argument should be an :class:`~concurrent.futures.Executor` - instance. The default executor is used if *executor* is ``None``. - - :ref:`Use functools.partial to pass keywords to the *func* - `. - - This method is a :ref:`coroutine `. - -.. method:: BaseEventLoop.set_default_executor(executor) - - Set the default executor used by :meth:`run_in_executor`. - - -Error Handling API ------------------- - -Allows to customize how exceptions are handled in the event loop. - -.. method:: BaseEventLoop.set_exception_handler(handler) - - Set *handler* as the new event loop exception handler. - - If *handler* is ``None``, the default exception handler will - be set. - - If *handler* is a callable object, it should have a - matching signature to ``(loop, context)``, where ``loop`` - will be a reference to the active event loop, ``context`` - will be a ``dict`` object (see :meth:`call_exception_handler` - documentation for details about context). - -.. method:: BaseEventLoop.default_exception_handler(context) - - Default exception handler. - - This is called when an exception occurs and no exception - handler is set, and can be called by a custom exception - handler that wants to defer to the default behavior. - - *context* parameter has the same meaning as in - :meth:`call_exception_handler`. - -.. method:: BaseEventLoop.call_exception_handler(context) - - Call the current event loop exception handler. - - *context* is a ``dict`` object containing the following keys - (new keys may be introduced later): - - * 'message': Error message; - * 'exception' (optional): Exception object; - * 'future' (optional): :class:`asyncio.Future` instance; - * 'handle' (optional): :class:`asyncio.Handle` instance; - * 'protocol' (optional): :ref:`Protocol ` instance; - * 'transport' (optional): :ref:`Transport ` instance; - * 'socket' (optional): :class:`socket.socket` instance. - - .. note:: - - Note: this method should not be overloaded in subclassed - event loops. For any custom exception handling, use - :meth:`set_exception_handler()` method. - -Debug mode ----------- - -.. method:: BaseEventLoop.get_debug() - - Get the debug mode (:class:`bool`) of the event loop. - - The default value is ``True`` if the environment variable - :envvar:`PYTHONASYNCIODEBUG` is set to a non-empty string, ``False`` - otherwise. - - .. versionadded:: 3.4.2 - -.. method:: BaseEventLoop.set_debug(enabled: bool) - - Set the debug mode of the event loop. - - .. versionadded:: 3.4.2 - -.. seealso:: - - The :ref:`debug mode of asyncio `. - -Server ------- - -.. class:: Server - - Server listening on sockets. - - Object created by the :meth:`BaseEventLoop.create_server` method and the - :func:`start_server` function. Don't instantiate the class directly. - - .. method:: close() - - Stop serving: close listening sockets and set the :attr:`sockets` - attribute to ``None``. - - The sockets that represent existing incoming client connections are left - open. - - The server is closed asynchronously, use the :meth:`wait_closed` - coroutine to wait until the server is closed. - - .. coroutinemethod:: wait_closed() - - Wait until the :meth:`close` method completes. - - This method is a :ref:`coroutine `. - - .. attribute:: sockets - - List of :class:`socket.socket` objects the server is listening to, or - ``None`` if the server is closed. - - -Handle ------- - -.. class:: Handle - - A callback wrapper object returned by :func:`BaseEventLoop.call_soon`, - :func:`BaseEventLoop.call_soon_threadsafe`, :func:`BaseEventLoop.call_later`, - and :func:`BaseEventLoop.call_at`. - - .. method:: cancel() - - Cancel the call. If the callback is already canceled or executed, - this method has no effect. - - -Event loop examples -------------------- - -.. _asyncio-hello-world-callback: - -Hello World with call_soon() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example using the :meth:`BaseEventLoop.call_soon` method to schedule a -callback. The callback displays ``"Hello World"`` and then stops the event -loop:: - - import asyncio - - def hello_world(loop): - print('Hello World') - loop.stop() - - loop = asyncio.get_event_loop() - - # Schedule a call to hello_world() - loop.call_soon(hello_world, loop) - - # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() - -.. seealso:: - - The :ref:`Hello World coroutine ` example - uses a :ref:`coroutine `. - - -.. _asyncio-date-callback: - -Display the current date with call_later() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of callback displaying the current date every second. The callback uses -the :meth:`BaseEventLoop.call_later` method to reschedule itself during 5 -seconds, and then stops the event loop:: - - import asyncio - import datetime - - def display_date(end_time, loop): - print(datetime.datetime.now()) - if (loop.time() + 1.0) < end_time: - loop.call_later(1, display_date, end_time, loop) - else: - loop.stop() - - loop = asyncio.get_event_loop() - - # Schedule the first call to display_date() - end_time = loop.time() + 5.0 - loop.call_soon(display_date, end_time, loop) - - # Blocking call interrupted by loop.stop() - loop.run_forever() - loop.close() - -.. seealso:: - - The :ref:`coroutine displaying the current date - ` example uses a :ref:`coroutine - `. - - -.. _asyncio-watch-read-event: - -Watch a file descriptor for read events -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Wait until a file descriptor received some data using the -:meth:`BaseEventLoop.add_reader` method and then close the event loop:: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - # Create a pair of connected file descriptors - rsock, wsock = socketpair() - loop = asyncio.get_event_loop() - - def reader(): - data = rsock.recv(100) - print("Received:", data.decode()) - # We are done: unregister the file descriptor - loop.remove_reader(rsock) - # Stop the event loop - loop.stop() - - # Register the file descriptor for read event - loop.add_reader(rsock, reader) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() - -.. seealso:: - - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`BaseEventLoop.create_connection` method. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. - - -Set signal handlers for SIGINT and SIGTERM -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM` using -the :meth:`BaseEventLoop.add_signal_handler` method:: - - import asyncio - import functools - import os - import signal - - def ask_exit(signame): - print("got signal %s: exit" % signame) - loop.stop() - - loop = asyncio.get_event_loop() - for signame in ('SIGINT', 'SIGTERM'): - loop.add_signal_handler(getattr(signal, signame), - functools.partial(ask_exit, signame)) - - print("Event loop running forever, press Ctrl+C to interrupt.") - print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid()) - try: - loop.run_forever() - finally: - loop.close() - -This example only works on UNIX. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-eventloops.rst --- a/Doc/library/asyncio-eventloops.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,200 +0,0 @@ -.. currentmodule:: asyncio - -Event loops -=========== - -Event loop functions --------------------- - -The following functions are convenient shortcuts to accessing the methods of the -global policy. Note that this provides access to the default policy, unless an -alternative policy was set by calling :func:`set_event_loop_policy` earlier in -the execution of the process. - -.. function:: get_event_loop() - - Equivalent to calling ``get_event_loop_policy().get_event_loop()``. - -.. function:: set_event_loop(loop) - - Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``. - -.. function:: new_event_loop() - - Equivalent to calling ``get_event_loop_policy().new_event_loop()``. - - -.. _asyncio-event-loops: - -Available event loops ---------------------- - -asyncio currently provides two implementations of event loops: -:class:`SelectorEventLoop` and :class:`ProactorEventLoop`. - -.. class:: SelectorEventLoop - - Event loop based on the :mod:`selectors` module. Subclass of - :class:`BaseEventLoop`. - - Use the most efficient selector available on the platform. - - On Windows, only sockets are supported (ex: pipes are not supported): - see the `MSDN documentation of select - `_. - -.. class:: ProactorEventLoop - - Proactor event loop for Windows using "I/O Completion Ports" aka IOCP. - Subclass of :class:`BaseEventLoop`. - - Availability: Windows. - - .. seealso:: - - `MSDN documentation on I/O Completion Ports - `_. - -Example to use a :class:`ProactorEventLoop` on Windows:: - - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. _asyncio-platform-support: - -Platform support ----------------- - -The :mod:`asyncio` module has been designed to be portable, but each platform -still has subtle differences and may not support all :mod:`asyncio` features. - -Windows -^^^^^^^ - -Common limits of Windows event loops: - -- :meth:`~BaseEventLoop.create_unix_connection` and - :meth:`~BaseEventLoop.create_unix_server` are not supported: the socket - family :data:`socket.AF_UNIX` is specific to UNIX -- :meth:`~BaseEventLoop.add_signal_handler` and - :meth:`~BaseEventLoop.remove_signal_handler` are not supported -- :meth:`EventLoopPolicy.set_child_watcher` is not supported. - :class:`ProactorEventLoop` supports subprocesses. It has only one - implementation to watch child processes, there is no need to configure it. - -:class:`SelectorEventLoop` specific limits: - -- :class:`~selectors.SelectSelector` is used which only supports sockets - and is limited to 512 sockets. -- :meth:`~BaseEventLoop.add_reader` and :meth:`~BaseEventLoop.add_writer` only - accept file descriptors of sockets -- Pipes are not supported - (ex: :meth:`~BaseEventLoop.connect_read_pipe`, - :meth:`~BaseEventLoop.connect_write_pipe`) -- :ref:`Subprocesses ` are not supported - (ex: :meth:`~BaseEventLoop.subprocess_exec`, - :meth:`~BaseEventLoop.subprocess_shell`) - -:class:`ProactorEventLoop` specific limits: - -- :meth:`~BaseEventLoop.create_datagram_endpoint` (UDP) is not supported -- :meth:`~BaseEventLoop.add_reader` and :meth:`~BaseEventLoop.add_writer` are - not supported - -The resolution of the monotonic clock on Windows is usually around 15.6 msec. -The best resolution is 0.5 msec. The resolution depends on the hardware -(availability of `HPET -`_) and on the Windows -configuration. See :ref:`asyncio delayed calls `. - -.. versionchanged:: 3.5 - - :class:`ProactorEventLoop` now supports SSL. - - -Mac OS X -^^^^^^^^ - -Character devices like PTY are only well supported since Mavericks (Mac OS -10.9). They are not supported at all on Mac OS 10.5 and older. - -On Mac OS 10.6, 10.7 and 10.8, the default event loop is -:class:`SelectorEventLoop` which uses :class:`selectors.KqueueSelector`. -:class:`selectors.KqueueSelector` does not support character devices on these -versions. The :class:`SelectorEventLoop` can be used with -:class:`~selectors.SelectSelector` or :class:`~selectors.PollSelector` to -support character devices on these versions of Mac OS X. Example:: - - import asyncio - import selectors - - selector = selectors.SelectSelector() - loop = asyncio.SelectorEventLoop(selector) - asyncio.set_event_loop(loop) - - -Event loop policies and the default policy ------------------------------------------- - -Event loop management is abstracted with a *policy* pattern, to provide maximal -flexibility for custom platforms and frameworks. Throughout the execution of a -process, a single global policy object manages the event loops available to the -process based on the calling context. A policy is an object implementing the -:class:`AbstractEventLoopPolicy` interface. - -For most users of :mod:`asyncio`, policies never have to be dealt with -explicitly, since the default global policy is sufficient. - -The default policy defines context as the current thread, and manages an event -loop per thread that interacts with :mod:`asyncio`. The module-level functions -:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to -event loops managed by the default policy. - - -Event loop policy interface ---------------------------- - -An event loop policy must implement the following interface: - -.. class:: AbstractEventLoopPolicy - - Event loop policy. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Returns an event loop object implementing the :class:`BaseEventLoop` - interface. - - Raises an exception in case no event loop has been set for the current - context and the current policy does not specify to create one. It must - never return ``None``. - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object according to this policy's - rules. - - If there's need to set this loop as the event loop for the current - context, :meth:`set_event_loop` must be called explicitly. - - -Access to the global loop policy --------------------------------- - -.. function:: get_event_loop_policy() - - Get the current event loop policy. - -.. function:: set_event_loop_policy(policy) - - Set the current event loop policy. If *policy* is ``None``, the default - policy is restored. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-protocol.rst --- a/Doc/library/asyncio-protocol.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,716 +0,0 @@ -.. currentmodule:: asyncio - -++++++++++++++++++++++++++++++++++++++++++++++ -Transports and protocols (callback based API) -++++++++++++++++++++++++++++++++++++++++++++++ - -.. _asyncio-transport: - -Transports -========== - -Transports are classes provided by :mod:`asyncio` in order to abstract -various kinds of communication channels. You generally won't instantiate -a transport yourself; instead, you will call a :class:`BaseEventLoop` method -which will create the transport and try to initiate the underlying -communication channel, calling you back when it succeeds. - -Once the communication channel is established, a transport is always -paired with a :ref:`protocol ` instance. The protocol can -then call the transport's methods for various purposes. - -:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and -subprocess pipes. The methods available on a transport depend on -the transport's kind. - -The transport classes are :ref:`not thread safe `. - - -BaseTransport -------------- - -.. class:: BaseTransport - - Base class for transports. - - .. method:: close(self) - - Close the transport. If the transport has a buffer for outgoing - data, buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's :meth:`connection_lost` method will be called with - :const:`None` as its argument. - - .. method:: is_closing(self) - - Return ``True`` if the transport is closing or is closed. - - .. versionadded:: 3.5.1 - - .. method:: get_extra_info(name, default=None) - - Return optional transport information. *name* is a string representing - the piece of transport-specific information to get, *default* is the - value to return if the information doesn't exist. - - This method allows transport implementations to easily expose - channel-specific information. - - * socket: - - - ``'peername'``: the remote address to which the socket is connected, - result of :meth:`socket.socket.getpeername` (``None`` on error) - - ``'socket'``: :class:`socket.socket` instance - - ``'sockname'``: the socket's own address, - result of :meth:`socket.socket.getsockname` - - * SSL socket: - - - ``'compression'``: the compression algorithm being used as a string, - or ``None`` if the connection isn't compressed; result of - :meth:`ssl.SSLSocket.compression` - - ``'cipher'``: a three-value tuple containing the name of the cipher - being used, the version of the SSL protocol that defines its use, and - the number of secret bits being used; result of - :meth:`ssl.SSLSocket.cipher` - - ``'peercert'``: peer certificate; result of - :meth:`ssl.SSLSocket.getpeercert` - - ``'sslcontext'``: :class:`ssl.SSLContext` instance - - ``'ssl_object'``: :class:`ssl.SSLObject` or :class:`ssl.SSLSocket` - instance - - * pipe: - - - ``'pipe'``: pipe object - - * subprocess: - - - ``'subprocess'``: :class:`subprocess.Popen` instance - - .. versionchanged:: 3.5.1 - ``'ssl_object'`` info was added to SSL sockets. - - -ReadTransport -------------- - -.. class:: ReadTransport - - Interface for read-only transports. - - .. method:: pause_reading() - - Pause the receiving end of the transport. No data will be passed to - the protocol's :meth:`data_received` method until :meth:`resume_reading` - is called. - - .. method:: resume_reading() - - Resume the receiving end. The protocol's :meth:`data_received` method - will be called once again if some data is available for reading. - - -WriteTransport --------------- - -.. class:: WriteTransport - - Interface for write-only transports. - - .. method:: abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. - - .. method:: get_write_buffer_size() - - Return the current size of the output buffer used by the transport. - - .. method:: get_write_buffer_limits() - - Get the *high*- and *low*-water limits for write flow control. Return a - tuple ``(low, high)`` where *low* and *high* are positive number of - bytes. - - Use :meth:`set_write_buffer_limits` to set the limits. - - .. versionadded:: 3.4.2 - - .. method:: set_write_buffer_limits(high=None, low=None) - - Set the *high*- and *low*-water limits for write flow control. - - These two values control when call the protocol's - :meth:`pause_writing` and :meth:`resume_writing` methods are called. - If specified, the low-water limit must be less than or equal to the - high-water limit. Neither *high* nor *low* can be negative. - - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to an - implementation-specific value less than or equal to the - high-water limit. Setting *high* to zero forces *low* to zero as - well, and causes :meth:`pause_writing` to be called whenever the - buffer becomes non-empty. Setting *low* to zero causes - :meth:`resume_writing` to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. - - Use :meth:`get_write_buffer_limits` to get the limits. - - .. method:: write(data) - - Write some *data* bytes to the transport. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - - .. method:: writelines(list_of_data) - - Write a list (or any iterable) of data bytes to the transport. - This is functionally equivalent to calling :meth:`write` on each - element yielded by the iterable, but may be implemented more efficiently. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data. - Data may still be received. - - This method can raise :exc:`NotImplementedError` if the transport - (e.g. SSL) doesn't support half-closes. - - -DatagramTransport ------------------ - -.. method:: DatagramTransport.sendto(data, addr=None) - - Send the *data* bytes to the remote peer given by *addr* (a - transport-dependent target address). If *addr* is :const:`None`, the - data is sent to the target address given on transport creation. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - -.. method:: DatagramTransport.abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - -BaseSubprocessTransport ------------------------ - -.. class:: BaseSubprocessTransport - - .. method:: get_pid() - - Return the subprocess process id as an integer. - - .. method:: get_pipe_transport(fd) - - Return the transport for the communication pipe corresponding to the - integer file descriptor *fd*: - - * ``0``: readable streaming transport of the standard input (*stdin*), - or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), - or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), - or :const:`None` if the subprocess was not created with ``stderr=PIPE`` - * other *fd*: :const:`None` - - .. method:: get_returncode() - - Return the subprocess returncode as an integer or :const:`None` - if it hasn't returned, similarly to the - :attr:`subprocess.Popen.returncode` attribute. - - .. method:: kill(self) - - Kill the subprocess, as in :meth:`subprocess.Popen.kill`. - - On POSIX systems, the function sends SIGKILL to the subprocess. - On Windows, this method is an alias for :meth:`terminate`. - - .. method:: send_signal(signal) - - Send the *signal* number to the subprocess, as in - :meth:`subprocess.Popen.send_signal`. - - .. method:: terminate() - - Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. - This method is an alias for the :meth:`close` method. - - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to - stop the subprocess. - - .. method:: close() - - Ask the subprocess to stop by calling the :meth:`terminate` method if the - subprocess hasn't returned yet, and close transports of all pipes - (*stdin*, *stdout* and *stderr*). - - -.. _asyncio-protocol: - -Protocols -========= - -:mod:`asyncio` provides base classes that you can subclass to implement -your network protocols. Those classes are used in conjunction with -:ref:`transports ` (see below): the protocol parses incoming -data and asks for the writing of outgoing data, while the transport is -responsible for the actual I/O and buffering. - -When subclassing a protocol class, it is recommended you override certain -methods. Those methods are callbacks: they will be called by the transport -on certain events (for example when some data is received); you shouldn't -call them yourself, unless you are implementing a transport. - -.. note:: - All callbacks have default implementations, which are empty. Therefore, - you only need to implement the callbacks for the events in which you - are interested. - - -Protocol classes ----------------- - -.. class:: Protocol - - The base class for implementing streaming protocols (for use with - e.g. TCP and SSL transports). - -.. class:: DatagramProtocol - - The base class for implementing datagram protocols (for use with - e.g. UDP transports). - -.. class:: SubprocessProtocol - - The base class for implementing protocols communicating with child - processes (through a set of unidirectional pipes). - - -Connection callbacks --------------------- - -These callbacks may be called on :class:`Protocol`, :class:`DatagramProtocol` -and :class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.connection_made(transport) - - Called when a connection is made. - - The *transport* argument is the transport representing the - connection. You are responsible for storing it somewhere - (e.g. as an attribute) if you need to. - -.. method:: BaseProtocol.connection_lost(exc) - - Called when the connection is lost or closed. - - The argument is either an exception object or :const:`None`. - The latter means a regular EOF is received, or the connection was - aborted or closed by this side of the connection. - -:meth:`~BaseProtocol.connection_made` and :meth:`~BaseProtocol.connection_lost` -are called exactly once per successful connection. All other callbacks will be -called between those two methods, which allows for easier resource management -in your protocol implementation. - -The following callbacks may be called only on :class:`SubprocessProtocol` -instances: - -.. method:: SubprocessProtocol.pipe_data_received(fd, data) - - Called when the child process writes data into its stdout or stderr pipe. - *fd* is the integer file descriptor of the pipe. *data* is a non-empty - bytes object containing the data. - -.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) - - Called when one of the pipes communicating with the child process - is closed. *fd* is the integer file descriptor that was closed. - -.. method:: SubprocessProtocol.process_exited() - - Called when the child process has exited. - - -Streaming protocols -------------------- - -The following callbacks are called on :class:`Protocol` instances: - -.. method:: Protocol.data_received(data) - - Called when some data is received. *data* is a non-empty bytes object - containing the incoming data. - - .. note:: - Whether the data is buffered, chunked or reassembled depends on - the transport. In general, you shouldn't rely on specific semantics - and instead make your parsing generic and flexible enough. However, - data is always received in the correct order. - -.. method:: Protocol.eof_received() - - Calls when the other end signals it won't send any more data - (for example by calling :meth:`write_eof`, if the other end also uses - asyncio). - - This method may return a false value (including None), in which case - the transport will close itself. Conversely, if this method returns a - true value, closing the transport is up to the protocol. Since the - default implementation returns None, it implicitly closes the connection. - - .. note:: - Some transports such as SSL don't support half-closed connections, - in which case returning true from this method will not prevent closing - the connection. - -:meth:`data_received` can be called an arbitrary number of times during -a connection. However, :meth:`eof_received` is called at most once -and, if called, :meth:`data_received` won't be called after it. - -State machine: - - start -> :meth:`~BaseProtocol.connection_made` - [-> :meth:`~Protocol.data_received` \*] - [-> :meth:`~Protocol.eof_received` ?] - -> :meth:`~BaseProtocol.connection_lost` -> end - - -Datagram protocols ------------------- - -The following callbacks are called on :class:`DatagramProtocol` instances. - -.. method:: DatagramProtocol.datagram_received(data, addr) - - Called when a datagram is received. *data* is a bytes object containing - the incoming data. *addr* is the address of the peer sending the data; - the exact format depends on the transport. - -.. method:: DatagramProtocol.error_received(exc) - - Called when a previous send or receive operation raises an - :class:`OSError`. *exc* is the :class:`OSError` instance. - - This method is called in rare conditions, when the transport (e.g. UDP) - detects that a datagram couldn't be delivered to its recipient. - In many conditions though, undeliverable datagrams will be silently - dropped. - - -Flow control callbacks ----------------------- - -These callbacks may be called on :class:`Protocol`, -:class:`DatagramProtocol` and :class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.pause_writing() - - Called when the transport's buffer goes over the high-water mark. - -.. method:: BaseProtocol.resume_writing() - - Called when the transport's buffer drains below the low-water mark. - - -:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- -:meth:`pause_writing` is called once when the buffer goes strictly over -the high-water mark (even if subsequent writes increases the buffer size -even more), and eventually :meth:`resume_writing` is called once when the -buffer size reaches the low-water mark. - -.. note:: - If the buffer size equals the high-water mark, - :meth:`pause_writing` is not called -- it must go strictly over. - Conversely, :meth:`resume_writing` is called when the buffer size is - equal or lower than the low-water mark. These end conditions - are important to ensure that things go as expected when either - mark is zero. - -.. note:: - On BSD systems (OS X, FreeBSD, etc.) flow control is not supported - for :class:`DatagramProtocol`, because send failures caused by - writing too many packets cannot be detected easily. The socket - always appears 'ready' and excess packets are dropped; an - :class:`OSError` with errno set to :const:`errno.ENOBUFS` may or - may not be raised; if it is raised, it will be reported to - :meth:`DatagramProtocol.error_received` but otherwise ignored. - - -Coroutines and protocols ------------------------- - -Coroutines can be scheduled in a protocol method using :func:`ensure_future`, -but there is no guarantee made about the execution order. Protocols are not -aware of coroutines created in protocol methods and so will not wait for them. - -To have a reliable execution order, use :ref:`stream objects ` in a -coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain` -coroutine can be used to wait until the write buffer is flushed. - - -Protocol examples -================= - -.. _asyncio-tcp-echo-client-protocol: - -TCP echo client protocol ------------------------- - -TCP echo client using the :meth:`BaseEventLoop.create_connection` method, send -data and wait until the connection is closed:: - - import asyncio - - class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, loop): - self.message = message - self.loop = loop - - def connection_made(self, transport): - transport.write(self.message.encode()) - print('Data sent: {!r}'.format(self.message)) - - def data_received(self, data): - print('Data received: {!r}'.format(data.decode())) - - def connection_lost(self, exc): - print('The server closed the connection') - print('Stop the event loop') - self.loop.stop() - - loop = asyncio.get_event_loop() - message = 'Hello World!' - coro = loop.create_connection(lambda: EchoClientProtocol(message, loop), - '127.0.0.1', 8888) - loop.run_until_complete(coro) - loop.run_forever() - loop.close() - -The event loop is running twice. The -:meth:`~BaseEventLoop.run_until_complete` method is preferred in this short -example to raise an exception if the server is not listening, instead of -having to write a short coroutine to handle the exception and stop the -running loop. At :meth:`~BaseEventLoop.run_until_complete` exit, the loop is -no longer running, so there is no need to stop the loop in case of an error. - -.. seealso:: - - The :ref:`TCP echo client using streams ` - example uses the :func:`asyncio.open_connection` function. - - -.. _asyncio-tcp-echo-server-protocol: - -TCP echo server protocol ------------------------- - -TCP echo server using the :meth:`BaseEventLoop.create_server` method, send back -received data and close the connection:: - - import asyncio - - class EchoServerClientProtocol(asyncio.Protocol): - def connection_made(self, transport): - peername = transport.get_extra_info('peername') - print('Connection from {}'.format(peername)) - self.transport = transport - - def data_received(self, data): - message = data.decode() - print('Data received: {!r}'.format(message)) - - print('Send: {!r}'.format(message)) - self.transport.write(data) - - print('Close the client socket') - self.transport.close() - - loop = asyncio.get_event_loop() - # Each client connection will create a new protocol instance - coro = loop.create_server(EchoServerClientProtocol, '127.0.0.1', 8888) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - -:meth:`Transport.close` can be called immediately after -:meth:`WriteTransport.write` even if data are not sent yet on the socket: both -methods are asynchronous. ``yield from`` is not needed because these transport -methods are not coroutines. - -.. seealso:: - - The :ref:`TCP echo server using streams ` - example uses the :func:`asyncio.start_server` function. - - -.. _asyncio-udp-echo-client-protocol: - -UDP echo client protocol ------------------------- - -UDP echo client using the :meth:`BaseEventLoop.create_datagram_endpoint` -method, send data and close the transport when we received the answer:: - - import asyncio - - class EchoClientProtocol: - def __init__(self, message, loop): - self.message = message - self.loop = loop - self.transport = None - - def connection_made(self, transport): - self.transport = transport - print('Send:', self.message) - self.transport.sendto(self.message.encode()) - - def datagram_received(self, data, addr): - print("Received:", data.decode()) - - print("Close the socket") - self.transport.close() - - def error_received(self, exc): - print('Error received:', exc) - - def connection_lost(self, exc): - print("Socket closed, stop the event loop") - loop = asyncio.get_event_loop() - loop.stop() - - loop = asyncio.get_event_loop() - message = "Hello World!" - connect = loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, loop), - remote_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(connect) - loop.run_forever() - transport.close() - loop.close() - - -.. _asyncio-udp-echo-server-protocol: - -UDP echo server protocol ------------------------- - -UDP echo server using the :meth:`BaseEventLoop.create_datagram_endpoint` -method, send back received data:: - - import asyncio - - class EchoServerProtocol: - def connection_made(self, transport): - self.transport = transport - - def datagram_received(self, data, addr): - message = data.decode() - print('Received %r from %s' % (message, addr)) - print('Send %r to %s' % (message, addr)) - self.transport.sendto(data, addr) - - loop = asyncio.get_event_loop() - print("Starting UDP server") - # One protocol instance will be created to serve all client requests - listen = loop.create_datagram_endpoint( - EchoServerProtocol, local_addr=('127.0.0.1', 9999)) - transport, protocol = loop.run_until_complete(listen) - - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - transport.close() - loop.close() - - -.. _asyncio-register-socket: - -Register an open socket to wait for data using a protocol ---------------------------------------------------------- - -Wait until a socket receives data using the -:meth:`BaseEventLoop.create_connection` method with a protocol, and then close -the event loop :: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - # Create a pair of connected sockets - rsock, wsock = socketpair() - loop = asyncio.get_event_loop() - - class MyProtocol(asyncio.Protocol): - transport = None - - def connection_made(self, transport): - self.transport = transport - - def data_received(self, data): - print("Received:", data.decode()) - - # We are done: close the transport (it will call connection_lost()) - self.transport.close() - - def connection_lost(self, exc): - # The socket has been closed, stop the event loop - loop.stop() - - # Register the socket to wait for data - connect_coro = loop.create_connection(MyProtocol, sock=rsock) - transport, protocol = loop.run_until_complete(connect_coro) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Run the event loop - loop.run_forever() - - # We are done, close sockets and the event loop - rsock.close() - wsock.close() - loop.close() - -.. seealso:: - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`BaseEventLoop.add_reader` method to register the file descriptor of a - socket. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-queue.rst --- a/Doc/library/asyncio-queue.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,158 +0,0 @@ -.. currentmodule:: asyncio - -Queues -====== - -Queues: - -* :class:`Queue` -* :class:`PriorityQueue` -* :class:`LifoQueue` - -asyncio queue API was designed to be close to classes of the :mod:`queue` -module (:class:`~queue.Queue`, :class:`~queue.PriorityQueue`, -:class:`~queue.LifoQueue`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. - -Queue ------ - -.. class:: Queue(maxsize=0, \*, loop=None) - - A queue, useful for coordinating producer and consumer coroutines. - - If *maxsize* is less than or equal to zero, the queue size is infinite. If - it is an integer greater than ``0``, then ``yield from put()`` will block - when the queue reaches *maxsize*, until an item is removed by :meth:`get`. - - Unlike the standard library :mod:`queue`, you can reliably know this Queue's - size with :meth:`qsize`, since your single-threaded asyncio application won't - be interrupted between calling :meth:`qsize` and doing an operation on the - Queue. - - This class is :ref:`not thread safe `. - - .. versionchanged:: 3.4.4 - New :meth:`join` and :meth:`task_done` methods. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: full() - - Return ``True`` if there are :attr:`maxsize` items in the queue. - - .. note:: - - If the Queue was initialized with ``maxsize=0`` (the default), then - :meth:`full()` is never ``True``. - - .. coroutinemethod:: get() - - Remove and return an item from the queue. If queue is empty, wait until - an item is available. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`empty` method. - - .. method:: get_nowait() - - Remove and return an item from the queue. - - Return an item if one is immediately available, else raise - :exc:`QueueEmpty`. - - .. coroutinemethod:: join() - - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer thread calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`join` unblocks. - - This method is a :ref:`coroutine `. - - .. versionadded:: 3.4.4 - - .. coroutinemethod:: put(item) - - Put an item into the queue. If the queue is full, wait until a free slot - is available before adding item. - - This method is a :ref:`coroutine `. - - .. seealso:: - - The :meth:`full` method. - - .. method:: put_nowait(item) - - Put an item into the queue without blocking. - - If no free slot is immediately available, raise :exc:`QueueFull`. - - .. method:: qsize() - - Number of items in the queue. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a - subsequent call to :meth:`task_done` tells the queue that the processing - on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items - have been processed (meaning that a :meth:`task_done` call was received - for every item that had been :meth:`~Queue.put` into the queue). - - Raises :exc:`ValueError` if called more times than there were items - placed in the queue. - - .. versionadded:: 3.4.4 - - .. attribute:: maxsize - - Number of items allowed in the queue. - - -PriorityQueue -------------- - -.. class:: PriorityQueue - - A subclass of :class:`Queue`; retrieves entries in priority order (lowest - first). - - Entries are typically tuples of the form: (priority number, data). - - -LifoQueue ---------- - -.. class:: LifoQueue - - A subclass of :class:`Queue` that retrieves most recently added entries - first. - - -Exceptions -^^^^^^^^^^ - -.. exception:: QueueEmpty - - Exception raised when the :meth:`~Queue.get_nowait` method is called on a - :class:`Queue` object which is empty. - - -.. exception:: QueueFull - - Exception raised when the :meth:`~Queue.put_nowait` method is called on a - :class:`Queue` object which is full. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-stream.rst --- a/Doc/library/asyncio-stream.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,431 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-streams: - -+++++++++++++++++++++++++++++ -Streams (coroutine based API) -+++++++++++++++++++++++++++++ - -Stream functions -================ - -.. note:: - - The top-level functions in this module are meant convenience wrappers - only; there's really nothing special there, and if they don't do - exactly what you want, feel free to copy their code. - - -.. coroutinefunction:: open_connection(host=None, port=None, \*, loop=None, limit=None, \*\*kwds) - - A wrapper for :meth:`~BaseEventLoop.create_connection()` returning a (reader, - writer) pair. - - The reader returned is a :class:`StreamReader` instance; the writer is - a :class:`StreamWriter` instance. - - The arguments are all the usual arguments to - :meth:`BaseEventLoop.create_connection` except *protocol_factory*; most - common are positional host and port, with various optional keyword arguments - following. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - :class:`StreamReader`). - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, \*\*kwds) - - Start a socket server, with a callback for each client connected. The return - value is the same as :meth:`~BaseEventLoop.create_server()`. - - The *client_connected_cb* parameter is called with two parameters: - *client_reader*, *client_writer*. *client_reader* is a - :class:`StreamReader` object, while *client_writer* is a - :class:`StreamWriter` object. The *client_connected_cb* parameter can - either be a plain callback function or a :ref:`coroutine function - `; if it is a coroutine function, it will be automatically - converted into a :class:`Task`. - - The rest of the arguments are all the usual arguments to - :meth:`~BaseEventLoop.create_server()` except *protocol_factory*; most - common are positional *host* and *port*, with various optional keyword - arguments following. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - :class:`StreamReader`). - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, limit=None, **kwds) - - A wrapper for :meth:`~BaseEventLoop.create_unix_connection()` returning - a (reader, writer) pair. - - See :func:`open_connection` for information about return value and other - details. - - This function is a :ref:`coroutine `. - - Availability: UNIX. - -.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, **kwds) - - Start a UNIX Domain Socket server, with a callback for each client connected. - - See :func:`start_server` for information about return value and other - details. - - This function is a :ref:`coroutine `. - - Availability: UNIX. - - -StreamReader -============ - -.. class:: StreamReader(limit=None, loop=None) - - This class is :ref:`not thread safe `. - - .. method:: exception() - - Get the exception. - - .. method:: feed_eof() - - Acknowledge the EOF. - - .. method:: feed_data(data) - - Feed *data* bytes in the internal buffer. Any operations waiting - for the data will be resumed. - - .. method:: set_exception(exc) - - Set the exception. - - .. method:: set_transport(transport) - - Set the transport. - - .. coroutinemethod:: read(n=-1) - - Read up to *n* bytes. If *n* is not provided, or set to ``-1``, - read until EOF and return all read bytes. - - If the EOF was received and the internal buffer is empty, - return an empty ``bytes`` object. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: readline() - - Read one line, where "line" is a sequence of bytes ending with ``\n``. - - If EOF is received, and ``\n`` was not found, the method will - return the partial read bytes. - - If the EOF was received and the internal buffer is empty, - return an empty ``bytes`` object. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: readexactly(n) - - Read exactly *n* bytes. Raise an :exc:`IncompleteReadError` if the end of - the stream is reached before *n* can be read, the - :attr:`IncompleteReadError.partial` attribute of the exception contains - the partial read bytes. - - This method is a :ref:`coroutine `. - - .. method:: at_eof() - - Return ``True`` if the buffer is empty and :meth:`feed_eof` was called. - - -StreamWriter -============ - -.. class:: StreamWriter(transport, protocol, reader, loop) - - Wraps a Transport. - - This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`, - :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds - :meth:`drain` which returns an optional :class:`Future` on which you can - wait for flow control. It also adds a transport attribute which references - the :class:`Transport` directly. - - This class is :ref:`not thread safe `. - - .. attribute:: transport - - Transport. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. See :meth:`WriteTransport.can_write_eof`. - - .. method:: close() - - Close the transport: see :meth:`BaseTransport.close`. - - .. coroutinemethod:: drain() - - Let the write buffer of the underlying transport a chance to be flushed. - - The intended use is to write:: - - w.write(data) - yield from w.drain() - - When the size of the transport buffer reaches the high-water limit (the - protocol is paused), block until the size of the buffer is drained down - to the low-water limit and the protocol is resumed. When there is nothing - to wait for, the yield-from continues immediately. - - Yielding from :meth:`drain` gives the opportunity for the loop to - schedule the write operation and flush the buffer. It should especially - be used when a possibly large amount of data is written to the transport, - and the coroutine does not yield-from between calls to :meth:`write`. - - This method is a :ref:`coroutine `. - - .. method:: get_extra_info(name, default=None) - - Return optional transport information: see - :meth:`BaseTransport.get_extra_info`. - - .. method:: write(data) - - Write some *data* bytes to the transport: see - :meth:`WriteTransport.write`. - - .. method:: writelines(data) - - Write a list (or any iterable) of data bytes to the transport: - see :meth:`WriteTransport.writelines`. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data: - see :meth:`WriteTransport.write_eof`. - - -StreamReaderProtocol -==================== - -.. class:: StreamReaderProtocol(stream_reader, client_connected_cb=None, loop=None) - - Trivial helper class to adapt between :class:`Protocol` and - :class:`StreamReader`. Sublclass of :class:`Protocol`. - - *stream_reader* is a :class:`StreamReader` instance, *client_connected_cb* - is an optional function called with (stream_reader, stream_writer) when a - connection is made, *loop* is the event loop instance to use. - - (This is a helper class instead of making :class:`StreamReader` itself a - :class:`Protocol` subclass, because the :class:`StreamReader` has other - potential uses, and to prevent the user of the :class:`StreamReader` from - accidentally calling inappropriate methods of the protocol.) - - -IncompleteReadError -=================== - -.. exception:: IncompleteReadError - - Incomplete read error, subclass of :exc:`EOFError`. - - .. attribute:: expected - - Total number of expected bytes (:class:`int`). - - .. attribute:: partial - - Read bytes string before the end of stream was reached (:class:`bytes`). - - -Stream examples -=============== - -.. _asyncio-tcp-echo-client-streams: - -TCP echo client using streams ------------------------------ - -TCP echo client using the :func:`asyncio.open_connection` function:: - - import asyncio - - @asyncio.coroutine - def tcp_echo_client(message, loop): - reader, writer = yield from asyncio.open_connection('127.0.0.1', 8888, - loop=loop) - - print('Send: %r' % message) - writer.write(message.encode()) - - data = yield from reader.read(100) - print('Received: %r' % data.decode()) - - print('Close the socket') - writer.close() - - message = 'Hello World!' - loop = asyncio.get_event_loop() - loop.run_until_complete(tcp_echo_client(message, loop)) - loop.close() - -.. seealso:: - - The :ref:`TCP echo client protocol ` - example uses the :meth:`BaseEventLoop.create_connection` method. - - -.. _asyncio-tcp-echo-server-streams: - -TCP echo server using streams ------------------------------ - -TCP echo server using the :func:`asyncio.start_server` function:: - - import asyncio - - @asyncio.coroutine - def handle_echo(reader, writer): - data = yield from reader.read(100) - message = data.decode() - addr = writer.get_extra_info('peername') - print("Received %r from %r" % (message, addr)) - - print("Send: %r" % message) - writer.write(data) - yield from writer.drain() - - print("Close the client socket") - writer.close() - - loop = asyncio.get_event_loop() - coro = asyncio.start_server(handle_echo, '127.0.0.1', 8888, loop=loop) - server = loop.run_until_complete(coro) - - # Serve requests until Ctrl+C is pressed - print('Serving on {}'.format(server.sockets[0].getsockname())) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - # Close the server - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - -.. seealso:: - - The :ref:`TCP echo server protocol ` - example uses the :meth:`BaseEventLoop.create_server` method. - - -Get HTTP headers ----------------- - -Simple example querying HTTP headers of the URL passed on the command line:: - - import asyncio - import urllib.parse - import sys - - @asyncio.coroutine - def print_http_headers(url): - url = urllib.parse.urlsplit(url) - if url.scheme == 'https': - connect = asyncio.open_connection(url.hostname, 443, ssl=True) - else: - connect = asyncio.open_connection(url.hostname, 80) - reader, writer = yield from connect - query = ('HEAD {path} HTTP/1.0\r\n' - 'Host: {hostname}\r\n' - '\r\n').format(path=url.path or '/', hostname=url.hostname) - writer.write(query.encode('latin-1')) - while True: - line = yield from reader.readline() - if not line: - break - line = line.decode('latin1').rstrip() - if line: - print('HTTP header> %s' % line) - - # Ignore the body, close the socket - writer.close() - - url = sys.argv[1] - loop = asyncio.get_event_loop() - task = asyncio.ensure_future(print_http_headers(url)) - loop.run_until_complete(task) - loop.close() - -Usage:: - - python example.py http://example.com/path/page.html - -or with HTTPS:: - - python example.py https://example.com/path/page.html - -.. _asyncio-register-socket-streams: - -Register an open socket to wait for data using streams ------------------------------------------------------- - -Coroutine waiting until a socket receives data using the -:func:`open_connection` function:: - - import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair - - @asyncio.coroutine - def wait_for_data(loop): - # Create a pair of connected sockets - rsock, wsock = socketpair() - - # Register the open socket to wait for data - reader, writer = yield from asyncio.open_connection(sock=rsock, loop=loop) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Wait for data - data = yield from reader.read(100) - - # Got data, we are done: close the socket - print("Received:", data.decode()) - writer.close() - - # Close the second socket - wsock.close() - - loop = asyncio.get_event_loop() - loop.run_until_complete(wait_for_data(loop)) - loop.close() - -.. seealso:: - - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol created by the - :meth:`BaseEventLoop.create_connection` method. - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`BaseEventLoop.add_reader` method to register the file descriptor of a - socket. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-subprocess.rst --- a/Doc/library/asyncio-subprocess.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,419 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-subprocess: - -Subprocess -========== - -Windows event loop ------------------- - -On Windows, the default event loop is :class:`SelectorEventLoop` which does not -support subprocesses. :class:`ProactorEventLoop` should be used instead. -Example to use it on Windows:: - - import asyncio, sys - - if sys.platform == 'win32': - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - -.. seealso:: - - :ref:`Available event loops ` and :ref:`Platform - support `. - - -Create a subprocess: high-level API using Process -------------------------------------------------- - -.. coroutinefunction:: create_subprocess_exec(\*args, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Create a subprocess. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`BaseEventLoop.subprocess_exec` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - This function is a :ref:`coroutine `. - -.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds) - - Run the shell command *cmd*. - - The *limit* parameter sets the buffer limit passed to the - :class:`StreamReader`. See :meth:`BaseEventLoop.subprocess_shell` for other - parameters. - - Return a :class:`~asyncio.subprocess.Process` instance. - - It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. - - This function is a :ref:`coroutine `. - -Use the :meth:`BaseEventLoop.connect_read_pipe` and -:meth:`BaseEventLoop.connect_write_pipe` methods to connect pipes. - - -Create a subprocess: low-level API using subprocess.Popen ---------------------------------------------------------- - -Run subprocesses asynchronously using the :mod:`subprocess` module. - -.. coroutinemethod:: BaseEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) - - Create a subprocess from one or more string arguments (character strings or - bytes strings encoded to the :ref:`filesystem encoding - `), where the first string - specifies the program to execute, and the remaining strings specify the - program's arguments. (Thus, together the string arguments form the - ``sys.argv`` value of the program, assuming it is a Python script.) This is - similar to the standard library :class:`subprocess.Popen` class called with - shell=False and the list of strings passed as the first argument; - however, where :class:`~subprocess.Popen` takes a single argument which is - list of strings, :func:`subprocess_exec` takes multiple string arguments. - - The *protocol_factory* must instanciate a subclass of the - :class:`asyncio.SubprocessProtocol` class. - - Other parameters: - - * *stdin*: Either a file-like object representing the pipe to be connected - to the subprocess's standard input stream using - :meth:`~BaseEventLoop.connect_write_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. - - * *stdout*: Either a file-like object representing the pipe to be connected - to the subprocess's standard output stream using - :meth:`~BaseEventLoop.connect_read_pipe`, or the constant - :const:`subprocess.PIPE` (the default). By default a new pipe will be - created and connected. - - * *stderr*: Either a file-like object representing the pipe to be connected - to the subprocess's standard error stream using - :meth:`~BaseEventLoop.connect_read_pipe`, or one of the constants - :const:`subprocess.PIPE` (the default) or :const:`subprocess.STDOUT`. - By default a new pipe will be created and connected. When - :const:`subprocess.STDOUT` is specified, the subprocess's standard error - stream will be connected to the same pipe as the standard output stream. - - * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines* and - *shell*, which should not be specified at all. - - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. - - This method is a :ref:`coroutine `. - - See the constructor of the :class:`subprocess.Popen` class for parameters. - -.. coroutinemethod:: BaseEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs) - - Create a subprocess from *cmd*, which is a character string or a bytes - string encoded to the :ref:`filesystem encoding `, - using the platform's "shell" syntax. This is similar to the standard library - :class:`subprocess.Popen` class called with ``shell=True``. - - The *protocol_factory* must instanciate a subclass of the - :class:`asyncio.SubprocessProtocol` class. - - See :meth:`~BaseEventLoop.subprocess_exec` for more details about - the remaining arguments. - - Returns a pair of ``(transport, protocol)``, where *transport* is an - instance of :class:`BaseSubprocessTransport`. - - It is the application's responsibility to ensure that all whitespace and - metacharacters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and shell metacharacters in strings that are going to be - used to construct shell commands. - - This method is a :ref:`coroutine `. - -.. seealso:: - - The :meth:`BaseEventLoop.connect_read_pipe` and - :meth:`BaseEventLoop.connect_write_pipe` methods. - - -Constants ---------- - -.. data:: asyncio.subprocess.PIPE - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that a pipe to the standard stream should be opened. - -.. data:: asyncio.subprocess.STDOUT - - Special value that can be used as the *stderr* argument to - :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that standard error should go into the same handle as standard - output. - -.. data:: asyncio.subprocess.DEVNULL - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and - indicates that the special file :data:`os.devnull` will be used. - - -Process -------- - -.. class:: asyncio.subprocess.Process - - A subprocess created by the :func:`create_subprocess_exec` or the - :func:`create_subprocess_shell` function. - - The API of the :class:`~asyncio.subprocess.Process` class was designed to be - close to the API of the :class:`subprocess.Popen` class, but there are some - differences: - - * There is no explicit :meth:`~subprocess.Popen.poll` method - * The :meth:`~subprocess.Popen.communicate` and - :meth:`~subprocess.Popen.wait` methods don't take a *timeout* parameter: - use the :func:`wait_for` function - * The *universal_newlines* parameter is not supported (only bytes strings - are supported) - * The :meth:`~asyncio.subprocess.Process.wait` method of - the :class:`~asyncio.subprocess.Process` class is asynchronous whereas the - :meth:`~subprocess.Popen.wait` method of the :class:`~subprocess.Popen` - class is implemented as a busy loop. - - This class is :ref:`not thread safe `. See also the - :ref:`Subprocess and threads ` section. - - .. coroutinemethod:: wait() - - Wait for child process to terminate. Set and return :attr:`returncode` - attribute. - - This method is a :ref:`coroutine `. - - .. note:: - - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` and - the child process generates enough output to a pipe such that it - blocks waiting for the OS pipe buffer to accept more data. Use the - :meth:`communicate` method when using pipes to avoid that. - - .. coroutinemethod:: communicate(input=None) - - Interact with process: Send data to stdin. Read data from stdout and - stderr, until end-of-file is reached. Wait for process to terminate. - The optional *input* argument should be data to be sent to the child - process, or ``None``, if no data should be sent to the child. The type - of *input* must be bytes. - - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - - If a :exc:`BrokenPipeError` or :exc:`ConnectionResetError` exception is - raised when writing *input* into stdin, the exception is ignored. It - occurs when the process exits before all data are written into stdin. - - Note that if you want to send data to the process's stdin, you need to - create the Process object with ``stdin=PIPE``. Similarly, to get anything - other than ``None`` in the result tuple, you need to give ``stdout=PIPE`` - and/or ``stderr=PIPE`` too. - - This method is a :ref:`coroutine `. - - .. note:: - - The data read is buffered in memory, so do not use this method if the - data size is large or unlimited. - - .. versionchanged:: 3.4.2 - The method now ignores :exc:`BrokenPipeError` and - :exc:`ConnectionResetError`. - - .. method:: send_signal(signal) - - Sends the signal *signal* to the child process. - - .. note:: - - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. - ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes - started with a *creationflags* parameter which includes - ``CREATE_NEW_PROCESS_GROUP``. - - .. method:: terminate() - - Stop the child. On Posix OSs the method sends :py:data:`signal.SIGTERM` - to the child. On Windows the Win32 API function - :c:func:`TerminateProcess` is called to stop the child. - - .. method:: kill() - - Kills the child. On Posix OSs the function sends :py:data:`SIGKILL` to - the child. On Windows :meth:`kill` is an alias for :meth:`terminate`. - - .. attribute:: stdin - - Standard input stream (:class:`StreamWriter`), ``None`` if the process - was created with ``stdin=None``. - - .. attribute:: stdout - - Standard output stream (:class:`StreamReader`), ``None`` if the process - was created with ``stdout=None``. - - .. attribute:: stderr - - Standard error stream (:class:`StreamReader`), ``None`` if the process - was created with ``stderr=None``. - - .. warning:: - - Use the :meth:`communicate` method rather than :attr:`.stdin.write - `, :attr:`.stdout.read ` or :attr:`.stderr.read ` - to avoid deadlocks due to streams pausing reading or writing and blocking - the child process. - - .. attribute:: pid - - The identifier of the process. - - Note that for processes created by the :func:`create_subprocess_shell` - function, this attribute is the process identifier of the spawned shell. - - .. attribute:: returncode - - Return code of the process when it exited. A ``None`` value indicates - that the process has not terminated yet. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (Unix only). - - -.. _asyncio-subprocess-threads: - -Subprocess and threads ----------------------- - -asyncio supports running subprocesses from different threads, but there -are limits: - -* An event loop must run in the main thread -* The child watcher must be instantiated in the main thread, before executing - subprocesses from other threads. Call the :func:`get_child_watcher` - function in the main thread to instantiate the child watcher. - -The :class:`asyncio.subprocess.Process` class is not thread safe. - -.. seealso:: - - The :ref:`Concurrency and multithreading in asyncio - ` section. - - -Subprocess examples -------------------- - -Subprocess using transport and protocol -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of a subprocess protocol using to get the output of a subprocess and to -wait for the subprocess exit. The subprocess is created by the -:meth:`BaseEventLoop.subprocess_exec` method:: - - import asyncio - import sys - - class DateProtocol(asyncio.SubprocessProtocol): - def __init__(self, exit_future): - self.exit_future = exit_future - self.output = bytearray() - - def pipe_data_received(self, fd, data): - self.output.extend(data) - - def process_exited(self): - self.exit_future.set_result(True) - - @asyncio.coroutine - def get_date(loop): - code = 'import datetime; print(datetime.datetime.now())' - exit_future = asyncio.Future(loop=loop) - - # Create the subprocess controlled by the protocol DateProtocol, - # redirect the standard output into a pipe - create = loop.subprocess_exec(lambda: DateProtocol(exit_future), - sys.executable, '-c', code, - stdin=None, stderr=None) - transport, protocol = yield from create - - # Wait for the subprocess exit using the process_exited() method - # of the protocol - yield from exit_future - - # Close the stdout pipe - transport.close() - - # Read the output which was collected by the pipe_data_received() - # method of the protocol - data = bytes(protocol.output) - return data.decode('ascii').rstrip() - - if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date(loop)) - print("Current date: %s" % date) - loop.close() - - -Subprocess using streams -^^^^^^^^^^^^^^^^^^^^^^^^ - -Example using the :class:`~asyncio.subprocess.Process` class to control the -subprocess and the :class:`StreamReader` class to read from the standard -output. The subprocess is created by the :func:`create_subprocess_exec` -function:: - - import asyncio.subprocess - import sys - - @asyncio.coroutine - def get_date(): - code = 'import datetime; print(datetime.datetime.now())' - - # Create the subprocess, redirect the standard output into a pipe - create = asyncio.create_subprocess_exec(sys.executable, '-c', code, - stdout=asyncio.subprocess.PIPE) - proc = yield from create - - # Read one line of output - data = yield from proc.stdout.readline() - line = data.decode('ascii').rstrip() - - # Wait for the subprocess exit - yield from proc.wait() - return line - - if sys.platform == "win32": - loop = asyncio.ProactorEventLoop() - asyncio.set_event_loop(loop) - else: - loop = asyncio.get_event_loop() - - date = loop.run_until_complete(get_date()) - print("Current date: %s" % date) - loop.close() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-sync.rst --- a/Doc/library/asyncio-sync.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,293 +0,0 @@ -.. currentmodule:: asyncio -.. _asyncio-sync: - -Synchronization primitives -========================== - -Locks: - -* :class:`Lock` -* :class:`Event` -* :class:`Condition` - -Semaphores: - -* :class:`Semaphore` -* :class:`BoundedSemaphore` - -asyncio lock API was designed to be close to classes of the :mod:`threading` -module (:class:`~threading.Lock`, :class:`~threading.Event`, -:class:`~threading.Condition`, :class:`~threading.Semaphore`, -:class:`~threading.BoundedSemaphore`), but it has no *timeout* parameter. The -:func:`asyncio.wait_for` function can be used to cancel a task after a timeout. - -Locks ------ - -Lock -^^^^ - -.. class:: Lock(\*, loop=None) - - Primitive lock objects. - - A primitive lock is a synchronization primitive that is not owned by a - particular coroutine when locked. A primitive lock is in one of two states, - 'locked' or 'unlocked'. - - It is created in the unlocked state. It has two basic methods, :meth:`acquire` - and :meth:`release`. When the state is unlocked, acquire() changes the state to - locked and returns immediately. When the state is locked, acquire() blocks - until a call to release() in another coroutine changes it to unlocked, then - the acquire() call resets it to locked and returns. The release() method - should only be called in the locked state; it changes the state to unlocked - and returns immediately. If an attempt is made to release an unlocked lock, - a :exc:`RuntimeError` will be raised. - - When more than one coroutine is blocked in acquire() waiting for the state - to turn to unlocked, only one coroutine proceeds when a release() call - resets the state to unlocked; first coroutine which is blocked in acquire() - is being processed. - - :meth:`acquire` is a coroutine and should be called with ``yield from``. - - Locks also support the context management protocol. ``(yield from lock)`` - should be used as context manager expression. - - This class is :ref:`not thread safe `. - - Usage:: - - lock = Lock() - ... - yield from lock - try: - ... - finally: - lock.release() - - Context manager usage:: - - lock = Lock() - ... - with (yield from lock): - ... - - Lock objects can be tested for locking state:: - - if not lock.locked(): - yield from lock - else: - # lock is acquired - ... - - .. method:: locked() - - Return ``True`` if the lock is acquired. - - .. coroutinemethod:: acquire() - - Acquire a lock. - - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. - - This method is a :ref:`coroutine `. - - .. method:: release() - - Release a lock. - - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - -Event -^^^^^ - -.. class:: Event(\*, loop=None) - - An Event implementation, asynchronous equivalent to :class:`threading.Event`. - - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`set` method and reset to false with the :meth:`clear` - method. The :meth:`wait` method blocks until the flag is true. The flag is - initially false. - - This class is :ref:`not thread safe `. - - .. method:: clear() - - Reset the internal flag to false. Subsequently, coroutines calling - :meth:`wait` will block until :meth:`set` is called to set the internal - flag to true again. - - .. method:: is_set() - - Return ``True`` if and only if the internal flag is true. - - .. method:: set() - - Set the internal flag to true. All coroutines waiting for it to become - true are awakened. Coroutine that call :meth:`wait` once the flag is true - will not block at all. - - .. coroutinemethod:: wait() - - Block until the internal flag is true. - - If the internal flag is true on entry, return ``True`` immediately. - Otherwise, block until another coroutine calls :meth:`set` to set the - flag to true, then return ``True``. - - This method is a :ref:`coroutine `. - - -Condition -^^^^^^^^^ - -.. class:: Condition(lock=None, \*, loop=None) - - A Condition implementation, asynchronous equivalent to - :class:`threading.Condition`. - - This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. - - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` - object, and it is used as the underlying lock. Otherwise, - a new :class:`Lock` object is created and used as the underlying lock. - - This class is :ref:`not thread safe `. - - .. coroutinemethod:: acquire() - - Acquire the underlying lock. - - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. - - This method is a :ref:`coroutine `. - - .. method:: notify(n=1) - - By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method wakes up at most *n* of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. - - .. note:: - - An awakened coroutine does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. - - .. method:: locked() - - Return ``True`` if the underlying lock is acquired. - - .. method:: notify_all() - - Wake up all coroutines waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting coroutines instead of one. If the - calling coroutine has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - .. method:: release() - - Release the underlying lock. - - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - .. coroutinemethod:: wait() - - Wait until notified. - - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another coroutine. Once awakened, it re-acquires - the lock and returns ``True``. - - This method is a :ref:`coroutine `. - - .. coroutinemethod:: wait_for(predicate) - - Wait until a predicate becomes true. - - The predicate should be a callable which result will be interpreted as a - boolean value. The final predicate value is the return value. - - This method is a :ref:`coroutine `. - - -Semaphores ----------- - -Semaphore -^^^^^^^^^ - -.. class:: Semaphore(value=1, \*, loop=None) - - A Semaphore implementation. - - A semaphore manages an internal counter which is decremented by each - :meth:`acquire` call and incremented by each :meth:`release` call. The - counter can never go below zero; when :meth:`acquire` finds that it is zero, - it blocks, waiting until some other coroutine calls :meth:`release`. - - Semaphores also support the context management protocol. - - The optional argument gives the initial value for the internal counter; it - defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` - is raised. - - This class is :ref:`not thread safe `. - - .. coroutinemethod:: acquire() - - Acquire a semaphore. - - If the internal counter is larger than zero on entry, decrement it by one - and return ``True`` immediately. If it is zero on entry, block, waiting - until some other coroutine has called :meth:`release` to make it larger - than ``0``, and then return ``True``. - - This method is a :ref:`coroutine `. - - .. method:: locked() - - Returns ``True`` if semaphore can not be acquired immediately. - - .. method:: release() - - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another coroutine is waiting for it to become - larger than zero again, wake up that coroutine. - - -BoundedSemaphore -^^^^^^^^^^^^^^^^ - -.. class:: BoundedSemaphore(value=1, \*, loop=None) - - A bounded semaphore implementation. Inherit from :class:`Semaphore`. - - This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would - increase the value above the initial value. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio-task.rst --- a/Doc/library/asyncio-task.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,743 +0,0 @@ -.. currentmodule:: asyncio - -Tasks and coroutines -==================== - -.. _coroutine: - -Coroutines ----------- - -Coroutines used with :mod:`asyncio` may be implemented using the -:keyword:`async def` statement, or by using :term:`generators `. -The :keyword:`async def` type of coroutine was added in Python 3.5, and -is recommended if there is no need to support older Python versions. - -Generator-based coroutines should be decorated with :func:`@asyncio.coroutine -`, although this is not strictly enforced. -The decorator enables compatibility with :keyword:`async def` coroutines, -and also serves as documentation. Generator-based -coroutines use the ``yield from`` syntax introduced in :pep:`380`, -instead of the original ``yield`` syntax. - -The word "coroutine", like the word "generator", is used for two -different (though related) concepts: - -- The function that defines a coroutine - (a function definition using :keyword:`async def` or - decorated with ``@asyncio.coroutine``). If disambiguation is needed - we will call this a *coroutine function* (:func:`iscoroutinefunction` - returns ``True``). - -- The object obtained by calling a coroutine function. This object - represents a computation or an I/O operation (usually a combination) - that will complete eventually. If disambiguation is needed we will - call it a *coroutine object* (:func:`iscoroutine` returns ``True``). - -Things a coroutine can do: - -- ``result = await future`` or ``result = yield from future`` -- - suspends the coroutine until the - future is done, then returns the future's result, or raises an - exception, which will be propagated. (If the future is cancelled, - it will raise a ``CancelledError`` exception.) Note that tasks are - futures, and everything said about futures also applies to tasks. - -- ``result = await coroutine`` or ``result = yield from coroutine`` -- - wait for another coroutine to - produce a result (or raise an exception, which will be propagated). - The ``coroutine`` expression must be a *call* to another coroutine. - -- ``return expression`` -- produce a result to the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -- ``raise exception`` -- raise an exception in the coroutine that is - waiting for this one using :keyword:`await` or ``yield from``. - -Calling a coroutine does not start its code running -- -the coroutine object returned by the call doesn't do anything until you -schedule its execution. There are two basic ways to start it running: -call ``await coroutine`` or ``yield from coroutine`` from another coroutine -(assuming the other coroutine is already running!), or schedule its execution -using the :func:`ensure_future` function or the :meth:`BaseEventLoop.create_task` -method. - - -Coroutines (and tasks) can only run when the event loop is running. - -.. decorator:: coroutine - - Decorator to mark generator-based coroutines. This enables - the generator use :keyword:`!yield from` to call :keyword:`async - def` coroutines, and also enables the generator to be called by - :keyword:`async def` coroutines, for instance using an - :keyword:`await` expression. - - There is no need to decorate :keyword:`async def` coroutines themselves. - - If the generator is not yielded from before it is destroyed, an error - message is logged. See :ref:`Detect coroutines never scheduled - `. - -.. note:: - - In this documentation, some methods are documented as coroutines, - even if they are plain Python functions returning a :class:`Future`. - This is intentional to have a freedom of tweaking the implementation - of these functions in the future. If such a function is needed to be - used in a callback-style code, wrap its result with :func:`ensure_future`. - - -.. _asyncio-hello-world-coroutine: - -Example: Hello World coroutine -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of coroutine displaying ``"Hello World"``:: - - import asyncio - - async def hello_world(): - print("Hello World!") - - loop = asyncio.get_event_loop() - # Blocking call which returns when the hello_world() coroutine is done - loop.run_until_complete(hello_world()) - loop.close() - -.. seealso:: - - The :ref:`Hello World with call_soon() ` - example uses the :meth:`BaseEventLoop.call_soon` method to schedule a - callback. - - -.. _asyncio-date-coroutine: - -Example: Coroutine displaying the current date -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of coroutine displaying the current date every second during 5 seconds -using the :meth:`sleep` function:: - - import asyncio - import datetime - - async def display_date(loop): - end_time = loop.time() + 5.0 - while True: - print(datetime.datetime.now()) - if (loop.time() + 1.0) >= end_time: - break - await asyncio.sleep(1) - - loop = asyncio.get_event_loop() - # Blocking call which returns when the display_date() coroutine is done - loop.run_until_complete(display_date(loop)) - loop.close() - -The same coroutine implemented using a generator:: - - @asyncio.coroutine - def display_date(loop): - end_time = loop.time() + 5.0 - while True: - print(datetime.datetime.now()) - if (loop.time() + 1.0) >= end_time: - break - yield from asyncio.sleep(1) - -.. seealso:: - - The :ref:`display the current date with call_later() - ` example uses a callback with the - :meth:`BaseEventLoop.call_later` method. - - -Example: Chain coroutines -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example chaining coroutines:: - - import asyncio - - async def compute(x, y): - print("Compute %s + %s ..." % (x, y)) - await asyncio.sleep(1.0) - return x + y - - async def print_sum(x, y): - result = await compute(x, y) - print("%s + %s = %s" % (x, y, result)) - - loop = asyncio.get_event_loop() - loop.run_until_complete(print_sum(1, 2)) - loop.close() - -``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits -until ``compute()`` is completed before returning its result. - -Sequence diagram of the example: - -.. image:: tulip_coro.png - :align: center - -The "Task" is created by the :meth:`BaseEventLoop.run_until_complete` method -when it gets a coroutine object instead of a task. - -The diagram shows the control flow, it does not describe exactly how things -work internally. For example, the sleep coroutine creates an internal future -which uses :meth:`BaseEventLoop.call_later` to wake up the task in 1 second. - - -InvalidStateError ------------------ - -.. exception:: InvalidStateError - - The operation is not allowed in this state. - - -TimeoutError ------------- - -.. exception:: TimeoutError - - The operation exceeded the given deadline. - -.. note:: - - This exception is different from the builtin :exc:`TimeoutError` exception! - - -Future ------- - -.. class:: Future(\*, loop=None) - - This class is *almost* compatible with :class:`concurrent.futures.Future`. - - Differences: - - - :meth:`result` and :meth:`exception` do not take a timeout argument and - raise an exception when the future isn't done yet. - - - Callbacks registered with :meth:`add_done_callback` are always called - via the event loop's :meth:`~BaseEventLoop.call_soon_threadsafe`. - - - This class is not compatible with the :func:`~concurrent.futures.wait` and - :func:`~concurrent.futures.as_completed` functions in the - :mod:`concurrent.futures` package. - - This class is :ref:`not thread safe `. - - .. method:: cancel() - - Cancel the future and schedule callbacks. - - If the future is already done or cancelled, return ``False``. Otherwise, - change the future's state to cancelled, schedule the callbacks and return - ``True``. - - .. method:: cancelled() - - Return ``True`` if the future was cancelled. - - .. method:: done() - - Return True if the future is done. - - Done means either that a result / exception are available, or that the - future was cancelled. - - .. method:: result() - - Return the result this future represents. - - If the future has been cancelled, raises :exc:`CancelledError`. If the - future's result isn't yet available, raises :exc:`InvalidStateError`. If - the future is done and has an exception set, this exception is raised. - - .. method:: exception() - - Return the exception that was set on this future. - - The exception (or ``None`` if no exception was set) is returned only if - the future is done. If the future has been cancelled, raises - :exc:`CancelledError`. If the future isn't done yet, raises - :exc:`InvalidStateError`. - - .. method:: add_done_callback(fn) - - Add a callback to be run when the future becomes done. - - The callback is called with a single argument - the future object. If the - future is already done when this is called, the callback is scheduled - with :meth:`~BaseEventLoop.call_soon`. - - :ref:`Use functools.partial to pass parameters to the callback - `. For example, - ``fut.add_done_callback(functools.partial(print, "Future:", - flush=True))`` will call ``print("Future:", fut, flush=True)``. - - .. method:: remove_done_callback(fn) - - Remove all instances of a callback from the "call when done" list. - - Returns the number of callbacks removed. - - .. method:: set_result(result) - - Mark the future done and set its result. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - .. method:: set_exception(exception) - - Mark the future done and set an exception. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - -Example: Future with run_until_complete() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example combining a :class:`Future` and a :ref:`coroutine function -`:: - - import asyncio - - @asyncio.coroutine - def slow_operation(future): - yield from asyncio.sleep(1) - future.set_result('Future is done!') - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - loop.run_until_complete(future) - print(future.result()) - loop.close() - -The coroutine function is responsible for the computation (which takes 1 second) -and it stores the result into the future. The -:meth:`~BaseEventLoop.run_until_complete` method waits for the completion of -the future. - -.. note:: - The :meth:`~BaseEventLoop.run_until_complete` method uses internally the - :meth:`~Future.add_done_callback` method to be notified when the future is - done. - - -Example: Future with run_forever() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The previous example can be written differently using the -:meth:`Future.add_done_callback` method to describe explicitly the control -flow:: - - import asyncio - - @asyncio.coroutine - def slow_operation(future): - yield from asyncio.sleep(1) - future.set_result('Future is done!') - - def got_result(future): - print(future.result()) - loop.stop() - - loop = asyncio.get_event_loop() - future = asyncio.Future() - asyncio.ensure_future(slow_operation(future)) - future.add_done_callback(got_result) - try: - loop.run_forever() - finally: - loop.close() - -In this example, the future is used to link ``slow_operation()`` to -``got_result()``: when ``slow_operation()`` is done, ``got_result()`` is called -with the result. - - -Task ----- - -.. class:: Task(coro, \*, loop=None) - - Schedule the execution of a :ref:`coroutine `: wrap it in a - future. A task is a subclass of :class:`Future`. - - A task is responsible for executing a coroutine object in an event loop. If - the wrapped coroutine yields from a future, the task suspends the execution - of the wrapped coroutine and waits for the completition of the future. When - the future is done, the execution of the wrapped coroutine restarts with the - result or the exception of the future. - - Event loops use cooperative scheduling: an event loop only runs one task at - a time. Other tasks may run in parallel if other event loops are - running in different threads. While a task waits for the completion of a - future, the event loop executes a new task. - - The cancellation of a task is different from the cancelation of a - future. Calling :meth:`cancel` will throw a - :exc:`~concurrent.futures.CancelledError` to the wrapped - coroutine. :meth:`~Future.cancelled` only returns ``True`` if the - wrapped coroutine did not catch the - :exc:`~concurrent.futures.CancelledError` exception, or raised a - :exc:`~concurrent.futures.CancelledError` exception. - - If a pending task is destroyed, the execution of its wrapped :ref:`coroutine - ` did not complete. It is probably a bug and a warning is - logged: see :ref:`Pending task destroyed `. - - Don't directly create :class:`Task` instances: use the :func:`ensure_future` - function or the :meth:`BaseEventLoop.create_task` method. - - This class is :ref:`not thread safe `. - - .. classmethod:: all_tasks(loop=None) - - Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - - .. classmethod:: current_task(loop=None) - - Return the currently running task in an event loop or ``None``. - - By default the current task for the current event loop is returned. - - ``None`` is returned when called not in the context of a :class:`Task`. - - .. method:: cancel() - - Request that this task cancel itself. - - This arranges for a :exc:`~concurrent.futures.CancelledError` to be - thrown into the wrapped coroutine on the next cycle through the event - loop. The coroutine then has a chance to clean up or even deny the - request using try/except/finally. - - Unlike :meth:`Future.cancel`, this does not guarantee that the task - will be cancelled: the exception might be caught and acted upon, delaying - cancellation of the task or preventing cancellation completely. The task - may also return a value or raise a different exception. - - Immediately after this method is called, :meth:`~Future.cancelled` will - not return ``True`` (unless the task was already cancelled). A task will - be marked as cancelled when the wrapped coroutine terminates with a - :exc:`~concurrent.futures.CancelledError` exception (even if - :meth:`cancel` was not called). - - .. method:: get_stack(\*, limit=None) - - Return the list of stack frames for this task's coroutine. - - If the coroutine is not done, this returns the stack where it is - suspended. If the coroutine has completed successfully or was - cancelled, this returns an empty list. If the coroutine was - terminated by an exception, this returns the list of traceback - frames. - - The frames are always ordered from oldest to newest. - - The optional limit gives the maximum number of frames to return; by - default all available frames are returned. Its meaning differs depending - on whether a stack or a traceback is returned: the newest frames of a - stack are returned, but the oldest frames of a traceback are returned. - (This matches the behavior of the traceback module.) - - For reasons beyond our control, only one stack frame is returned for a - suspended coroutine. - - .. method:: print_stack(\*, limit=None, file=None) - - Print the stack or traceback for this task's coroutine. - - This produces output similar to that of the traceback module, for the - frames retrieved by get_stack(). The limit argument is passed to - get_stack(). The file argument is an I/O stream to which the output - is written; by default output is written to sys.stderr. - - -Example: Parallel execution of tasks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example executing 3 tasks (A, B, C) in parallel:: - - import asyncio - - @asyncio.coroutine - def factorial(name, number): - f = 1 - for i in range(2, number+1): - print("Task %s: Compute factorial(%s)..." % (name, i)) - yield from asyncio.sleep(1) - f *= i - print("Task %s: factorial(%s) = %s" % (name, number, f)) - - loop = asyncio.get_event_loop() - tasks = [ - asyncio.ensure_future(factorial("A", 2)), - asyncio.ensure_future(factorial("B", 3)), - asyncio.ensure_future(factorial("C", 4))] - loop.run_until_complete(asyncio.wait(tasks)) - loop.close() - -Output:: - - Task A: Compute factorial(2)... - Task B: Compute factorial(2)... - Task C: Compute factorial(2)... - Task A: factorial(2) = 2 - Task B: Compute factorial(3)... - Task C: Compute factorial(3)... - Task B: factorial(3) = 6 - Task C: Compute factorial(4)... - Task C: factorial(4) = 24 - -A task is automatically scheduled for execution when it is created. The event -loop stops when all tasks are done. - - -Task functions --------------- - -.. note:: - - In the functions below, the optional *loop* argument allows to explicitly set - the event loop object used by the underlying task or coroutine. If it's - not provided, the default event loop is used. - -.. function:: as_completed(fs, \*, loop=None, timeout=None) - - Return an iterator whose values, when waited for, are :class:`Future` - instances. - - Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures - are done. - - Example:: - - for f in as_completed(fs): - result = yield from f # The 'yield from' may raise - # Use result - - .. note:: - - The futures ``f`` are not necessarily members of fs. - -.. function:: ensure_future(coro_or_future, \*, loop=None) - - Schedule the execution of a :ref:`coroutine object `: wrap it in - a future. Return a :class:`Task` object. - - If the argument is a :class:`Future`, it is returned directly. - - .. versionadded:: 3.4.4 - - .. versionchanged:: 3.5.1 - The function accepts any :term:`awaitable` object. - - .. seealso:: - - The :meth:`BaseEventLoop.create_task` method. - -.. function:: async(coro_or_future, \*, loop=None) - - A deprecated alias to :func:`ensure_future`. - - .. deprecated:: 3.4.4 - -.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) - - Return a future aggregating results from the given coroutine objects or - futures. - - All futures must share the same event loop. If all the tasks are done - successfully, the returned future's result is the list of results (in the - order of the original sequence, not necessarily the order of results - arrival). If *return_exceptions* is True, exceptions in the tasks are - treated the same as successful results, and gathered in the result list; - otherwise, the first raised exception will be immediately propagated to the - returned future. - - Cancellation: if the outer Future is cancelled, all children (that have not - completed yet) are also cancelled. If any child is cancelled, this is - treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the - outer Future is *not* cancelled in this case. (This is to prevent the - cancellation of one child to cause other children to be cancelled.) - -.. function:: iscoroutine(obj) - - Return ``True`` if *obj* is a :ref:`coroutine object `, - which may be based on a generator or an :keyword:`async def` coroutine. - -.. function:: iscoroutinefunction(func) - - Return ``True`` if *func* is determined to be a :ref:`coroutine function - `, which may be a decorated generator function or an - :keyword:`async def` function. - -.. function:: run_coroutine_threadsafe(coro, loop) - - Submit a :ref:`coroutine object ` to a given event loop. - - Return a :class:`concurrent.futures.Future` to access the result. - - This function is meant to be called from a different thread than the one - where the event loop is running. Usage:: - - # Create a coroutine - coro = asyncio.sleep(1, result=3) - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 - - If an exception is raised in the coroutine, the returned future will be - notified. It can also be used to cancel the task in the event loop:: - - try: - result = future.result(timeout) - except asyncio.TimeoutError: - print('The coroutine took too long, cancelling the task...') - future.cancel() - except Exception as exc: - print('The coroutine raised an exception: {!r}'.format(exc)) - else: - print('The coroutine returned: {!r}'.format(result)) - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - .. note:: - - Unlike other functions from the module, - :func:`run_coroutine_threadsafe` requires the *loop* argument to - be passed explicitely. - - .. versionadded:: 3.5.1 - -.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) - - Create a :ref:`coroutine ` that completes after a given - time (in seconds). If *result* is provided, it is produced to the caller - when the coroutine completes. - - The resolution of the sleep depends on the :ref:`granularity of the event - loop `. - - This function is a :ref:`coroutine `. - -.. function:: shield(arg, \*, loop=None) - - Wait for a future, shielding it from cancellation. - - The statement:: - - res = yield from shield(something()) - - is exactly equivalent to the statement:: - - res = yield from something() - - *except* that if the coroutine containing it is cancelled, the task running - in ``something()`` is not cancelled. From the point of view of - ``something()``, the cancellation did not happen. But its caller is still - cancelled, so the yield-from expression still raises - :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is - cancelled by other means this will still cancel ``shield()``. - - If you want to completely ignore cancellation (not recommended) you can - combine ``shield()`` with a try/except clause, as follows:: - - try: - res = yield from shield(something()) - except CancelledError: - res = None - -.. function:: timeout(timeout, \*, loop=None) - - Return a context manager that cancels a block on *timeout* expiring:: - - with timeout(1.5): - yield from inner() - - 1. If ``inner()`` is executed faster than in ``1.5`` seconds - nothing happens. - 2. Otherwise ``inner()`` is cancelled internally but - :exc:`asyncio.TimeoutError` is raised outside of - context manager scope. - -.. coroutinefunction:: wait(futures, \*, loop=None, timeout=None,\ - return_when=ALL_COMPLETED) - - Wait for the Futures and coroutine objects given by the sequence *futures* - to complete. Coroutines will be wrapped in Tasks. Returns two sets of - :class:`Future`: (done, pending). - - The sequence *futures* must not be empty. - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - - *return_when* indicates when this function should return. It must be one of - the following constants of the :mod:`concurrent.futures` module: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - - This function is a :ref:`coroutine `. - - Usage:: - - done, pending = yield from asyncio.wait(fs) - - .. note:: - - This does not raise :exc:`asyncio.TimeoutError`! Futures that aren't done - when the timeout occurs are returned in the second set. - - -.. coroutinefunction:: wait_for(fut, timeout, \*, loop=None) - - Wait for the single :class:`Future` or :ref:`coroutine object ` - to complete with timeout. If *timeout* is ``None``, block until the future - completes. - - Coroutine will be wrapped in :class:`Task`. - - Returns result of the Future or coroutine. When a timeout occurs, it - cancels the task and raises :exc:`asyncio.TimeoutError`. To avoid the task - cancellation, wrap it in :func:`shield`. - - If the wait is cancelled, the future *fut* is also cancelled. - - This function is a :ref:`coroutine `, usage:: - - result = yield from asyncio.wait_for(fut, 60.0) - - .. versionchanged:: 3.4.3 - If the wait is cancelled, the future *fut* is now also cancelled. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncio.rst --- a/Doc/library/asyncio.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,73 +0,0 @@ -:mod:`asyncio` -- Asynchronous I/O, event loop, coroutines and tasks -==================================================================== - -.. module:: asyncio - :synopsis: Asynchronous I/O, event loop, coroutines and tasks. - -.. note:: - - The asyncio package has been included in the standard library on a - :term:`provisional basis `. Backwards incompatible - changes (up to and including removal of the module) may occur if deemed - necessary by the core developers. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/asyncio/` - --------------- - -This module provides infrastructure for writing single-threaded concurrent -code using coroutines, multiplexing I/O access over sockets and other -resources, running network clients and servers, and other related primitives. -Here is a more detailed list of the package contents: - -* a pluggable :ref:`event loop ` with various system-specific - implementations; - -* :ref:`transport ` and :ref:`protocol ` abstractions - (similar to those in `Twisted `_); - -* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and - others (some may be system-dependent); - -* a :class:`Future` class that mimics the one in the :mod:`concurrent.futures` - module, but adapted for use with the event loop; - -* coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write - concurrent code in a sequential fashion; - -* cancellation support for :class:`Future`\s and coroutines; - -* :ref:`synchronization primitives ` for use between coroutines in - a single thread, mimicking those in the :mod:`threading` module; - -* an interface for passing work off to a threadpool, for times when - you absolutely, positively have to use a library that makes blocking - I/O calls. - -Asynchronous programming is more complex than classical "sequential" -programming: see the :ref:`Develop with asyncio ` page which lists -common traps and explains how to avoid them. :ref:`Enable the debug mode -` during development to detect common issues. - -Table of contents: - -.. toctree:: - :maxdepth: 3 - - asyncio-eventloop.rst - asyncio-eventloops.rst - asyncio-task.rst - asyncio-protocol.rst - asyncio-stream.rst - asyncio-subprocess.rst - asyncio-sync.rst - asyncio-queue.rst - asyncio-dev.rst - -.. seealso:: - - The :mod:`asyncio` module was designed in :PEP:`3156`. For a - motivational primer on transports and protocols, see :PEP:`3153`. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/asyncore.rst --- a/Doc/library/asyncore.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/asyncore.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,11 +13,6 @@ -------------- -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - This module provides the basic infrastructure for writing asynchronous socket service clients and servers. @@ -58,10 +53,10 @@ channels have been closed. All arguments are optional. The *count* parameter defaults to None, resulting in the loop terminating only when all channels have been closed. The *timeout* argument sets the timeout - parameter for the appropriate :func:`~select.select` or :func:`~select.poll` - call, measured in seconds; the default is 30 seconds. The *use_poll* - parameter, if true, indicates that :func:`~select.poll` should be used in - preference to :func:`~select.select` (the default is ``False``). + parameter for the appropriate :func:`select` or :func:`poll` call, measured + in seconds; the default is 30 seconds. The *use_poll* parameter, if true, + indicates that :func:`poll` should be used in preference to :func:`select` + (the default is ``False``). The *map* parameter is a dictionary whose items are the channels to watch. As channels are closed they are deleted from their map. If *map* is @@ -213,12 +208,7 @@ .. method:: recv(buffer_size) Read at most *buffer_size* bytes from the socket's remote end-point. An - empty bytes object implies that the channel has been closed from the - other end. - - Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though - :func:`select.select` or :func:`select.poll` has reported the socket - ready for reading. + empty string implies that the channel has been closed from the other end. .. method:: listen(backlog) @@ -287,7 +277,7 @@ Here is a very basic HTTP client that uses the :class:`dispatcher` class to implement its socket handling:: - import asyncore + import asyncore, socket class HTTPClient(asyncore.dispatcher): @@ -327,6 +317,7 @@ connections and dispatches the incoming connections to a handler:: import asyncore + import socket class EchoHandler(asyncore.dispatcher_with_send): @@ -350,3 +341,4 @@ server = EchoServer('localhost', 8080) asyncore.loop() + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/atexit.rst --- a/Doc/library/atexit.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/atexit.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,14 +9,13 @@ The :mod:`atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* -order in which they were registered; if you register ``A``, ``B``, and ``C``, -at interpreter termination time they will be run in the order ``C``, ``B``, -``A``. +interpreter termination. The order in which the functions are called is not +defined; if you have cleanup operations that depend on each other, you should +wrap them in a function and register that one. This keeps :mod:`atexit` simple. -**Note:** The functions registered via this module are not called when the -program is killed by a signal not handled by Python, when a Python fatal -internal error is detected, or when :func:`os._exit` is called. +Note: the functions registered via this module are not called when the program +is killed by a signal not handled by Python, when a Python fatal internal error +is detected, or when :func:`os._exit` is called. .. function:: register(func, *args, **kargs) @@ -68,7 +67,7 @@ making an explicit call into this module at termination. :: try: - with open("counterfile") as infile: + with open("/tmp/counter") as infile: _count = int(infile.read()) except FileNotFoundError: _count = 0 @@ -78,7 +77,7 @@ _count = _count + n def savecounter(): - with open("counterfile", "w") as outfile: + with open("/tmp/counter", "w") as outfile: outfile.write("%d" % _count) import atexit diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/audioop.rst --- a/Doc/library/audioop.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/audioop.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,14 +6,9 @@ The :mod:`audioop` module contains some useful operations on sound fragments. -It operates on sound fragments consisting of signed integer samples 8, 16, 24 -or 32 bits wide, stored in :term:`bytes-like object`\ s. All scalar items are -integers, unless specified otherwise. - -.. versionchanged:: 3.4 - Support for 24-bit samples was added. - All functions now accept any :term:`bytes-like object`. - String input now results in an immediate error. +It operates on sound fragments consisting of signed integer samples 8, 16 or 32 +bits wide, stored in Python strings. All scalar items are integers, unless +specified otherwise. .. index:: single: Intel/DVI ADPCM @@ -40,8 +35,8 @@ .. function:: add(fragment1, fragment2, width) Return a fragment which is the addition of the two samples passed as parameters. - *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both - fragments should have the same length. Samples are truncated in case of overflow. + *width* is the sample width in bytes, either ``1``, ``2`` or ``4``. Both + fragments should have the same length. .. function:: adpcm2lin(adpcmfragment, width, state) @@ -72,15 +67,7 @@ .. function:: bias(fragment, width, bias) Return a fragment that is the original fragment with a bias added to each - sample. Samples wrap around in case of overflow. - - -.. function:: byteswap(fragment, width) - - "Byteswap" all samples in a fragment and returns the modified fragment. - Converts big-endian samples to little-endian and vice versa. - - .. versionadded:: 3.4 + sample. .. function:: cross(fragment, width) @@ -139,56 +126,56 @@ .. function:: lin2alaw(fragment, width) Convert samples in the audio fragment to a-LAW encoding and return this as a - bytes object. a-LAW is an audio encoding format whereby you get a dynamic + Python string. a-LAW is an audio encoding format whereby you get a dynamic range of about 13 bits using only 8 bit samples. It is used by the Sun audio hardware, among others. .. function:: lin2lin(fragment, width, newwidth) - Convert samples between 1-, 2-, 3- and 4-byte formats. + Convert samples between 1-, 2- and 4-byte formats. .. note:: - In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are + In some audio formats, such as .WAV files, 16 and 32 bit samples are signed, but 8 bit samples are unsigned. So when converting to 8 bit wide samples for these formats, you need to also add 128 to the result:: new_frames = audioop.lin2lin(frames, old_width, 1) new_frames = audioop.bias(new_frames, 1, 128) - The same, in reverse, has to be applied when converting from 8 to 16, 24 - or 32 bit width samples. + The same, in reverse, has to be applied when converting from 8 to 16 or 32 + bit width samples. .. function:: lin2ulaw(fragment, width) Convert samples in the audio fragment to u-LAW encoding and return this as a - bytes object. u-LAW is an audio encoding format whereby you get a dynamic + Python string. u-LAW is an audio encoding format whereby you get a dynamic range of about 14 bits using only 8 bit samples. It is used by the Sun audio hardware, among others. -.. function:: max(fragment, width) - - Return the maximum of the *absolute value* of all samples in a fragment. - - -.. function:: maxpp(fragment, width) - - Return the maximum peak-peak value in the sound fragment. - - .. function:: minmax(fragment, width) Return a tuple consisting of the minimum and maximum values of all samples in the sound fragment. +.. function:: max(fragment, width) + + Return the maximum of the *absolute value* of all samples in a fragment. + + +.. function:: maxpp(fragment, width) + + Return the maximum peak-peak value in the sound fragment. + + .. function:: mul(fragment, width, factor) Return a fragment that has all samples in the original fragment multiplied by - the floating-point value *factor*. Samples are truncated in case of overflow. + the floating-point value *factor*. Overflow is silently ignored. .. function:: ratecv(fragment, width, nchannels, inrate, outrate, state[, weightA[, weightB]]) @@ -254,7 +241,7 @@ transmit the data but also the state. Note that you should send the *initial* state (the one you passed to :func:`lin2adpcm`) along to the decoder, not the final state (as returned by the coder). If you want to use -:class:`struct.Struct` to store the state in binary you can code the first +:func:`struct.struct` to store the state in binary you can code the first element (the predicted value) in 16 bits and the second (the delta index) in 8. The ADPCM coders have never been tried against other ADPCM coders, only against diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/base64.rst --- a/Doc/library/base64.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/base64.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,72 +1,56 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== +:mod:`base64` --- RFC 3548: Base16, Base32, Base64 Data Encodings +================================================================= .. module:: base64 - :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings; - Base85 and Ascii85 + :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings .. index:: pair: base64; encoding single: MIME; base64 encoding -This module provides functions for encoding binary data to printable -ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`3548`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`3548` encodings are suitable for encoding binary data so that it can +This module provides data encoding and decoding as specified in :rfc:`3548`. +This standard defines the Base16, Base32, and Base64 algorithms for encoding +and decoding arbitrary binary strings into ASCII-only byte strings that can be safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. The encoding algorithm is not the same as the :program:`uuencode` program. There are two interfaces provided by this module. The modern interface -supports encoding :term:`bytes-like objects ` to ASCII -:class:`bytes`, and decoding :term:`bytes-like objects ` or -strings containing ASCII to :class:`bytes`. All three :rfc:`3548` defined -alphabets (normal, URL-safe, and filesystem-safe) are supported. - -The legacy interface does not support decoding from strings, but it does -provide functions for encoding and decoding to and from :term:`file objects -`. It only supports the Base64 standard alphabet, and it adds -newlines every 76 characters as per :rfc:`2045`. Note that if you are looking -for :rfc:`2045` support you probably want to be looking at the :mod:`email` -package instead. - +supports encoding and decoding ASCII byte string objects using all three +alphabets. Additionally, the decoding functions of the modern interface also +accept Unicode strings containing only ASCII characters. The legacy interface +provides for encoding and decoding to and from file-like objects as well as +byte strings, but only using the Base64 standard alphabet. .. versionchanged:: 3.3 ASCII-only Unicode strings are now accepted by the decoding functions of the modern interface. -.. versionchanged:: 3.4 - Any :term:`bytes-like object`\ s are now accepted by all - encoding and decoding functions in this module. Ascii85/Base85 support added. - The modern interface provides: .. function:: b64encode(s, altchars=None) - Encode the :term:`bytes-like object` *s* using Base64 and return the encoded - :class:`bytes`. + Encode a byte string using Base64. - Optional *altchars* must be a :term:`bytes-like object` of at least + *s* is the string to encode. Optional *altchars* must be a string of at least length 2 (additional characters are ignored) which specifies an alternative alphabet for the ``+`` and ``/`` characters. This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. + The encoded byte string is returned. + .. function:: b64decode(s, altchars=None, validate=False) - Decode the Base64 encoded :term:`bytes-like object` or ASCII string - *s* and return the decoded :class:`bytes`. + Decode a Base64 encoded byte string. - Optional *altchars* must be a :term:`bytes-like object` or ASCII string of + *s* is the byte string to decode. Optional *altchars* must be a string of at least length 2 (additional characters are ignored) which specifies the alternative alphabet used instead of the ``+`` and ``/`` characters. - A :exc:`binascii.Error` exception is raised + The decoded string is returned. A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. If *validate* is ``False`` (the default), non-base64-alphabet characters are @@ -77,44 +61,38 @@ .. function:: standard_b64encode(s) - Encode :term:`bytes-like object` *s* using the standard Base64 alphabet - and return the encoded :class:`bytes`. + Encode byte string *s* using the standard Base64 alphabet. .. function:: standard_b64decode(s) - Decode :term:`bytes-like object` or ASCII string *s* using the standard - Base64 alphabet and return the decoded :class:`bytes`. + Decode byte string *s* using the standard Base64 alphabet. .. function:: urlsafe_b64encode(s) - Encode :term:`bytes-like object` *s* using a URL-safe alphabet, which - substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the - standard Base64 alphabet, and return the encoded :class:`bytes`. The result + Encode byte string *s* using a URL-safe alphabet, which substitutes ``-`` instead of + ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet. The result can still contain ``=``. .. function:: urlsafe_b64decode(s) - Decode :term:`bytes-like object` or ASCII string *s* using a URL-safe - alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of - ``/`` in the standard Base64 alphabet, and return the decoded - :class:`bytes`. + Decode byte string *s* using a URL-safe alphabet, which substitutes ``-`` instead of + ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet. .. function:: b32encode(s) - Encode the :term:`bytes-like object` *s* using Base32 and return the - encoded :class:`bytes`. + Encode a byte string using Base32. *s* is the string to encode. The encoded string + is returned. .. function:: b32decode(s, casefold=False, map01=None) - Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. + Decode a Base32 encoded byte string. - Optional *casefold* is a flag specifying + *s* is the byte string to decode. Optional *casefold* is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. @@ -125,99 +103,29 @@ digit 0 is always mapped to the letter O). For security purposes the default is ``None``, so that 0 and 1 are not allowed in the input. - A :exc:`binascii.Error` is raised if *s* is + The decoded byte string is returned. A :exc:`TypeError` is raised if *s* were incorrectly padded or if there are non-alphabet characters present in the - input. + string. .. function:: b16encode(s) - Encode the :term:`bytes-like object` *s* using Base16 and return the - encoded :class:`bytes`. + Encode a byte string using Base16. + + *s* is the string to encode. The encoded byte string is returned. .. function:: b16decode(s, casefold=False) - Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. + Decode a Base16 encoded byte string. - Optional *casefold* is a flag specifying whether a + *s* is the string to decode. Optional *casefold* is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. - A :exc:`TypeError` is raised if *s* is + The decoded byte string is returned. A :exc:`TypeError` is raised if *s* were incorrectly padded or if there are non-alphabet characters present in the - input. - - -.. function:: a85encode(s, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) - - Encode the :term:`bytes-like object` *s* using Ascii85 and return the - encoded :class:`bytes`. - - *foldspaces* is an optional flag that uses the special short sequence 'y' - instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. - - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long. - - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. - - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. - - .. versionadded:: 3.4 - - -.. function:: a85decode(s, *, foldspaces=False, adobe=False, ignorechars=b' \\t\\n\\r\\v') - - Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. - - *foldspaces* is a flag that specifies whether the 'y' short sequence - should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. - - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). - - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by - default contains all whitespace characters in ASCII. - - .. versionadded:: 3.4 - - -.. function:: b85encode(s, pad=False) - - Encode the :term:`bytes-like object` *s* using base85 (as used in e.g. - git-style binary diffs) and return the encoded :class:`bytes`. - - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - - .. versionadded:: 3.4 - - -.. function:: b85decode(b) - - Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. - - .. versionadded:: 3.4 - - -.. note:: - Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or - Ascii85 characters can encode 4 binary bytes), while the better-known - Base64 has an expansion factor of 6 to 4. They are therefore more - efficient when space expensive. They differ by details such as the - character map used for encoding. + string. The legacy interface: @@ -226,38 +134,34 @@ Decode the contents of the binary *input* file and write the resulting binary data to the *output* file. *input* and *output* must be :term:`file objects - `. *input* will be read until ``input.readline()`` returns an - empty bytes object. + `. *input* will be read until ``input.read()`` returns an empty + bytes object. .. function:: decodebytes(s) decodestring(s) - Decode the :term:`bytes-like object` *s*, which must contain one or more - lines of base64 encoded data, and return the decoded :class:`bytes`. + Decode the byte string *s*, which must contain one or more lines of base64 + encoded data, and return a byte string containing the resulting binary data. ``decodestring`` is a deprecated alias. - .. versionadded:: 3.1 - .. function:: encode(input, output) Encode the contents of the binary *input* file and write the resulting base64 encoded data to the *output* file. *input* and *output* must be :term:`file objects `. *input* will be read until ``input.read()`` returns - an empty bytes object. :func:`encode` inserts a newline character (``b'\n'``) - after every 76 bytes of the output, as well as ensuring that the output - always ends with a newline, as per :rfc:`2045` (MIME). + an empty bytes object. :func:`encode` returns the encoded data plus a trailing + newline character (``b'\n'``). .. function:: encodebytes(s) encodestring(s) - Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary - data, and return :class:`bytes` containing the base64-encoded data, with newlines - (``b'\n'``) inserted after every 76 bytes of output, and ensuring that - there is a trailing newline, as per :rfc:`2045` (MIME). - + Encode the byte string *s*, which can contain arbitrary binary data, and + return a byte string containing one or more lines of base64-encoded data. + :func:`encodebytes` returns a string containing one or more lines of + base64-encoded data always including an extra trailing newline (``b'\n'``). ``encodestring`` is a deprecated alias. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/bdb.rst --- a/Doc/library/bdb.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/bdb.rst Mon Jan 25 17:05:13 2016 +0100 @@ -194,17 +194,17 @@ .. method:: user_line(frame) This method is called from :meth:`dispatch_line` when either - :meth:`stop_here` or :meth:`break_here` yields ``True``. + :meth:`stop_here` or :meth:`break_here` yields True. .. method:: user_return(frame, return_value) This method is called from :meth:`dispatch_return` when :meth:`stop_here` - yields ``True``. + yields True. .. method:: user_exception(frame, exc_info) This method is called from :meth:`dispatch_exception` when - :meth:`stop_here` yields ``True``. + :meth:`stop_here` yields True. .. method:: do_clear(arg) @@ -231,7 +231,7 @@ .. method:: set_until(frame) Stop when the line with the line no greater than the current one is - reached or when returning from current frame. + reached or when returning from current frame .. method:: set_trace([frame]) @@ -245,7 +245,7 @@ .. method:: set_quit() - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in + Set the :attr:`quitting` attribute to True. This raises :exc:`BdbQuit` in the next call to one of the :meth:`dispatch_\*` methods. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/binary.rst --- a/Doc/library/binary.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,23 +0,0 @@ -.. _binaryservices: - -******************** -Binary Data Services -******************** - -The modules described in this chapter provide some basic services operations -for manipulation of binary data. Other operations on binary data, specifically -in relation to file formats and network protocols, are described in the -relevant sections. - -Some libraries described under :ref:`textservices` also work with either -ASCII-compatible binary formats (for example, :mod:`re`) or all binary data -(for example, :mod:`difflib`). - -In addition, see the documentation for Python's built-in binary data types in -:ref:`binaryseq`. - -.. toctree:: - - struct.rst - codecs.rst - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/binascii.rst --- a/Doc/library/binascii.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/binascii.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,9 +21,8 @@ .. note:: ``a2b_*`` functions accept Unicode strings containing only ASCII characters. - Other functions only accept :term:`bytes-like object`\ s (such as - :class:`bytes`, :class:`bytearray` and other objects that support the buffer - protocol). + Other functions only accept bytes and bytes-compatible objects (such as + bytearray objects and other objects implementing the buffer API). .. versionchanged:: 3.3 ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. @@ -52,22 +51,22 @@ than one line may be passed at a time. -.. function:: b2a_base64(data, \*, newline=True) +.. function:: b2a_base64(data) Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. + value is the converted line, including a newline char. The length of *data* + should be at most 57 to adhere to the base64 standard. - .. versionchanged:: 3.6 - Added the *newline* parameter. - -.. function:: a2b_qp(data, header=False) +.. function:: a2b_qp(string, header=False) Convert a block of quoted-printable data back to binary and return the binary data. More than one line may be passed at a time. If the optional argument *header* is present and true, underscores will be decoded as spaces. + .. versionchanged:: 3.2 + Accept only bytestring or bytearray objects as input. + .. function:: b2a_qp(data, quotetabs=False, istext=True, header=False) @@ -113,16 +112,15 @@ possibly the last fragment). -.. function:: crc_hqx(data, value) +.. function:: crc_hqx(data, crc) - Compute the binhex4 crc value of *data*, starting with *value* as the - initial crc, and return the result. + Compute the binhex4 crc value of *data*, starting with an initial *crc* and + returning the result. -.. function:: crc32(data[, value]) +.. function:: crc32(data[, crc]) - Compute CRC-32, the 32-bit checksum of *data*, starting with an - initial CRC of *value*. The default initial CRC is zero. The algorithm + Compute CRC-32, the 32-bit checksum of data, starting with an initial crc. This is consistent with the ZIP file checksum. Since the algorithm is designed for use as a checksum algorithm, it is not suitable for use as a general hash algorithm. Use as follows:: @@ -130,13 +128,15 @@ print(binascii.crc32(b"hello world")) # Or, in two pieces: crc = binascii.crc32(b"hello") - crc = binascii.crc32(b" world", crc) + crc = binascii.crc32(b" world", crc) & 0xffffffff print('crc32 = {:#010x}'.format(crc)) - .. versionchanged:: 3.0 - The result is always unsigned. - To generate the same numeric value across all Python versions and - platforms, use ``crc32(data) & 0xffffffff``. +.. note:: + To generate the same numeric value across all Python versions and + platforms use crc32(data) & 0xffffffff. If you are only using + the checksum in packed binary format this is not necessary as the + return value is the correct 32bit binary representation + regardless of sign. .. function:: b2a_hex(data) @@ -144,7 +144,7 @@ Return the hexadecimal representation of the binary *data*. Every byte of *data* is converted into the corresponding 2-digit hex representation. The - returned bytes object is therefore twice as long as the length of *data*. + resulting string is therefore twice as long as the length of *data*. .. function:: a2b_hex(hexstr) @@ -155,6 +155,9 @@ of hexadecimal digits (which can be upper or lower case), otherwise a :exc:`TypeError` is raised. + .. versionchanged:: 3.2 + Accept only bytestring or bytearray objects as input. + .. exception:: Error @@ -170,8 +173,7 @@ .. seealso:: Module :mod:`base64` - Support for RFC compliant base64-style encoding in base 16, 32, 64, - and 85. + Support for base64 encoding used in MIME email messages. Module :mod:`binhex` Support for the binhex format used on the Macintosh. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/bz2.rst --- a/Doc/library/bz2.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/bz2.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,8 +14,7 @@ The :mod:`bz2` module contains: -* The :func:`.open` function and :class:`BZ2File` class for reading and - writing compressed files. +* The :class:`BZ2File` class for reading and writing compressed files. * The :class:`BZ2Compressor` and :class:`BZ2Decompressor` classes for incremental (de)compression. * The :func:`compress` and :func:`decompress` functions for one-shot @@ -27,51 +26,16 @@ (De)compression of files ------------------------ -.. function:: open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None) +.. class:: BZ2File(filename=None, mode='r', buffering=None, compresslevel=9, \*, fileobj=None) - Open a bzip2-compressed file in binary or text mode, returning a :term:`file - object`. + Open a bzip2-compressed file. - As with the constructor for :class:`BZ2File`, the *filename* argument can be - an actual filename (a :class:`str` or :class:`bytes` object), or an existing - file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``, - ``'x'``, ``'xb'``, ``'a'`` or ``'ab'`` for binary mode, or ``'rt'``, - ``'wt'``, ``'xt'``, or ``'at'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 1 to 9, as for the - :class:`BZ2File` constructor. - - For binary mode, this function is equivalent to the :class:`BZ2File` - constructor: ``BZ2File(filename, mode, compresslevel=compresslevel)``. In - this case, the *encoding*, *errors* and *newline* arguments must not be - provided. - - For text mode, a :class:`BZ2File` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - -.. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9) - - Open a bzip2-compressed file in binary mode. - - If *filename* is a :class:`str` or :class:`bytes` object, open the named file - directly. Otherwise, *filename* should be a :term:`file object`, which will - be used to read or write the compressed data. + The :class:`BZ2File` can wrap an existing :term:`file object` (given by + *fileobj*), or operate directly on a named file (named by *filename*). + Exactly one of these two parameters should be provided. The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for - overwriting, ``'x'`` for exclusive creation, or ``'a'`` for appending. These - can equivalently be given as ``'rb'``, ``'wb'``, ``'xb'`` and ``'ab'`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of + overwriting, or ``'a'`` for appending. If *fileobj* is provided, a mode of ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. The *buffering* argument is ignored. Its use is deprecated. @@ -95,11 +59,6 @@ byte of data will be returned (unless at EOF). The exact number of bytes returned is unspecified. - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`BZ2File`, it may change the position of the underlying file - object (e.g. if the :class:`BZ2File` was constructed by passing a file - object for *filename*). - .. versionadded:: 3.3 .. versionchanged:: 3.1 @@ -110,20 +69,12 @@ :meth:`read1` and :meth:`readinto` methods were added. .. versionchanged:: 3.3 - Support was added for *filename* being a :term:`file object` instead of an - actual filename. + The *fileobj* argument to the constructor was added. .. versionchanged:: 3.3 The ``'a'`` (append) mode was added, along with support for reading multi-stream files. - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - Incremental (de)compression --------------------------- @@ -166,36 +117,19 @@ you need to decompress a multi-stream input with :class:`BZ2Decompressor`, you must use a new decompressor for each stream. - .. method:: decompress(data, max_length=-1) + .. method:: decompress(data) - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. + Provide data to the decompressor object. Returns a chunk of decompressed + data if possible, or an empty byte string otherwise. - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + Attempting to decompress data after the end of the current stream is + reached raises an :exc:`EOFError`. If any data is found after the end of + the stream, it is ignored and saved in the :attr:`unused_data` attribute. - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an `EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. .. attribute:: eof - ``True`` if the end-of-stream marker has been reached. + True if the end-of-stream marker has been reached. .. versionadded:: 3.3 @@ -207,13 +141,6 @@ If this attribute is accessed before the end of the stream has been reached, its value will be ``b''``. - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 - One-shot (de)compression ------------------------ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/calendar.rst --- a/Doc/library/calendar.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/calendar.rst Mon Jan 25 17:05:13 2016 +0100 @@ -272,11 +272,10 @@ .. function:: timegm(tuple) - An unrelated but handy function that takes a time tuple such as returned by - the :func:`~time.gmtime` function in the :mod:`time` module, and returns the - corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' - inverse. + An unrelated but handy function that takes a time tuple such as returned by the + :func:`gmtime` function in the :mod:`time` module, and returns the corresponding + Unix timestamp value, assuming an epoch of 1970, and the POSIX encoding. In + fact, :func:`time.gmtime` and :func:`timegm` are each others' inverse. The :mod:`calendar` module exports the following data attributes: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/cgi.rst --- a/Doc/library/cgi.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/cgi.rst Mon Jan 25 17:05:13 2016 +0100 @@ -79,25 +79,22 @@ instead, with code like this:: import cgitb - cgitb.enable(display=0, logdir="/path/to/logdir") + cgitb.enable(display=0, logdir="/tmp") It's very helpful to use this feature during script development. The reports produced by :mod:`cgitb` provide information that can save you a lot of time in tracking down bugs. You can always remove the ``cgitb`` line later when you have tested your script and are confident that it works correctly. -To get at submitted form data, use the :class:`FieldStorage` class. If the form -contains non-ASCII characters, use the *encoding* keyword parameter set to the -value of the encoding defined for the document. It is usually contained in the -META tag in the HEAD section of the HTML document or by the -:mailheader:`Content-Type` header). This reads the form contents from the -standard input or the environment (depending on the value of various -environment variables set according to the CGI standard). Since it may consume -standard input, it should be instantiated only once. +To get at submitted form data, use the :class:`FieldStorage` class. Instantiate +it exactly once, without arguments. This reads the form contents from standard +input or the environment (depending on the value of various environment +variables set according to the CGI standard). Since it may consume standard +input, it should be instantiated only once. The :class:`FieldStorage` instance can be indexed like a Python dictionary. It allows membership testing with the :keyword:`in` operator, and also supports -the standard dictionary method :meth:`~dict.keys` and the built-in function +the standard dictionary method :meth:`keys` and the built-in function :func:`len`. Form fields containing empty strings are ignored and do not appear in the dictionary; to keep such values, provide a true value for the optional *keep_blank_values* keyword parameter when creating the :class:`FieldStorage` @@ -119,34 +116,30 @@ Here the fields, accessed through ``form[key]``, are themselves instances of :class:`FieldStorage` (or :class:`MiniFieldStorage`, depending on the form -encoding). The :attr:`~FieldStorage.value` attribute of the instance yields -the string value of the field. The :meth:`~FieldStorage.getvalue` method -returns this string value directly; it also accepts an optional second argument -as a default to return if the requested key is not present. +encoding). The :attr:`value` attribute of the instance yields the string value +of the field. The :meth:`getvalue` method returns this string value directly; +it also accepts an optional second argument as a default to return if the +requested key is not present. If the submitted form data contains more than one field with the same name, the object retrieved by ``form[key]`` is not a :class:`FieldStorage` or :class:`MiniFieldStorage` instance but a list of such instances. Similarly, in this situation, ``form.getvalue(key)`` would return a list of strings. If you expect this possibility (when your HTML form contains multiple fields with the -same name), use the :meth:`~FieldStorage.getlist` method, which always returns -a list of values (so that you do not need to special-case the single item -case). For example, this code concatenates any number of username fields, -separated by commas:: +same name), use the :func:`getlist` function, which always returns a list of +values (so that you do not need to special-case the single item case). For +example, this code concatenates any number of username fields, separated by +commas:: value = form.getlist("username") usernames = ",".join(value) If a field represents an uploaded file, accessing the value via the -:attr:`~FieldStorage.value` attribute or the :meth:`~FieldStorage.getvalue` -method reads the entire file in memory as bytes. This may not be what you -want. You can test for an uploaded file by testing either the -:attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file` -attribute. You can then read the data from the :attr:`!file` -attribute before it is automatically closed as part of the garbage collection of -the :class:`FieldStorage` instance -(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will -return bytes):: +:attr:`value` attribute or the :func:`getvalue` method reads the entire file in +memory as a string. This may not be what you want. You can test for an uploaded +file by testing either the :attr:`filename` attribute or the :attr:`!file` +attribute. You can then read the data at leisure from the :attr:`!file` +attribute:: fileitem = form["userfile"] if fileitem.file: @@ -157,13 +150,10 @@ if not line: break linecount = linecount + 1 -:class:`FieldStorage` objects also support being used in a :keyword:`with` -statement, which will automatically close them when done. - If an error is encountered when obtaining the contents of an uploaded file (for example, when the user interrupts the form submission by clicking on -a Back or Cancel button) the :attr:`~FieldStorage.done` attribute of the -object for the field will be set to the value -1. +a Back or Cancel button) the :attr:`done` attribute of the object for the +field will be set to the value -1. The file upload draft standard entertains the possibility of uploading multiple files from one field (using a recursive :mimetype:`multipart/\*` encoding). @@ -181,15 +171,6 @@ A form submitted via POST that also has a query string will contain both :class:`FieldStorage` and :class:`MiniFieldStorage` items. -.. versionchanged:: 3.4 - The :attr:`~FieldStorage.file` attribute is automatically closed upon the - garbage collection of the creating :class:`FieldStorage` instance. - -.. versionchanged:: 3.5 - Added support for the context management protocol to the - :class:`FieldStorage` class. - - Higher Level Interface ---------------------- @@ -239,8 +220,8 @@ code which checks whether the obtained value is a single value or a list of values. That's annoying and leads to less readable scripts. -A more convenient approach is to use the methods :meth:`~FieldStorage.getfirst` -and :meth:`~FieldStorage.getlist` provided by this higher level interface. +A more convenient approach is to use the methods :meth:`getfirst` and +:meth:`getlist` provided by this higher level interface. .. method:: FieldStorage.getfirst(name, default=None) @@ -292,7 +273,7 @@ .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) - This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` + This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. .. function:: parse_multipart(fp, pdict) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/chunk.rst --- a/Doc/library/chunk.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/chunk.rst Mon Jan 25 17:05:13 2016 +0100 @@ -47,16 +47,15 @@ the :class:`Chunk` class defined here is to instantiate an instance at the start of each chunk and read from the instance until it reaches the end, after which a new instance can be instantiated. At the end of the file, creating a new -instance will fail with an :exc:`EOFError` exception. +instance will fail with a :exc:`EOFError` exception. .. class:: Chunk(file, align=True, bigendian=True, inclheader=False) Class which represents a chunk. The *file* argument is expected to be a file-like object. An instance of this class is specifically allowed. The - only method that is needed is :meth:`~io.IOBase.read`. If the methods - :meth:`~io.IOBase.seek` and :meth:`~io.IOBase.tell` are present and don't - raise an exception, they are also used. + only method that is needed is :meth:`read`. If the methods :meth:`seek` and + :meth:`tell` are present and don't raise an exception, they are also used. If these methods are present and raise an exception, they are expected to not have altered the object. If the optional argument *align* is true, chunks are assumed to be aligned on 2-byte boundaries. If *align* is false, no @@ -113,15 +112,15 @@ Read at most *size* bytes from the chunk (less if the read hits the end of the chunk before obtaining *size* bytes). If the *size* argument is - negative or omitted, read all data until the end of the chunk. An empty - bytes object is returned when the end of the chunk is encountered - immediately. + negative or omitted, read all data until the end of the chunk. The bytes + are returned as a string object. An empty string is returned when the end + of the chunk is encountered immediately. .. method:: skip() Skip to the end of the chunk. All further calls to :meth:`read` for the - chunk will return ``b''``. If you are not interested in the contents of + chunk will return ``''``. If you are not interested in the contents of the chunk, this method should be called so that the file points to the start of the next chunk. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/cmath.rst --- a/Doc/library/cmath.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/cmath.rst Mon Jan 25 17:05:13 2016 +0100 @@ -149,13 +149,13 @@ .. function:: acosh(x) - Return the inverse hyperbolic cosine of *x*. There is one branch cut, - extending left from 1 along the real axis to -∞, continuous from above. + Return the hyperbolic arc cosine of *x*. There is one branch cut, extending left + from 1 along the real axis to -∞, continuous from above. .. function:: asinh(x) - Return the inverse hyperbolic sine of *x*. There are two branch cuts: + Return the hyperbolic arc sine of *x*. There are two branch cuts: One extends from ``1j`` along the imaginary axis to ``∞j``, continuous from the right. The other extends from ``-1j`` along the imaginary axis to ``-∞j``, continuous from the left. @@ -163,7 +163,7 @@ .. function:: atanh(x) - Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One + Return the hyperbolic arc tangent of *x*. There are two branch cuts: One extends from ``1`` along the real axis to ``∞``, continuous from below. The other extends from ``-1`` along the real axis to ``-∞``, continuous from above. @@ -207,38 +207,6 @@ and ``False`` otherwise. -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - Constants --------- diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/cmd.rst --- a/Doc/library/cmd.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/cmd.rst Mon Jan 25 17:05:13 2016 +0100 @@ -148,8 +148,8 @@ Hook method executed once when :meth:`cmdloop` is about to return. This method is a stub in :class:`Cmd`; it exists to be overridden by subclasses. +Instances of :class:`Cmd` subclasses have some public instance variables: -Instances of :class:`Cmd` subclasses have some public instance variables: .. attribute:: Cmd.prompt @@ -166,13 +166,6 @@ The last nonempty command prefix seen. -.. attribute:: Cmd.cmdqueue - - A list of queued input lines. The cmdqueue list is checked in - :meth:`cmdloop` when new input is needed; if it is nonempty, its elements - will be processed in order, as if entered at the prompt. - - .. attribute:: Cmd.intro A string to issue as an intro or banner. May be overridden by giving the @@ -259,7 +252,7 @@ 'Move turtle to an absolute position with changing orientation. GOTO 100 200' goto(*parse(arg)) def do_home(self, arg): - 'Return turtle to the home position: HOME' + 'Return turtle to the home postion: HOME' home() def do_circle(self, arg): 'Draw circle with given radius an options extent and steps: CIRCLE 50' @@ -283,7 +276,7 @@ print('Thank you for using Turtle') self.close() bye() - return True + sys.exit(0) # ----- record and playback ----- def do_record(self, arg): diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/code.rst --- a/Doc/library/code.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/code.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,6 @@ .. module:: code :synopsis: Facilities to implement read-eval-print loops. -**Source code:** :source:`Lib/code.py` The ``code`` module provides facilities to implement read-eval-print loops in Python. Two classes and convenience functions are included which can be used to @@ -30,13 +29,13 @@ .. function:: interact(banner=None, readfunc=None, local=None) - Convenience function to run a read-eval-print loop. This creates a new - instance of :class:`InteractiveConsole` and sets *readfunc* to be used as - the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is - provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` - method of the instance is then run with *banner* passed as the banner to - use, if provided. The console object is discarded after use. + Convenience function to run a read-eval-print loop. This creates a new instance + of :class:`InteractiveConsole` and sets *readfunc* to be used as the + :meth:`raw_input` method, if provided. If *local* is provided, it is passed to + the :class:`InteractiveConsole` constructor for use as the default namespace for + the interpreter loop. The :meth:`interact` method of the instance is then run + with *banner* passed as the banner to use, if provided. The console object is + discarded after use. .. function:: compile_command(source, filename="", symbol="single") @@ -114,9 +113,6 @@ because it is within the interpreter object implementation. The output is written by the :meth:`write` method. - .. versionchanged:: 3.5 The full chained traceback is displayed instead - of just the primary traceback. - .. method:: InteractiveInterpreter.write(data) @@ -136,15 +132,12 @@ .. method:: InteractiveConsole.interact(banner=None) - Closely emulate the interactive Python console. The optional *banner* argument + Closely emulate the interactive Python console. The optional banner argument specify the banner to print before the first interaction; by default it prints a banner similar to the one printed by the standard Python interpreter, followed by the class name of the console object in parentheses (so as not to confuse this with the real interpreter -- since it's so close!). - .. versionchanged:: 3.4 - To suppress printing any banner, pass an empty string. - .. method:: InteractiveConsole.push(line) @@ -169,3 +162,4 @@ newline. When the user enters the EOF key sequence, :exc:`EOFError` is raised. The base implementation reads from ``sys.stdin``; a subclass may replace this with a different implementation. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/codecs.rst --- a/Doc/library/codecs.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/codecs.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,11 +3,10 @@ .. module:: codecs :synopsis: Encode and decode data and streams. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg +.. moduleauthor:: Marc-Andre Lemburg +.. sectionauthor:: Marc-Andre Lemburg .. sectionauthor:: Martin v. Löwis -**Source code:** :source:`Lib/codecs.py` .. index:: single: Unicode @@ -18,90 +17,88 @@ pair: stackable; streams This module defines base classes for standard Python codecs (encoders and -decoders) and provides access to the internal Python codec registry, which -manages the codec and error handling lookup process. Most standard codecs -are :term:`text encodings `, which encode text to bytes, -but there are also codecs provided that encode text to text, and bytes to -bytes. Custom codecs may encode and decode between arbitrary types, but some -module features are restricted to use specifically with -:term:`text encodings `, or with codecs that encode to -:class:`bytes`. +decoders) and provides access to the internal Python codec registry which +manages the codec and error handling lookup process. -The module defines the following functions for encoding and decoding with -any codec: +It defines the following functions: -.. function:: encode(obj, encoding='utf-8', errors='strict') - Encodes *obj* using the codec registered for *encoding*. +.. function:: register(search_function) - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that encoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeEncodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. + Register a codec search function. Search functions are expected to take one + argument, the encoding name in all lower case letters, and return a + :class:`CodecInfo` object having the following attributes: -.. function:: decode(obj, encoding='utf-8', errors='strict') + * ``name`` The name of the encoding; - Decodes *obj* using the codec registered for *encoding*. + * ``encode`` The stateless encoding function; - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that decoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. + * ``decode`` The stateless decoding function; -The full details for each codec can also be looked up directly: + * ``incrementalencoder`` An incremental encoder class or factory function; + + * ``incrementaldecoder`` An incremental decoder class or factory function; + + * ``streamwriter`` A stream writer class or factory function; + + * ``streamreader`` A stream reader class or factory function. + + The various functions or classes take the following arguments: + + *encode* and *decode*: These must be functions or methods which have the same + interface as the :meth:`encode`/:meth:`decode` methods of Codec instances (see + Codec Interface). The functions/methods are expected to work in a stateless + mode. + + *incrementalencoder* and *incrementaldecoder*: These have to be factory + functions providing the following interface: + + ``factory(errors='strict')`` + + The factory functions must return objects providing the interfaces defined by + the base classes :class:`IncrementalEncoder` and :class:`IncrementalDecoder`, + respectively. Incremental codecs can maintain state. + + *streamreader* and *streamwriter*: These have to be factory functions providing + the following interface: + + ``factory(stream, errors='strict')`` + + The factory functions must return objects providing the interfaces defined by + the base classes :class:`StreamWriter` and :class:`StreamReader`, respectively. + Stream codecs can maintain state. + + Possible values for errors are + + * ``'strict'``: raise an exception in case of an encoding error + * ``'replace'``: replace malformed data with a suitable replacement marker, + such as ``'?'`` or ``'\ufffd'`` + * ``'ignore'``: ignore malformed data and continue without further notice + * ``'xmlcharrefreplace'``: replace with the appropriate XML character + reference (for encoding only) + * ``'backslashreplace'``: replace with backslashed escape sequences (for + encoding only) + * ``'surrogateescape'``: replace with surrogate U+DCxx, see :pep:`383` + + as well as any other error handling name defined via :func:`register_error`. + + In case a search function cannot find a given encoding, it should return + ``None``. + .. function:: lookup(encoding) Looks up the codec info in the Python codec registry and returns a - :class:`CodecInfo` object as defined below. + :class:`CodecInfo` object as defined above. Encodings are first looked up in the registry's cache. If not found, the list of registered search functions is scanned. If no :class:`CodecInfo` object is found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object is stored in the cache and returned to the caller. -.. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) +To simplify access to the various codecs, the module provides these additional +functions which use :func:`lookup` for the codec lookup: - Codec details when looking up the codec registry. The constructor - arguments are stored in attributes of the same name: - - - .. attribute:: name - - The name of the encoding. - - - .. attribute:: encode - decode - - The stateless encoding and decoding functions. These must be - functions or methods which have the same interface as - the :meth:`~Codec.encode` and :meth:`~Codec.decode` methods of Codec - instances (see :ref:`Codec Interface `). - The functions or methods are expected to work in a stateless mode. - - - .. attribute:: incrementalencoder - incrementaldecoder - - Incremental encoder and decoder classes or factory functions. - These have to provide the interface defined by the base classes - :class:`IncrementalEncoder` and :class:`IncrementalDecoder`, - respectively. Incremental codecs can maintain state. - - - .. attribute:: streamwriter - streamreader - - Stream writer and reader classes or factory functions. These have to - provide the interface defined by the base classes - :class:`StreamWriter` and :class:`StreamReader`, respectively. - Stream codecs can maintain state. - -To simplify access to the various codec components, the module provides -these additional functions which use :func:`lookup` for the codec lookup: .. function:: getencoder(encoding) @@ -150,43 +147,87 @@ Raises a :exc:`LookupError` in case the encoding cannot be found. -Custom codecs are made available by registering a suitable codec search -function: -.. function:: register(search_function) +.. function:: register_error(name, error_handler) - Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters, and return a - :class:`CodecInfo` object. In case a search function cannot find - a given encoding, it should return ``None``. + Register the error handling function *error_handler* under the name *name*. + *error_handler* will be called during encoding and decoding in case of an error, + when *name* is specified as the errors parameter. + + For encoding *error_handler* will be called with a :exc:`UnicodeEncodeError` + instance, which contains information about the location of the error. The error + handler must either raise this or a different exception or return a tuple with a + replacement for the unencodable part of the input and a position where encoding + should continue. The encoder will encode the replacement and continue encoding + the original input at the specified position. Negative position values will be + treated as being relative to the end of the input string. If the resulting + position is out of bound an :exc:`IndexError` will be raised. + + Decoding and translating works similar, except :exc:`UnicodeDecodeError` or + :exc:`UnicodeTranslateError` will be passed to the handler and that the + replacement from the error handler will be put into the output directly. + + +.. function:: lookup_error(name) + + Return the error handler previously registered under the name *name*. + + Raises a :exc:`LookupError` in case the handler cannot be found. + + +.. function:: strict_errors(exception) + + Implements the ``strict`` error handling: each encoding or decoding error + raises a :exc:`UnicodeError`. + + +.. function:: replace_errors(exception) + + Implements the ``replace`` error handling: malformed data is replaced with a + suitable replacement character such as ``'?'`` in bytestrings and + ``'\ufffd'`` in Unicode strings. + + +.. function:: ignore_errors(exception) + + Implements the ``ignore`` error handling: malformed data is ignored and + encoding or decoding is continued without further notice. + + +.. function:: xmlcharrefreplace_errors(exception) + + Implements the ``xmlcharrefreplace`` error handling (for encoding only): the + unencodable character is replaced by an appropriate XML character reference. + + +.. function:: backslashreplace_errors(exception) + + Implements the ``backslashreplace`` error handling (for encoding only): the + unencodable character is replaced by a backslashed escape sequence. + +To simplify working with encoded files or stream, the module also defines these +utility functions: + + +.. function:: open(filename, mode[, encoding[, errors[, buffering]]]) + + Open an encoded file using the given *mode* and return a wrapped version + providing transparent encoding/decoding. The default file mode is ``'r'`` + meaning to open the file in read mode. .. note:: - Search function registration is not currently reversible, - which may cause problems in some cases, such as unit testing or - module reloading. - -While the builtin :func:`open` and the associated :mod:`io` module are the -recommended approach for working with encoded text files, this module -provides additional utility functions and classes that allow the use of a -wider range of codecs when working with binary files: - -.. function:: open(filename, mode='r', encoding=None, errors='strict', buffering=1) - - Open an encoded file using the given *mode* and return an instance of - :class:`StreamReaderWriter`, providing transparent encoding/decoding. - The default file mode is ``'r'``, meaning to open the file in read mode. + The wrapped version's methods will accept and return strings only. Bytes + arguments will be rejected. .. note:: - Underlying encoded files are always opened in binary mode. - No automatic conversion of ``'\n'`` is done on reading and writing. - The *mode* argument may be any binary mode acceptable to the built-in - :func:`open` function; the ``'b'`` is automatically added. + Files are always opened in binary mode, even if no binary mode was + specified. This is done to avoid data loss due to encodings using 8-bit + values. This means that no automatic conversion of ``b'\n'`` is done + on reading and writing. *encoding* specifies the encoding which is to be used for the file. - Any encoding that encodes to and decodes from bytes is allowed, and - the data types supported by the file methods depend on the codec used. *errors* may be given to define the error handling. It defaults to ``'strict'`` which causes a :exc:`ValueError` to be raised in case an encoding error occurs. @@ -197,15 +238,12 @@ .. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict') - Return a :class:`StreamRecoder` instance, a wrapped version of *file* - which provides transparent transcoding. The original file is closed - when the wrapped version is closed. + Return a wrapped version of file which provides transparent encoding + translation. - Data written to the wrapped file is decoded according to the given - *data_encoding* and then written to the original file as bytes using - *file_encoding*. Bytes read from the original file are decoded - according to *file_encoding*, and the result is encoded - using *data_encoding*. + Bytes written to the wrapped file are interpreted according to the given + *data_encoding* and then written to the original file as bytes using the + *file_encoding*. If *file_encoding* is not given, it defaults to *data_encoding*. @@ -217,16 +255,14 @@ .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. This function is a :term:`generator`. *errors* (as well as any other keyword argument) is passed through to the incremental encoder. .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. This function is a :term:`generator`. *errors* (as well as any other keyword argument) is passed through to the incremental decoder. @@ -245,10 +281,9 @@ BOM_UTF32_BE BOM_UTF32_LE - These constants define various byte sequences, - being Unicode byte order marks (BOMs) for several encodings. They are - used in UTF-16 and UTF-32 data streams to indicate the byte order used, - and in UTF-8 as a Unicode signature. :const:`BOM_UTF16` is either + These constants define various encodings of the Unicode byte order mark (BOM) + used in UTF-16 and UTF-32 data streams to indicate the byte order used in the + stream or file and in UTF-8 as a Unicode signature. :const:`BOM_UTF16` is either :const:`BOM_UTF16_BE` or :const:`BOM_UTF16_LE` depending on the platform's native byte order, :const:`BOM` is an alias for :const:`BOM_UTF16`, :const:`BOM_LE` for :const:`BOM_UTF16_LE` and :const:`BOM_BE` for @@ -262,200 +297,82 @@ ------------------ The :mod:`codecs` module defines a set of base classes which define the -interfaces for working with codec objects, and can also be used as the basis -for custom codec implementations. +interface and can also be used to easily write your own codecs for use in +Python. Each codec has to define four interfaces to make it usable as codec in Python: stateless encoder, stateless decoder, stream reader and stream writer. The stream reader and writers typically reuse the stateless encoder/decoder to -implement the file protocols. Codec authors also need to define how the -codec will handle encoding and decoding errors. +implement the file protocols. +The :class:`Codec` class defines the interface for stateless encoders/decoders. -.. _surrogateescape: -.. _error-handlers: - -Error Handlers -^^^^^^^^^^^^^^ - -To simplify and standardize error handling, -codecs may implement different error handling schemes by -accepting the *errors* string argument. The following string values are -defined and implemented by all standard Python codecs: - -.. tabularcolumns:: |l|L| +To simplify and standardize error handling, the :meth:`encode` and +:meth:`decode` methods may implement different error handling schemes by +providing the *errors* string argument. The following string values are defined +and implemented by all standard Python codecs: +-------------------------+-----------------------------------------------+ | Value | Meaning | +=========================+===============================================+ | ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass); | -| | this is the default. Implemented in | -| | :func:`strict_errors`. | +| | this is the default. | +-------------------------+-----------------------------------------------+ -| ``'ignore'`` | Ignore the malformed data and continue | -| | without further notice. Implemented in | -| | :func:`ignore_errors`. | +| ``'ignore'`` | Ignore the character and continue with the | +| | next. | ++-------------------------+-----------------------------------------------+ +| ``'replace'`` | Replace with a suitable replacement | +| | character; Python will use the official | +| | U+FFFD REPLACEMENT CHARACTER for the built-in | +| | Unicode codecs on decoding and '?' on | +| | encoding. | ++-------------------------+-----------------------------------------------+ +| ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | +| | reference (only for encoding). | ++-------------------------+-----------------------------------------------+ +| ``'backslashreplace'`` | Replace with backslashed escape sequences | +| | (only for encoding). | ++-------------------------+-----------------------------------------------+ +| ``'surrogateescape'`` | Replace byte with surrogate U+DCxx, as defined| +| | in :pep:`383`. | +-------------------------+-----------------------------------------------+ -The following error handlers are only applicable to -:term:`text encodings `: +In addition, the following error handlers are specific to a single codec: -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'replace'`` | Replace with a suitable replacement | -| | marker; Python will use the official | -| | ``U+FFFD`` REPLACEMENT CHARACTER for the | -| | built-in codecs on decoding, and '?' on | -| | encoding. Implemented in | -| | :func:`replace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'xmlcharrefreplace'`` | Replace with the appropriate XML character | -| | reference (only for encoding). Implemented | -| | in :func:`xmlcharrefreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'backslashreplace'`` | Replace with backslashed escape sequences. | -| | Implemented in | -| | :func:`backslashreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences | -| | (only for encoding). Implemented in | -| | :func:`namereplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'surrogateescape'`` | On decoding, replace byte with individual | -| | surrogate code ranging from ``U+DC80`` to | -| | ``U+DCFF``. This code will then be turned | -| | back into the same byte when the | -| | ``'surrogateescape'`` error handler is used | -| | when encoding the data. (See :pep:`383` for | -| | more.) | -+-------------------------+-----------------------------------------------+ - -In addition, the following error handler is specific to the given codecs: - -+-------------------+------------------------+-------------------------------------------+ -| Value | Codecs | Meaning | -+===================+========================+===========================================+ -|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate | -| | utf-16-be, utf-16-le, | codes. These codecs normally treat the | -| | utf-32-be, utf-32-le | presence of surrogates as an error. | -+-------------------+------------------------+-------------------------------------------+ ++-------------------+---------+-------------------------------------------+ +| Value | Codec | Meaning | ++===================+=========+===========================================+ +|``'surrogatepass'``| utf-8 | Allow encoding and decoding of surrogate | +| | | codes in UTF-8. | ++-------------------+---------+-------------------------------------------+ .. versionadded:: 3.1 The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. -.. versionchanged:: 3.4 - The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs. - -.. versionadded:: 3.5 - The ``'namereplace'`` error handler. - -.. versionchanged:: 3.5 - The ``'backslashreplace'`` error handlers now works with decoding and - translating. - -The set of allowed values can be extended by registering a new named error -handler: - -.. function:: register_error(name, error_handler) - - Register the error handling function *error_handler* under the name *name*. - The *error_handler* argument will be called during encoding and decoding - in case of an error, when *name* is specified as the errors parameter. - - For encoding, *error_handler* will be called with a :exc:`UnicodeEncodeError` - instance, which contains information about the location of the error. The - error handler must either raise this or a different exception, or return a - tuple with a replacement for the unencodable part of the input and a position - where encoding should continue. The replacement may be either :class:`str` or - :class:`bytes`. If the replacement is bytes, the encoder will simply copy - them into the output buffer. If the replacement is a string, the encoder will - encode the replacement. Encoding continues on original input at the - specified position. Negative position values will be treated as being - relative to the end of the input string. If the resulting position is out of - bound an :exc:`IndexError` will be raised. - - Decoding and translating works similarly, except :exc:`UnicodeDecodeError` or - :exc:`UnicodeTranslateError` will be passed to the handler and that the - replacement from the error handler will be put into the output directly. - - -Previously registered error handlers (including the standard error handlers) -can be looked up by name: - -.. function:: lookup_error(name) - - Return the error handler previously registered under the name *name*. - - Raises a :exc:`LookupError` in case the handler cannot be found. - -The following standard error handlers are also made available as module level -functions: - -.. function:: strict_errors(exception) - - Implements the ``'strict'`` error handling: each encoding or - decoding error raises a :exc:`UnicodeError`. - - -.. function:: replace_errors(exception) - - Implements the ``'replace'`` error handling (for :term:`text encodings - ` only): substitutes ``'?'`` for encoding errors - (to be encoded by the codec), and ``'\ufffd'`` (the Unicode replacement - character) for decoding errors. - - -.. function:: ignore_errors(exception) - - Implements the ``'ignore'`` error handling: malformed data is ignored and - encoding or decoding is continued without further notice. - - -.. function:: xmlcharrefreplace_errors(exception) - - Implements the ``'xmlcharrefreplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by an appropriate XML character reference. - - -.. function:: backslashreplace_errors(exception) - - Implements the ``'backslashreplace'`` error handling (for - :term:`text encodings ` only): malformed data is - replaced by a backslashed escape sequence. - -.. function:: namereplace_errors(exception) - - Implements the ``'namereplace'`` error handling (for encoding with - :term:`text encodings ` only): the - unencodable character is replaced by a ``\N{...}`` escape sequence. - - .. versionadded:: 3.5 +The set of allowed values can be extended via :meth:`register_error`. .. _codec-objects: -Stateless Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Codec Objects +^^^^^^^^^^^^^ -The base :class:`Codec` class defines these methods which also define the -function interfaces of the stateless encoder and decoder: +The :class:`Codec` class defines these methods which also define the function +interfaces of the stateless encoder and decoder: .. method:: Codec.encode(input[, errors]) Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular + Encoding converts a string object to a bytes object using a particular character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + *errors* defines the error handling to apply. It defaults to ``'strict'`` + handling. The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. + :class:`StreamCodec` for codecs which have to keep state in order to make + encoding/decoding efficient. The encoder must be able to handle zero length input and return an empty object of the output object type in this situation. @@ -464,53 +381,45 @@ .. method:: Codec.decode(input[, errors]) Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular + consumed). Decoding converts a bytes object encoded using a particular character set encoding to a string object. - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only + *input* must be a bytes object or one which provides the read-only character buffer interface -- for example, buffer objects and memory mapped files. - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + *errors* defines the error handling to apply. It defaults to ``'strict'`` + handling. The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. + :class:`StreamCodec` for codecs which have to keep state in order to make + encoding/decoding efficient. The decoder must be able to handle zero length input and return an empty object of the output object type in this situation. - -Incremental Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - The :class:`IncrementalEncoder` and :class:`IncrementalDecoder` classes provide the basic interface for incremental encoding and decoding. Encoding/decoding the input isn't done with one call to the stateless encoder/decoder function, but -with multiple calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method of -the incremental encoder/decoder. The incremental encoder/decoder keeps track of -the encoding/decoding process during method calls. +with multiple calls to the :meth:`encode`/:meth:`decode` method of the +incremental encoder/decoder. The incremental encoder/decoder keeps track of the +encoding/decoding process during method calls. -The joined output of calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method is -the same as if all the single inputs were joined into one, and this input was +The joined output of calls to the :meth:`encode`/:meth:`decode` method is the +same as if all the single inputs were joined into one, and this input was encoded/decoded with the stateless encoder/decoder. .. _incremental-encoder-objects: IncrementalEncoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`IncrementalEncoder` class is used for encoding an input in multiple steps. It defines the following methods which every incremental encoder must define in order to be compatible with the Python codec registry. -.. class:: IncrementalEncoder(errors='strict') +.. class:: IncrementalEncoder([errors]) Constructor for an :class:`IncrementalEncoder` instance. @@ -519,14 +428,26 @@ the Python codec registry. The :class:`IncrementalEncoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. + by providing the *errors* keyword argument. These parameters are predefined: + + * ``'strict'`` Raise :exc:`ValueError` (or a subclass); this is the default. + + * ``'ignore'`` Ignore the character and continue with the next. + + * ``'replace'`` Replace with a suitable replacement character + + * ``'xmlcharrefreplace'`` Replace with the appropriate XML character reference + + * ``'backslashreplace'`` Replace with backslashed escape sequences. The *errors* argument will be assigned to an attribute of the same name. Assigning to this attribute makes it possible to switch between different error handling strategies during the lifetime of the :class:`IncrementalEncoder` object. + The set of allowed values for the *errors* argument can be extended with + :func:`register_error`. + .. method:: encode(object[, final]) @@ -538,8 +459,7 @@ .. method:: reset() Reset the encoder to the initial state. The output is discarded: call - ``.encode(object, final=True)``, passing an empty byte or text string - if necessary, to reset the encoder and to get the output. + ``.encode('', final=True)`` to reset the encoder and to get the output. .. method:: IncrementalEncoder.getstate() @@ -560,14 +480,14 @@ .. _incremental-decoder-objects: IncrementalDecoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`IncrementalDecoder` class is used for decoding an input in multiple steps. It defines the following methods which every incremental decoder must define in order to be compatible with the Python codec registry. -.. class:: IncrementalDecoder(errors='strict') +.. class:: IncrementalDecoder([errors]) Constructor for an :class:`IncrementalDecoder` instance. @@ -576,14 +496,22 @@ the Python codec registry. The :class:`IncrementalDecoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. + by providing the *errors* keyword argument. These parameters are predefined: + + * ``'strict'`` Raise :exc:`ValueError` (or a subclass); this is the default. + + * ``'ignore'`` Ignore the character and continue with the next. + + * ``'replace'`` Replace with a suitable replacement character. The *errors* argument will be assigned to an attribute of the same name. Assigning to this attribute makes it possible to switch between different error handling strategies during the lifetime of the :class:`IncrementalDecoder` object. + The set of allowed values for the *errors* argument can be extended with + :func:`register_error`. + .. method:: decode(object[, final]) @@ -622,10 +550,6 @@ returned by :meth:`getstate`. -Stream Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - The :class:`StreamWriter` and :class:`StreamReader` classes provide generic working interfaces which can be used to implement new encoding submodules very easily. See :mod:`encodings.utf_8` for an example of how this is done. @@ -634,14 +558,14 @@ .. _stream-writer-objects: StreamWriter Objects -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ The :class:`StreamWriter` class is a subclass of :class:`Codec` and defines the following methods which every stream writer must define in order to be compatible with the Python codec registry. -.. class:: StreamWriter(stream, errors='strict') +.. class:: StreamWriter(stream[, errors]) Constructor for a :class:`StreamWriter` instance. @@ -649,17 +573,29 @@ additional keyword arguments, but only the ones defined here are used by the Python codec registry. - The *stream* argument must be a file-like object open for writing - text or binary data, as appropriate for the specific codec. + *stream* must be a file-like object open for writing binary data. The :class:`StreamWriter` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. + providing the *errors* keyword argument. These parameters are predefined: + + * ``'strict'`` Raise :exc:`ValueError` (or a subclass); this is the default. + + * ``'ignore'`` Ignore the character and continue with the next. + + * ``'replace'`` Replace with a suitable replacement character + + * ``'xmlcharrefreplace'`` Replace with the appropriate XML character reference + + * ``'backslashreplace'`` Replace with backslashed escape sequences. The *errors* argument will be assigned to an attribute of the same name. Assigning to this attribute makes it possible to switch between different error handling strategies during the lifetime of the :class:`StreamWriter` object. + The set of allowed values for the *errors* argument can be extended with + :func:`register_error`. + + .. method:: write(object) Writes the object's contents encoded to the stream. @@ -668,8 +604,7 @@ .. method:: writelines(list) Writes the concatenated list of strings to the stream (possibly by reusing - the :meth:`write` method). The standard bytes-to-bytes codecs - do not support this method. + the :meth:`write` method). .. method:: reset() @@ -688,14 +623,14 @@ .. _stream-reader-objects: StreamReader Objects -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ The :class:`StreamReader` class is a subclass of :class:`Codec` and defines the following methods which every stream reader must define in order to be compatible with the Python codec registry. -.. class:: StreamReader(stream, errors='strict') +.. class:: StreamReader(stream[, errors]) Constructor for a :class:`StreamReader` instance. @@ -703,12 +638,16 @@ additional keyword arguments, but only the ones defined here are used by the Python codec registry. - The *stream* argument must be a file-like object open for reading - text or binary data, as appropriate for the specific codec. + *stream* must be a file-like object open for reading (binary) data. The :class:`StreamReader` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. + providing the *errors* keyword argument. These parameters are defined: + + * ``'strict'`` Raise :exc:`ValueError` (or a subclass); this is the default. + + * ``'ignore'`` Ignore the character and continue with the next. + + * ``'replace'`` Replace with a suitable replacement character. The *errors* argument will be assigned to an attribute of the same name. Assigning to this attribute makes it possible to switch between different error @@ -722,20 +661,17 @@ Decodes data from the stream and returns the resulting object. - The *chars* argument indicates the number of decoded - code points or bytes to return. The :func:`read` method will - never return more data than requested, but it might return less, - if there is not enough available. + *chars* indicates the number of characters to read from the + stream. :func:`read` will never return more than *chars* characters, but + it might return less, if there are not enough characters available. - The *size* argument indicates the approximate maximum - number of encoded bytes or code points to read - for decoding. The decoder can modify this setting as + *size* indicates the approximate maximum number of bytes to read from the + stream for decoding purposes. The decoder can modify this setting as appropriate. The default value -1 indicates to read and decode as much as - possible. This parameter is intended to - prevent having to decode huge files in one step. + possible. *size* is intended to prevent having to decode huge files in + one step. - The *firstline* flag indicates that - it would be sufficient to only return the first + *firstline* indicates that it would be sufficient to only return the first line, if there are decoding errors on later lines. The method should use a greedy read strategy meaning that it should read @@ -749,7 +685,7 @@ Read one line from the input stream and return the decoded data. *size*, if given, is passed as size argument to the stream's - :meth:`read` method. + :meth:`readline` method. If *keepends* is false line-endings will be stripped from the lines returned. @@ -778,13 +714,17 @@ In addition to the above methods, the :class:`StreamReader` must also inherit all other methods and attributes from the underlying stream. +The next two base classes are included for convenience. They are not needed by +the codec registry, but may provide useful in practice. + + .. _stream-reader-writer: StreamReaderWriter Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :class:`StreamReaderWriter` is a convenience class that allows wrapping -streams which work in both read and write modes. +The :class:`StreamReaderWriter` allows wrapping streams which work in both read +and write modes. The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. @@ -805,9 +745,9 @@ .. _stream-recoder-objects: StreamRecoder Objects -~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^ -The :class:`StreamRecoder` translates data from one encoding to another, +The :class:`StreamRecoder` provide a frontend - backend view of encoding data which is sometimes useful when dealing with different encoding environments. The design is such that one can use the factory functions returned by the @@ -817,20 +757,22 @@ .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors) Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* - work on the backend — the data in *stream*. + *encode* and *decode* work on the frontend (the input to :meth:`read` and output + of :meth:`write`) while *Reader* and *Writer* work on the backend (reading and + writing to the stream). - You can use these objects to do transparent transcodings from e.g. Latin-1 + You can use these objects to do transparent direct recodings from e.g. Latin-1 to UTF-8 and back. - The *stream* argument must be a file-like object. + *stream* must be a file-like object. - The *encode* and *decode* arguments must - adhere to the :class:`Codec` interface. *Reader* and + *encode*, *decode* must adhere to the :class:`Codec` interface. *Reader*, *Writer* must be factory functions or classes providing objects of the :class:`StreamReader` and :class:`StreamWriter` interface respectively. + *encode* and *decode* are needed for the frontend translation, *Reader* and + *Writer* for the backend translation. + Error handling is done in the same way as defined for the stream readers and writers. @@ -845,34 +787,31 @@ Encodings and Unicode --------------------- -Strings are stored internally as sequences of code points in -range ``0x0``-``0x10FFFF``. (See :pep:`393` for -more details about the implementation.) -Once a string object is used outside of CPU and memory, endianness -and how these arrays are stored as bytes become an issue. As with other -codecs, serialising a string into a sequence of bytes is known as *encoding*, -and recreating the string from the sequence of bytes is known as *decoding*. -There are a variety of different text serialisation codecs, which are -collectivity referred to as :term:`text encodings `. - -The simplest text encoding (called ``'latin-1'`` or ``'iso-8859-1'``) maps -the code points 0-255 to the bytes ``0x0``-``0xff``, which means that a string -object that contains code points above ``U+00FF`` can't be encoded with this -codec. Doing so will raise a :exc:`UnicodeEncodeError` that looks -like the following (although the details of the error message may differ): -``UnicodeEncodeError: 'latin-1' codec can't encode character '\u1234' in -position 3: ordinal not in range(256)``. +Strings are stored internally as sequences of codepoints in range ``0 - 10FFFF`` +(see :pep:`393` for more details about the implementation). +Once a string object is used outside of CPU and memory, CPU endianness +and how these arrays are stored as bytes become an issue. Transforming a +string object into a sequence of bytes is called encoding and recreating the +string object from the sequence of bytes is known as decoding. There are many +different methods for how this transformation can be done (these methods are +also called encodings). The simplest method is to map the codepoints 0-255 to +the bytes ``0x0``-``0xff``. This means that a string object that contains +codepoints above ``U+00FF`` can't be encoded with this method (which is called +``'latin-1'`` or ``'iso-8859-1'``). :func:`str.encode` will raise a +:exc:`UnicodeEncodeError` that looks like this: ``UnicodeEncodeError: 'latin-1' +codec can't encode character '\u1234' in position 3: ordinal not in +range(256)``. There's another group of encodings (the so called charmap encodings) that choose -a different subset of all Unicode code points and how these code points are +a different subset of all Unicode code points and how these codepoints are mapped to the bytes ``0x0``-``0xff``. To see how this is done simply open e.g. :file:`encodings/cp1252.py` (which is an encoding that is used primarily on Windows). There's a string constant with 256 characters that shows you which character is mapped to which byte value. -All of these encodings can only encode 256 of the 1114112 code points +All of these encodings can only encode 256 of the 1114112 codepoints defined in Unicode. A simple and straightforward way that can store each Unicode -code point, is to store each code point as four consecutive bytes. There are two +code point, is to store each codepoint as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you @@ -984,8 +923,6 @@ * an IBM PC code page, which is ASCII compatible -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| - +-----------------+--------------------------------+--------------------------------+ | Codec | Aliases | Languages | +=================+================================+================================+ @@ -997,10 +934,6 @@ +-----------------+--------------------------------+--------------------------------+ | cp037 | IBM037, IBM039 | English | +-----------------+--------------------------------+--------------------------------+ -| cp273 | 273, IBM273, csIBM273 | German | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ | cp424 | EBCDIC-CP-HE, IBM424 | Hebrew | +-----------------+--------------------------------+--------------------------------+ | cp437 | 437, IBM437 | English | @@ -1057,10 +990,6 @@ +-----------------+--------------------------------+--------------------------------+ | cp1026 | ibm1026 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp1125 | 1125, ibm1125, cp866u, ruscii | Ukrainian | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ | cp1140 | ibm1140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | @@ -1148,8 +1077,6 @@ +-----------------+--------------------------------+--------------------------------+ | iso8859_10 | iso-8859-10, latin6, L6 | Nordic languages | +-----------------+--------------------------------+--------------------------------+ -| iso8859_11 | iso-8859-11, thai | Thai languages | -+-----------------+--------------------------------+--------------------------------+ | iso8859_13 | iso-8859-13, latin7, L7 | Baltic languages | +-----------------+--------------------------------+--------------------------------+ | iso8859_14 | iso-8859-14, latin8, L8 | Celtic languages | @@ -1162,16 +1089,8 @@ +-----------------+--------------------------------+--------------------------------+ | koi8_r | | Russian | +-----------------+--------------------------------+--------------------------------+ -| koi8_t | | Tajik | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ | koi8_u | | Ukrainian | +-----------------+--------------------------------+--------------------------------+ -| kz1048 | kz_1048, strk1048_2002, rk1048 | Kazakh | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ | mac_cyrillic | maccyrillic | Bulgarian, Byelorussian, | | | | Macedonian, Russian, Serbian | +-----------------+--------------------------------+--------------------------------+ @@ -1216,40 +1135,14 @@ | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ -.. versionchanged:: 3.4 - The utf-16\* and utf-32\* encoders no longer allow surrogate code points - (``U+D800``--``U+DFFF``) to be encoded. - The utf-32\* decoders no longer decode - byte sequences that correspond to surrogate code points. - - -Python Specific Encodings -------------------------- - -A number of predefined codecs are specific to Python, so their codec names have -no meaning outside Python. These are listed in the tables below based on the -expected input and output types (note that while text encodings are the most -common use case for codecs, the underlying codec infrastructure supports -arbitrary data transforms rather than just text encodings). For asymmetric -codecs, the stated purpose describes the encoding direction. - -Text Encodings -^^^^^^^^^^^^^^ - -The following codecs provide :class:`str` to :class:`bytes` encoding and -:term:`bytes-like object` to :class:`str` decoding, similar to the Unicode text -encodings. - -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| +.. XXX fix here, should be in above table +--------------------+---------+---------------------------+ | Codec | Aliases | Purpose | +====================+=========+===========================+ | idna | | Implements :rfc:`3490`, | | | | see also | -| | | :mod:`encodings.idna`. | -| | | Only ``errors='strict'`` | -| | | is supported. | +| | | :mod:`encodings.idna` | +--------------------+---------+---------------------------+ | mbcs | dbcs | Windows only: Encode | | | | operand according to the | @@ -1257,125 +1150,70 @@ +--------------------+---------+---------------------------+ | palmos | | Encoding of PalmOS 3.5 | +--------------------+---------+---------------------------+ -| punycode | | Implements :rfc:`3492`. | -| | | Stateful codecs are not | -| | | supported. | +| punycode | | Implements :rfc:`3492` | +--------------------+---------+---------------------------+ -| raw_unicode_escape | | Latin-1 encoding with | -| | | ``\uXXXX`` and | -| | | ``\UXXXXXXXX`` for other | -| | | code points. Existing | -| | | backslashes are not | -| | | escaped in any way. | -| | | It is used in the Python | -| | | pickle protocol. | +| raw_unicode_escape | | Produce a string that is | +| | | suitable as raw Unicode | +| | | literal in Python source | +| | | code | +--------------------+---------+---------------------------+ | undefined | | Raise an exception for | -| | | all conversions, even | -| | | empty strings. The error | -| | | handler is ignored. | +| | | all conversions. Can be | +| | | used as the system | +| | | encoding if no automatic | +| | | coercion between byte and | +| | | Unicode strings is | +| | | desired. | +--------------------+---------+---------------------------+ -| unicode_escape | | Encoding suitable as the | -| | | contents of a Unicode | -| | | literal in ASCII-encoded | -| | | Python source code, | -| | | except that quotes are | -| | | not escaped. Decodes from | -| | | Latin-1 source code. | -| | | Beware that Python source | -| | | code actually uses UTF-8 | -| | | by default. | +| unicode_escape | | Produce a string that is | +| | | suitable as Unicode | +| | | literal in Python source | +| | | code | +--------------------+---------+---------------------------+ | unicode_internal | | Return the internal | | | | representation of the | -| | | operand. Stateful codecs | -| | | are not supported. | +| | | operand | | | | | | | | .. deprecated:: 3.3 | -| | | This representation is | -| | | obsoleted by | -| | | :pep:`393`. | +--------------------+---------+---------------------------+ -.. _binary-transforms: +The following codecs provide bytes-to-bytes mappings. -Binary Transforms -^^^^^^^^^^^^^^^^^ ++--------------------+---------------------------+---------------------------+ +| Codec | Aliases | Purpose | ++====================+===========================+===========================+ +| base64_codec | base64, base-64 | Convert operand to MIME | +| | | base64 | ++--------------------+---------------------------+---------------------------+ +| bz2_codec | bz2 | Compress the operand | +| | | using bz2 | ++--------------------+---------------------------+---------------------------+ +| hex_codec | hex | Convert operand to | +| | | hexadecimal | +| | | representation, with two | +| | | digits per byte | ++--------------------+---------------------------+---------------------------+ +| quopri_codec | quopri, quoted-printable, | Convert operand to MIME | +| | quotedprintable | quoted printable | ++--------------------+---------------------------+---------------------------+ +| uu_codec | uu | Convert the operand using | +| | | uuencode | ++--------------------+---------------------------+---------------------------+ +| zlib_codec | zip, zlib | Compress the operand | +| | | using gzip | ++--------------------+---------------------------+---------------------------+ -The following codecs provide binary transforms: :term:`bytes-like object` -to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` -(which only produces :class:`str` output). +The following codecs provide string-to-string mappings. - -.. tabularcolumns:: |l|L|L|L| - -+----------------------+------------------+------------------------------+------------------------------+ -| Codec | Aliases | Purpose | Encoder / decoder | -+======================+==================+==============================+==============================+ -| base64_codec [#b64]_ | base64, base_64 | Convert operand to multiline | :meth:`base64.encodebytes` / | -| | | MIME base64 (the result | :meth:`base64.decodebytes` | -| | | always includes a trailing | | -| | | ``'\n'``) | | -| | | | | -| | | .. versionchanged:: 3.4 | | -| | | accepts any | | -| | | :term:`bytes-like object` | | -| | | as input for encoding and | | -| | | decoding | | -+----------------------+------------------+------------------------------+------------------------------+ -| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / | -| | | using bz2 | :meth:`bz2.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ -| hex_codec | hex | Convert operand to | :meth:`binascii.b2a_hex` / | -| | | hexadecimal | :meth:`binascii.a2b_hex` | -| | | representation, with two | | -| | | digits per byte | | -+----------------------+------------------+------------------------------+------------------------------+ -| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encode` with | -| | quotedprintable, | quoted printable | ``quotetabs=True`` / | -| | quoted_printable | | :meth:`quopri.decode` | -+----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode | :meth:`uu.decode` | -+----------------------+------------------+------------------------------+------------------------------+ -| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / | -| | | using gzip | :meth:`zlib.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ - -.. [#b64] In addition to :term:`bytes-like objects `, - ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for - decoding ++--------------------+---------------------------+---------------------------+ +| Codec | Aliases | Purpose | ++====================+===========================+===========================+ +| rot_13 | rot13 | Returns the Caesar-cypher | +| | | encryption of the operand | ++--------------------+---------------------------+---------------------------+ .. versionadded:: 3.2 - Restoration of the binary transforms. - -.. versionchanged:: 3.4 - Restoration of the aliases for the binary transforms. - - -.. _text-transforms: - -Text Transforms -^^^^^^^^^^^^^^^ - -The following codec provides a text transform: a :class:`str` to :class:`str` -mapping. It is not supported by :meth:`str.encode` (which only produces -:class:`bytes` output). - -.. tabularcolumns:: |l|l|L| - -+--------------------+---------+---------------------------+ -| Codec | Aliases | Purpose | -+====================+=========+===========================+ -| rot_13 | rot13 | Returns the Caesar-cypher | -| | | encryption of the operand | -+--------------------+---------+---------------------------+ - -.. versionadded:: 3.2 - Restoration of the ``rot_13`` text transform. - -.. versionchanged:: 3.4 - Restoration of the ``rot13`` alias. + bytes-to-bytes and string-to-string codecs. :mod:`encodings.idna` --- Internationalized Domain Names in Applications @@ -1472,3 +1310,4 @@ BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this is only done once (on the first write to the byte stream). For decoding an optional UTF-8 encoded BOM at the start of the data will be skipped. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/collections.abc.rst --- a/Doc/library/collections.abc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/collections.abc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -15,7 +15,7 @@ import itertools __name__ = '' -**Source code:** :source:`Lib/_collections_abc.py` +**Source code:** :source:`Lib/collections/abc.py` -------------- @@ -31,48 +31,39 @@ The collections module offers the following :term:`ABCs `: -.. tabularcolumns:: |l|L|L|L| - -========================== ====================== ======================= ==================================================== +========================= ===================== ====================== ==================================================== ABC Inherits from Abstract Methods Mixin Methods -========================== ====================== ======================= ==================================================== +========================= ===================== ====================== ==================================================== :class:`Container` ``__contains__`` :class:`Hashable` ``__hash__`` :class:`Iterable` ``__iter__`` :class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` -:class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` :class:`Sized` ``__len__`` :class:`Callable` ``__call__`` -:class:`Sequence` :class:`Sized`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Iterable`, ``__len__`` ``index``, and ``count`` +:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``, + :class:`Iterable`, ``index``, and ``count`` :class:`Container` -:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` - ``__len__``, - ``insert`` +:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and + ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``, + ``insert`` ``remove``, ``clear``, and ``__iadd__`` -:class:`Set` :class:`Sized`, ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, - :class:`Iterable`, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - :class:`Container` ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` +:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, + :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, + :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint`` -:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and - ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, - ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` - ``add``, - ``discard`` +:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and + ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``, + ``__iand__``, ``__ixor__``, and ``__isub__`` -:class:`Mapping` :class:`Sized`, ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, - :class:`Iterable`, ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` - :class:`Container` ``__len__`` +:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``, + :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__`` + :class:`Container` -:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and - ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, - ``__delitem__``, and ``setdefault`` - ``__iter__``, - ``__len__`` +:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and + ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``, + and ``setdefault`` :class:`MappingView` :class:`Sized` ``__len__`` @@ -81,11 +72,7 @@ :class:`KeysView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` :class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__`` -:class:`Awaitable` ``__await__`` -:class:`Coroutine` :class:`Awaitable` ``send``, ``throw`` ``close`` -:class:`AsyncIterable` ``__aiter__`` -:class:`AsyncIterator` :class:`AsyncIterable` ``__anext__`` ``__aiter__`` -========================== ====================== ======================= ==================================================== +========================= ===================== ====================== ==================================================== .. class:: Container @@ -103,38 +90,14 @@ .. class:: Iterator - ABC for classes that provide the :meth:`~iterator.__iter__` and - :meth:`~iterator.__next__` methods. See also the definition of - :term:`iterator`. - -.. class:: Generator - - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, - :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. - - .. versionadded:: 3.5 + ABC for classes that provide the :meth:`__iter__` and :meth:`next` methods. + See also the definition of :term:`iterator`. .. class:: Sequence MutableSequence ABCs for read-only and mutable :term:`sequences `. - Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`__getitem__` method. - Consequently, if :meth:`__getitem__` is implemented with constant - access speed, the mixin methods will have linear performance; - however, if the underlying method is linear (as it would be with a - linked list), the mixins will have quadratic performance and will - likely need to be overridden. - - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. - - .. class:: Set MutableSet @@ -150,74 +113,24 @@ KeysView ValuesView - ABCs for mapping, items, keys, and values :term:`views `. - -.. class:: Awaitable - - ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. - - :term:`Coroutine` objects and instances of the - :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: Coroutine - - ABC for coroutine compatible classes. These implement the - following methods, defined in :ref:`coroutine-objects`: - :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and - :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: AsyncIterable - - ABC for classes that provide ``__aiter__`` method. See also the - definition of :term:`asynchronous iterable`. - - .. versionadded:: 3.5 - -.. class:: AsyncIterator - - ABC for classes that provide ``__aiter__`` and ``__anext__`` - methods. See also the definition of :term:`asynchronous iterator`. - - .. versionadded:: 3.5 + ABCs for mapping, items, keys, and values :term:`views `. These ABCs allow us to ask classes or instances if they provide particular functionality, for example:: size = None - if isinstance(myvar, collections.abc.Sized): + if isinstance(myvar, collections.Sized): size = len(myvar) Several of the ABCs are also useful as mixins that make it easier to develop classes supporting container APIs. For example, to write a class supporting -the full :class:`Set` API, it is only necessary to supply the three underlying +the full :class:`Set` API, it only necessary to supply the three underlying abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: +:meth:`isdisjoint` :: - class ListBasedSet(collections.abc.Set): + class ListBasedSet(collections.Set): ''' Alternate set implementation favoring space over speed and not requiring the set elements to be hashable. ''' def __init__(self, iterable): @@ -251,13 +164,13 @@ (2) To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, + semantics are fixed), redefine :meth:`__le__` and then the other operations will automatically follow suit. (3) The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value for the set; however, :meth:`__hash__` is not defined because not all sets - are hashable or immutable. To add set hashability using mixins, + are hashable or immutable. To add set hashabilty using mixins, inherit from both :meth:`Set` and :meth:`Hashable`, then define ``__hash__ = Set._hash``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/collections.rst --- a/Doc/library/collections.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/collections.rst Mon Jan 25 17:05:13 2016 +0100 @@ -2,15 +2,15 @@ ========================================== .. module:: collections - :synopsis: Container datatypes + :synopsis: Container datatypes .. moduleauthor:: Raymond Hettinger .. sectionauthor:: Raymond Hettinger .. testsetup:: * - from collections import * - import itertools - __name__ = '' + from collections import * + import itertools + __name__ = '' **Source code:** :source:`Lib/collections/__init__.py` @@ -33,16 +33,14 @@ ===================== ==================================================================== .. versionchanged:: 3.3 - Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. - For backwards compatibility, they continue to be visible in this module - as well. + Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. + For backwards compatibility, they continue to be visible in this module + as well. :class:`ChainMap` objects ------------------------- -.. versionadded:: 3.3 - A :class:`ChainMap` class is provided for quickly linking a number of mappings so they can be treated as a single unit. It is often much faster than creating a new dictionary and running multiple :meth:`~dict.update` calls. @@ -51,153 +49,107 @@ .. class:: ChainMap(*maps) - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. + A :class:`ChainMap` groups multiple dicts or other mappings together to + create a single, updateable view. If no *maps* are specified, a single empty + dictionary is provided so that a new chain always has at least one mapping. - The underlying mappings are stored in a list. That list is public and can - accessed or updated using the *maps* attribute. There is no other state. + The underlying mappings are stored in a list. That list is public and can + accessed or updated using the *maps* attribute. There is no other state. - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. + Lookups search the underlying mappings successively until a key is found. In + contrast, writes, updates, and deletions only operate on the first mapping. - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. + A :class:`ChainMap` incorporates the underlying mappings by reference. So, if + one of the underlying mappings gets updated, those changes will be reflected + in :class:`ChainMap`. - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: + All of the usual dictionary methods are supported. In addition, there is a + *maps* attribute, a method for creating new subcontexts, and a property for + accessing all but the first mapping: - .. attribute:: maps + .. attribute:: maps - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. + A user updateable list of mappings. The list is ordered from + first-searched to last-searched. It is the only stored state and can + be modified to change which mappings are searched. The list should + always contain at least one mapping. - .. method:: new_child(m=None) + .. method:: new_child() - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. + Returns a new :class:`ChainMap` containing a new :class:`dict` followed by + all of the maps in the current instance. A call to ``d.new_child()`` is + equivalent to: ``ChainMap({}, *d.maps)``. This method is used for + creating subcontexts that can be updated without altering values in any + of the parent mappings. - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. + .. method:: parents() - .. attribute:: parents + Returns a new :class:`ChainMap` containing all of the maps in the current + instance except the first one. This is useful for skipping the first map + in the search. The use-cases are similar to those for the + :keyword:`nonlocal` keyword used in :term:`nested scopes `. + The use-cases also parallel those for the builtin :func:`super` function. + A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. + .. versionadded:: 3.3 + Example of simulating Python's internal lookup chain:: -.. seealso:: + import builtins + pylookup = ChainMap(locals(), globals(), vars(builtins)) - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. + Example of letting user specified values take precedence over environment + variables which in turn take precedence over default values:: - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :meth:`~collections.ChainMap.parents` property. + import os, argparse + defaults = {'color': 'red', 'user': guest} + parser = argparse.ArgumentParser() + parser.add_argument('-u', '--user') + parser.add_argument('-c', '--color') + user_specified = vars(parser.parse_args()) + combined = ChainMap(user_specified, os.environ, defaults) - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. + Example patterns for using the :class:`ChainMap` class to simulate nested + contexts:: - * A `greatly simplified read-only version of Chainmap - `_. + c = ChainMap() # Create root context + d = c.new_child() # Create nested child context + e = c.new_child() # Child of c, independent from d + e.maps[0] # Current context dictionary -- like Python's locals() + e.maps[-1] # Root context -- like Python's globals() + e.parents # Enclosing context chain -- like Python's nonlocals + d['x'] # Get first key in the chain of contexts + d['x'] = 1 # Set value in current context + del['x'] # Delete from current context + list(d) # All nested values + k in d # Check all nested values + len(d) # Number of nested values + d.items() # All nested items + dict(d) # Flatten into a regular dictionary -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + .. seealso:: -This section shows various approaches to working with chained maps. + * The `MultiContext class + `_ + in the Enthought `CodeTools package + `_ has options to support + writing to any mapping in the chain. + * Django's `Context class + `_ + for templating is a read-only chain of mappings. It also features + pushing and popping of contexts similar to the + :meth:`~collections.ChainMap.new_child` method and the + :meth:`~collections.ChainMap.parents` property. -Example of simulating Python's internal lookup chain:: + * The `Nested Contexts recipe + `_ has options to control + whether writes and other mutations apply only to the first mapping or to + any mapping in the chain. - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k:v for k, v in vars(namespace).items() if v} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] # Get first key in the chain of contexts - d['x'] = 1 # Set value in current context - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) + * A `greatly simplified read-only version of Chainmap + `_. :class:`Counter` objects @@ -215,93 +167,92 @@ >>> # Find the ten most common words in Hamlet >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) + >>> words = re.findall('\w+', open('hamlet.txt').read().lower()) >>> Counter(words).most_common(10) [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] .. class:: Counter([iterable-or-mapping]) - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. - It is an unordered collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. + A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + It is an unordered collection where elements are stored as dictionary keys + and their counts are stored as dictionary values. Counts are allowed to be + any integer value including zero or negative counts. The :class:`Counter` + class is similar to bags or multisets in other languages. - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): + Elements are counted from an *iterable* or initialized from another + *mapping* (or counter): >>> c = Counter() # a new, empty counter >>> c = Counter('gallahad') # a new counter from an iterable >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: + Counter objects have a dictionary interface except that they return a zero + count for missing items instead of raising a :exc:`KeyError`: >>> c = Counter(['eggs', 'ham']) >>> c['bacon'] # count of a missing element is zero 0 - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: + Setting a count to zero does not remove an element from a counter. + Use ``del`` to remove it entirely: >>> c['sausage'] = 0 # counter entry with a zero count >>> del c['sausage'] # del actually removes the entry - .. versionadded:: 3.1 + .. versionadded:: 3.1 - Counter objects support three methods beyond those available for all - dictionaries: + Counter objects support three methods beyond those available for all + dictionaries: - .. method:: elements() + .. method:: elements() - Return an iterator over elements repeating each as many times as its - count. Elements are returned in arbitrary order. If an element's count - is less than one, :meth:`elements` will ignore it. + Return an iterator over elements repeating each as many times as its + count. Elements are returned in arbitrary order. If an element's count + is less than one, :meth:`elements` will ignore it. >>> c = Counter(a=4, b=2, c=0, d=-2) >>> list(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common([n]) - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :func:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered arbitrarily: + Return a list of the *n* most common elements and their counts from the + most common to the least. If *n* is not specified, :func:`most_common` + returns *all* elements in the counter. Elements with equal counts are + ordered arbitrarily: >>> Counter('abracadabra').most_common(3) [('a', 5), ('r', 2), ('b', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract([iterable-or-mapping]) - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. + Elements are subtracted from an *iterable* or from another *mapping* + (or counter). Like :meth:`dict.update` but subtracts counts instead + of replacing them. Both inputs and outputs may be zero or negative. >>> c = Counter(a=4, b=2, c=0, d=-2) >>> d = Counter(a=1, b=2, c=3, d=4) >>> c.subtract(d) - >>> c Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - .. versionadded:: 3.2 + .. versionadded:: 3.2 - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + The usual dictionary methods are available for :class:`Counter` objects + except for two which work differently for counters. - .. method:: fromkeys(iterable) + .. method:: fromkeys(iterable) - This class method is not implemented for :class:`Counter` objects. + This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update([iterable-or-mapping]) - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. + Elements are counted from an *iterable* or added-in from another + *mapping* (or counter). Like :meth:`dict.update` but adds counts + instead of replacing them. Also, the *iterable* is expected to be a + sequence of elements, not a sequence of ``(key, value)`` pairs. Common patterns for working with :class:`Counter` objects:: @@ -312,7 +263,7 @@ dict(c) # convert to a regular dictionary c.items() # convert to a list of (elem, cnt) pairs Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements + c.most_common()[:-n:-1] # n least common elements +c # remove zero and negative counts Several mathematical operations are provided for combining :class:`Counter` @@ -333,7 +284,7 @@ >>> c | d # union: max(c[x], d[x]) Counter({'a': 3, 'b': 2}) -Unary addition and subtraction are shortcuts for adding an empty counter +Unary addition and substraction are shortcuts for adding an empty counter or subtracting from an empty counter. >>> c = Counter(a=2, b=-4) @@ -343,37 +294,41 @@ Counter({'b': 4}) .. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. + Added support for unary plus, unary minus, and in-place multiset operations. .. note:: - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. + Counters were primarily designed to work with positive integers to represent + running counts; however, care was taken to not unnecessarily preclude use + cases needing other types or negative values. To help with those use cases, + this section documents the minimum range and type restrictions. - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. + * The :class:`Counter` class itself is a dictionary subclass with no + restrictions on its keys and values. The values are intended to be numbers + representing counts, but you *could* store anything in the value field. - * The :meth:`most_common` method requires only that the values be orderable. + * The :meth:`most_common` method requires only that the values be orderable. - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`update` and :meth:`subtract` which allow negative and zero values - for both inputs and outputs. + * For in-place operations such as ``c[key] += 1``, the value type need only + support addition and subtraction. So fractions, floats, and decimals would + work and negative values are supported. The same is also true for + :meth:`update` and :meth:`subtract` which allow negative and zero values + for both inputs and outputs. - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. + * The multiset methods are designed only for use cases with positive values. + The inputs may be negative or zero, but only outputs with positive values + are created. There are no type restrictions, but the value type needs to + support addition, subtraction, and comparison. - * The :meth:`elements` method requires integer counts. It ignores zero and - negative counts. + * The :meth:`elements` method requires integer counts. It ignores zero and + negative counts. .. seealso:: + * `Counter class `_ + adapted for Python 2.5 and an early `Bag recipe + `_ for Python 2.4. + * `Bag class `_ in Smalltalk. @@ -387,9 +342,9 @@ Section 4.6.3, Exercise 19*. * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`: + elements, see :func:`itertools.combinations_with_replacement`. - map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC + map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC :class:`deque` objects @@ -397,128 +352,105 @@ .. class:: deque([iterable, [maxlen]]) - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. + Returns a new deque object initialized left-to-right (using :meth:`append`) with + data from *iterable*. If *iterable* is not specified, the new deque is empty. - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. + Deques are a generalization of stacks and queues (the name is pronounced "deck" + and is short for "double-ended queue"). Deques support thread-safe, memory + efficient appends and pops from either side of the deque with approximately the + same O(1) performance in either direction. - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. + Though :class:`list` objects support similar operations, they are optimized for + fast fixed-length operations and incur O(n) memory movement costs for + ``pop(0)`` and ``insert(0, v)`` operations which change both the size and + position of the underlying data representation. - If *maxlen* is not specified or is *None*, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. + If *maxlen* is not specified or is *None*, deques may grow to an + arbitrary length. Otherwise, the deque is bounded to the specified maximum + length. Once a bounded length deque is full, when new items are added, a + corresponding number of items are discarded from the opposite end. Bounded + length deques provide functionality similar to the ``tail`` filter in + Unix. They are also useful for tracking transactions and other pools of data + where only the most recent activity is of interest. - Deque objects support the following methods: + Deque objects support the following methods: - .. method:: append(x) + .. method:: append(x) - Add *x* to the right side of the deque. + Add *x* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(x) - Add *x* to the left side of the deque. + Add *x* to the left side of the deque. - .. method:: clear() + .. method:: clear() - Remove all elements from the deque leaving it with length 0. + Remove all elements from the deque leaving it with length 0. - .. method:: copy() + .. method:: count(x) - Create a shallow copy of the deque. + Count the number of deque elements equal to *x*. - .. versionadded:: 3.5 + .. versionadded:: 3.2 - .. method:: count(x) + .. method:: extend(iterable) - Count the number of deque elements equal to *x*. + Extend the right side of the deque by appending elements from the iterable + argument. - .. versionadded:: 3.2 + .. method:: extendleft(iterable) - .. method:: extend(iterable) + Extend the left side of the deque by appending elements from *iterable*. + Note, the series of left appends results in reversing the order of + elements in the iterable argument. - Extend the right side of the deque by appending elements from the iterable - argument. + .. method:: pop() - .. method:: extendleft(iterable) + Remove and return an element from the right side of the deque. If no + elements are present, raises an :exc:`IndexError`. - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. + .. method:: popleft() - .. method:: index(x[, start[, stop]]) + Remove and return an element from the left side of the deque. If no + elements are present, raises an :exc:`IndexError`. - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - .. versionadded:: 3.5 + .. method:: remove(value) + Removed the first occurrence of *value*. If not found, raises a + :exc:`ValueError`. - .. method:: insert(i, x) - Insert *x* into the deque at position *i*. + .. method:: reverse() - .. versionadded:: 3.5 + Reverse the elements of the deque in-place and then return ``None``. + .. versionadded:: 3.2 - .. method:: pop() - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. + .. method:: rotate(n) + Rotate the deque *n* steps to the right. If *n* is negative, rotate to + the left. Rotating one step to the right is equivalent to: + ``d.appendleft(d.pop())``. - .. method:: popleft() - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. + Deque objects also provide one read-only attribute: + .. attribute:: maxlen - .. method:: remove(value) + Maximum size of a deque or *None* if unbounded. - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate to - the left. Rotating one step to the right is equivalent to: - ``d.appendleft(d.pop())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or *None* if unbounded. - - .. versionadded:: 3.1 + .. versionadded:: 3.1 In addition to the above, deques support iteration, pickling, ``len(d)``, @@ -527,63 +459,60 @@ access is O(1) at both ends but slows to O(n) in the middle. For fast random access, use lists instead. -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - Example: .. doctest:: - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I + >>> from collections import deque + >>> d = deque('ghi') # make a new deque with three items + >>> for elem in d: # iterate over the deque's elements + ... print(elem.upper()) + G + H + I - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) + >>> d.append('j') # add a new entry to the right side + >>> d.appendleft('f') # add a new entry to the left side + >>> d # show the representation of the deque + deque(['f', 'g', 'h', 'i', 'j']) - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' + >>> d.pop() # return and remove the rightmost item + 'j' + >>> d.popleft() # return and remove the leftmost item + 'f' + >>> list(d) # list the contents of the deque + ['g', 'h', 'i'] + >>> d[0] # peek at leftmost item + 'g' + >>> d[-1] # peek at rightmost item + 'i' - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) + >>> list(reversed(d)) # list the contents of a deque in reverse + ['i', 'h', 'g'] + >>> 'h' in d # search the deque + True + >>> d.extend('jkl') # add multiple elements at once + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + >>> d.rotate(1) # right rotation + >>> d + deque(['l', 'g', 'h', 'i', 'j', 'k']) + >>> d.rotate(-1) # left rotation + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque + >>> deque(reversed(d)) # make a new deque in reverse order + deque(['l', 'k', 'j', 'i', 'h', 'g']) + >>> d.clear() # empty the deque + >>> d.pop() # cannot pop from an empty deque + Traceback (most recent call last): + File "", line 1, in -toplevel- + d.pop() + IndexError: pop from an empty deque - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) + >>> d.extendleft('abc') # extendleft() reverses the input order + >>> d + deque(['c', 'b', 'a']) :class:`deque` Recipes @@ -594,10 +523,10 @@ Bounded length deques provide functionality similar to the ``tail`` filter in Unix:: - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) + def tail(filename, n=10): + 'Return the last n lines of a file' + with open(filename) as f: + return deque(f, n) Another approach to using deques is to maintain a sequence of recently added elements by appending to the right and popping to the left:: @@ -618,10 +547,10 @@ deletion. For example, a pure Python implementation of ``del d[n]`` relies on the :meth:`rotate` method to position elements to be popped:: - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) + def delete_nth(d, n): + d.rotate(-n) + d.popleft() + d.rotate(n) To implement :class:`deque` slicing, use a similar approach applying :meth:`rotate` to bring a target element to the left side of the deque. Remove @@ -637,50 +566,50 @@ .. class:: defaultdict([default_factory[, ...]]) - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. + Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the + built-in :class:`dict` class. It overrides one method and adds one writable + instance variable. The remaining functionality is the same as for the + :class:`dict` class and is not documented here. - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. + The first argument provides the initial value for the :attr:`default_factory` + attribute; it defaults to ``None``. All remaining arguments are treated the same + as if they were passed to the :class:`dict` constructor, including keyword + arguments. - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: + :class:`defaultdict` objects support the following method in addition to the + standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key) - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. + If the :attr:`default_factory` attribute is ``None``, this raises a + :exc:`KeyError` exception with the *key* as argument. - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. + If :attr:`default_factory` is not ``None``, it is called without arguments + to provide a default value for the given *key*, this value is inserted in + the dictionary for the *key*, and returned. - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. + If calling :attr:`default_factory` raises an exception this exception is + propagated unchanged. - This method is called by the :meth:`__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. + This method is called by the :meth:`__getitem__` method of the + :class:`dict` class when the requested key is not found; whatever it + returns or raises is then returned or raised by :meth:`__getitem__`. - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. + Note that :meth:`__missing__` is *not* called for any operations besides + :meth:`__getitem__`. This means that :meth:`get` will, like normal + dictionaries, return ``None`` as a default rather than using + :attr:`default_factory`. - :class:`defaultdict` objects support the following instance variable: + :class:`defaultdict` objects support the following instance variable: - .. attribute:: default_factory + .. attribute:: default_factory - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. + This attribute is used by the :meth:`__missing__` method; it is + initialized from the first argument to the constructor, if present, or to + ``None``, if absent. :class:`defaultdict` Examples @@ -689,13 +618,13 @@ Using :class:`list` as the :attr:`default_factory`, it is easy to group a sequence of key-value pairs into a dictionary of lists: - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> list(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] + >>> d = defaultdict(list) + >>> for k, v in s: + ... d[k].append(v) + ... + >>> list(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`default_factory` @@ -705,24 +634,24 @@ :meth:`list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> list(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + >>> d = {} + >>> for k, v in s: + ... d.setdefault(k, []).append(v) + ... + >>> list(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] Setting the :attr:`default_factory` to :class:`int` makes the :class:`defaultdict` useful for counting (like a bag or multiset in other languages): - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> list(d.items()) - [('i', 4), ('p', 2), ('s', 4), ('m', 1)] + >>> s = 'mississippi' + >>> d = defaultdict(int) + >>> for k in s: + ... d[k] += 1 + ... + >>> list(d.items()) + [('i', 4), ('p', 2), ('s', 4), ('m', 1)] When a letter is first encountered, it is missing from the mapping, so the :attr:`default_factory` function calls :func:`int` to supply a default count of @@ -733,23 +662,23 @@ is to use a lambda function which can supply any constant value (not just zero): - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' + >>> def constant_factory(value): + ... return lambda: value + >>> d = defaultdict(constant_factory('')) + >>> d.update(name='John', action='ran') + >>> '%(name)s %(action)s to %(object)s' % d + 'John ran to ' Setting the :attr:`default_factory` to :class:`set` makes the :class:`defaultdict` useful for building a dictionary of sets: - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> list(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] + >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] + >>> d = defaultdict(set) + >>> for k, v in s: + ... d[k].add(v) + ... + >>> list(d.items()) + [('blue', {2, 4}), ('red', {1, 3})] :func:`namedtuple` Factory Function for Tuples with Named Fields @@ -761,69 +690,69 @@ .. function:: namedtuple(typename, field_names, verbose=False, rename=False) - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. + Returns a new tuple subclass named *typename*. The new subclass is used to + create tuple-like objects that have fields accessible by attribute lookup as + well as being indexable and iterable. Instances of the subclass also have a + helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` + method which lists the tuple contents in a ``name=value`` format. - The *field_names* are a single string with each fieldname separated by whitespace - and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* - can be a sequence of strings such as ``['x', 'y']``. + The *field_names* are a single string with each fieldname separated by whitespace + and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names* + can be a sequence of strings such as ``['x', 'y']``. - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. + Any valid Python identifier may be used for a fieldname except for names + starting with an underscore. Valid identifiers consist of letters, digits, + and underscores but do not start with a digit or underscore and cannot be + a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, + or *raise*. - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. + If *rename* is true, invalid fieldnames are automatically replaced + with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is + converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword + ``def`` and the duplicate fieldname ``abc``. - If *verbose* is true, the class definition is printed after it is - built. This option is outdated; instead, it is simpler to print the - :attr:`_source` attribute. + If *verbose* is true, the class definition is printed after it is + built. This option is outdated; instead, it is simpler to print the + :attr:`_source` attribute. - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. + Named tuple instances do not have per-instance dictionaries, so they are + lightweight and require no more memory than regular tuples. - .. versionchanged:: 3.1 - Added support for *rename*. + .. versionchanged:: 3.1 + Added support for *rename*. .. doctest:: - :options: +NORMALIZE_WHITESPACE + :options: +NORMALIZE_WHITESPACE - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) + >>> # Basic example + >>> Point = namedtuple('Point', ['x', 'y']) + >>> p = Point(11, y=22) # instantiate with positional or keyword arguments + >>> p[0] + p[1] # indexable like the plain tuple (11, 22) + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> p # readable __repr__ with a name=value style + Point(x=11, y=22) Named tuples are especially useful for assigning field names to result tuples returned by the :mod:`csv` or :mod:`sqlite3` modules:: - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') + EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) + import csv + for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): + print(emp.name, emp.title) - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) + import sqlite3 + conn = sqlite3.connect('/companydata') + cursor = conn.cursor() + cursor.execute('SELECT name, age, title, department, paygrade FROM employees') + for emp in map(EmployeeRecord._make, cursor.fetchall()): + print(emp.name, emp.title) In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with @@ -831,61 +760,62 @@ .. classmethod:: somenamedtuple._make(iterable) - Class method that makes a new instance from an existing sequence or iterable. + Class method that makes a new instance from an existing sequence or iterable. - .. doctest:: +.. doctest:: - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) + >>> t = [11, 22] + >>> Point._make(t) + Point(x=11, y=22) .. method:: somenamedtuple._asdict() - Return a new :class:`OrderedDict` which maps field names to their corresponding - values:: + Return a new :class:`OrderedDict` which maps field names to their corresponding + values:: - >>> p = Point(x=11, y=22) - >>> p._asdict() - OrderedDict([('x', 11), ('y', 22)]) + >>> p._asdict() + OrderedDict([('x', 11), ('y', 22)]) - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. + .. versionchanged:: 3.1 + Returns an :class:`OrderedDict` instead of a regular :class:`dict`. .. method:: somenamedtuple._replace(kwargs) - Return a new instance of the named tuple replacing specified fields with new - values:: + Return a new instance of the named tuple replacing specified fields with new + values: - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) +:: - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + >>> p = Point(x=11, y=22) + >>> p._replace(x=33) + Point(x=33, y=22) + + >>> for partnum, record in inventory.items(): + ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) .. attribute:: somenamedtuple._source - A string with the pure Python source code used to create the named - tuple class. The source makes the named tuple self-documenting. - It can be printed, executed using :func:`exec`, or saved to a file - and imported. + A string with the pure Python source code used to create the named + tuple class. The source makes the named tuple self-documenting. + It can be printed, executed using :func:`exec`, or saved to a file + and imported. - .. versionadded:: 3.3 + .. versionadded:: 3.3 .. attribute:: somenamedtuple._fields - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. + Tuple of strings listing the field names. Useful for introspection + and for creating new named tuple types from existing named tuples. - .. doctest:: +.. doctest:: - >>> p._fields # view the field names - ('x', 'y') + >>> p._fields # view the field names + ('x', 'y') - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) + >>> Color = namedtuple('Color', 'red green blue') + >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) + >>> Pixel(11, 22, 128, 255, 0) + Pixel(x=11, y=22, red=128, green=255, blue=0) To retrieve a field whose name is stored in a string, use the :func:`getattr` function: @@ -896,9 +826,9 @@ To convert a dictionary to a named tuple, use the double-star-operator (as described in :ref:`tut-unpacking-arguments`): - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) + >>> d = {'x': 11, 'y': 22} + >>> Point(**d) + Point(x=11, y=22) Since a named tuple is a regular Python class, it is easy to add or change functionality with a subclass. Here is how to add a calculated field and @@ -925,18 +855,6 @@ >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - Default values can be implemented by using :meth:`_replace` to customize a prototype instance: @@ -945,18 +863,26 @@ >>> johns_account = default_account._replace(owner='John') >>> janes_account = default_account._replace(owner='Jane') +Enumerated constants can be implemented with named tuples, but it is simpler +and more efficient to use a simple class declaration: + + >>> Status = namedtuple('Status', 'open pending closed')._make(range(3)) + >>> Status.open, Status.pending, Status.closed + (0, 1, 2) + >>> class Status: + open, pending, closed = range(3) .. seealso:: - * `Recipe for named tuple abstract base class with a metaclass mix-in - `_ - by Jan Kaliszewski. Besides providing an :term:`abstract base class` for - named tuples, it also supports an alternate :term:`metaclass`-based - constructor that is convenient for use cases where named tuples are being - subclassed. + * `Named tuple recipe `_ + adapted for Python 2.4. - * :meth:`types.SimpleNamespace` for a mutable namespace based on an underlying - dictionary instead of a tuple. + * `Recipe for named tuple abstract base class with a metaclass mix-in + `_ + by Jan Kaliszewski. Besides providing an :term:`abstract base class` for + named tuples, it also supports an alternate :term:`metaclass`-based + constructor that is convenient for use cases where named tuples are being + subclassed. :class:`OrderedDict` objects @@ -968,36 +894,36 @@ .. class:: OrderedDict([items]) - Return an instance of a dict subclass, supporting the usual :class:`dict` - methods. An *OrderedDict* is a dict that remembers the order that keys - were first inserted. If a new entry overwrites an existing entry, the - original insertion position is left unchanged. Deleting an entry and - reinserting it will move it to the end. + Return an instance of a dict subclass, supporting the usual :class:`dict` + methods. An *OrderedDict* is a dict that remembers the order that keys + were first inserted. If a new entry overwrites an existing entry, the + original insertion position is left unchanged. Deleting an entry and + reinserting it will move it to the end. - .. versionadded:: 3.1 + .. versionadded:: 3.1 - .. method:: popitem(last=True) + .. method:: popitem(last=True) - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in LIFO order if *last* is true - or FIFO order if false. + The :meth:`popitem` method for ordered dictionaries returns and removes a + (key, value) pair. The pairs are returned in LIFO order if *last* is true + or FIFO order if false. - .. method:: move_to_end(key, last=True) + .. method:: move_to_end(key, last=True) - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist:: + Move an existing *key* to either end of an ordered dictionary. The item + is moved to the right end if *last* is true (the default) or to the + beginning if *last* is false. Raises :exc:`KeyError` if the *key* does + not exist:: - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d.keys()) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d.keys()) - 'bacde' + >>> d = OrderedDict.fromkeys('abcde') + >>> d.move_to_end('b') + >>> ''.join(d.keys()) + 'acdeb' + >>> d.move_to_end('b', last=False) + >>> ''.join(d.keys()) + 'bacde' - .. versionadded:: 3.2 + .. versionadded:: 3.2 In addition to the usual mapping methods, ordered dictionaries also support reverse iteration using :func:`reversed`. @@ -1005,23 +931,24 @@ Equality tests between :class:`OrderedDict` objects are order-sensitive and are implemented as ``list(od1.items())==list(od2.items())``. Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. +:class:`Mapping` objects are order-insensitive like regular dictionaries. +This allows :class:`OrderedDict` objects to be substituted anywhere a +regular dictionary is used. The :class:`OrderedDict` constructor and :meth:`update` method both accept keyword arguments, but their order is lost because Python's function call -semantics pass in keyword arguments using a regular unordered dictionary. +semantics pass-in keyword arguments using a regular unordered dictionary. -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. +.. seealso:: + + `Equivalent OrderedDict recipe `_ + that runs on Python 2.4 or later. :class:`OrderedDict` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since an ordered dictionary remembers its insertion order, it can be used -in conjunction with sorting to make a sorted dictionary:: +in conjuction with sorting to make a sorted dictionary:: >>> # regular unsorted dictionary >>> d = {'banana': 3, 'apple':4, 'pear': 1, 'orange': 2} @@ -1043,7 +970,7 @@ to the end and the sort is not maintained. It is also straight-forward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. +that the remembers the order the keys were *last* inserted. If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: @@ -1058,7 +985,7 @@ An ordered dictionary can be combined with the :class:`Counter` class so that the counter remembers the order elements are first encountered:: - class OrderedCounter(Counter, OrderedDict): + class OrderedCounter(Counter, OrderedDict): 'Counter that remembers the order elements are first encountered' def __repr__(self): @@ -1079,19 +1006,19 @@ .. class:: UserDict([initialdata]) - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. + Class that simulates a dictionary. The instance's contents are kept in a + regular dictionary, which is accessible via the :attr:`data` attribute of + :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is + initialized with its contents; note that a reference to *initialdata* will not + be kept, allowing it be used for other purposes. - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + In addition to supporting the methods and operations of mappings, + :class:`UserDict` instances provide the following attribute: - .. attribute:: data + .. attribute:: data - A real dictionary used to store the contents of the :class:`UserDict` - class. + A real dictionary used to store the contents of the :class:`UserDict` + class. @@ -1109,21 +1036,21 @@ .. class:: UserList([list]) - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. + Class that simulates a list. The instance's contents are kept in a regular + list, which is accessible via the :attr:`data` attribute of :class:`UserList` + instances. The instance's contents are initially set to a copy of *list*, + defaulting to the empty list ``[]``. *list* can be any iterable, for + example a real Python list or a :class:`UserList` object. - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: + In addition to supporting the methods and operations of mutable sequences, + :class:`UserList` instances provide the following attribute: - .. attribute:: data + .. attribute:: data - A real :class:`list` object used to store the contents of the - :class:`UserList` class. + A real :class:`list` object used to store the contents of the + :class:`UserList` class. -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to +**Subclassing requirements:** Subclasses of :class:`UserList` are expect to offer a constructor which can be called with either no arguments or one argument. List operations which return a new sequence attempt to create an instance of the actual implementation class. To do so, it assumes that the @@ -1146,14 +1073,10 @@ .. class:: UserString([sequence]) - Class that simulates a string or a Unicode string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *sequence*. The *sequence* can - be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a - subclass) or an arbitrary sequence which can be converted into a string using - the built-in :func:`str` function. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. + Class that simulates a string or a Unicode string object. The instance's + content is kept in a regular string object, which is accessible via the + :attr:`data` attribute of :class:`UserString` instances. The instance's + contents are initially set to a copy of *sequence*. The *sequence* can + be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a + subclass) or an arbitrary sequence which can be converted into a string using + the built-in :func:`str` function. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/colorsys.rst --- a/Doc/library/colorsys.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/colorsys.rst Mon Jan 25 17:05:13 2016 +0100 @@ -58,7 +58,7 @@ Example:: >>> import colorsys - >>> colorsys.rgb_to_hsv(0.2, 0.4, 0.4) - (0.5, 0.5, 0.4) - >>> colorsys.hsv_to_rgb(0.5, 0.5, 0.4) - (0.2, 0.4, 0.4) + >>> colorsys.rgb_to_hsv(.3, .4, .2) + (0.25, 0.5, 0.4) + >>> colorsys.hsv_to_rgb(0.25, 0.5, 0.4) + (0.3, 0.4, 0.2) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/compileall.rst --- a/Doc/library/compileall.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/compileall.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,6 @@ .. module:: compileall :synopsis: Tools for byte-compiling all Python source files in a directory tree. -**Source code:** :source:`Lib/compileall.py` - --------------- - This module provides some utility functions to support installing Python libraries. These functions compile Python source files in a directory tree. @@ -24,8 +20,7 @@ .. program:: compileall -.. cmdoption:: directory ... - file ... +.. cmdoption:: [directory|file]... Positional arguments are files to compile or directories that contain source files, traversed recursively. If no argument is given, behave as if @@ -42,8 +37,7 @@ .. cmdoption:: -q - Do not print the list of files compiled. If passed once, error messages will - still be printed. If passed twice (``-qq``), all output is suppressed. + Do not print the list of files compiled, print only error messages. .. cmdoption:: -d destdir @@ -71,28 +65,9 @@ is to write files to their :pep:`3147` locations and names, which allows byte-code files from multiple versions of Python to coexist. -.. cmdoption:: -r - - Control the maximum recursion level for subdirectories. - If this is given, then ``-l`` option will not be taken into account. - :program:`python -m compileall -r 0` is equivalent to - :program:`python -m compileall -l`. - -.. cmdoption:: -j N - - Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.cpu_count()` - will be used. - .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. -.. versionchanged:: 3.5 - Added the ``-j``, ``-r``, and ``-qq`` options. ``-q`` option - was changed to a multilevel value. ``-b`` will always produce a - byte-code file ending in ``.pyc``, never ``.pyo``. - - There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already provides the option: :program:`python -O -m compileall`. @@ -100,11 +75,10 @@ Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` - files along the way. Return a true value if all the files compiled successfully, - and a false value otherwise. + files along the way. The *maxlevels* parameter is used to limit the depth of the recursion; it defaults to ``10``. @@ -122,9 +96,8 @@ file considered for compilation, and if it returns a true value, the file is skipped. - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. + If *quiet* is true, nothing is printed to the standard output unless errors + occur. If *legacy* is true, byte-code files are written to their legacy locations and names, which may overwrite byte-code files created by another version of @@ -135,29 +108,13 @@ *optimize* specifies the optimization level for the compiler. It is passed to the built-in :func:`compile` function. - The argument *workers* specifies how many workers are used to - compile files in parallel. The default is to not use multiple workers. - If the platform can't use multiple workers and *workers* argument is given, - then sequential compilation will be used as a fallback. If *workers* is - lower than ``0``, a :exc:`ValueError` will be raised. - .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. - .. versionchanged:: 3.5 - Added the *workers* parameter. - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1) - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1) - - Compile the file with path *fullname*. Return a true value if the file - compiled successfully, and a false value otherwise. + Compile the file with path *fullname*. If *ddir* is given, it is prepended to the path to the file being compiled for use in compilation time tracebacks, and is also compiled in to the @@ -169,9 +126,8 @@ file being compiled, and if it returns a true value, the file is not compiled and ``True`` is returned. - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. + If *quiet* is true, nothing is printed to the standard output unless errors + occur. If *legacy* is true, byte-code files are written to their legacy locations and names, which may overwrite byte-code files created by another version of @@ -184,19 +140,11 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, legacy=False, optimize=-1) -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1) - - Byte-compile all the :file:`.py` files found along ``sys.path``. Return a - true value if all the files compiled successfully, and a false value otherwise. - - If *skip_curdir* is true (the default), the current directory is not included + Byte-compile all the :file:`.py` files found along ``sys.path``. If + *skip_curdir* is true (the default), the current directory is not included in the search. All other parameters are passed to the :func:`compile_dir` function. Note that unlike the other compile functions, ``maxlevels`` defaults to ``0``. @@ -204,12 +152,6 @@ .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: @@ -220,7 +162,7 @@ # Perform same compilation, excluding files in .svn directories. import re - compileall.compile_dir('Lib/', rx=re.compile(r'[/\\][.]svn'), force=True) + compileall.compile_dir('Lib/', rx=re.compile('/[.]svn'), force=True) .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/concurrency.rst --- a/Doc/library/concurrency.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,31 +0,0 @@ -.. _concurrency: - -******************** -Concurrent Execution -******************** - -The modules described in this chapter provide support for concurrent -execution of code. The appropriate choice of tool will depend on the -task to be executed (CPU bound vs IO bound) and preferred style of -development (event driven cooperative multitasking vs preemptive -multitasking). Here's an overview: - - -.. toctree:: - - threading.rst - multiprocessing.rst - concurrent.rst - concurrent.futures.rst - subprocess.rst - sched.rst - queue.rst - - -The following are support modules for some of the above services: - -.. toctree:: - - dummy_threading.rst - _thread.rst - _dummy_thread.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/concurrent.futures.rst --- a/Doc/library/concurrent.futures.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/concurrent.futures.rst Mon Jan 25 17:05:13 2016 +0100 @@ -38,26 +38,16 @@ future = executor.submit(pow, 323, 1235) print(future.result()) - .. method:: map(func, *iterables, timeout=None, chunksize=1) + .. method:: map(func, *iterables, timeout=None) - Equivalent to :func:`map(func, *iterables) ` except *func* is executed + Equivalent to ``map(func, *iterables)`` except *func* is executed asynchronously and several calls to *func* may be made concurrently. The - returned iterator raises a :exc:`concurrent.futures.TimeoutError` if - :meth:`~iterator.__next__` is called and the result isn't available - after *timeout* seconds from the original call to :meth:`Executor.map`. - *timeout* can be an int or a float. If *timeout* is not specified or - ``None``, there is no limit to the wait time. If a call raises an - exception, then that exception will be raised when its value is - retrieved from the iterator. When using :class:`ProcessPoolExecutor`, this - method chops *iterables* into a number of chunks which it submits to the - pool as separate tasks. The (approximate) size of these chunks can be - specified by setting *chunksize* to a positive integer. For very long - iterables, using a large value for *chunksize* can significantly improve - performance compared to the default size of 1. With :class:`ThreadPoolExecutor`, - *chunksize* has no effect. - - .. versionchanged:: 3.5 - Added the *chunksize* argument. + returned iterator raises a :exc:`TimeoutError` if :meth:`__next__()` is + called and the result isn't available after *timeout* seconds from the + original call to :meth:`Executor.map`. *timeout* can be an int or a + float. If *timeout* is not specified or ``None``, there is no limit to + the wait time. If a call raises an exception, then that exception will + be raised when its value is retrieved from the iterator. .. method:: shutdown(wait=True) @@ -84,13 +74,13 @@ e.submit(shutil.copy, 'src1.txt', 'dest1.txt') e.submit(shutil.copy, 'src2.txt', 'dest2.txt') e.submit(shutil.copy, 'src3.txt', 'dest3.txt') - e.submit(shutil.copy, 'src4.txt', 'dest4.txt') + e.submit(shutil.copy, 'src3.txt', 'dest4.txt') ThreadPoolExecutor ------------------ -:class:`ThreadPoolExecutor` is an :class:`Executor` subclass that uses a pool of +:class:`ThreadPoolExecutor` is a :class:`Executor` subclass that uses a pool of threads to execute calls asynchronously. Deadlocks can occur when the callable associated with a :class:`Future` waits on @@ -124,19 +114,11 @@ executor.submit(wait_on_future) -.. class:: ThreadPoolExecutor(max_workers=None) +.. class:: ThreadPoolExecutor(max_workers) An :class:`Executor` subclass that uses a pool of at most *max_workers* threads to execute calls asynchronously. - .. versionchanged:: 3.5 - If *max_workers* is ``None`` or - not given, it will default to the number of processors on the machine, - multiplied by ``5``, assuming that :class:`ThreadPoolExecutor` is often - used to overlap I/O instead of CPU work and the number of workers - should be higher than the number of workers - for :class:`ProcessPoolExecutor`. - .. _threadpoolexecutor-example: @@ -153,23 +135,20 @@ 'http://www.bbc.co.uk/', 'http://some-made-up-domain.com/'] - # Retrieve a single page and report the url and contents def load_url(url, timeout): - with urllib.request.urlopen(url, timeout=timeout) as conn: - return conn.read() + return urllib.request.urlopen(url, timeout=timeout).read() - # We can use a with statement to ensure threads are cleaned up promptly with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Start the load operations and mark each future with its URL - future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} + future_to_url = dict((executor.submit(load_url, url, 60), url) + for url in URLS) + for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] - try: - data = future.result() - except Exception as exc: - print('%r generated an exception: %s' % (url, exc)) + if future.exception() is not None: + print('%r generated an exception: %s' % (url, + future.exception())) else: - print('%r page is %d bytes' % (url, len(data))) + print('%r page is %d bytes' % (url, len(future.result()))) ProcessPoolExecutor @@ -181,9 +160,6 @@ allows it to side-step the :term:`Global Interpreter Lock` but also means that only picklable objects can be executed and returned. -The ``__main__`` module must be importable by worker subprocesses. This means -that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. - Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. @@ -192,8 +168,6 @@ An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` - will be raised. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a @@ -274,12 +248,11 @@ Return the value returned by the call. If the call hasn't yet completed then this method will wait up to *timeout* seconds. If the call hasn't - completed in *timeout* seconds, then a - :exc:`concurrent.futures.TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. + completed in *timeout* seconds, then a :exc:`TimeoutError` will be + raised. *timeout* can be an int or float. If *timeout* is not specified + or ``None``, there is no limit to the wait time. - If the future is cancelled before completing then :exc:`.CancelledError` + If the future is cancelled before completing then :exc:`CancelledError` will be raised. If the call raised, this method will raise the same exception. @@ -288,12 +261,11 @@ Return the exception raised by the call. If the call hasn't yet completed then this method will wait up to *timeout* seconds. If the - call hasn't completed in *timeout* seconds, then a - :exc:`concurrent.futures.TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. + call hasn't completed in *timeout* seconds, then a :exc:`TimeoutError` + will be raised. *timeout* can be an int or float. If *timeout* is not + specified or ``None``, there is no limit to the wait time. - If the future is cancelled before completing then :exc:`.CancelledError` + If the future is cancelled before completing then :exc:`CancelledError` will be raised. If the call completed without raising, ``None`` is returned. @@ -306,7 +278,7 @@ Added callables are called in the order that they were added and are always called in a thread belonging to the process that added them. If - the callable raises an :exc:`Exception` subclass, it will be logged and + the callable raises a :exc:`Exception` subclass, it will be logged and ignored. If the callable raises a :exc:`BaseException` subclass, the behavior is undefined. @@ -370,8 +342,6 @@ *return_when* indicates when this function should return. It must be one of the following constants: - .. tabularcolumns:: |l|L| - +-----------------------------+----------------------------------------+ | Constant | Description | +=============================+========================================+ @@ -392,13 +362,12 @@ Returns an iterator over the :class:`Future` instances (possibly created by different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or were cancelled). Any futures given by *fs* that - are duplicated will be returned once. Any futures that completed before - :func:`as_completed` is called will be yielded first. The returned iterator - raises a :exc:`concurrent.futures.TimeoutError` if :meth:`~iterator.__next__` - is called and the result isn't available after *timeout* seconds from the - original call to :func:`as_completed`. *timeout* can be an int or float. If - *timeout* is not specified or ``None``, there is no limit to the wait time. + they complete (finished or were cancelled). Any futures that completed + before :func:`as_completed` is called will be yielded first. The returned + iterator raises a :exc:`TimeoutError` if :meth:`__next__` is called and the + result isn't available after *timeout* seconds from the original call to + :func:`as_completed`. *timeout* can be an int or float. If *timeout* is not + specified or ``None``, there is no limit to the wait time. .. seealso:: @@ -411,18 +380,6 @@ Exception classes ----------------- -.. currentmodule:: concurrent.futures - -.. exception:: CancelledError - - Raised when a future is cancelled. - -.. exception:: TimeoutError - - Raised when a future operation exceeds the given timeout. - -.. currentmodule:: concurrent.futures.process - .. exception:: BrokenProcessPool Derived from :exc:`RuntimeError`, this exception class is raised when diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/concurrent.rst --- a/Doc/library/concurrent.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6 +0,0 @@ -The :mod:`concurrent` package -============================= - -Currently, there is only one module in this package: - -* :mod:`concurrent.futures` -- Launching parallel tasks diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/configparser.rst --- a/Doc/library/configparser.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/configparser.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,8 +11,6 @@ .. sectionauthor:: Christopher G. Petrilli .. sectionauthor:: Łukasz Langa -**Source code:** :source:`Lib/configparser.py` - .. index:: pair: .ini; file pair: configuration; file @@ -144,13 +142,12 @@ >>> float(topsecret['CompressionLevel']) 9.0 -Since this task is so common, config parsers provide a range of handy getter -methods to handle integers, floats and booleans. The last one is the most -interesting because simply passing the value to ``bool()`` would do no good -since ``bool('False')`` is still ``True``. This is why config parsers also -provide :meth:`getboolean`. This method is case-insensitive and recognizes -Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, -``'true'``/``'false'`` and ``'1'``/``'0'`` [1]_. For example: +Extracting Boolean values is not that simple, though. Passing the value +to ``bool()`` would do no good since ``bool('False')`` is still +``True``. This is why config parsers also provide :meth:`getboolean`. +This method is case-insensitive and recognizes Boolean values from +``'yes'``/``'no'``, ``'on'``/``'off'`` and ``'1'``/``'0'`` [1]_. +For example: .. doctest:: @@ -162,8 +159,10 @@ True Apart from :meth:`getboolean`, config parsers also provide equivalent -:meth:`getint` and :meth:`getfloat` methods. You can register your own -converters and customize the provided ones. [1]_ +:meth:`getint` and :meth:`getfloat` methods, but these are far less +useful since conversion using :func:`int` and :func:`float` is +sufficient for these types. + Fallback Values --------------- @@ -318,11 +317,11 @@ .. class:: ExtendedInterpolation() An alternative handler for interpolation which implements a more advanced - syntax, used for instance in ``zc.buildout``. Extended interpolation is + syntax, used for instance in ``zc.buildout``. Extended interpolation is using ``${section:option}`` to denote a value from a foreign section. - Interpolation can span multiple levels. For convenience, if the - ``section:`` part is omitted, interpolation defaults to the current section - (and possibly the default values from the special section). + Interpolation can span multiple levels. For convenience, if the ``section:`` + part is omitted, interpolation defaults to the current section (and possibly + the default values from the special section). For example, the configuration specified above with basic interpolation, would look like this with extended interpolation: @@ -372,8 +371,7 @@ parser. :mod:`configparser` objects behave as close to actual dictionaries as possible. -The mapping interface is complete and adheres to the -:class:`~collections.abc.MutableMapping` ABC. +The mapping interface is complete and adheres to the ``MutableMapping`` ABC. However, there are a few differences that should be taken into account: * By default, all keys in sections are accessible in a case-insensitive manner @@ -387,26 +385,20 @@ * All sections include ``DEFAULTSECT`` values as well which means that ``.clear()`` on a section may not leave the section visibly empty. This is because default values cannot be deleted from the section (because technically - they are not there). If they are overridden in the section, deleting causes + they are not there). If they are overriden in the section, deleting causes the default value to be visible again. Trying to delete a default value causes a ``KeyError``. -* ``DEFAULTSECT`` cannot be removed from the parser: - - * trying to delete it raises ``ValueError``, - - * ``parser.clear()`` leaves it intact, - - * ``parser.popitem()`` never returns it. +* Trying to delete the ``DEFAULTSECT`` raises ``ValueError``. * ``parser.get(section, option, **kwargs)`` - the second argument is **not** - a fallback value. Note however that the section-level ``get()`` methods are + a fallback value. Note however that the section-level ``get()`` methods are compatible both with the mapping protocol and the classic configparser API. * ``parser.items()`` is compatible with the mapping protocol (returns a list of *section_name*, *section_proxy* pairs including the DEFAULTSECT). However, this method can also be invoked with arguments: ``parser.items(section, raw, - vars)``. The latter call returns a list of *option*, *value* pairs for + vars)``. The latter call returns a list of *option*, *value* pairs for a specified ``section``, with all interpolations expanded (unless ``raw=True`` is provided). @@ -540,9 +532,9 @@ * *delimiters*, default value: ``('=', ':')`` - Delimiters are substrings that delimit keys from values within a section. - The first occurrence of a delimiting substring on a line is considered - a delimiter. This means values (but not keys) can contain the delimiters. + Delimiters are substrings that delimit keys from values within a section. The + first occurence of a delimiting substring on a line is considered a delimiter. + This means values (but not keys) can contain the delimiters. See also the *space_around_delimiters* argument to :meth:`ConfigParser.write`. @@ -554,7 +546,7 @@ Comment prefixes are strings that indicate the start of a valid comment within a config file. *comment_prefixes* are used only on otherwise empty lines (optionally indented) whereas *inline_comment_prefixes* can be used after - every valid value (e.g. section names, options and empty lines as well). By + every valid value (e.g. section names, options and empty lines as well). By default inline comments are disabled and ``'#'`` and ``';'`` are used as prefixes for whole line comments. @@ -564,10 +556,10 @@ Please note that config parsers don't support escaping of comment prefixes so using *inline_comment_prefixes* may prevent users from specifying option - values with characters used as comment prefixes. When in doubt, avoid - setting *inline_comment_prefixes*. In any circumstances, the only way of - storing comment prefix characters at the beginning of a line in multiline - values is to interpolate the prefix, for example:: + values with characters used as comment prefixes. When in doubt, avoid setting + *inline_comment_prefixes*. In any circumstances, the only way of storing + comment prefix characters at the beginning of a line in multiline values is to + interpolate the prefix, for example:: >>> from configparser import ConfigParser, ExtendedInterpolation >>> parser = ConfigParser(interpolation=ExtendedInterpolation()) @@ -612,7 +604,7 @@ When set to ``True``, the parser will not allow for any section or option duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict + :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict parsers in new applications. .. versionchanged:: 3.2 @@ -647,12 +639,12 @@ The convention of allowing a special section of default values for other sections or interpolation purposes is a powerful concept of this library, - letting users create complex declarative configurations. This section is + letting users create complex declarative configurations. This section is normally called ``"DEFAULT"`` but this can be customized to point to any - other valid section name. Some typical values include: ``"general"`` or - ``"common"``. The name provided is used for recognizing default sections - when reading from any source and is used when writing configuration back to - a file. Its current value can be retrieved using the + other valid section name. Some typical values include: ``"general"`` or + ``"common"``. The name provided is used for recognizing default sections when + reading from any source and is used when writing configuration back to + a file. Its current value can be retrieved using the ``parser_instance.default_section`` attribute and may be modified at runtime (i.e. to convert files from one format to another). @@ -661,30 +653,14 @@ Interpolation behaviour may be customized by providing a custom handler through the *interpolation* argument. ``None`` can be used to turn off interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the + advanced variant inspired by ``zc.buildout``. More on the subject in the `dedicated documentation section <#interpolation-of-values>`_. :class:`RawConfigParser` has a default value of ``None``. -* *converters*, default value: not set - - Config parsers provide option value getters that perform type conversion. By - default :meth:`getint`, :meth:`getfloat`, and :meth:`getboolean` are - implemented. Should other getters be desirable, users may define them in - a subclass or pass a dictionary where each key is a name of the converter and - each value is a callable implementing said conversion. For instance, passing - ``{'decimal': decimal.Decimal}`` would add :meth:`getdecimal` on both the - parser object and all section proxies. In other words, it will be possible - to write both ``parser_instance.getdecimal('section', 'key', fallback=0)`` - and ``parser_instance['section'].getdecimal('key', 0)``. - - If the converter needs to access the state of the parser, it can be - implemented as a method on a config parser subclass. If the name of this - method starts with ``get``, it will be available on all section proxies, in - the dict-compatible form (see the ``getdecimal()`` example above). More advanced customization may be achieved by overriding default values of -these parser attributes. The defaults are defined on the classes, so they may -be overridden by subclasses or by attribute assignment. +these parser attributes. The defaults are defined on the classes, so they +may be overriden by subclasses or by attribute assignment. .. attribute:: BOOLEAN_STATES @@ -742,11 +718,10 @@ .. attribute:: SECTCRE - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: + A compiled regular expression used to parse section headers. The default + matches ``[section]`` to the name ``"section"``. Whitespace is considered part + of the section name, thus ``[ larch ]`` will be read as a section of name + ``" larch "``. Override this attribute if that's unsuitable. For example: .. doctest:: @@ -795,9 +770,9 @@ # values using the mapping protocol or ConfigParser's set() does not allow # such assignments to take place. config.add_section('Section1') - config.set('Section1', 'an_int', '15') - config.set('Section1', 'a_bool', 'true') - config.set('Section1', 'a_float', '3.1415') + config.set('Section1', 'int', '15') + config.set('Section1', 'bool', 'true') + config.set('Section1', 'float', '3.1415') config.set('Section1', 'baz', 'fun') config.set('Section1', 'bar', 'Python') config.set('Section1', 'foo', '%(bar)s is %(baz)s!') @@ -815,13 +790,13 @@ # getfloat() raises an exception if the value is not a float # getint() and getboolean() also do this for their respective types - a_float = config.getfloat('Section1', 'a_float') - an_int = config.getint('Section1', 'an_int') - print(a_float + an_int) + float = config.getfloat('Section1', 'float') + int = config.getint('Section1', 'int') + print(float + int) # Notice that the next output does not interpolate '%(bar)s' or '%(baz)s'. # This is because we are using a RawConfigParser(). - if config.getboolean('Section1', 'a_bool'): + if config.getboolean('Section1', 'bool'): print(config.get('Section1', 'foo')) To get interpolation, use :class:`ConfigParser`:: @@ -877,7 +852,7 @@ ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) +.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation()) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -887,8 +862,8 @@ When *delimiters* is given, it is used as the set of substrings that divide keys from values. When *comment_prefixes* is given, it will be used as the set of substrings that prefix comments in otherwise empty lines. - Comments can be indented. When *inline_comment_prefixes* is given, it will - be used as the set of substrings that prefix comments in non-empty lines. + Comments can be indented. When *inline_comment_prefixes* is given, it will be + used as the set of substrings that prefix comments in non-empty lines. When *strict* is ``True`` (the default), the parser won't allow for any section or option duplicates while reading from a single source (file, @@ -902,13 +877,13 @@ When *default_section* is given, it specifies the name for the special section holding default values for other sections and interpolation purposes - (normally named ``"DEFAULT"``). This value can be retrieved and changed on + (normally named ``"DEFAULT"``). This value can be retrieved and changed on runtime using the ``default_section`` instance attribute. Interpolation behaviour may be customized by providing a custom handler through the *interpolation* argument. ``None`` can be used to turn off interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the + advanced variant inspired by ``zc.buildout``. More on the subject in the `dedicated documentation section <#interpolation-of-values>`_. All option names used in interpolation will be passed through the @@ -917,12 +892,6 @@ converts option names to lower case), the values ``foo %(bar)s`` and ``foo %(BAR)s`` are equivalent. - When *converters* is given, it should be a dictionary where each key - represents the name of a type converter and each value is a callable - implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser - object and section proxies. - .. versionchanged:: 3.1 The default *dict_type* is :class:`collections.OrderedDict`. @@ -931,9 +900,6 @@ *empty_lines_in_values*, *default_section* and *interpolation* were added. - .. versionchanged:: 3.5 - The *converters* argument was added. - .. method:: defaults() @@ -971,7 +937,7 @@ .. method:: has_option(section, option) If the given *section* exists, and contains the given *option*, return - :const:`True`; otherwise return :const:`False`. If the specified + :const:`True`; otherwise return :const:`False`. If the specified *section* is :const:`None` or an empty string, DEFAULT is assumed. @@ -1041,7 +1007,7 @@ .. versionadded:: 3.2 - .. method:: get(section, option, *, raw=False, vars=None[, fallback]) + .. method:: get(section, option, raw=False, [vars, fallback]) Get an *option* value for the named *section*. If *vars* is provided, it must be a dictionary. The *option* is looked up in *vars* (if provided), @@ -1059,21 +1025,21 @@ (especially when using the mapping protocol). - .. method:: getint(section, option, *, raw=False, vars=None[, fallback]) + .. method:: getint(section, option, raw=False, [vars, fallback]) A convenience method which coerces the *option* in the specified *section* to an integer. See :meth:`get` for explanation of *raw*, *vars* and *fallback*. - .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) + .. method:: getfloat(section, option, raw=False, [vars, fallback]) A convenience method which coerces the *option* in the specified *section* to a floating point number. See :meth:`get` for explanation of *raw*, *vars* and *fallback*. - .. method:: getboolean(section, option, *, raw=False, vars=None[, fallback]) + .. method:: getboolean(section, option, raw=False, [vars, fallback]) A convenience method which coerces the *option* in the specified *section* to a Boolean value. Note that the accepted values for the option are @@ -1085,8 +1051,7 @@ *fallback*. - .. method:: items(raw=False, vars=None) - items(section, raw=False, vars=None) + .. method:: items([section], raw=False, vars=None) When *section* is not given, return a list of *section_name*, *section_proxy* pairs, including DEFAULTSECT. @@ -1096,7 +1061,7 @@ :meth:`get` method. .. versionchanged:: 3.2 - Items present in *vars* no longer appear in the result. The previous + Items present in *vars* no longer appear in the result. The previous behaviour mixed actual parser options with variables provided for interpolation. @@ -1184,20 +1149,14 @@ RawConfigParser Objects ----------------------- -.. class:: RawConfigParser(defaults=None, dict_type=collections.OrderedDict, \ - allow_no_value=False, *, delimiters=('=', ':'), \ - comment_prefixes=('#', ';'), \ - inline_comment_prefixes=None, strict=True, \ - empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) +.. class:: RawConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configaparser.DEFAULTSECT, interpolation=None) Legacy variant of the :class:`ConfigParser` with interpolation disabled by default and unsafe ``add_section`` and ``set`` methods. .. note:: Consider using :class:`ConfigParser` instead which checks types of - the values to be stored internally. If you don't want interpolation, you + the values to be stored internally. If you don't want interpolation, you can use ``ConfigParser(interpolation=None)``. @@ -1208,7 +1167,7 @@ *default section* name is passed, :exc:`ValueError` is raised. Type of *section* is not checked which lets users create non-string named - sections. This behaviour is unsupported and may cause internal errors. + sections. This behaviour is unsupported and may cause internal errors. .. method:: set(section, option, value) @@ -1309,4 +1268,3 @@ .. [1] Config parsers allow for heavy customization. If you are interested in changing the behaviour outlined by the footnote reference, consult the `Customizing Parser Behaviour`_ section. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/constants.rst --- a/Doc/library/constants.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/constants.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,30 +19,16 @@ .. data:: None - The sole value of the type ``NoneType``. ``None`` is frequently used to + The sole value of :attr:`types.NoneType`. ``None`` is frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. .. data:: NotImplemented - Special value which should be returned by the binary special methods - (e.g. :meth:`__eq__`, :meth:`__lt__`, :meth:`__add__`, :meth:`__rsub__`, - etc.) to indicate that the operation is not implemented with respect to - the other type; may be returned by the in-place binary special methods - (e.g. :meth:`__imul__`, :meth:`__iand__`, etc.) for the same purpose. - Its truth value is true. - -.. note:: - - When ``NotImplemented`` is returned, the interpreter will then try the - reflected operation on the other type, or some other fallback, depending - on the operator. If all attempted operations return ``NotImplemented``, the - interpreter will raise an appropriate exception. - - See - :ref:`implementing-the-arithmetic-operations` - for more details. + Special value which can be returned by the "rich comparison" special methods + (:meth:`__eq__`, :meth:`__lt__`, and friends), to indicate that the comparison + is not implemented with respect to the other type. .. data:: Ellipsis diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/contextlib.rst --- a/Doc/library/contextlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/contextlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,11 +12,8 @@ statement. For more information see also :ref:`typecontextmanager` and :ref:`context-managers`. +Functions provided: -Utilities ---------- - -Functions and classes provided: .. decorator:: contextmanager @@ -95,93 +92,6 @@ ``page.close()`` will be called when the :keyword:`with` block is exited. -.. function:: suppress(*exceptions) - - Return a context manager that suppresses any of the specified exceptions - if they occur in the body of a with statement and then resumes execution - with the first statement following the end of the with statement. - - As with any other mechanism that completely suppresses exceptions, this - context manager should be used only to cover very specific errors where - silently continuing with program execution is known to be the right - thing to do. - - For example:: - - from contextlib import suppress - - with suppress(FileNotFoundError): - os.remove('somefile.tmp') - - with suppress(FileNotFoundError): - os.remove('someotherfile.tmp') - - This code is equivalent to:: - - try: - os.remove('somefile.tmp') - except FileNotFoundError: - pass - - try: - os.remove('someotherfile.tmp') - except FileNotFoundError: - pass - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stdout(new_target) - - Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. - - This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. - - For example, the output of :func:`help` normally is sent to *sys.stdout*. - You can capture that output in a string by redirecting the output to an - :class:`io.StringIO` object:: - - f = io.StringIO() - with redirect_stdout(f): - help(pow) - s = f.getvalue() - - To send the output of :func:`help` to a file on disk, redirect the output - to a regular file:: - - with open('help.txt', 'w') as f: - with redirect_stdout(f): - help(pow) - - To send the output of :func:`help` to *sys.stderr*:: - - with redirect_stdout(sys.stderr): - help(pow) - - Note that the global side effect on :data:`sys.stdout` means that this - context manager is not suitable for use in library code and most threaded - applications. It also has no effect on the output of subprocesses. - However, it is still a useful approach for many utility scripts. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stderr(new_target) - - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.5 - - .. class:: ContextDecorator() A base class that enables a context manager to also be used as a decorator. @@ -258,502 +168,9 @@ .. versionadded:: 3.2 -.. class:: ExitStack() - - A context manager that is designed to make it easy to programmatically - combine other context managers and cleanup functions, especially those - that are optional or otherwise driven by input data. - - For example, a set of files may easily be handled in a single with - statement as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception - - Each instance maintains a stack of registered callbacks that are called in - reverse order when the instance is closed (either explicitly or implicitly - at the end of a :keyword:`with` statement). Note that callbacks are *not* - invoked implicitly when the context stack instance is garbage collected. - - This stack model is used so that context managers that acquire their - resources in their ``__init__`` method (such as file objects) can be - handled correctly. - - Since registered callbacks are invoked in the reverse order of - registration, this ends up behaving as if multiple nested :keyword:`with` - statements had been used with the registered set of callbacks. This even - extends to exception handling - if an inner callback suppresses or replaces - an exception, then outer callbacks will be passed arguments based on that - updated state. - - This is a relatively low level API that takes care of the details of - correctly unwinding the stack of exit callbacks. It provides a suitable - foundation for higher level context managers that manipulate the exit - stack in application specific ways. - - .. versionadded:: 3.3 - - .. method:: enter_context(cm) - - Enters a new context manager and adds its :meth:`__exit__` method to - the callback stack. The return value is the result of the context - manager's own :meth:`__enter__` method. - - These context managers may suppress exceptions just as they normally - would if used directly as part of a :keyword:`with` statement. - - .. method:: push(exit) - - Adds a context manager's :meth:`__exit__` method to the callback stack. - - As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`__enter__` implementation with a context manager's own - :meth:`__exit__` method. - - If passed an object that is not a context manager, this method assumes - it is a callback with the same signature as a context manager's - :meth:`__exit__` method and adds it directly to the callback stack. - - By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`__exit__` methods can. - - The passed in object is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: callback(callback, *args, **kwds) - - Accepts an arbitrary callback function and arguments and adds it to - the callback stack. - - Unlike the other methods, callbacks added this way cannot suppress - exceptions (as they are never passed the exception details). - - The passed in callback is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: pop_all() - - Transfers the callback stack to a fresh :class:`ExitStack` instance - and returns it. No callbacks are invoked by this operation - instead, - they will now be invoked when the new stack is closed (either - explicitly or implicitly at the end of a :keyword:`with` statement). - - For example, a group of files can be opened as an "all or nothing" - operation as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # Hold onto the close method, but don't call it yet. - close_files = stack.pop_all().close - # If opening any file fails, all previously opened files will be - # closed automatically. If all files are opened successfully, - # they will remain open even after the with statement ends. - # close_files() can then be invoked explicitly to close them all. - - .. method:: close() - - Immediately unwinds the callback stack, invoking callbacks in the - reverse order of registration. For any context managers and exit - callbacks registered, the arguments passed in will indicate that no - exception occurred. - - -Examples and Recipes --------------------- - -This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. - - -Supporting a variable number of context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The primary use case for :class:`ExitStack` is the one given in the class -documentation: supporting a variable number of context managers and other -cleanup operations in a single :keyword:`with` statement. The variability -may come from the number of context managers needed being driven by user -input (such as opening a user specified collection of files), or from -some of the context managers being optional:: - - with ExitStack() as stack: - for resource in resources: - stack.enter_context(resource) - if need_special_resource(): - special = acquire_special_resource() - stack.callback(release_special_resource, special) - # Perform operations that use the acquired resources - -As shown, :class:`ExitStack` also makes it quite easy to use :keyword:`with` -statements to manage arbitrary resources that don't natively support the -context management protocol. - - -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - -Catching exceptions from ``__enter__`` methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is occasionally desirable to catch exceptions from an ``__enter__`` -method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` -method. By using :class:`ExitStack` the steps in the context management -protocol can be separated slightly in order to allow this:: - - stack = ExitStack() - try: - x = stack.enter_context(cm) - except Exception: - # handle __enter__ exception - else: - with stack: - # Handle normal case - -Actually needing to do this is likely to indicate that the underlying API -should be providing a direct resource management interface for use with -:keyword:`try`/:keyword:`except`/:keyword:`finally` statements, but not -all APIs are well designed in that regard. When a context manager is the -only resource management API provided, then :class:`ExitStack` can make it -easier to handle various situations that can't be handled directly in a -:keyword:`with` statement. - - -Cleaning up in an ``__enter__`` implementation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As noted in the documentation of :meth:`ExitStack.push`, this -method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`__enter__` implementation fail. - -Here's an example of doing this for a context manager that accepts resource -acquisition and release functions, along with an optional validation function, -and maps them to the context management protocol:: - - from contextlib import contextmanager, ExitStack - - class ResourceManager: - - def __init__(self, acquire_resource, release_resource, check_resource_ok=None): - self.acquire_resource = acquire_resource - self.release_resource = release_resource - if check_resource_ok is None: - def check_resource_ok(resource): - return True - self.check_resource_ok = check_resource_ok - - @contextmanager - def _cleanup_on_error(self): - with ExitStack() as stack: - stack.push(self) - yield - # The validation check passed and didn't raise an exception - # Accordingly, we want to keep the resource, and pass it - # back to our caller - stack.pop_all() - - def __enter__(self): - resource = self.acquire_resource() - with self._cleanup_on_error(): - if not self.check_resource_ok(resource): - msg = "Failed validation for {!r}" - raise RuntimeError(msg.format(resource)) - return resource - - def __exit__(self, *exc_details): - # We don't need to duplicate any of our resource release logic - self.release_resource() - - -Replacing any use of ``try-finally`` and flag variables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A pattern you will sometimes see is a ``try-finally`` statement with a flag -variable to indicate whether or not the body of the ``finally`` clause should -be executed. In its simplest form (that can't already be handled just by -using an ``except`` clause instead), it looks something like this:: - - cleanup_needed = True - try: - result = perform_operation() - if result: - cleanup_needed = False - finally: - if cleanup_needed: - cleanup_resources() - -As with any ``try`` statement based code, this can cause problems for -development and review, because the setup code and the cleanup code can end -up being separated by arbitrarily long sections of code. - -:class:`ExitStack` makes it possible to instead register a callback for -execution at the end of a ``with`` statement, and then later decide to skip -executing that callback:: - - from contextlib import ExitStack - - with ExitStack() as stack: - stack.callback(cleanup_resources) - result = perform_operation() - if result: - stack.pop_all() - -This allows the intended cleanup up behaviour to be made explicit up front, -rather than requiring a separate flag variable. - -If a particular application uses this pattern a lot, it can be simplified -even further by means of a small helper class:: - - from contextlib import ExitStack - - class Callback(ExitStack): - def __init__(self, callback, *args, **kwds): - super(Callback, self).__init__() - self.callback(callback, *args, **kwds) - - def cancel(self): - self.pop_all() - - with Callback(cleanup_resources) as cb: - result = perform_operation() - if result: - cb.cancel() - -If the resource cleanup isn't already neatly bundled into a standalone -function, then it is still possible to use the decorator form of -:meth:`ExitStack.callback` to declare the resource cleanup in -advance:: - - from contextlib import ExitStack - - with ExitStack() as stack: - @stack.callback - def cleanup_resources(): - ... - result = perform_operation() - if result: - stack.pop_all() - -Due to the way the decorator protocol works, a callback function -declared this way cannot take any parameters. Instead, any resources to -be released must be accessed as closure variables. - - -Using a context manager as a function decorator -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`ContextDecorator` makes it possible to use a context manager in -both an ordinary ``with`` statement and also as a function decorator. - -For example, it is sometimes useful to wrap functions or groups of statements -with a logger that can track the time of entry and time of exit. Rather than -writing both a function decorator and a context manager for the task, -inheriting from :class:`ContextDecorator` provides both capabilities in a -single definition:: - - from contextlib import ContextDecorator - import logging - - logging.basicConfig(level=logging.INFO) - - class track_entry_and_exit(ContextDecorator): - def __init__(self, name): - self.name = name - - def __enter__(self): - logging.info('Entering: {}'.format(self.name)) - - def __exit__(self, exc_type, exc, exc_tb): - logging.info('Exiting: {}'.format(self.name)) - -Instances of this class can be used as both a context manager:: - - with track_entry_and_exit('widget loader'): - print('Some time consuming activity goes here') - load_widget() - -And also as a function decorator:: - - @track_entry_and_exit('widget loader') - def activity(): - print('Some time consuming activity goes here') - load_widget() - -Note that there is one additional limitation when using context managers -as function decorators: there's no way to access the return value of -:meth:`__enter__`. If that value is needed, then it is still necessary to use -an explicit ``with`` statement. - .. seealso:: :pep:`0343` - The "with" statement The specification, background, and examples for the Python :keyword:`with` statement. -.. _single-use-reusable-and-reentrant-cms: - -Single use, reusable and reentrant context managers ---------------------------------------------------- - -Most context managers are written in a way that means they can only be -used effectively in a :keyword:`with` statement once. These single use -context managers must be created afresh each time they're used - -attempting to use them a second time will trigger an exception or -otherwise not work correctly. - -This common limitation means that it is generally advisable to create -context managers directly in the header of the :keyword:`with` statement -where they are used (as shown in all of the usage examples above). - -Files are an example of effectively single use context managers, since -the first :keyword:`with` statement will close the file, preventing any -further IO operations using that file object. - -Context managers created using :func:`contextmanager` are also single use -context managers, and will complain about the underlying generator failing -to yield if an attempt is made to use them a second time:: - - >>> from contextlib import contextmanager - >>> @contextmanager - ... def singleuse(): - ... print("Before") - ... yield - ... print("After") - ... - >>> cm = singleuse() - >>> with cm: - ... pass - ... - Before - After - >>> with cm: - ... pass - ... - Traceback (most recent call last): - ... - RuntimeError: generator didn't yield - - -.. _reentrant-cms: - -Reentrant context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -More sophisticated context managers may be "reentrant". These context -managers can not only be used in multiple :keyword:`with` statements, -but may also be used *inside* a :keyword:`with` statement that is already -using the same context manager. - -:class:`threading.RLock` is an example of a reentrant context manager, as are -:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of -reentrant use:: - - >>> from contextlib import redirect_stdout - >>> from io import StringIO - >>> stream = StringIO() - >>> write_to_stream = redirect_stdout(stream) - >>> with write_to_stream: - ... print("This is written to the stream rather than stdout") - ... with write_to_stream: - ... print("This is also written to the stream") - ... - >>> print("This is written directly to stdout") - This is written directly to stdout - >>> print(stream.getvalue()) - This is written to the stream rather than stdout - This is also written to the stream - -Real world examples of reentrancy are more likely to involve multiple -functions calling each other and hence be far more complicated than this -example. - -Note also that being reentrant is *not* the same thing as being thread safe. -:func:`redirect_stdout`, for example, is definitely not thread safe, as it -makes a global modification to the system state by binding :data:`sys.stdout` -to a different stream. - - -.. _reusable-cms: - -Reusable context managers -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Distinct from both single use and reentrant context managers are "reusable" -context managers (or, to be completely explicit, "reusable, but not -reentrant" context managers, since reentrant context managers are also -reusable). These context managers support being used multiple times, but -will fail (or otherwise not work correctly) if the specific context manager -instance has already been used in a containing with statement. - -:class:`threading.Lock` is an example of a reusable, but not reentrant, -context manager (for a reentrant lock, it is necessary to use -:class:`threading.RLock` instead). - -Another example of a reusable, but not reentrant, context manager is -:class:`ExitStack`, as it invokes *all* currently registered callbacks -when leaving any with statement, regardless of where those callbacks -were added:: - - >>> from contextlib import ExitStack - >>> stack = ExitStack() - >>> with stack: - ... stack.callback(print, "Callback: from first context") - ... print("Leaving first context") - ... - Leaving first context - Callback: from first context - >>> with stack: - ... stack.callback(print, "Callback: from second context") - ... print("Leaving second context") - ... - Leaving second context - Callback: from second context - >>> with stack: - ... stack.callback(print, "Callback: from outer context") - ... with stack: - ... stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Callback: from outer context - Leaving outer context - -As the output from the example shows, reusing a single stack object across -multiple with statements works correctly, but attempting to nest them -will cause the stack to be cleared at the end of the innermost with -statement, which is unlikely to be desirable behaviour. - -Using separate :class:`ExitStack` instances instead of reusing a single -instance avoids that problem:: - - >>> from contextlib import ExitStack - >>> with ExitStack() as outer_stack: - ... outer_stack.callback(print, "Callback: from outer context") - ... with ExitStack() as inner_stack: - ... inner_stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Leaving outer context - Callback: from outer context diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/copy.rst --- a/Doc/library/copy.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/copy.rst Mon Jan 25 17:05:13 2016 +0100 @@ -67,8 +67,8 @@ Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered -pickle functions from the :mod:`copyreg` module. +methods. The :mod:`copy` module does not use the :mod:`copyreg` registration +module. .. index:: single: __copy__() (copy protocol) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/copyreg.rst --- a/Doc/library/copyreg.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/copyreg.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,10 +9,9 @@ module: pickle module: copy -The :mod:`copyreg` module offers a way to define functions used while pickling -specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions -when pickling/copying those objects. The module provides configuration -information about object constructors which are not classes. +The :mod:`copyreg` module provides support for the :mod:`pickle` module. The +:mod:`copy` module is likely to use this in the future as well. It provides +configuration information about object constructors which are not classes. Such constructors may be factory functions or class instances. @@ -38,25 +37,3 @@ :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler object or subclass of :class:`pickle.Pickler` can also be used for declaring reduction functions. - -Example -------- - -The example below would like to show how to register a pickle function and how -it will be used: - - >>> import copyreg, copy, pickle - >>> class C(object): - ... def __init__(self, a): - ... self.a = a - ... - >>> def pickle_c(c): - ... print("pickling a C instance...") - ... return C, (c.a,) - ... - >>> copyreg.pickle(C, pickle_c) - >>> c = C(1) - >>> d = copy.copy(c) - pickling a C instance... - >>> p = pickle.dumps(c) - pickling a C instance... diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/crypt.rst --- a/Doc/library/crypt.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/crypt.rst Mon Jan 25 17:05:13 2016 +0100 @@ -64,7 +64,7 @@ A list of available password hashing algorithms, as ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest. + weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``. Module Functions @@ -121,14 +121,11 @@ Examples -------- -A simple example illustrating typical use (a constant-time comparison -operation is needed to limit exposure to timing attacks. -:func:`hmac.compare_digest` is suitable for this purpose):: +A simple example illustrating typical use:: import pwd import crypt import getpass - from hmac import compare_digest as compare_hash def login(): username = input('Python login: ') @@ -137,7 +134,7 @@ if cryptedpasswd == 'x' or cryptedpasswd == '*': raise ValueError('no support for shadow passwords') cleartext = getpass.getpass() - return compare_hash(crypt.crypt(cleartext, cryptedpasswd), cryptedpasswd) + return crypt.crypt(cleartext, cryptedpasswd) == cryptedpasswd else: return True @@ -145,8 +142,7 @@ check it against the original:: import crypt - from hmac import compare_digest as compare_hash hashed = crypt.crypt(plaintext) - if not compare_hash(hashed, crypt.crypt(plaintext, hashed)): + if hashed != crypt.crypt(plaintext, hashed): raise ValueError("hashed version doesn't validate against original") diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/crypto.rst --- a/Doc/library/crypto.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/crypto.rst Mon Jan 25 17:05:13 2016 +0100 @@ -8,7 +8,6 @@ The modules described in this chapter implement various algorithms of a cryptographic nature. They are available at the discretion of the installation. -On Unix systems, the :mod:`crypt` module may also be available. Here's an overview: @@ -16,3 +15,15 @@ hashlib.rst hmac.rst + +.. index:: + pair: AES; algorithm + single: cryptography + single: Kuchling, Andrew + +Hardcore cypherpunks will probably find the cryptographic modules written by +A.M. Kuchling of further interest; the package contains modules for various +encryption algorithms, most notably AES. These modules are not distributed with +Python but available separately. See the URL +http://www.pycrypto.org for more information. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/csv.rst --- a/Doc/library/csv.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/csv.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,6 @@ :synopsis: Write and read tabular data to and from delimited files. .. sectionauthor:: Skip Montanaro -**Source code:** :source:`Lib/csv.py` .. index:: single: csv @@ -47,9 +46,6 @@ The :mod:`csv` module defines the following functions: -.. index:: - single: universal newlines; csv.reader function - .. function:: reader(csvfile, dialect='excel', **fmtparams) Return a reader object which will iterate over lines in the given *csvfile*. @@ -72,10 +68,9 @@ A short usage example:: >>> import csv - >>> with open('eggs.csv', newline='') as csvfile: - ... spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|') - ... for row in spamreader: - ... print(', '.join(row)) + >>> spamReader = csv.reader(open('eggs.csv', newline=''), delimiter=' ', quotechar='|') + >>> for row in spamReader: + ... print(', '.join(row)) Spam, Spam, Spam, Spam, Spam, Baked Beans Spam, Lovely Spam, Wonderful Spam @@ -101,15 +96,14 @@ A short usage example:: - import csv - with open('eggs.csv', 'w', newline='') as csvfile: - spamwriter = csv.writer(csvfile, delimiter=' ', - quotechar='|', quoting=csv.QUOTE_MINIMAL) - spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) - spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) + >>> import csv + >>> spamWriter = csv.writer(open('eggs.csv', 'w', newline=''), delimiter=' ', + ... quotechar='|', quoting=csv.QUOTE_MINIMAL) + >>> spamWriter.writerow(['Spam'] * 5 + ['Baked Beans']) + >>> spamWriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) -.. function:: register_dialect(name[, dialect[, **fmtparams]]) +.. function:: register_dialect(name[, dialect], **fmtparams) Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or @@ -143,68 +137,36 @@ The :mod:`csv` module defines the following classes: -.. class:: DictReader(csvfile, fieldnames=None, restkey=None, restval=None, \ - dialect='excel', *args, **kwds) +.. class:: DictReader(csvfile, fieldnames=None, restkey=None, restval=None, dialect='excel', *args, **kwds) - Create an object which operates like a regular reader but maps the - information read into a dict whose keys are given by the optional - *fieldnames* parameter. The *fieldnames* parameter is a :mod:`sequence - ` whose elements are associated with the fields of the - input data in order. These elements become the keys of the resulting - dictionary. If the *fieldnames* parameter is omitted, the values in the - first row of the *csvfile* will be used as the fieldnames. If the row read - has more fields than the fieldnames sequence, the remaining data is added as - a sequence keyed by the value of *restkey*. If the row read has fewer - fields than the fieldnames sequence, the remaining keys take the value of - the optional *restval* parameter. Any other optional or keyword arguments - are passed to the underlying :class:`reader` instance. + Create an object which operates like a regular reader but maps the information + read into a dict whose keys are given by the optional *fieldnames* parameter. + If the *fieldnames* parameter is omitted, the values in the first row of the + *csvfile* will be used as the fieldnames. If the row read has more fields + than the fieldnames sequence, the remaining data is added as a sequence + keyed by the value of *restkey*. If the row read has fewer fields than the + fieldnames sequence, the remaining keys take the value of the optional + *restval* parameter. Any other optional or keyword arguments are passed to + the underlying :class:`reader` instance. - A short usage example:: - >>> import csv - >>> with open('names.csv') as csvfile: - ... reader = csv.DictReader(csvfile) - ... for row in reader: - ... print(row['first_name'], row['last_name']) - ... - Baked Beans - Lovely Spam - Wonderful Spam +.. class:: DictWriter(csvfile, fieldnames, restval='', extrasaction='raise', dialect='excel', *args, **kwds) + Create an object which operates like a regular writer but maps dictionaries onto + output rows. The *fieldnames* parameter identifies the order in which values in + the dictionary passed to the :meth:`writerow` method are written to the + *csvfile*. The optional *restval* parameter specifies the value to be written + if the dictionary is missing a key in *fieldnames*. If the dictionary passed to + the :meth:`writerow` method contains a key not found in *fieldnames*, the + optional *extrasaction* parameter indicates what action to take. If it is set + to ``'raise'`` a :exc:`ValueError` is raised. If it is set to ``'ignore'``, + extra values in the dictionary are ignored. Any other optional or keyword + arguments are passed to the underlying :class:`writer` instance. -.. class:: DictWriter(csvfile, fieldnames, restval='', extrasaction='raise', \ - dialect='excel', *args, **kwds) - - Create an object which operates like a regular writer but maps dictionaries - onto output rows. The *fieldnames* parameter is a :mod:`sequence - ` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to the - *csvfile*. The optional *restval* parameter specifies the value to be - written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in - *fieldnames*, the optional *extrasaction* parameter indicates what action to - take. If it is set to ``'raise'`` a :exc:`ValueError` is raised. If it is - set to ``'ignore'``, extra values in the dictionary are ignored. Any other - optional or keyword arguments are passed to the underlying :class:`writer` - instance. - - Note that unlike the :class:`DictReader` class, the *fieldnames* parameter - of the :class:`DictWriter` is not optional. Since Python's :class:`dict` - objects are not ordered, there is not enough information available to deduce - the order in which the row should be written to the *csvfile*. - - A short usage example:: - - import csv - - with open('names.csv', 'w') as csvfile: - fieldnames = ['first_name', 'last_name'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - - writer.writeheader() - writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'}) - writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) - writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) + Note that unlike the :class:`DictReader` class, the *fieldnames* parameter of + the :class:`DictWriter` is not optional. Since Python's :class:`dict` objects + are not ordered, there is not enough information available to deduce the order + in which the row should be written to the *csvfile*. .. class:: Dialect @@ -256,11 +218,11 @@ An example for :class:`Sniffer` use:: - with open('example.csv') as csvfile: - dialect = csv.Sniffer().sniff(csvfile.read(1024)) - csvfile.seek(0) - reader = csv.reader(csvfile, dialect) - # ... process CSV file contents here ... + csvfile = open("example.csv") + dialect = csv.Sniffer().sniff(csvfile.read(1024)) + csvfile.seek(0) + reader = csv.reader(csvfile, dialect) + # ... process CSV file contents here ... The :mod:`csv` module defines the following constants: @@ -325,7 +287,7 @@ .. attribute:: Dialect.doublequote - Controls how instances of *quotechar* appearing inside a field should + Controls how instances of *quotechar* appearing inside a field should be themselves be quoted. When :const:`True`, the character is doubled. When :const:`False`, the *escapechar* is used as a prefix to the *quotechar*. It defaults to :const:`True`. @@ -374,11 +336,6 @@ The default is :const:`False`. -.. attribute:: Dialect.strict - - When ``True``, raise exception :exc:`Error` on bad CSV input. - The default is ``False``. - Reader Objects -------------- @@ -419,7 +376,7 @@ :class:`Writer` objects (:class:`DictWriter` instances and objects returned by the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary +a sequence of strings or numbers for :class:`Writer` objects and a dictionary mapping fieldnames to strings or numbers (by passing them through :func:`str` first) for :class:`DictWriter` objects. Note that complex numbers are written out surrounded by parens. This may cause some problems for other programs which @@ -431,8 +388,6 @@ Write the *row* parameter to the writer's file object, formatted according to the current dialect. - .. versionchanged:: 3.5 - Added support of arbitrary iterables. .. method:: csvwriter.writerows(rows) @@ -531,5 +486,4 @@ .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields will not be interpreted correctly, and on platforms that use ``\r\n`` linendings on write an extra ``\r`` will be added. It should always be safe to specify - ``newline=''``, since the csv module does its own - (:term:`universal `) newline handling. + ``newline=''``, since the csv module does its own (universal) newline handling. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ctypes.rst --- a/Doc/library/ctypes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/ctypes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -39,7 +39,7 @@ convention, while *windll* libraries call functions using the ``stdcall`` calling convention. *oledll* also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`HRESULT` error code. The error -code is used to automatically raise an :class:`OSError` exception when the +code is used to automatically raise a :class:`OSError` exception when the function call fails. .. versionchanged:: 3.3 @@ -198,9 +198,7 @@ >>> There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +should be careful anyway. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -218,7 +216,7 @@ Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: +:mod:`ctypes` defines a number of primitive C compatible data types : +----------------------+------------------------------------------+----------------------------+ | ctypes type | C type | Python type | @@ -250,11 +248,6 @@ | :class:`c_ulonglong` | :c:type:`unsigned __int64` or | int | | | :c:type:`unsigned long long` | | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:type:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ | :class:`c_float` | :c:type:`float` | float | +----------------------+------------------------------------------+----------------------------+ | :class:`c_double` | :c:type:`double` | float | @@ -576,8 +569,8 @@ ValueError: too many initializers >>> -You can, however, build much more complicated structures. A structure can -itself contain other structures by using a structure as a field type. +You can, however, build much more complicated structures. Structures can itself +contain other structures by using a structure as a field type. Here is a RECT structure which contains two POINTs named *upperleft* and *lowerright*:: @@ -610,13 +603,6 @@ .. _ctypes-structureunion-alignment-byte-order: -.. warning:: - - :mod:`ctypes` does not support passing unions or structures with bit-fields - to functions by value. While this may work on 32-bit x86, it's not - guaranteed by the library to work in the general case. Unions and - structures with bit-fields should always be passed to functions by pointer. - Structure/union alignment and byte order ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -831,11 +817,6 @@ 3 >>> -In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed -type (``c_int`` in this case) can be passed to the function. ctypes will apply -the required :func:`byref` conversion in this case automatically. - To set a POINTER type field to ``NULL``, you can assign ``None``:: >>> bar.values = None @@ -1022,18 +1003,12 @@ 1 5 7 33 99 >>> -.. note:: - - Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be - garbage collected, crashing your program when a callback is made. - - Also, note that if the callback function is called in a thread created - outside of Python's control (e.g. by the foreign code that calls the - callback), ctypes creates a new dummy Python thread on every invocation. This - behavior is correct for most purposes, but it means that values stored with - :class:`threading.local` will *not* survive across different callbacks, even when - those calls are made from the same C thread. +**Important note for callback functions:** + +Make sure you keep references to :func:`CFUNCTYPE` objects as long as they are +used from C code. :mod:`ctypes` doesn't, and if you don't, they may be garbage +collected, crashing your program when a callback is made. + .. _ctypes-accessing-values-exported-from-dlls: @@ -1114,8 +1089,8 @@ Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other -than what actually happens. +There are some edges in :mod:`ctypes` where you may be expect something else than +what actually happens. Consider the following example:: @@ -1282,7 +1257,7 @@ like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper +the shared library name at development type, and hardcode that into the wrapper module instead of using :func:`find_library` to locate the library at runtime. @@ -1291,7 +1266,7 @@ Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ -There are several ways to load shared libraries into the Python process. One +There are several ways to loaded shared libraries into the Python process. One way is to instantiate one of the following classes: @@ -1386,16 +1361,11 @@ The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time: - - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False +Instances of these classes have no public methods, however :meth:`__getattr__` +and :meth:`__getitem__` have special behavior: functions exported by the shared +library can be accessed as attributes of by index. Please note that both +:meth:`__getattr__` and :meth:`__getitem__` cache their result, so calling them +repeatedly returns the same object each time. The following public attributes are available, their name starts with an underscore to not clash with exported function names: @@ -1662,7 +1632,7 @@ WINUSERAPI int WINAPI MessageBoxA( - HWND hWnd, + HWND hWnd , LPCSTR lpText, LPCSTR lpCaption, UINT uType); @@ -1833,7 +1803,7 @@ .. function:: find_msvcrt() :module: ctypes.util - Windows only: return the filename of the VC runtime library used by Python, + Windows only: return the filename of the VC runtype library used by Python, and by the extension modules. If the name of the library cannot be determined, ``None`` is returned. @@ -1920,8 +1890,8 @@ .. function:: sizeof(obj_or_type) - Returns the size in bytes of a ctypes type or instance memory buffer. - Does the same as the C ``sizeof`` operator. + Returns the size in bytes of a ctypes type or instance memory buffer. Does the + same as the C ``sizeof()`` function. .. function:: string_at(address, size=-1) @@ -2253,7 +2223,7 @@ .. class:: c_bool Represent the C :c:type:`bool` datatype (more accurately, :c:type:`_Bool` from - C99). Its value can be ``True`` or ``False``, and the constructor accepts any object + C99). Its value can be True or False, and the constructor accepts any object that has a truth value. @@ -2335,6 +2305,11 @@ and so on). Later assignments to the :attr:`_fields_` class variable will raise an AttributeError. + Structure and union subclass constructors accept both positional and named + arguments. Positional arguments are used to initialize the fields in the + same order as they appear in the :attr:`_fields_` definition, named + arguments are used to initialize the fields with the corresponding name. + It is possible to defined sub-subclasses of structure types, they inherit the fields of the base class plus the :attr:`_fields_` defined in the sub-subclass, if any. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/curses.rst --- a/Doc/library/curses.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/curses.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,7 +12,7 @@ de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available -for Windows, DOS, and possibly other systems as well. This extension module is +for DOS, OS/2, and possibly other systems as well. This extension module is designed to match the API of ncurses, an open-source curses library hosted on Linux and the BSD variants of Unix. @@ -45,7 +45,7 @@ Tutorial material on using curses with Python, by Andrew Kuchling and Eric Raymond. - The :source:`Tools/demo/` directory in the Python source distribution contains + The :file:`Tools/demo/` directory in the Python source distribution contains some example programs using the curses bindings provided by this module. @@ -377,8 +377,7 @@ is to be displayed. -.. function:: newwin(nlines, ncols) - newwin(nlines, ncols, begin_y, begin_x) +.. function:: newwin([nlines, ncols,] begin_y, begin_x) Return a new window, whose left-upper corner is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. @@ -599,13 +598,6 @@ Only one *ch* can be pushed before :meth:`getch` is called. -.. function:: update_lines_cols() - - Update :envvar:`LINES` and :envvar:`COLS`. Useful for detecting manual screen resize. - - .. versionadded:: 3.5 - - .. function:: unget_wch(ch) Push *ch* so the next :meth:`get_wch` will return it. @@ -664,8 +656,7 @@ the following methods and attributes: -.. method:: window.addch(ch[, attr]) - window.addch(y, x, ch[, attr]) +.. method:: window.addch([y, x,] ch[, attr]) .. note:: @@ -679,15 +670,13 @@ position and attributes are the current settings for the window object. -.. method:: window.addnstr(str, n[, attr]) - window.addnstr(y, x, str, n[, attr]) +.. method:: window.addnstr([y, x,] str, n[, attr]) Paint at most *n* characters of the string *str* at ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. -.. method:: window.addstr(str[, attr]) - window.addstr(y, x, str[, attr]) +.. method:: window.addstr([y, x,] str[, attr]) Paint the string *str* at ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. @@ -774,10 +763,7 @@ *bs* are *horch*. The default corner characters are always used by this function. -.. method:: window.chgat(attr) - window.chgat(num, attr) - window.chgat(y, x, attr) - window.chgat(y, x, num, attr) +.. method:: window.chgat([y, x, ] [num,] attr) Set the attributes of *num* characters at the current cursor position, or at position ``(y, x)`` if supplied. If no value of *num* is given or *num* = -1, @@ -826,8 +812,7 @@ Delete the line under the cursor. All following lines are moved up by one line. -.. method:: window.derwin(begin_y, begin_x) - window.derwin(nlines, ncols, begin_y, begin_x) +.. method:: window.derwin([nlines, ncols,] begin_y, begin_x) An abbreviation for "derive window", :meth:`derwin` is the same as calling :meth:`subwin`, except that *begin_y* and *begin_x* are relative to the origin @@ -852,7 +837,7 @@ .. attribute:: window.encoding Encoding used to encode method arguments (Unicode strings and characters). - The encoding attribute is inherited from the parent window when a subwindow + The encoding attribute is inherited from by parent window when a subwindow is created, for example with :meth:`window.subwin`. By default, the locale encoding is used (see :func:`locale.getpreferredencoding`). @@ -884,8 +869,8 @@ .. method:: window.get_wch([y, x]) - Get a wide character. Return a character for most keys, or an integer for - function keys, keypad keys, and other special keys. + Get a wide character. Like :meth:`getch`, but the integer returned is the + Unicode code point for the key pressed, so it can be passed to :func:`chr`. .. versionadded:: 3.3 @@ -893,9 +878,8 @@ .. method:: window.getkey([y, x]) Get a character, returning a string instead of an integer, as :meth:`getch` - does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, an exception is raised if - there is no input. + does. Function keys, keypad keys and so on return a multibyte string containing + the key name. In no-delay mode, an exception is raised if there is no input. .. method:: window.getmaxyx() @@ -921,8 +905,7 @@ upper-left corner. -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) +.. method:: window.hline([y, x,] ch, n) Display a horizontal line starting at ``(y, x)`` with length *n* consisting of the character *ch*. @@ -956,8 +939,7 @@ the character proper, and upper bits are the attributes. -.. method:: window.insch(ch[, attr]) - window.insch(y, x, ch[, attr]) +.. method:: window.insch([y, x,] ch[, attr]) Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from position *x* right by one character. @@ -978,8 +960,7 @@ line. -.. method:: window.insnstr(str, n[, attr]) - window.insnstr(y, x, str, n[, attr]) +.. method:: window.insnstr([y, x,] str, n [, attr]) Insert a character string (as many characters as will fit on the line) before the character under the cursor, up to *n* characters. If *n* is zero or @@ -988,8 +969,7 @@ The cursor position does not change (after moving to *y*, *x*, if specified). -.. method:: window.insstr(str[, attr]) - window.insstr(y, x, str[, attr]) +.. method:: window.insstr([y, x, ] str [, attr]) Insert a character string (as many characters as will fit on the line) before the character under the cursor. All characters to the right of the cursor are @@ -997,8 +977,7 @@ position does not change (after moving to *y*, *x*, if specified). -.. method:: window.instr([n]) - window.instr(y, x[, n]) +.. method:: window.instr([y, x] [, n]) Return a string of characters, extracted from the window starting at the current cursor position, or at *y*, *x* if specified. Attributes are stripped @@ -1173,15 +1152,13 @@ Turn on attribute *A_STANDOUT*. -.. method:: window.subpad(begin_y, begin_x) - window.subpad(nlines, ncols, begin_y, begin_x) +.. method:: window.subpad([nlines, ncols,] begin_y, begin_x) Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. -.. method:: window.subwin(begin_y, begin_x) - window.subwin(nlines, ncols, begin_y, begin_x) +.. method:: window.subwin([nlines, ncols,] begin_y, begin_x) Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. @@ -1238,8 +1215,7 @@ :meth:`refresh`. -.. method:: window.vline(ch, n) - window.vline(y, x, ch, n) +.. method:: window.vline([y, x,] ch, n) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the character *ch*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/datatypes.rst --- a/Doc/library/datatypes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/datatypes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -25,9 +25,10 @@ heapq.rst bisect.rst array.rst + sched.rst + queue.rst weakref.rst types.rst copy.rst pprint.rst reprlib.rst - enum.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/datetime.rst --- a/Doc/library/datetime.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/datetime.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,41 +7,33 @@ .. sectionauthor:: Tim Peters .. sectionauthor:: A.M. Kuchling -**Source code:** :source:`Lib/datetime.py` - .. XXX what order should the types be discussed in? The :mod:`datetime` module supplies classes for manipulating dates and times in both simple and complex ways. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output -formatting and manipulation. For related functionality, see also the -:mod:`time` and :mod:`calendar` modules. +formatting and manipulation. For related +functionality, see also the :mod:`time` and :mod:`calendar` modules. -There are two kinds of date and time objects: "naive" and "aware". +There are two kinds of date and time objects: "naive" and "aware". This +distinction refers to whether the object has any notion of time zone, daylight +saving time, or other kind of algorithmic or political time adjustment. Whether +a naive :class:`.datetime` object represents Coordinated Universal Time (UTC), +local time, or time in some other timezone is purely up to the program, just +like it's up to the program whether a particular number represents metres, +miles, or mass. Naive :class:`.datetime` objects are easy to understand and to +work with, at the cost of ignoring some aspects of reality. -An aware object has sufficient knowledge of applicable algorithmic and -political time adjustments, such as time zone and daylight saving time -information, to locate itself relative to other aware objects. An aware object -is used to represent a specific moment in time that is not open to -interpretation [#]_. - -A naive object does not contain enough information to unambiguously locate -itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is -purely up to the program, just like it is up to the program whether a -particular number represents metres, miles, or mass. Naive objects are easy to -understand and to work with, at the cost of ignoring some aspects of reality. - -For applications requiring aware objects, :class:`.datetime` and :class:`.time` -objects have an optional time zone information attribute, :attr:`tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. -These :class:`tzinfo` objects capture information about the offset from UTC -time, the time zone name, and whether Daylight Saving Time is in effect. Note -that only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offset from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of -detail is up to the application. The rules for time adjustment across the +For applications requiring more, :class:`.datetime` and :class:`.time` objects +have an optional time zone information attribute, :attr:`tzinfo`, that can be +set to an instance of a subclass of the abstract :class:`tzinfo` class. These +:class:`tzinfo` objects capture information about the offset from UTC time, the +time zone name, and whether Daylight Saving Time is in effect. Note that only +one concrete :class:`tzinfo` class, the :class:`timezone` class, is supplied by the +:mod:`datetime` module. The :class:`timezone` class can represent simple +timezones with fixed offset from UTC such as UTC itself or North American EST and +EDT timezones. Supporting timezones at whatever level of detail is +required is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. @@ -122,13 +114,10 @@ Objects of the :class:`date` type are always naive. -An object of type :class:`.time` or :class:`.datetime` may be naive or aware. -A :class:`.datetime` object *d* is aware if ``d.tzinfo`` is not ``None`` and -``d.tzinfo.utcoffset(d)`` does not return ``None``. If ``d.tzinfo`` is -``None``, or if ``d.tzinfo`` is not ``None`` but ``d.tzinfo.utcoffset(d)`` -returns ``None``, *d* is naive. A :class:`.time` object *t* is aware -if ``t.tzinfo`` is not ``None`` and ``t.tzinfo.utcoffset(None)`` does not return -``None``. Otherwise, *t* is naive. +An object *d* of type :class:`.time` or :class:`.datetime` may be naive or aware. +*d* is aware if ``d.tzinfo`` is not ``None`` and ``d.tzinfo.utcoffset(d)`` does +not return ``None``. If ``d.tzinfo`` is ``None``, or if ``d.tzinfo`` is not +``None`` but ``d.tzinfo.utcoffset(d)`` returns ``None``, *d* is naive. The distinction between naive and aware doesn't apply to :class:`timedelta` objects. @@ -172,12 +161,10 @@ * ``0 <= seconds < 3600*24`` (the number of seconds in one day) * ``-999999999 <= days <= 999999999`` - If any argument is a float and there are fractional microseconds, - the fractional microseconds left over from all arguments are - combined and their sum is rounded to the nearest microsecond using - round-half-to-even tiebreaker. If no argument is a float, the - conversion and normalization processes are exact (no information is - lost). + If any argument is a float and there are fractional microseconds, the fractional + microseconds left over from all arguments are combined and their sum is rounded + to the nearest microsecond. If no argument is a float, the conversion and + normalization processes are exact (no information is lost). If the normalized value of days lies outside the indicated range, :exc:`OverflowError` is raised. @@ -560,7 +547,7 @@ Return a 3-tuple, (ISO year, ISO week number, ISO weekday). The ISO calendar is a widely used variant of the Gregorian calendar. See - http://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm for a good + http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm for a good explanation. The ISO year consists of 52 or 53 full weeks, and where a week starts on a @@ -597,17 +584,8 @@ .. method:: date.strftime(format) Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: date.__format__(format) - - Same as :meth:`.date.strftime`. This makes it possible to specify format - string for a :class:`.date` object when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + Format codes referring to hours, minutes or seconds will see 0 values. See + section :ref:`strftime-strptime-behavior`. Example of counting days to an event:: @@ -660,14 +638,12 @@ '11/03/02' >>> d.strftime("%A %d. %B %Y") 'Monday 11. March 2002' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") - 'The day is 11, the month is March.' .. _datetime-datetime: -:class:`.datetime` Objects --------------------------- +:class:`datetime` Objects +------------------------- A :class:`.datetime` object is a single object containing all the information from a :class:`date` object and a :class:`.time` object. Like a :class:`date` @@ -759,19 +735,24 @@ :attr:`tzinfo` ``None``. This may raise :exc:`OverflowError`, if the timestamp is out of the range of values supported by the platform C :c:func:`gmtime` function, and :exc:`OSError` on :c:func:`gmtime` failure. - It's common for this to be restricted to years in 1970 through 2038. + It's common for this to be restricted to years in 1970 through 2038. See also + :meth:`fromtimestamp`. - To get an aware :class:`.datetime` object, call :meth:`fromtimestamp`:: + On the POSIX compliant platforms, ``utcfromtimestamp(timestamp)`` + is equivalent to the following expression:: - datetime.fromtimestamp(timestamp, timezone.utc) + datetime(1970, 1, 1) + timedelta(seconds=timestamp) - On the POSIX compliant platforms, it is equivalent to the following - expression:: + There is no method to obtain the timestamp from a :class:`datetime` + instance, but POSIX timestamp corresponding to a :class:`datetime` + instance ``dt`` can be easily calculated as follows. For a naive + ``dt``:: - datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=timestamp) + timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) - except the latter formula always supports the full years range: between - :const:`MINYEAR` and :const:`MAXYEAR` inclusive. + And for an aware ``dt``:: + + timestamp = (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)) / timedelta(seconds=1) .. versionchanged:: 3.3 Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp @@ -805,8 +786,7 @@ *format*. This is equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + time tuple. See section :ref:`strftime-strptime-behavior`. @@ -923,20 +903,13 @@ *datetime1* is considered less than *datetime2* when *datetime1* precedes *datetime2* in time. - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - + If one comparand is naive and the other is aware, :exc:`TypeError` is raised. If both comparands are aware, and have the same :attr:`tzinfo` attribute, the common :attr:`tzinfo` attribute is ignored and the base datetimes are compared. If both comparands are aware and have different :attr:`tzinfo` attributes, the comparands are first adjusted by subtracting their UTC offsets (obtained from ``self.utcoffset()``). - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`datetime` - instances don't raise :exc:`TypeError`. - .. note:: In order to stop comparison from falling back to the default scheme of comparing @@ -979,22 +952,17 @@ datetime with no conversion of date and time data. -.. method:: datetime.astimezone(tz=None) +.. method:: datetime.astimezone(tz) - Return a :class:`datetime` object with new :attr:`tzinfo` attribute *tz*, + Return a :class:`.datetime` object with new :attr:`tzinfo` attribute *tz*, adjusting the date and time data so the result is the same UTC time as *self*, but in *tz*'s local time. - If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its + *tz* must be an instance of a :class:`tzinfo` subclass, and its :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. *self* must be aware (``self.tzinfo`` must not be ``None``, and ``self.utcoffset()`` must not return ``None``). - If called without arguments (or with ``tz=None``) the system local - timezone is assumed. The ``tzinfo`` attribute of the converted - datetime instance will be set to an instance of :class:`timezone` - with the zone name and offset obtained from the OS. - If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no adjustment of date or time data is performed. Else the result is local time in time zone *tz*, representing the same UTC time as *self*: after @@ -1021,9 +989,6 @@ # Convert from UTC to tz's local time. return tz.fromutc(utc) - .. versionchanged:: 3.3 - *tz* now can be omitted. - .. method:: datetime.utcoffset() @@ -1080,39 +1045,6 @@ Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. -.. method:: datetime.timestamp() - - Return POSIX timestamp corresponding to the :class:`datetime` - instance. The return value is a :class:`float` similar to that - returned by :func:`time.time`. - - Naive :class:`datetime` instances are assumed to represent local - time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`datetime` - supports wider range of values than :c:func:`mktime` on many - platforms, this method may raise :exc:`OverflowError` for times far - in the past or far in the future. - - For aware :class:`datetime` instances, the return value is computed - as:: - - (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - - .. versionadded:: 3.3 - - .. note:: - - There is no method to obtain the POSIX timestamp directly from a - naive :class:`datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not - set to UTC, you can obtain the POSIX timestamp by supplying - ``tzinfo=timezone.utc``:: - - timestamp = dt.replace(tzinfo=timezone.utc).timestamp() - - or by calculating the timestamp directly:: - - timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) .. method:: datetime.weekday() @@ -1173,16 +1105,7 @@ .. method:: datetime.strftime(format) Return a string representing the date and time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: datetime.__format__(format) - - Same as :meth:`.datetime.strftime`. This makes it possible to specify format - string for a :class:`.datetime` object when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + string. See section :ref:`strftime-strptime-behavior`. Examples of working with datetime objects: @@ -1229,21 +1152,19 @@ >>> # Formatting datetime >>> dt.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") - 'The day is 21, the month is November, the time is 04:30PM.' Using datetime with tzinfo: >>> from datetime import timedelta, datetime, tzinfo >>> class GMT1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) + self.dst(dt) - ... def dst(self, dt): - ... # DST starts last Sunday in March + ... def __init__(self): # DST starts last Sunday in March ... d = datetime(dt.year, 4, 1) # ends last Sunday in October ... self.dston = d - timedelta(days=d.weekday() + 1) ... d = datetime(dt.year, 11, 1) ... self.dstoff = d - timedelta(days=d.weekday() + 1) + ... def utcoffset(self, dt): + ... return timedelta(hours=1) + self.dst(dt) + ... def dst(self, dt): ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: ... return timedelta(hours=1) ... else: @@ -1252,15 +1173,16 @@ ... return "GMT +1" ... >>> class GMT2(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=2) + self.dst(dt) - ... def dst(self, dt): + ... def __init__(self): ... d = datetime(dt.year, 4, 1) ... self.dston = d - timedelta(days=d.weekday() + 1) ... d = datetime(dt.year, 11, 1) ... self.dstoff = d - timedelta(days=d.weekday() + 1) + ... def utcoffset(self, dt): + ... return timedelta(hours=1) + self.dst(dt) + ... def dst(self, dt): ... if self.dston <= dt.replace(tzinfo=None) < self.dstoff: - ... return timedelta(hours=1) + ... return timedelta(hours=2) ... else: ... return timedelta(0) ... def tzname(self,dt): @@ -1363,10 +1285,7 @@ * comparison of :class:`.time` to :class:`.time`, where *a* is considered less than *b* when *a* precedes *b* in time. If one comparand is naive and the other - is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have + is aware, :exc:`TypeError` is raised. If both comparands are aware, and have the same :attr:`tzinfo` attribute, the common :attr:`tzinfo` attribute is ignored and the base times are compared. If both comparands are aware and have different :attr:`tzinfo` attributes, the comparands are first adjusted by @@ -1376,21 +1295,14 @@ different type, :exc:`TypeError` is raised unless the comparison is ``==`` or ``!=``. The latter cases return :const:`False` or :const:`True`, respectively. - .. versionchanged:: 3.3 - Equality comparisons between naive and aware :class:`time` instances - don't raise :exc:`TypeError`. - * hash, use as dict key * efficient pickling -In boolean contexts, a :class:`.time` object is always considered to be true. +* in Boolean contexts, a :class:`.time` object is considered to be true if and + only if, after converting it to minutes and subtracting :meth:`utcoffset` (or + ``0`` if that's ``None``), the result is non-zero. -.. versionchanged:: 3.5 - Before Python 3.5, a :class:`.time` object was considered to be false if it - represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. Instance methods: @@ -1417,17 +1329,8 @@ .. method:: time.strftime(format) - Return a string representing the time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. - - -.. method:: time.__format__(format) - - Same as :meth:`.time.strftime`. This makes it possible to specify format string - for a :class:`.time` object when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + Return a string representing the time, controlled by an explicit format string. + See section :ref:`strftime-strptime-behavior`. .. method:: time.utcoffset() @@ -1476,8 +1379,6 @@ 'Europe/Prague' >>> t.strftime("%H:%M:%S %Z") '12:10:30 Europe/Prague' - >>> 'The {} is {:%H:%M}.'.format("time", t) - 'The time is 12:10.' .. _datetime-tzinfo: @@ -1489,7 +1390,7 @@ instantiated directly. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`datetime` module supplies -a simple concrete subclass of :class:`tzinfo` :class:`timezone` which can represent +a simple concrete subclass of :class:`tzinfo` :class:`timezone` which can reprsent timezones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -1656,6 +1557,7 @@ .. literalinclude:: ../includes/tzinfo-examples.py + Note that there are unavoidable subtleties twice per year in a :class:`tzinfo` subclass accounting for both standard and daylight time, at the DST transition points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the @@ -1674,7 +1576,7 @@ 3:00. A wall time of the form 2:MM doesn't really make sense on that day, so ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. In order for :meth:`astimezone` to make this guarantee, the -:meth:`tzinfo.dst` method must consider times in the "missing hour" (2:MM for +:meth:`rzinfo.dst` method must consider times in the "missing hour" (2:MM for Eastern) to be in daylight time. When DST ends (the "end" line), there's a potentially worse problem: there's an @@ -1695,22 +1597,6 @@ or any other fixed-offset :class:`tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). -.. seealso:: - - `pytz `_ - The standard library has :class:`timezone` class for handling arbitrary - fixed offsets from UTC and :attr:`timezone.utc` as UTC timezone instance. - - *pytz* library brings the *IANA timezone database* (also known as the - Olson database) to Python and its usage is recommended. - - `IANA timezone database `_ - The Time Zone Database (often called tz or zoneinfo) contains code and - data that represent the history of local time for many representative - locations around the globe. It is updated periodically to reflect changes - made by political bodies to time zone boundaries, UTC offsets, and - daylight-saving rules. - .. _datetime-timezone: @@ -1734,9 +1620,10 @@ otherwise :exc:`ValueError` is raised. The *name* argument is optional. If specified it must be a string that - will be used as the value returned by the :meth:`datetime.tzname` method. - - .. versionadded:: 3.2 + is used as the value returned by the ``tzname(dt)`` method. Otherwise, + ``tzname(dt)`` returns a string 'UTCsHH:MM', where s is the sign of + *offset*, HH and MM are two digits of ``offset.hours`` and + ``offset.minutes`` respectively. .. method:: timezone.utcoffset(dt) @@ -1747,19 +1634,11 @@ .. method:: timezone.tzname(dt) - Return the fixed value specified when the :class:`timezone` instance - is constructed. If *name* is not provided in the constructor, the - name returned by ``tzname(dt)`` is generated from the value of the - ``offset`` as follows. If *offset* is ``timedelta(0)``, the name - is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign - of ``offset``, HH and MM are two digits of ``offset.hours`` and + Return the fixed value specified when the :class:`timezone` instance is + constructed or a string 'UTCsHH:MM', where s is the sign of + *offset*, HH and MM are two digits of ``offset.hours`` and ``offset.minutes`` respectively. - .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. - - .. method:: timezone.dst(dt) Always returns ``None``. @@ -1800,157 +1679,141 @@ microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. +For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty +strings. + +For an aware object: + +``%z`` + :meth:`utcoffset` is transformed into a 5-character string of the form +HHMM or + -HHMM, where HH is a 2-digit string giving the number of UTC offset hours, and + MM is a 2-digit string giving the number of UTC offset minutes. For example, if + :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is + replaced with the string ``'-0330'``. + +``%Z`` + If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty string. + Otherwise ``%Z`` is replaced by the returned value, which must be a string. + The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform -variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. +variations are common. The following is a list of all the format codes that the C standard (1989 version) requires, and these work on all platforms with a standard C implementation. Note that the 1999 version of the C standard added additional format codes. -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | -| | abbreviated name. | (en_US); | | -| | || So, Mo, ..., Sa | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | -| | | Saturday (en_US); | | -| | || Sonntag, Montag, ..., | | -| | | Samstag (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | -| | name. | (en_US); | | -| | || Jan, Feb, ..., Dez | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | -| | | ..., December (en_US);| | -| | || Januar, Februar, ..., | | -| | | Dezember (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | -+-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE) | \(3) | -+-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4) | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded on the | 999999 | | -| | left. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(6) | -| | or -HHMM (empty string if the | +1030 | | -| | the object is naive). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | -| | if the object is naive). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Sunday as the first day of | | | -| | the week) as a zero padded | | | -| | decimal number. All days in a | | | -| | new year preceding the first | | | -| | Sunday are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7) | -| | (Monday as the first day of | | | -| | the week) as a decimal number. | | | -| | All days in a new year | | | -| | preceding the first Monday | | | -| | are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | -| | representation. || 08/16/1988 (en_US); | | -| | || 16.08.1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | -| | representation. || 21:30:00 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | -+-----------+--------------------------------+------------------------+-------+ - -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. These -may not be available on all platforms when used with the :meth:`strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | -| | number with Monday as | | | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | -+-----------+--------------------------------+------------------------+-------+ - -.. versionadded:: 3.6 - ``%G``, ``%u`` and ``%V`` were added. ++-----------+--------------------------------+-------+ +| Directive | Meaning | Notes | ++===========+================================+=======+ +| ``%a`` | Locale's abbreviated weekday | | +| | name. | | ++-----------+--------------------------------+-------+ +| ``%A`` | Locale's full weekday name. | | ++-----------+--------------------------------+-------+ +| ``%b`` | Locale's abbreviated month | | +| | name. | | ++-----------+--------------------------------+-------+ +| ``%B`` | Locale's full month name. | | ++-----------+--------------------------------+-------+ +| ``%c`` | Locale's appropriate date and | | +| | time representation. | | ++-----------+--------------------------------+-------+ +| ``%d`` | Day of the month as a decimal | | +| | number [01,31]. | | ++-----------+--------------------------------+-------+ +| ``%f`` | Microsecond as a decimal | \(1) | +| | number [0,999999], zero-padded | | +| | on the left | | ++-----------+--------------------------------+-------+ +| ``%H`` | Hour (24-hour clock) as a | | +| | decimal number [00,23]. | | ++-----------+--------------------------------+-------+ +| ``%I`` | Hour (12-hour clock) as a | | +| | decimal number [01,12]. | | ++-----------+--------------------------------+-------+ +| ``%j`` | Day of the year as a decimal | | +| | number [001,366]. | | ++-----------+--------------------------------+-------+ +| ``%m`` | Month as a decimal number | | +| | [01,12]. | | ++-----------+--------------------------------+-------+ +| ``%M`` | Minute as a decimal number | | +| | [00,59]. | | ++-----------+--------------------------------+-------+ +| ``%p`` | Locale's equivalent of either | \(2) | +| | AM or PM. | | ++-----------+--------------------------------+-------+ +| ``%S`` | Second as a decimal number | \(3) | +| | [00,59]. | | ++-----------+--------------------------------+-------+ +| ``%U`` | Week number of the year | \(4) | +| | (Sunday as the first day of | | +| | the week) as a decimal number | | +| | [00,53]. All days in a new | | +| | year preceding the first | | +| | Sunday are considered to be in | | +| | week 0. | | ++-----------+--------------------------------+-------+ +| ``%w`` | Weekday as a decimal number | | +| | [0(Sunday),6]. | | ++-----------+--------------------------------+-------+ +| ``%W`` | Week number of the year | \(4) | +| | (Monday as the first day of | | +| | the week) as a decimal number | | +| | [00,53]. All days in a new | | +| | year preceding the first | | +| | Monday are considered to be in | | +| | week 0. | | ++-----------+--------------------------------+-------+ +| ``%x`` | Locale's appropriate date | | +| | representation. | | ++-----------+--------------------------------+-------+ +| ``%X`` | Locale's appropriate time | | +| | representation. | | ++-----------+--------------------------------+-------+ +| ``%y`` | Year without century as a | | +| | decimal number [00,99]. | | ++-----------+--------------------------------+-------+ +| ``%Y`` | Year with century as a decimal | \(5) | +| | number [0001,9999]. | | ++-----------+--------------------------------+-------+ +| ``%z`` | UTC offset in the form +HHMM | \(6) | +| | or -HHMM (empty string if the | | +| | the object is naive). | | ++-----------+--------------------------------+-------+ +| ``%Z`` | Time zone name (empty string | | +| | if the object is naive). | | ++-----------+--------------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | | ++-----------+--------------------------------+-------+ Notes: (1) - Because the format depends on the current locale, care should be taken when - making assumptions about the output value. Field orderings will vary (for - example, "month/day/year" versus "day/month/year"), and the output may - contain Unicode characters encoded using the locale's default encoding (for - example, if the current locale is ``ja_JP``, the default encoding could be - any one of ``eucJP``, ``SJIS``, or ``utf-8``; use :meth:`locale.getlocale` - to determine the current locale's encoding). + When used with the :meth:`strptime` method, the ``%f`` directive + accepts from one to six digits and zero pads on the right. ``%f`` is + an extension to the set of format characters in the C standard (but + implemented separately in datetime objects, and therefore always + available). (2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but - years < 1000 must be zero-filled to 4-digit width. + When used with the :meth:`strptime` method, the ``%p`` directive only affects + the output hour field if the ``%I`` directive is used to parse the hour. + +(3) + Unlike :mod:`time` module, :mod:`datetime` module does not support + leap seconds. + +(4) + When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used in + calculations when the day of the week and the year are specified. + +(5) + The :meth:`strptime` method can + parse years in the full [1, 9999] range, but years < 1000 must be + zero-filled to 4-digit width. .. versionchanged:: 3.2 In previous versions, :meth:`strftime` method was restricted to @@ -1960,56 +1823,11 @@ In version 3.2, :meth:`strftime` method was restricted to years >= 1000. -(3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects - the output hour field if the ``%I`` directive is used to parse the hour. +(6) + For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, + ``%z`` is replaced with the string ``'-0330'``. -(4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support - leap seconds. - -(5) - When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is - an extension to the set of format characters in the C standard (but - implemented separately in datetime objects, and therefore always - available). - -(6) - For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty - strings. - - For an aware object: - - ``%z`` - :meth:`utcoffset` is transformed into a 5-character string of the form - +HHMM or -HHMM, where HH is a 2-digit string giving the number of UTC - offset hours, and MM is a 2-digit string giving the number of UTC offset - minutes. For example, if :meth:`utcoffset` returns - ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string - ``'-0330'``. - - ``%Z`` - If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty - string. Otherwise ``%Z`` is replaced by the returned value, which must - be a string. - - .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an - aware :class:`.datetime` object will be produced. The ``tzinfo`` of the - result will be set to a :class:`timezone` instance. - -(7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used - in calculations when the day of the week and the calendar year (``%Y``) - are specified. - -(8) - Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the - day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not - interchangable. - -.. rubric:: Footnotes - -.. [#] If, that is, we ignore the effects of Relativity +.. versionchanged:: 3.2 + When the ``%z`` directive is provided to the :meth:`strptime` method, an + aware :class:`.datetime` object will be produced. The ``tzinfo`` of the + result will be set to a :class:`timezone` instance. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/dbm.rst --- a/Doc/library/dbm.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/dbm.rst Mon Jan 25 17:05:13 2016 +0100 @@ -73,39 +73,33 @@ strings are used they are implicitly converted to the default encoding before being stored. -These objects also support being used in a :keyword:`with` statement, which -will automatically close them when done. - -.. versionchanged:: 3.4 - Added native support for the context management protocol to the objects - returned by :func:`.open`. - The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: import dbm # Open database, creating it if necessary. - with dbm.open('cache', 'c') as db: + db = dbm.open('cache', 'c') - # Record some values - db[b'hello'] = b'there' - db['www.python.org'] = 'Python Website' - db['www.cnn.com'] = 'Cable News Network' + # Record some values + db[b'hello'] = b'there' + db['www.python.org'] = 'Python Website' + db['www.cnn.com'] = 'Cable News Network' - # Note that the keys are considered bytes now. - assert db[b'www.python.org'] == b'Python Website' - # Notice how the value is now in bytes. - assert db['www.cnn.com'] == b'Cable News Network' + # Note that the keys are considered bytes now. + assert db[b'www.python.org'] == b'Python Website' + # Notice how the value is now in bytes. + assert db['www.cnn.com'] == b'Cable News Network' - # Often-used methods of the dict interface work too. - print(db.get('python.org', b'not present')) + # Often-used methods of the dict interface work too. + print(db.get('python.org', b'not present')) - # Storing a non-string key or value will raise an exception (most - # likely a TypeError). - db['www.yahoo.com'] = 4 + # Storing a non-string key or value will raise an exception (most + # likely a TypeError). + db['www.yahoo.com'] = 4 - # db is automatically closed when leaving the with statement. + # Close when done. + db.close() .. seealso:: @@ -222,9 +216,6 @@ When the database has been opened in fast mode, this method forces any unwritten data to be written to the disk. - .. method:: gdbm.close() - - Close the ``gdbm`` database. :mod:`dbm.ndbm` --- Interface based on ndbm ------------------------------------------- @@ -256,7 +247,7 @@ .. function:: open(filename[, flag[, mode]]) - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the + Open a dbm database and return a ``dbm`` object. The *filename* argument is the name of the database file (without the :file:`.dir` or :file:`.pag` extensions). The optional *flag* argument must be one of these values: @@ -281,12 +272,6 @@ database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). - In addition to the dictionary-like methods, ``ndbm`` objects - provide the following method: - - .. method:: ndbm.close() - - Close the ``ndbm`` database. :mod:`dbm.dumb` --- Portable DBM implementation @@ -325,28 +310,17 @@ dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions are created. - The optional *flag* argument supports only the semantics of ``'c'`` - and ``'n'`` values. Other values will default to database being always - opened for update, and will be created if it does not exist. + The optional *flag* argument is currently ignored; the database is always opened + for update, and will be created if it does not exist. The optional *mode* argument is the Unix mode of the file, used only when the database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). - .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. - - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: + In addition to the methods provided by the :class:`collections.MutableMapping` class, + :class:`dumbdbm` objects provide the following method: .. method:: dumbdbm.sync() Synchronize the on-disk directory and data files. This method is called by the :meth:`Shelve.sync` method. - - .. method:: dumbdbm.close() - - Close the ``dumbdbm`` database. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/debug.rst --- a/Doc/library/debug.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/debug.rst Mon Jan 25 17:05:13 2016 +0100 @@ -15,4 +15,3 @@ profile.rst timeit.rst trace.rst - tracemalloc.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/decimal.rst --- a/Doc/library/decimal.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/decimal.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,8 +12,6 @@ .. moduleauthor:: Stefan Krah .. sectionauthor:: Raymond D. Hettinger -**Source code:** :source:`Lib/decimal.py` - .. import modules for testing inline doctests with the Sphinx doctest builder .. testsetup:: * @@ -263,7 +261,7 @@ Context() constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`Decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -402,29 +400,6 @@ compared, sorted, and coerced to another type (such as :class:`float` or :class:`int`). - There are some small differences between arithmetic on Decimal objects and - arithmetic on integers and floats. When the remainder operator ``%`` is - applied to Decimal objects, the sign of the result is the sign of the - *dividend* rather than the sign of the divisor:: - - >>> (-7) % 4 - 1 - >>> Decimal(-7) % Decimal(4) - Decimal('-3') - - The integer division operator ``//`` behaves analogously, returning the - integer part of the true quotient (truncating towards zero) rather than its - floor, so as to preserve the usual identity ``x == (x // y) * y + x % y``:: - - >>> -7 // 4 - -2 - >>> Decimal(-7) // Decimal(4) - Decimal('-1') - - The ``%`` and ``//`` operators implement the ``remainder`` and - ``divide-integer`` operations (respectively) as described in the - specification. - Decimal objects cannot generally be combined with floats or instances of :class:`fractions.Fraction` in arithmetic operations: an attempt to add a :class:`Decimal` to a :class:`float`, for @@ -448,19 +423,6 @@ ``Decimal('321e+5').adjusted()`` returns seven. Used for determining the position of the most significant digit with respect to the decimal point. - .. method:: as_integer_ratio() - - Return a pair ``(n, d)`` of integers that represent the given - :class:`Decimal` instance as a fraction, in lowest terms and - with a positive denominator:: - - >>> Decimal('-3.14').as_integer_ratio() - (-157, 50) - - The conversion is exact. Raise OverflowError on infinities and ValueError - on NaNs. - - .. versionadded:: 3.6 .. method:: as_tuple() @@ -474,7 +436,7 @@ a :class:`Decimal` instance is always canonical, so this operation returns its argument unchanged. - .. method:: compare(other, context=None) + .. method:: compare(other[, context]) Compare the values of two Decimal instances. :meth:`compare` returns a Decimal instance, and if either operand is a NaN then the result is a @@ -485,13 +447,13 @@ a == b ==> Decimal('0') a > b ==> Decimal('1') - .. method:: compare_signal(other, context=None) + .. method:: compare_signal(other[, context]) This operation is identical to the :meth:`compare` method, except that all NaNs signal. That is, if neither operand is a signaling NaN then any quiet NaN operand is treated as though it were a signaling NaN. - .. method:: compare_total(other, context=None) + .. method:: compare_total(other) Compare two operands using their abstract representation rather than their numerical value. Similar to the :meth:`compare` method, but the result @@ -509,21 +471,13 @@ higher in the total order than the second operand. See the specification for details of the total order. - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: compare_total_mag(other, context=None) + .. method:: compare_total_mag(other) Compare two operands using their abstract representation rather than their value as in :meth:`compare_total`, but ignoring the sign of each operand. ``x.compare_total_mag(y)`` is equivalent to ``x.copy_abs().compare_total(y.copy_abs())``. - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - .. method:: conjugate() Just returns self, this method is only to comply with the Decimal @@ -540,7 +494,7 @@ Return the negation of the argument. This operation is unaffected by the context and is quiet: no flags are changed and no rounding is performed. - .. method:: copy_sign(other, context=None) + .. method:: copy_sign(other) Return a copy of the first operand with the sign set to be the same as the sign of the second operand. For example: @@ -548,11 +502,10 @@ >>> Decimal('2.3').copy_sign(Decimal('-1.5')) Decimal('-2.3') - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: exp(context=None) + This operation is unaffected by the context and is quiet: no flags are + changed and no rounding is performed. + + .. method:: exp([context]) Return the value of the (natural) exponential function ``e**x`` at the given number. The result is correctly rounded using the @@ -589,7 +542,7 @@ .. versionadded:: 3.1 - .. method:: fma(other, third, context=None) + .. method:: fma(other, third[, context]) Fused multiply-add. Return self*other+third with no rounding of the intermediate product self*other. @@ -618,7 +571,7 @@ Return :const:`True` if the argument is a (quiet or signaling) NaN and :const:`False` otherwise. - .. method:: is_normal(context=None) + .. method:: is_normal() Return :const:`True` if the argument is a *normal* finite number. Return :const:`False` if the argument is zero, subnormal, infinite or a NaN. @@ -638,7 +591,7 @@ Return :const:`True` if the argument is a signaling NaN and :const:`False` otherwise. - .. method:: is_subnormal(context=None) + .. method:: is_subnormal() Return :const:`True` if the argument is subnormal, and :const:`False` otherwise. @@ -648,17 +601,17 @@ Return :const:`True` if the argument is a (positive or negative) zero and :const:`False` otherwise. - .. method:: ln(context=None) + .. method:: ln([context]) Return the natural (base e) logarithm of the operand. The result is correctly rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - .. method:: log10(context=None) + .. method:: log10([context]) Return the base ten logarithm of the operand. The result is correctly rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - .. method:: logb(context=None) + .. method:: logb([context]) For a nonzero number, return the adjusted exponent of its operand as a :class:`Decimal` instance. If the operand is a zero then @@ -666,73 +619,73 @@ is raised. If the operand is an infinity then ``Decimal('Infinity')`` is returned. - .. method:: logical_and(other, context=None) + .. method:: logical_and(other[, context]) :meth:`logical_and` is a logical operation which takes two *logical operands* (see :ref:`logical_operands_label`). The result is the digit-wise ``and`` of the two operands. - .. method:: logical_invert(context=None) + .. method:: logical_invert([context]) :meth:`logical_invert` is a logical operation. The result is the digit-wise inversion of the operand. - .. method:: logical_or(other, context=None) + .. method:: logical_or(other[, context]) :meth:`logical_or` is a logical operation which takes two *logical operands* (see :ref:`logical_operands_label`). The result is the digit-wise ``or`` of the two operands. - .. method:: logical_xor(other, context=None) + .. method:: logical_xor(other[, context]) :meth:`logical_xor` is a logical operation which takes two *logical operands* (see :ref:`logical_operands_label`). The result is the digit-wise exclusive or of the two operands. - .. method:: max(other, context=None) + .. method:: max(other[, context]) Like ``max(self, other)`` except that the context rounding rule is applied before returning and that :const:`NaN` values are either signaled or ignored (depending on the context and whether they are signaling or quiet). - .. method:: max_mag(other, context=None) + .. method:: max_mag(other[, context]) Similar to the :meth:`.max` method, but the comparison is done using the absolute values of the operands. - .. method:: min(other, context=None) + .. method:: min(other[, context]) Like ``min(self, other)`` except that the context rounding rule is applied before returning and that :const:`NaN` values are either signaled or ignored (depending on the context and whether they are signaling or quiet). - .. method:: min_mag(other, context=None) + .. method:: min_mag(other[, context]) Similar to the :meth:`.min` method, but the comparison is done using the absolute values of the operands. - .. method:: next_minus(context=None) + .. method:: next_minus([context]) Return the largest number representable in the given context (or in the current thread's context if no context is given) that is smaller than the given operand. - .. method:: next_plus(context=None) + .. method:: next_plus([context]) Return the smallest number representable in the given context (or in the current thread's context if no context is given) that is larger than the given operand. - .. method:: next_toward(other, context=None) + .. method:: next_toward(other[, context]) If the two operands are unequal, return the number closest to the first operand in the direction of the second operand. If both operands are numerically equal, return a copy of the first operand with the sign set to be the same as the sign of the second operand. - .. method:: normalize(context=None) + .. method:: normalize([context]) Normalize the number by stripping the rightmost trailing zeros and converting any result equal to :const:`Decimal('0')` to @@ -741,7 +694,7 @@ ``Decimal('0.321000e+2')`` both normalize to the equivalent value ``Decimal('32.1')``. - .. method:: number_class(context=None) + .. method:: number_class([context]) Return a string describing the *class* of the operand. The returned value is one of the following ten strings. @@ -757,7 +710,7 @@ * ``"NaN"``, indicating that the operand is a quiet NaN (Not a Number). * ``"sNaN"``, indicating that the operand is a signaling NaN. - .. method:: quantize(exp, rounding=None, context=None) + .. method:: quantize(exp[, rounding[, context[, watchexp]]]) Return a value equal to the first operand after rounding and having the exponent of the second operand. @@ -780,8 +733,9 @@ ``context`` argument; if neither argument is given the rounding mode of the current thread's context is used. - An error is returned whenever the resulting exponent is greater than - :attr:`Emax` or less than :attr:`Etiny`. + If *watchexp* is set (default), then an error is returned whenever the + resulting exponent is greater than :attr:`Emax` or less than + :attr:`Etiny`. .. method:: radix() @@ -789,25 +743,16 @@ class does all its arithmetic. Included for compatibility with the specification. - .. method:: remainder_near(other, context=None) - - Return the remainder from dividing *self* by *other*. This differs from - ``self % other`` in that the sign of the remainder is chosen so as to - minimize its absolute value. More precisely, the return value is - ``self - n * other`` where ``n`` is the integer nearest to the exact - value of ``self / other``, and if two integers are equally near then the - even one is chosen. - - If the result is zero then its sign will be the sign of *self*. - - >>> Decimal(18).remainder_near(Decimal(10)) - Decimal('-2') - >>> Decimal(25).remainder_near(Decimal(10)) - Decimal('5') - >>> Decimal(35).remainder_near(Decimal(10)) - Decimal('-5') - - .. method:: rotate(other, context=None) + .. method:: remainder_near(other[, context]) + + Compute the modulo as either a positive or negative value depending on + which is closest to zero. For instance, ``Decimal(10).remainder_near(6)`` + returns ``Decimal('-2')`` which is closer to zero than ``Decimal('4')``. + + If both are equally close, the one chosen will have the same sign as + *self*. + + .. method:: rotate(other[, context]) Return the result of rotating the digits of the first operand by an amount specified by the second operand. The second operand must be an integer in @@ -818,22 +763,18 @@ length precision if necessary. The sign and exponent of the first operand are unchanged. - .. method:: same_quantum(other, context=None) + .. method:: same_quantum(other[, context]) Test whether self and other have the same exponent or whether both are :const:`NaN`. - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: scaleb(other, context=None) + .. method:: scaleb(other[, context]) Return the first operand with exponent adjusted by the second. Equivalently, return the first operand multiplied by ``10**other``. The second operand must be an integer. - .. method:: shift(other, context=None) + .. method:: shift(other[, context]) Return the result of shifting the digits of the first operand by an amount specified by the second operand. The second operand must be an integer in @@ -843,25 +784,25 @@ right. Digits shifted into the coefficient are zeros. The sign and exponent of the first operand are unchanged. - .. method:: sqrt(context=None) + .. method:: sqrt([context]) Return the square root of the argument to full precision. - .. method:: to_eng_string(context=None) + .. method:: to_eng_string([context]) Convert to an engineering-type string. Engineering notation has an exponent which is a multiple of 3, so there are up to 3 digits left of the decimal place. For example, converts - ``Decimal('123E+1')`` to ``Decimal('1.23E+3')``. - - .. method:: to_integral(rounding=None, context=None) + ``Decimal('123E+1')`` to ``Decimal('1.23E+3')`` + + .. method:: to_integral([rounding[, context]]) Identical to the :meth:`to_integral_value` method. The ``to_integral`` name has been kept for compatibility with older versions. - .. method:: to_integral_exact(rounding=None, context=None) + .. method:: to_integral_exact([rounding[, context]]) Round to the nearest integer, signaling :const:`Inexact` or :const:`Rounded` as appropriate if rounding occurs. The rounding mode is @@ -869,7 +810,7 @@ ``context``. If neither parameter is given then the rounding mode of the current context is used. - .. method:: to_integral_value(rounding=None, context=None) + .. method:: to_integral_value([rounding[, context]]) Round to the nearest integer without signaling :const:`Inexact` or :const:`Rounded`. If given, applies *rounding*; otherwise, uses the @@ -915,10 +856,10 @@ You can also use the :keyword:`with` statement and the :func:`localcontext` function to temporarily change the active context. -.. function:: localcontext(ctx=None) +.. function:: localcontext([c]) Return a context manager that will set the current context for the active thread - to a copy of *ctx* on entry to the with-statement and restore the previous context + to a copy of *c* on entry to the with-statement and restore the previous context when exiting the with-statement. If no context is specified, a copy of the current context is used. @@ -1192,52 +1133,52 @@ .. method:: is_canonical(x) - Returns ``True`` if *x* is canonical; otherwise returns ``False``. + Returns True if *x* is canonical; otherwise returns False. .. method:: is_finite(x) - Returns ``True`` if *x* is finite; otherwise returns ``False``. + Returns True if *x* is finite; otherwise returns False. .. method:: is_infinite(x) - Returns ``True`` if *x* is infinite; otherwise returns ``False``. + Returns True if *x* is infinite; otherwise returns False. .. method:: is_nan(x) - Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. + Returns True if *x* is a qNaN or sNaN; otherwise returns False. .. method:: is_normal(x) - Returns ``True`` if *x* is a normal number; otherwise returns ``False``. + Returns True if *x* is a normal number; otherwise returns False. .. method:: is_qnan(x) - Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. + Returns True if *x* is a quiet NaN; otherwise returns False. .. method:: is_signed(x) - Returns ``True`` if *x* is negative; otherwise returns ``False``. + Returns True if *x* is negative; otherwise returns False. .. method:: is_snan(x) - Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. + Returns True if *x* is a signaling NaN; otherwise returns False. .. method:: is_subnormal(x) - Returns ``True`` if *x* is subnormal; otherwise returns ``False``. + Returns True if *x* is subnormal; otherwise returns False. .. method:: is_zero(x) - Returns ``True`` if *x* is a zero; otherwise returns ``False``. + Returns True if *x* is a zero; otherwise returns False. .. method:: ln(x) @@ -1337,7 +1278,7 @@ identity operation. - .. method:: power(x, y, modulo=None) + .. method:: power(x, y[, modulo]) Return ``x`` to the power of ``y``, reduced modulo ``modulo`` if given. @@ -1399,7 +1340,7 @@ .. method:: same_quantum(x, y) - Returns ``True`` if the two operands have the same exponent. + Returns True if the two operands have the same exponent. .. method:: scaleb (x, y) @@ -1461,9 +1402,9 @@ .. data:: HAVE_THREADS - The default value is ``True``. If Python is compiled without threads, the + The default value is True. If Python is compiled without threads, the C version automatically disables the expensive thread local context - machinery. In this case, the value is ``False``. + machinery. In this case, the value is False. Rounding modes -------------- @@ -2101,3 +2042,4 @@ >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/development.rst --- a/Doc/library/development.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/development.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,11 +16,14 @@ .. toctree:: - typing.rst pydoc.rst doctest.rst unittest.rst unittest.mock.rst + unittest.mock-patch.rst + unittest.mock-magicmethods.rst + unittest.mock-helpers.rst + unittest.mock-getting-started.rst unittest.mock-examples.rst 2to3.rst test.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/difflib.rst --- a/Doc/library/difflib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/difflib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,8 +7,6 @@ .. sectionauthor:: Tim Peters .. Markup by Fred L. Drake, Jr. -**Source code:** :source:`Lib/difflib.py` - .. testsetup:: import sys @@ -27,9 +25,7 @@ little fancier than, an algorithm published in the late 1980's by Ratcliff and Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to find the longest contiguous matching subsequence that contains no "junk" - elements; these "junk" elements are ones that are uninteresting in some - sense, such as blank lines or whitespace. (Handling junk is an - extension to the Ratcliff and Obershelp algorithm.) The same + elements (the Ratcliff and Obershelp algorithm doesn't address junk). The same idea is then applied recursively to the pieces of the sequences to the left and to the right of the matching subsequence. This does not yield minimal edit sequences, but does tend to yield matches that "look right" to people. @@ -98,14 +94,13 @@ *wrapcolumn* is an optional keyword to specify column number where lines are broken and wrapped, defaults to ``None`` where lines are not wrapped. - *linejunk* and *charjunk* are optional keyword arguments passed into :func:`ndiff` + *linejunk* and *charjunk* are optional keyword arguments passed into ``ndiff()`` (used by :class:`HtmlDiff` to generate the side by side HTML differences). See - :func:`ndiff` documentation for argument default values and descriptions. + ``ndiff()`` documentation for argument default values and descriptions. The following methods are public: - .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, \ - numlines=5, *, charset='utf-8') + .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5) Compares *fromlines* and *tolines* (lists of strings) and returns a string which is a complete HTML file containing a table showing line by line differences with @@ -124,10 +119,6 @@ the next difference highlight at the top of the browser without any leading context). - .. versionchanged:: 3.5 - *charset* keyword-only argument was added. The default charset of - HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``. - .. method:: make_table(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5) Compares *fromlines* and *tolines* (lists of strings) and returns a string which @@ -152,8 +143,8 @@ By default, the diff control lines (those with ``***`` or ``---``) are created with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing + :func:`file.readlines` result in diffs that are suitable for use with + :func:`file.writelines` since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the *lineterm* argument to @@ -217,7 +208,7 @@ Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style delta (a :term:`generator` generating the delta lines). - Optional keyword parameters *linejunk* and *charjunk* are filtering functions + Optional keyword parameters *linejunk* and *charjunk* are for filter functions (or ``None``): *linejunk*: A function that accepts a single string argument, and returns @@ -231,12 +222,12 @@ *charjunk*: A function that accepts a character (a string of length 1), and returns if the character is junk, or false if not. The default is module-level function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a - blank or tab; it's a bad idea to include newline in this!). + blank or tab; note: bad idea to include newline in this!). :file:`Tools/scripts/ndiff.py` is a command-line front-end to this function. - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), + ... 'ore\ntree\nemu\n'.splitlines(1)) >>> print(''.join(diff), end="") - one ? ^ @@ -259,8 +250,8 @@ Example: - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(1), + ... 'ore\ntree\nemu\n'.splitlines(1)) >>> diff = list(diff) # materialize the generated delta into a list >>> print(''.join(restore(diff, 1)), end="") one @@ -278,14 +269,14 @@ generating the delta lines) in unified diff format. Unified diffs are a compact way of showing just the lines that have changed plus - a few lines of context. The changes are shown in an inline style (instead of + a few lines of context. The changes are shown in a inline style (instead of separate before/after blocks). The number of context lines is set by *n* which defaults to three. By default, the diff control lines (those with ``---``, ``+++``, or ``@@``) are created with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing + :func:`file.readlines` result in diffs that are suitable for use with + :func:`file.writelines` since both the inputs and outputs have trailing newlines. For inputs that do not have trailing newlines, set the *lineterm* argument to @@ -315,21 +306,6 @@ See :ref:`difflib-interface` for a more detailed example. -.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\\n') - - Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a - sequence of delta lines (also bytes) in the format returned by *dfunc*. - *dfunc* must be a callable, typically either :func:`unified_diff` or - :func:`context_diff`. - - Allows you to compare data with unknown or inconsistent encoding. All - inputs except *n* must be bytes objects, not str. Works by losslessly - converting all inputs (except *n*) to str, and calling ``dfunc(a, b, - fromfile, tofile, fromfiledate, tofiledate, n, lineterm)``. The output of - *dfunc* is then converted back to bytes, so the delta lines that you - receive have the same unknown/inconsistent encodings as *a* and *b*. - - .. versionadded:: 3.5 .. function:: IS_LINE_JUNK(line) @@ -347,9 +323,9 @@ .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in `Dr. Dobb's Journal `_ in July, 1988. .. _sequence-matcher: @@ -383,7 +359,7 @@ The *autojunk* parameter. SequenceMatcher objects get three data attributes: *bjunk* is the - set of elements of *b* for which *isjunk* is ``True``; *bpopular* is the set of + set of elements of *b* for which *isjunk* is True; *bpopular* is the set of non-junk elements considered popular by the heuristic (if it is not disabled); *b2j* is a dict mapping the remaining elements of *b* to a list of positions where they occur. All three are reset whenever *b* is reset @@ -646,12 +622,6 @@ length 1), and returns true if the character is junk. The default is ``None``, meaning that no character is considered junk. - These junk-filtering functions speed up matching to find - differences and do not cause any differing lines or characters to - be ignored. Read the description of the - :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* - parameter for an explanation. - :class:`Differ` objects are used (deltas generated) via a single method: @@ -659,12 +629,10 @@ Compare two sequences of lines, and generate the delta (a sequence of lines). - Each sequence must contain individual single-line strings ending with - newlines. Such sequences can be obtained from the - :meth:`~io.IOBase.readlines` method of file-like objects. The delta - generated also consists of newline-terminated strings, ready to be - printed as-is via the :meth:`~io.IOBase.writelines` method of a - file-like object. + Each sequence must contain individual single-line strings ending with newlines. + Such sequences can be obtained from the :meth:`readlines` method of file-like + objects. The delta generated also consists of newline-terminated strings, ready + to be printed as-is via the :meth:`writelines` method of a file-like object. .. _differ-examples: @@ -674,13 +642,13 @@ This example compares two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be -obtained from the :meth:`~io.BaseIO.readlines` method of file-like objects): +obtained from the :meth:`readlines` method of file-like objects): >>> text1 = ''' 1. Beautiful is better than ugly. ... 2. Explicit is better than implicit. ... 3. Simple is better than complex. ... 4. Complex is better than complicated. - ... '''.splitlines(keepends=True) + ... '''.splitlines(1) >>> len(text1) 4 >>> text1[0][-1] @@ -689,7 +657,7 @@ ... 3. Simple is better than complex. ... 4. Complicated is better than complex. ... 5. Flat is better than nested. - ... '''.splitlines(keepends=True) + ... '''.splitlines(1) Next we instantiate a Differ object: @@ -743,4 +711,65 @@ It is also contained in the Python source distribution, as :file:`Tools/scripts/diff.py`. -.. literalinclude:: ../../Tools/scripts/diff.py +.. testcode:: + + """ Command line interface to difflib.py providing diffs in four formats: + + * ndiff: lists every line and highlights interline changes. + * context: highlights clusters of changes in a before/after format. + * unified: highlights clusters of changes in an inline format. + * html: generates side by side comparison with change highlights. + + """ + + import sys, os, time, difflib, optparse + + def main(): + # Configure the option parser + usage = "usage: %prog [options] fromfile tofile" + parser = optparse.OptionParser(usage) + parser.add_option("-c", action="store_true", default=False, + help='Produce a context format diff (default)') + parser.add_option("-u", action="store_true", default=False, + help='Produce a unified format diff') + hlp = 'Produce HTML side by side diff (can use -c and -l in conjunction)' + parser.add_option("-m", action="store_true", default=False, help=hlp) + parser.add_option("-n", action="store_true", default=False, + help='Produce a ndiff format diff') + parser.add_option("-l", "--lines", type="int", default=3, + help='Set number of context lines (default 3)') + (options, args) = parser.parse_args() + + if len(args) == 0: + parser.print_help() + sys.exit(1) + if len(args) != 2: + parser.error("need to specify both a fromfile and tofile") + + n = options.lines + fromfile, tofile = args # as specified in the usage string + + # we're passing these as arguments to the diff function + fromdate = time.ctime(os.stat(fromfile).st_mtime) + todate = time.ctime(os.stat(tofile).st_mtime) + with open(fromlines) as fromf, open(tofile) as tof: + fromlines, tolines = list(fromf), list(tof) + + if options.u: + diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile, + fromdate, todate, n=n) + elif options.n: + diff = difflib.ndiff(fromlines, tolines) + elif options.m: + diff = difflib.HtmlDiff().make_file(fromlines, tolines, fromfile, + tofile, context=options.c, + numlines=n) + else: + diff = difflib.context_diff(fromlines, tolines, fromfile, tofile, + fromdate, todate, n=n) + + # we're using writelines because diff is a generator + sys.stdout.writelines(diff) + + if __name__ == '__main__': + main() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/dis.rst --- a/Doc/library/dis.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/dis.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,9 +9,9 @@ -------------- The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by -disassembling it. The CPython bytecode which this module takes as an input is -defined in the file :file:`Include/opcode.h` and used by the compiler and the -interpreter. +disassembling it. The CPython bytecode which this module takes as an +input is defined in the file :file:`Include/opcode.h` and used by the compiler +and the interpreter. .. impl-detail:: @@ -26,8 +26,7 @@ def myfunc(alist): return len(alist) -the following command can be used to display the disassembly of -:func:`myfunc`:: +the following command can be used to get the disassembly of :func:`myfunc`:: >>> dis.dis(myfunc) 2 0 LOAD_GLOBAL 0 (len) @@ -37,80 +36,13 @@ (The "2" is a line number). -Bytecode analysis ------------------ +The :mod:`dis` module defines the following functions and constants: -.. versionadded:: 3.4 - -The bytecode analysis API allows pieces of Python code to be wrapped in a -:class:`Bytecode` object that provides easy access to details of the compiled -code. - -.. class:: Bytecode(x, *, first_line=None, current_offset=None) - - - Analyse the bytecode corresponding to a function, generator, method, string - of source code, or a code object (as returned by :func:`compile`). - - This is a convenience wrapper around many of the functions listed below, most - notably :func:`get_instructions`, as iterating over a :class:`Bytecode` - instance yields the bytecode operations as :class:`Instruction` instances. - - If *first_line* is not None, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - If *current_offset* is not None, it refers to an instruction offset in the - disassembled code. Setting this means :meth:`.dis` will display a "current - instruction" marker against the specified opcode. - - .. classmethod:: from_traceback(tb) - - Construct a :class:`Bytecode` instance from the given traceback, setting - *current_offset* to the instruction responsible for the exception. - - .. data:: codeobj - - The compiled code object. - - .. data:: first_line - - The first source line of the code object (if available) - - .. method:: dis() - - Return a formatted view of the bytecode operations (the same as printed by - :func:`dis.dis`, but returned as a multi-line string). - - .. method:: info() - - Return a formatted multi-line string with detailed information about the - code object, like :func:`code_info`. - -Example:: - - >>> bytecode = dis.Bytecode(myfunc) - >>> for instr in bytecode: - ... print(instr.opname) - ... - LOAD_GLOBAL - LOAD_FAST - CALL_FUNCTION - RETURN_VALUE - - -Analysis functions ------------------- - -The :mod:`dis` module also defines the following analysis functions that convert -the input directly to the desired output. They can be useful if only a single -operation is being performed, so the intermediate analysis object isn't useful: .. function:: code_info(x) Return a formatted multi-line string with detailed code object information - for the supplied function, generator, method, source code string or code object. + for the supplied function, method, source code string or code object. Note that the exact contents of code info strings are highly implementation dependent and they may change arbitrarily across Python VMs or Python @@ -119,54 +51,37 @@ .. versionadded:: 3.2 -.. function:: show_code(x, *, file=None) +.. function:: show_code(x) Print detailed code object information for the supplied function, method, - source code string or code object to *file* (or ``sys.stdout`` if *file* - is not specified). + source code string or code object to stdout. - This is a convenient shorthand for ``print(code_info(x), file=file)``, - intended for interactive exploration at the interpreter prompt. + This is a convenient shorthand for ``print(code_info(x))``, intended for + interactive exploration at the interpreter prompt. .. versionadded:: 3.2 - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: dis(x=None, *, file=None) +.. function:: dis(x=None) Disassemble the *x* object. *x* can denote either a module, a class, a - method, a function, a generator, a code object, a string of source code or - a byte sequence of raw bytecode. For a module, it disassembles all functions. - For a class, it disassembles all methods. For a code object or sequence of - raw bytecode, it prints one line per bytecode instruction. Strings are first - compiled to code objects with the :func:`compile` built-in function before being + method, a function, a code object, a string of source code or a byte sequence + of raw bytecode. For a module, it disassembles all functions. For a class, + it disassembles all methods. For a code object or sequence of raw bytecode, + it prints one line per bytecode instruction. Strings are first compiled to + code objects with the :func:`compile` built-in function before being disassembled. If no object is provided, this function disassembles the last traceback. - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: distb(tb=None, *, file=None) +.. function:: distb(tb=None) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is indicated. - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: disassemble(code, lasti=-1, *, file=None) - disco(code, lasti=-1, *, file=None) +.. function:: disassemble(code, lasti=-1) + disco(code, lasti=-1) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -182,28 +97,6 @@ The parameter interpretation recognizes local and global variable names, constant values, branch targets, and compare operators. - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: get_instructions(x, *, first_line=None) - - Return an iterator over the instructions in the supplied function, method, - source code string or code object. - - The iterator generates a series of :class:`Instruction` named tuples giving - the details of each operation in the supplied code. - - If *first_line* is not None, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - .. versionadded:: 3.4 - .. function:: findlinestarts(code) @@ -218,67 +111,61 @@ return a list of these offsets. -.. function:: stack_effect(opcode, [oparg]) +.. data:: opname - Compute the stack effect of *opcode* with argument *oparg*. + Sequence of operation names, indexable using the bytecode. - .. versionadded:: 3.4 + +.. data:: opmap + + Dictionary mapping operation names to bytecodes. + + +.. data:: cmp_op + + Sequence of all compare operation names. + + +.. data:: hasconst + + Sequence of bytecodes that have a constant parameter. + + +.. data:: hasfree + + Sequence of bytecodes that access a free variable. + + +.. data:: hasname + + Sequence of bytecodes that access an attribute by name. + + +.. data:: hasjrel + + Sequence of bytecodes that have a relative jump target. + + +.. data:: hasjabs + + Sequence of bytecodes that have an absolute jump target. + + +.. data:: haslocal + + Sequence of bytecodes that access a local variable. + + +.. data:: hascompare + + Sequence of bytecodes of Boolean operations. + .. _bytecodes: Python Bytecode Instructions ---------------------------- -The :func:`get_instructions` function and :class:`Bytecode` class provide -details of bytecode instructions as :class:`Instruction` instances: - -.. class:: Instruction - - Details for a bytecode operation - - .. data:: opcode - - numeric code for operation, corresponding to the opcode values listed - below and the bytecode values in the :ref:`opcode_collections`. - - - .. data:: opname - - human readable name for operation - - - .. data:: arg - - numeric argument to operation (if any), otherwise None - - - .. data:: argval - - resolved arg value (if known), otherwise same as arg - - - .. data:: argrepr - - human readable description of operation argument - - - .. data:: offset - - start index of operation within bytecode sequence - - - .. data:: starts_line - - line started by this opcode (if any), otherwise None - - - .. data:: is_jump_target - - ``True`` if other code jumps to here, otherwise ``False`` - - .. versionadded:: 3.4 - - The Python compiler currently generates the following bytecode instructions. @@ -346,14 +233,6 @@ Implements ``TOS = iter(TOS)``. -.. opcode:: GET_YIELD_FROM_ITER - - If ``TOS`` is a :term:`generator iterator` or :term:`coroutine` object - it is left as is. Otherwise, implements ``TOS = iter(TOS)``. - - .. versionadded:: 3.5 - - **Binary operations** Binary operations remove the top of the stack (TOS) and the second top-most @@ -370,13 +249,6 @@ Implements ``TOS = TOS1 * TOS``. -.. opcode:: BINARY_MATRIX_MULTIPLY - - Implements ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - .. opcode:: BINARY_FLOOR_DIVIDE Implements ``TOS = TOS1 // TOS``. @@ -449,13 +321,6 @@ Implements in-place ``TOS = TOS1 * TOS``. -.. opcode:: INPLACE_MATRIX_MULTIPLY - - Implements in-place ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - .. opcode:: INPLACE_FLOOR_DIVIDE Implements in-place ``TOS = TOS1 // TOS``. @@ -516,47 +381,13 @@ Implements ``del TOS1[TOS]``. -**Coroutine opcodes** - -.. opcode:: GET_AWAITABLE - - Implements ``TOS = get_awaitable(TOS)``, where ``get_awaitable(o)`` - returns ``o`` if ``o`` is a coroutine object or a generator object with - the CO_ITERABLE_COROUTINE flag, or resolves - ``o.__await__``. - - -.. opcode:: GET_AITER - - Implements ``TOS = get_awaitable(TOS.__aiter__())``. See ``GET_AWAITABLE`` - for details about ``get_awaitable`` - - -.. opcode:: GET_ANEXT - - Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE`` - for details about ``get_awaitable`` - - -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the - stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack. - - -.. opcode:: SETUP_ASYNC_WITH - - Creates a new frame object. - - - **Miscellaneous opcodes** .. opcode:: PRINT_EXPR Implements the expression statement for the interactive mode. TOS is removed - from the stack and printed. In non-interactive mode, an expression statement - is terminated with :opcode:`POP_TOP`. + from the stack and printed. In non-interactive mode, an expression statement is + terminated with ``POP_STACK``. .. opcode:: BREAK_LOOP @@ -567,7 +398,7 @@ .. opcode:: CONTINUE_LOOP (target) Continues a loop due to a :keyword:`continue` statement. *target* is the - address to jump to (which should be a :opcode:`FOR_ITER` instruction). + address to jump to (which should be a ``FOR_ITER`` instruction). .. opcode:: SET_ADD (i) @@ -585,10 +416,9 @@ Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict comprehensions. -For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` -instructions, while the added value or key/value pair is popped off, the -container object remains on the stack so that it is available for further -iterations of the loop. +For all of the SET_ADD, LIST_APPEND and MAP_ADD instructions, while the +added value or key/value pair is popped off, the container object remains on +the stack so that it is available for further iterations of the loop. .. opcode:: RETURN_VALUE @@ -598,35 +428,35 @@ .. opcode:: YIELD_VALUE - Pops TOS and yields it from a :term:`generator`. + Pops ``TOS`` and yields it from a :term:`generator`. .. opcode:: YIELD_FROM - Pops TOS and delegates to it as a subiterator from a :term:`generator`. + Pops ``TOS`` and delegates to it as a subiterator from a :term:`generator`. .. versionadded:: 3.3 .. opcode:: IMPORT_STAR - Loads all symbols not starting with ``'_'`` directly from the module TOS to - the local namespace. The module is popped after loading all names. This - opcode implements ``from module import *``. + Loads all symbols not starting with ``'_'`` directly from the module TOS to the + local namespace. The module is popped after loading all names. This opcode + implements ``from module import *``. .. opcode:: POP_BLOCK - Removes one block from the block stack. Per frame, there is a stack of - blocks, denoting nested loops, try statements, and such. + Removes one block from the block stack. Per frame, there is a stack of blocks, + denoting nested loops, try statements, and such. .. opcode:: POP_EXCEPT Removes one block from the block stack. The popped block must be an exception - handler block, as implicitly created when entering an except handler. In - addition to popping extraneous values from the frame stack, the last three - popped values are used to restore the exception state. + handler block, as implicitly created when entering an except handler. + In addition to popping extraneous values from the frame stack, the + last three popped values are used to restore the exception state. .. opcode:: END_FINALLY @@ -639,7 +469,7 @@ .. opcode:: LOAD_BUILD_CLASS Pushes :func:`builtins.__build_class__` onto the stack. It is later called - by :opcode:`CALL_FUNCTION` to construct a class. + by ``CALL_FUNCTION`` to construct a class. .. opcode:: SETUP_WITH (delta) @@ -654,11 +484,11 @@ :opcode:`UNPACK_SEQUENCE`). -.. opcode:: WITH_CLEANUP_START +.. opcode:: WITH_CLEANUP - Cleans up the stack when a :keyword:`with` statement block exits. TOS is the - context manager's :meth:`__exit__` bound method. Below TOS are 1--3 values - indicating how/why the finally clause was entered: + Cleans up the stack when a :keyword:`with` statement block exits. TOS is + the context manager's :meth:`__exit__` bound method. Below TOS are 1--3 + values indicating how/why the finally clause was entered: * SECOND = ``None`` * (SECOND, THIRD) = (``WHY_{RETURN,CONTINUE}``), retval @@ -666,30 +496,30 @@ * (SECOND, THIRD, FOURTH) = exc_info() In the last case, ``TOS(SECOND, THIRD, FOURTH)`` is called, otherwise - ``TOS(None, None, None)``. Pushes SECOND and result of the call - to the stack. + ``TOS(None, None, None)``. In addition, TOS is removed from the stack. - -.. opcode:: WITH_CLEANUP_FINISH - - Pops exception type and result of 'exit' function call from the stack. - - If the stack represents an exception, *and* the function call returns a - 'true' value, this information is "zapped" and replaced with a single - ``WHY_SILENCED`` to prevent :opcode:`END_FINALLY` from re-raising the - exception. (But non-local gotos will still be resumed.) + If the stack represents an exception, *and* the function call returns + a 'true' value, this information is "zapped" and replaced with a single + ``WHY_SILENCED`` to prevent ``END_FINALLY`` from re-raising the exception. + (But non-local gotos will still be resumed.) .. XXX explain the WHY stuff! +.. opcode:: STORE_LOCALS + + Pops TOS from the stack and stores it as the current frame's ``f_locals``. + This is used in class construction. + + All of the following opcodes expect arguments. An argument is two bytes, with the more significant byte last. .. opcode:: STORE_NAME (namei) Implements ``name = TOS``. *namei* is the index of *name* in the attribute - :attr:`co_names` of the code object. The compiler tries to use - :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. + :attr:`co_names` of the code object. The compiler tries to use ``STORE_FAST`` + or ``STORE_GLOBAL`` if possible. .. opcode:: DELETE_NAME (namei) @@ -729,12 +559,12 @@ .. opcode:: STORE_GLOBAL (namei) - Works as :opcode:`STORE_NAME`, but stores the name as a global. + Works as ``STORE_NAME``, but stores the name as a global. .. opcode:: DELETE_GLOBAL (namei) - Works as :opcode:`DELETE_NAME`, but deletes a global name. + Works as ``DELETE_NAME``, but deletes a global name. .. opcode:: LOAD_CONST (consti) @@ -749,18 +579,18 @@ .. opcode:: BUILD_TUPLE (count) - Creates a tuple consuming *count* items from the stack, and pushes the - resulting tuple onto the stack. + Creates a tuple consuming *count* items from the stack, and pushes the resulting + tuple onto the stack. .. opcode:: BUILD_LIST (count) - Works as :opcode:`BUILD_TUPLE`, but creates a list. + Works as ``BUILD_TUPLE``, but creates a list. .. opcode:: BUILD_SET (count) - Works as :opcode:`BUILD_TUPLE`, but creates a set. + Works as ``BUILD_TUPLE``, but creates a set. .. opcode:: BUILD_MAP (count) @@ -784,8 +614,8 @@ Imports the module ``co_names[namei]``. TOS and TOS1 are popped and provide the *fromlist* and *level* arguments of :func:`__import__`. The module - object is pushed onto the stack. The current namespace is not affected: for - a proper import statement, a subsequent :opcode:`STORE_FAST` instruction + object is pushed onto the stack. The current namespace is not affected: + for a proper import statement, a subsequent ``STORE_FAST`` instruction modifies the namespace. @@ -793,7 +623,7 @@ Loads the attribute ``co_names[namei]`` from the module found in TOS. The resulting object is pushed onto the stack, to be subsequently stored by a - :opcode:`STORE_FAST` instruction. + ``STORE_FAST`` instruction. .. opcode:: JUMP_FORWARD (delta) @@ -813,14 +643,14 @@ .. opcode:: JUMP_IF_TRUE_OR_POP (target) - If TOS is true, sets the bytecode counter to *target* and leaves TOS on the - stack. Otherwise (TOS is false), TOS is popped. + If TOS is true, sets the bytecode counter to *target* and leaves TOS + on the stack. Otherwise (TOS is false), TOS is popped. .. opcode:: JUMP_IF_FALSE_OR_POP (target) - If TOS is false, sets the bytecode counter to *target* and leaves TOS on the - stack. Otherwise (TOS is true), TOS is popped. + If TOS is false, sets the bytecode counter to *target* and leaves + TOS on the stack. Otherwise (TOS is true), TOS is popped. .. opcode:: JUMP_ABSOLUTE (target) @@ -830,10 +660,10 @@ .. opcode:: FOR_ITER (delta) - TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If - this yields a new value, push it on the stack (leaving the iterator below - it). If the iterator indicates it is exhausted TOS is popped, and the byte - code counter is incremented by *delta*. + ``TOS`` is an :term:`iterator`. Call its :meth:`__next__` method. If this + yields a new value, push it on the stack (leaving the iterator below it). If + the iterator indicates it is exhausted ``TOS`` is popped, and the byte code + counter is incremented by *delta*. .. opcode:: LOAD_GLOBAL (namei) @@ -849,15 +679,19 @@ .. opcode:: SETUP_EXCEPT (delta) - Pushes a try block from a try-except clause onto the block stack. *delta* - points to the first except block. + Pushes a try block from a try-except clause onto the block stack. *delta* points + to the first except block. .. opcode:: SETUP_FINALLY (delta) - Pushes a try block from a try-except clause onto the block stack. *delta* - points to the finally block. + Pushes a try block from a try-except clause onto the block stack. *delta* points + to the finally block. +.. opcode:: STORE_MAP + + Store a key and value pair in a dictionary. Pops the key and value while leaving + the dictionary on the stack. .. opcode:: LOAD_FAST (var_num) @@ -877,8 +711,8 @@ .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot *i* of the cell and free - variable storage. The name of the variable is ``co_cellvars[i]`` if *i* is - less than the length of *co_cellvars*. Otherwise it is ``co_freevars[i - + variable storage. The name of the variable is ``co_cellvars[i]`` if *i* is + less than the length of *co_cellvars*. Otherwise it is ``co_freevars[i - len(co_cellvars)]``. @@ -888,13 +722,6 @@ Pushes a reference to the object the cell contains on the stack. -.. opcode:: LOAD_CLASSDEREF (i) - - Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before - consulting the cell. This is used for loading free variables in class - bodies. - - .. opcode:: STORE_DEREF (i) Stores TOS into the cell contained in slot *i* of the cell and free variable @@ -918,36 +745,27 @@ Calls a function. The low byte of *argc* indicates the number of positional parameters, the high byte the number of keyword parameters. On the stack, the - opcode finds the keyword parameters first. For each keyword argument, the - value is on top of the key. Below the keyword parameters, the positional - parameters are on the stack, with the right-most parameter on top. Below the - parameters, the function object to call is on the stack. Pops all function - arguments, and the function itself off the stack, and pushes the return - value. + opcode finds the keyword parameters first. For each keyword argument, the value + is on top of the key. Below the keyword parameters, the positional parameters + are on the stack, with the right-most parameter on top. Below the parameters, + the function object to call is on the stack. Pops all function arguments, and + the function itself off the stack, and pushes the return value. .. opcode:: MAKE_FUNCTION (argc) - Pushes a new function object on the stack. From bottom to top, the consumed - stack must consist of - - * ``argc & 0xFF`` default argument objects in positional order - * ``(argc >> 8) & 0xFF`` pairs of name and default argument, with the name - just below the object on the stack, for keyword-only parameters - * ``(argc >> 16) & 0x7FFF`` parameter annotation objects - * a tuple listing the parameter names for the annotations (only if there are - ony annotation objects) - * the code associated with the function (at TOS1) - * the :term:`qualified name` of the function (at TOS) + Pushes a new function object on the stack. TOS is the + :term:`qualified name` of the function; TOS1 is the code associated with + the function. The function object is defined to have *argc* default parameters, + which are found below TOS1. .. opcode:: MAKE_CLOSURE (argc) Creates a new function object, sets its *__closure__* slot, and pushes it on - the stack. TOS is the :term:`qualified name` of the function, TOS1 is the - code associated with the function, and TOS2 is the tuple containing cells for - the closure's free variables. *argc* is interpreted as in ``MAKE_FUNCTION``; - the annotations and defaults are also in the same order below TOS2. + the stack. TOS is the code associated with the function, TOS1 the tuple + containing cells for the closure's free variables. The function also has + *argc* default parameters, which are found below the cells. .. opcode:: BUILD_SLICE (argc) @@ -963,116 +781,34 @@ Prefixes any opcode which has an argument too big to fit into the default two bytes. *ext* holds two additional bytes which, taken together with the - subsequent opcode's argument, comprise a four-byte argument, *ext* being the - two most-significant bytes. + subsequent opcode's argument, comprise a four-byte argument, *ext* being the two + most-significant bytes. .. opcode:: CALL_FUNCTION_VAR (argc) - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the variable argument list, followed by - keyword and positional arguments. + Calls a function. *argc* is interpreted as in ``CALL_FUNCTION``. The top element + on the stack contains the variable argument list, followed by keyword and + positional arguments. .. opcode:: CALL_FUNCTION_KW (argc) - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by explicit keyword and positional arguments. + Calls a function. *argc* is interpreted as in ``CALL_FUNCTION``. The top element + on the stack contains the keyword arguments dictionary, followed by explicit + keyword and positional arguments. .. opcode:: CALL_FUNCTION_VAR_KW (argc) - Calls a function. *argc* is interpreted as in :opcode:`CALL_FUNCTION`. The - top element on the stack contains the keyword arguments dictionary, followed - by the variable-arguments tuple, followed by explicit keyword and positional - arguments. - - -.. opcode:: FORMAT_VALUE (flags) - - Used for implementing formatted literal strings (f-strings). Pops - an optional *fmt_spec* from the stack, then a required *value*. - *flags* is interpreted as follows: - - * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. - * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before - formatting it. - * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before - formatting it. - * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before - formatting it. - * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use - it, else use an empty *fmt_spec*. - - Formatting is performed using :c:func:`PyObject_Format`. The - result is pushed on the stack. - - .. versionadded:: 3.6 + Calls a function. *argc* is interpreted as in ``CALL_FUNCTION``. The top + element on the stack contains the keyword arguments dictionary, followed by the + variable-arguments tuple, followed by explicit keyword and positional arguments. .. opcode:: HAVE_ARGUMENT - This is not really an opcode. It identifies the dividing line between - opcodes which don't take arguments ``< HAVE_ARGUMENT`` and those which do - ``>= HAVE_ARGUMENT``. + This is not really an opcode. It identifies the dividing line between opcodes + which don't take arguments ``< HAVE_ARGUMENT`` and those which do ``>= + HAVE_ARGUMENT``. -.. _opcode_collections: - -Opcode collections ------------------- - -These collections are provided for automatic introspection of bytecode -instructions: - -.. data:: opname - - Sequence of operation names, indexable using the bytecode. - - -.. data:: opmap - - Dictionary mapping operation names to bytecodes. - - -.. data:: cmp_op - - Sequence of all compare operation names. - - -.. data:: hasconst - - Sequence of bytecodes that have a constant parameter. - - -.. data:: hasfree - - Sequence of bytecodes that access a free variable (note that 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes). - - -.. data:: hasname - - Sequence of bytecodes that access an attribute by name. - - -.. data:: hasjrel - - Sequence of bytecodes that have a relative jump target. - - -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. - - -.. data:: haslocal - - Sequence of bytecodes that access a local variable. - - -.. data:: hascompare - - Sequence of bytecodes of Boolean operations. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/distribution.rst --- a/Doc/library/distribution.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,15 +0,0 @@ -*********************************** -Software Packaging and Distribution -*********************************** - -These libraries help you with publishing and installing Python software. -While these modules are designed to work in conjunction with the -`Python Package Index `__, they can also be used -with a local index server, or without any index server at all. - -.. toctree:: - - distutils.rst - ensurepip.rst - venv.rst - zipapp.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/distutils.rst --- a/Doc/library/distutils.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/distutils.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,31 +12,26 @@ 100%-pure Python, or may be extension modules written in C, or may be collections of Python packages which include modules coded in both Python and C. -Most Python users will *not* want to use this module directly, but instead -use the cross-version tools maintained by the Python Packaging Authority. In -particular, -`setuptools `__ is an -enhanced alternative to :mod:`distutils` that provides: +.. deprecated:: 3.3 + :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and + :ref:`packaging-install-index`. -* support for declaring project dependencies -* additional mechanisms for configuring which files to include in source - releases (including plugins for integration with version control systems) -* the ability to declare project "entry points", which can be used as the - basis for application plugin systems -* the ability to automatically generate Windows command line executables at - installation time rather than needing to prebuild them -* consistent behaviour across all supported Python versions -The recommended `pip `__ installer runs all -``setup.py`` scripts with ``setuptools``, even if the script itself only -imports ``distutils``. Refer to the -`Python Packaging User Guide `_ for more -information. +User documentation and API reference are provided in another document: -For the benefits of packaging tool authors and users seeking a deeper -understanding of the details of the current packaging and distribution -system, the legacy :mod:`distutils` based user documentation and API -reference remain available: +.. seealso:: -* :ref:`install-index` -* :ref:`distutils-index` + :ref:`distutils-index` + The manual for developers and packagers of Python modules. This describes + how to prepare :mod:`distutils`\ -based packages so that they may be + easily installed into an existing Python installation. If also contains + instructions for end-users wanting to install a distutils-based package, + :ref:`install-index`. + + +.. trick to silence a Sphinx warning + +.. toctree:: + :hidden: + + ../distutils/index diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/doctest.rst --- a/Doc/library/doctest.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/doctest.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,3 @@ -:keepdoctest: - :mod:`doctest` --- Test interactive Python examples =================================================== @@ -278,10 +276,6 @@ Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. -.. impl-detail:: - Prior to version 3.4, extension modules written in C were not fully - searched by doctest. - .. _doctest-finding-examples: @@ -324,8 +318,7 @@ Tabs in output generated by the tested code are not modified. Because any hard tabs in the sample output *are* expanded, this means that if the code output includes hard tabs, the only way the doctest can pass is if the - :const:`NORMALIZE_WHITESPACE` option or :ref:`directive ` - is in effect. + :const:`NORMALIZE_WHITESPACE` option or directive is in effect. Alternatively, the test can be rewritten to capture the output and compare it to an expected value as part of the test. This handling of tabs in the source was arrived at through trial and error, and has proven to be the least @@ -345,7 +338,7 @@ Backslashes in a raw docstring: m\n Otherwise, the backslash will be interpreted as part of the string. For example, - the ``\n`` above would be interpreted as a newline character. Alternatively, you + the "\\" above would be interpreted as a newline character. Alternatively, you can double each backslash in the doctest version (and not use a raw string):: >>> def f(x): @@ -490,20 +483,15 @@ SyntaxError: invalid syntax -.. _option-flags-and-directives: .. _doctest-options: -Option Flags -^^^^^^^^^^^^ +Option Flags and Directives +^^^^^^^^^^^^^^^^^^^^^^^^^^^ A number of option flags control various aspects of doctest's behavior. Symbolic names for the flags are supplied as module constants, which can be or'ed together and passed to various functions. The names can also be used in -:ref:`doctest directives `, and may be passed to the -doctest command line interface via the ``-o`` option. - -.. versionadded:: 3.4 - The ``-o`` command line option. +doctest directives (see below). The first group of options define test semantics, controlling aspects of how doctest decides whether actual output matches an example's expected output: @@ -557,14 +545,14 @@ :exc:`TypeError` is raised. It will also ignore the module name used in Python 3 doctest reports. Hence - both of these variations will work with the flag specified, regardless of - whether the test is run under Python 2.7 or Python 3.2 (or later versions):: + both these variations will work regardless of whether the test is run under + Python 2.7 or Python 3.2 (or later versions): - >>> raise CustomError('message') + >>> raise CustomError('message') #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CustomError: message - >>> raise CustomError('message') + >>> raise CustomError('message') #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): my_module.CustomError: message @@ -574,16 +562,15 @@ exception name. Using :const:`IGNORE_EXCEPTION_DETAIL` and the details from Python 2.3 is also the only clear way to write a doctest that doesn't care about the exception detail yet continues to pass under Python 2.3 or - earlier (those releases do not support :ref:`doctest directives - ` and ignore them as irrelevant comments). For example:: + earlier (those releases do not support doctest directives and ignore them + as irrelevant comments). For example, :: - >>> (1, 2)[3] = 'moo' + >>> (1, 2)[3] = 'moo' #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): File "", line 1, in ? TypeError: object doesn't support item assignment - passes under Python 2.3 and later Python versions with the flag specified, - even though the detail + passes under Python 2.3 and later Python versions, even though the detail changed in Python 2.4 to say "does not" instead of "doesn't". .. versionchanged:: 3.2 @@ -641,47 +628,13 @@ the output is suppressed. -.. data:: FAIL_FAST - - When specified, exit after the first failing example and don't attempt to run - the remaining examples. Thus, the number of failures reported will be at most - 1. This flag may be useful during debugging, since examples after the first - failure won't even produce debugging output. - - The doctest command line accepts the option ``-f`` as a shorthand for ``-o - FAIL_FAST``. - - .. versionadded:: 3.4 - - .. data:: REPORTING_FLAGS A bitmask or'ing together all the reporting flags above. - -There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: - - -.. function:: register_optionflag(name) - - Create a new option flag with a given name, and return the new flag's integer - value. :func:`register_optionflag` can be used when subclassing - :class:`OutputChecker` or :class:`DocTestRunner` to create new options that are - supported by your subclasses. :func:`register_optionflag` should always be - called using the following idiom:: - - MY_FLAG = register_optionflag('MY_FLAG') - - -.. _doctest-directives: - -Directives -^^^^^^^^^^ - -Doctest directives may be used to modify the :ref:`option flags -` for an individual example. Doctest directives are -special Python comments following an example's source code: +"Doctest directives" may be used to modify the option flags for individual +examples. Doctest directives are expressed as a special Python comment +following an example's source code: .. productionlist:: doctest directive: "#" "doctest:" `directive_options` @@ -699,7 +652,7 @@ For example, this test passes:: - >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE + >>> print(list(range(20))) #doctest: +NORMALIZE_WHITESPACE [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] @@ -711,8 +664,7 @@ >>> print(list(range(20))) # doctest: +ELLIPSIS [0, 1, ..., 18, 19] -Multiple directives can be used on a single physical line, separated by -commas:: +Multiple directives can be used on a single physical line, separated by commas:: >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE [0, 1, ..., 18, 19] @@ -738,6 +690,20 @@ functions that run doctests, establishing different defaults. In such cases, disabling an option via ``-`` in a directive can be useful. +There's also a way to register new option flag names, although this isn't useful +unless you intend to extend :mod:`doctest` internals via subclassing: + + +.. function:: register_optionflag(name) + + Create a new option flag with a given name, and return the new flag's integer + value. :func:`register_optionflag` can be used when subclassing + :class:`OutputChecker` or :class:`DocTestRunner` to create new options that are + supported by your subclasses. :func:`register_optionflag` should always be + called using the following idiom:: + + MY_FLAG = register_optionflag('MY_FLAG') + .. _doctest-warnings: @@ -865,8 +831,8 @@ nothing at the end. In verbose mode, the summary is detailed, else the summary is very brief (in fact, empty if all tests passed). - Optional argument *optionflags* (default value 0) takes the bitwise-or of - option flags. See section :ref:`doctest-options`. + Optional argument *optionflags* or's together option flags. See section + :ref:`doctest-options`. Optional argument *raise_on_error* defaults to false. If true, an exception is raised upon the first failure or unexpected exception in an example. This @@ -914,10 +880,15 @@ above, except that *globs* defaults to ``m.__dict__``. +There's also a function to run the doctests associated with a single object. +This function is provided for backward compatibility. There are no plans to +deprecate it, but it's rarely useful: + + .. function:: run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0) - Test examples associated with object *f*; for example, *f* may be a string, - a module, a function, or a class object. + Test examples associated with object *f*; for example, *f* may be a module, + function, or class object. A shallow copy of dictionary argument *globs* is used for the execution context. @@ -1053,10 +1024,6 @@ This function uses the same search technique as :func:`testmod`. - .. versionchanged:: 3.5 - :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* - contains no docstrings instead of raising :exc:`ValueError`. - Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`doctest.DocTestCase` instances, and :class:`DocTestCase` is a @@ -1086,7 +1053,7 @@ Set the :mod:`doctest` reporting flags to use. - Argument *flags* takes the bitwise-or of option flags. See section + Argument *flags* or's together option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. This is a module-global setting, and affects all future doctests run by module @@ -1279,8 +1246,9 @@ A processing class used to extract the :class:`DocTest`\ s that are relevant to a given object, from its docstring and the docstrings of its contained objects. - :class:`DocTest`\ s can be extracted from modules, classes, functions, - methods, staticmethods, classmethods, and properties. + :class:`DocTest`\ s can currently be extracted from the following object types: + modules, functions, classes, methods, staticmethods, classmethods, and + properties. The optional argument *verbose* can be used to display the objects searched by the finder. It defaults to ``False`` (no output). @@ -1810,27 +1778,6 @@ * Define a ``__test__`` dictionary mapping from regression test topics to docstrings containing test cases. -When you have placed your tests in a module, the module can itself be the test -runner. When a test fails, you can arrange for your test runner to re-run only -the failing doctest while you debug the problem. Here is a minimal example of -such a test runner:: - - if __name__ == '__main__': - import doctest - flags = doctest.REPORT_NDIFF|doctest.FAIL_FAST - if len(sys.argv) > 1: - name = sys.argv[1] - if name in globals(): - obj = globals()[name] - else: - obj = __test__[name] - doctest.run_docstring_examples(obj, globals(), name=name, - optionflags=flags) - else: - fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) - - .. rubric:: Footnotes .. [#] Examples containing both expected output and an exception are not supported. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/dummy_threading.rst --- a/Doc/library/dummy_threading.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/dummy_threading.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ try: import threading except ImportError: - import dummy_threading as threading + import dummy_threading Be careful to not use this module where deadlock might occur from a thread being created that blocks waiting for another thread to be created. This often occurs diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email-examples.rst --- a/Doc/library/email-examples.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email-examples.rst Mon Jan 25 17:05:13 2016 +0100 @@ -40,36 +40,6 @@ .. literalinclude:: ../includes/email-alternative.py -.. _email-contentmanager-api-examples: - -Examples using the Provisional API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Here is a reworking of the last example using the provisional API. To make -things a bit more interesting, we include a related image in the html part, and -we save a copy of what we are going to send to disk, as well as sending it. - -This example also shows how easy it is to include non-ASCII, and simplifies the -sending of the message using the :meth:`.send_message` method of the -:mod:`smtplib` module. - -.. literalinclude:: ../includes/email-alternative-new-api.py - -If we were instead sent the message from the last example, here is one -way we could process it: - -.. literalinclude:: ../includes/email-read-alternative-new-api.py - -Up to the prompt, the output from the above is:: - - To: Penelope Pussycat <"penelope@example.com">, Fabrette Pussycat <"fabrette@example.com"> - From: Pepé Le Pew - Subject: Ayons asperges pour le déjeuner - - Salut! - - Cela ressemble à un excellent recipie[1] déjeuner. - - .. rubric:: Footnotes .. [1] Thanks to Matthew Dixon Cowles for the original inspiration and examples. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.charset.rst --- a/Doc/library/email.charset.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.charset.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- +:mod:`email`: Representing character sets +----------------------------------------- .. module:: email.charset :synopsis: Character Sets @@ -234,5 +234,5 @@ *charset* is the canonical name of a character set. *codecname* is the name of a Python codec, as appropriate for the second argument to the :class:`str`'s - :meth:`~str.encode` method. + :func:`decode` method diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.contentmanager.rst --- a/Doc/library/email.contentmanager.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,434 +0,0 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- - -.. module:: email.contentmanager - :synopsis: Storing and Retrieving Content from MIME Parts - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - - -.. note:: - - The contentmanager module has been included in the standard library on a - :term:`provisional basis `. Backwards incompatible - changes (up to and including removal of the module) may occur if deemed - necessary by the core developers. - -.. versionadded:: 3.4 - as a :term:`provisional module `. - -The :mod:`~email.message` module provides a class that can represent an -arbitrary email message. That basic message model has a useful and flexible -API, but it provides only a lower-level API for interacting with the generic -parts of a message (the headers, generic header parameters, and the payload, -which may be a list of sub-parts). This module provides classes and tools -that provide an enhanced and extensible API for dealing with various specific -types of content, including the ability to retrieve the content of the message -as a specialized object type rather than as a simple bytes object. The module -automatically takes care of the RFC-specified MIME details (required headers -and parameters, etc.) for the certain common content types content properties, -and support for additional types can be added by an application using the -extension mechanisms. - -This module defines the eponymous "Content Manager" classes. The base -:class:`.ContentManager` class defines an API for registering content -management functions which extract data from ``Message`` objects or insert data -and headers into ``Message`` objects, thus providing a way of converting -between ``Message`` objects containing data and other representations of that -data (Python data types, specialized Python objects, external files, etc). The -module also defines one concrete content manager: :data:`raw_data_manager` -converts between MIME content types and ``str`` or ``bytes`` data. It also -provides a convenient API for managing the MIME parameters when inserting -content into ``Message``\ s. It also handles inserting and extracting -``Message`` objects when dealing with the ``message/rfc822`` content type. - -Another part of the enhanced interface is subclasses of -:class:`~email.message.Message` that provide new convenience API functions, -including convenience methods for calling the Content Managers derived from -this module. - -.. note:: - - Although :class:`.EmailMessage` and :class:`.MIMEPart` are currently - documented in this module because of the provisional nature of the code, the - implementation lives in the :mod:`email.message` module. - -.. currentmodule:: email.message - -.. class:: EmailMessage(policy=default) - - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to udpate and serialize the representation - of the message. If *policy* is not set, use the - :class:`~email.policy.default` policy, which follows the rules of the email - RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses - the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. - - This class is a subclass of :class:`~email.message.Message`. It adds - the following methods: - - - .. method:: is_attachment - - Return ``True`` if there is a :mailheader:`Content-Disposition` header - and its (case insensitive) value is ``attachment``, ``False`` otherwise. - - .. versionchanged:: 3.4.2 - is_attachment is now a method instead of a property, for consistency - with :meth:`~email.message.Message.is_multipart`. - - - .. method:: get_body(preferencelist=('related', 'html', 'plain')) - - Return the MIME part that is the best candidate to be the "body" of the - message. - - *preferencelist* must be a sequence of strings from the set ``related``, - ``html``, and ``plain``, and indicates the order of preference for the - content type of the part returned. - - Start looking for candidate matches with the object on which the - ``get_body`` method is called. - - If ``related`` is not included in *preferencelist*, consider the root - part (or subpart of the root part) of any related encountered as a - candidate if the (sub-)part matches a preference. - - When encountering a ``multipart/related``, check the ``start`` parameter - and if a part with a matching :mailheader:`Content-ID` is found, consider - only it when looking for candidate matches. Otherwise consider only the - first (default root) part of the ``multipart/related``. - - If a part has a :mailheader:`Content-Disposition` header, only consider - the part a candidate match if the value of the header is ``inline``. - - If none of the candidates matches any of the preferences in - *preferneclist*, return ``None``. - - Notes: (1) For most applications the only *preferencelist* combinations - that really make sense are ``('plain',)``, ``('html', 'plain')``, and the - default, ``('related', 'html', 'plain')``. (2) Because matching starts - with the object on which ``get_body`` is called, calling ``get_body`` on - a ``multipart/related`` will return the object itself unless - *preferencelist* has a non-default value. (3) Messages (or message parts) - that do not specify a :mailheader:`Content-Type` or whose - :mailheader:`Content-Type` header is invalid will be treated as if they - are of type ``text/plain``, which may occasionally cause ``get_body`` to - return unexpected results. - - - .. method:: iter_attachments() - - Return an iterator over all of the parts of the message that are not - candidate "body" parts. That is, skip the first occurrence of each of - ``text/plain``, ``text/html``, ``multipart/related``, or - ``multipart/alternative`` (unless they are explicitly marked as - attachments via :mailheader:`Content-Disposition: attachment`), and - return all remaining parts. When applied directly to a - ``multipart/related``, return an iterator over the all the related parts - except the root part (ie: the part pointed to by the ``start`` parameter, - or the first part if there is no ``start`` parameter or the ``start`` - parameter doesn't match the :mailheader:`Content-ID` of any of the - parts). When applied directly to a ``multipart/alternative`` or a - non-``multipart``, return an empty iterator. - - - .. method:: iter_parts() - - Return an iterator over all of the immediate sub-parts of the message, - which will be empty for a non-``multipart``. (See also - :meth:`~email.message.walk`.) - - - .. method:: get_content(*args, content_manager=None, **kw) - - Call the ``get_content`` method of the *content_manager*, passing self - as the message object, and passing along any other arguments or keywords - as additional arguments. If *content_manager* is not specified, use - the ``content_manager`` specified by the current :mod:`~email.policy`. - - - .. method:: set_content(*args, content_manager=None, **kw) - - Call the ``set_content`` method of the *content_manager*, passing self - as the message object, and passing along any other arguments or keywords - as additional arguments. If *content_manager* is not specified, use - the ``content_manager`` specified by the current :mod:`~email.policy`. - - - .. method:: make_related(boundary=None) - - Convert a non-``multipart`` message into a ``multipart/related`` message, - moving any existing :mailheader:`Content-` headers and payload into a - (new) first part of the ``multipart``. If *boundary* is specified, use - it as the boundary string in the multipart, otherwise leave the boundary - to be automatically created when it is needed (for example, when the - message is serialized). - - - .. method:: make_alternative(boundary=None) - - Convert a non-``multipart`` or a ``multipart/related`` into a - ``multipart/alternative``, moving any existing :mailheader:`Content-` - headers and payload into a (new) first part of the ``multipart``. If - *boundary* is specified, use it as the boundary string in the multipart, - otherwise leave the boundary to be automatically created when it is - needed (for example, when the message is serialized). - - - .. method:: make_mixed(boundary=None) - - Convert a non-``multipart``, a ``multipart/related``, or a - ``multipart-alternative`` into a ``multipart/mixed``, moving any existing - :mailheader:`Content-` headers and payload into a (new) first part of the - ``multipart``. If *boundary* is specified, use it as the boundary string - in the multipart, otherwise leave the boundary to be automatically - created when it is needed (for example, when the message is serialized). - - - .. method:: add_related(*args, content_manager=None, **kw) - - If the message is a ``multipart/related``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, - and :meth:`~email.message.Message.attach` it to the ``multipart``. If - the message is a non-``multipart``, call :meth:`make_related` and then - proceed as above. If the message is any other type of ``multipart``, - raise a :exc:`TypeError`. If *content_manager* is not specified, use - the ``content_manager`` specified by the current :mod:`~email.policy`. - If the added part has no :mailheader:`Content-Disposition` header, - add one with the value ``inline``. - - - .. method:: add_alternative(*args, content_manager=None, **kw) - - If the message is a ``multipart/alternative``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart`` or ``multipart/related``, call - :meth:`make_alternative` and then proceed as above. If the message is - any other type of ``multipart``, raise a :exc:`TypeError`. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: add_attachment(*args, content_manager=None, **kw) - - If the message is a ``multipart/mixed``, create a new message object, - pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart``, ``multipart/related``, or - ``multipart/alternative``, call :meth:`make_mixed` and then proceed as - above. If *content_manager* is not specified, use the ``content_manager`` - specified by the current :mod:`~email.policy`. If the added part - has no :mailheader:`Content-Disposition` header, add one with the value - ``attachment``. This method can be used both for explicit attachments - (:mailheader:`Content-Disposition: attachment` and ``inline`` attachments - (:mailheader:`Content-Disposition: inline`), by passing appropriate - options to the ``content_manager``. - - - .. method:: clear() - - Remove the payload and all of the headers. - - - .. method:: clear_content() - - Remove the payload and all of the :exc:`Content-` headers, leaving - all other headers intact and in their original order. - - -.. class:: MIMEPart(policy=default) - - This class represents a subpart of a MIME message. It is identical to - :class:`EmailMessage`, except that no :mailheader:`MIME-Version` headers are - added when :meth:`~EmailMessage.set_content` is called, since sub-parts do - not need their own :mailheader:`MIME-Version` headers. - - -.. currentmodule:: email.contentmanager - -.. class:: ContentManager() - - Base class for content managers. Provides the standard registry mechanisms - to register converters between MIME content and other representations, as - well as the ``get_content`` and ``set_content`` dispatch methods. - - - .. method:: get_content(msg, *args, **kw) - - Look up a handler function based on the ``mimetype`` of *msg* (see next - paragraph), call it, passing through all arguments, and return the result - of the call. The expectation is that the handler will extract the - payload from *msg* and return an object that encodes information about - the extracted data. - - To find the handler, look for the following keys in the registry, - stopping with the first one found: - - * the string representing the full MIME type (``maintype/subtype``) - * the string representing the ``maintype`` - * the empty string - - If none of these keys produce a handler, raise a :exc:`KeyError` for the - full MIME type. - - - .. method:: set_content(msg, obj, *args, **kw) - - If the ``maintype`` is ``multipart``, raise a :exc:`TypeError`; otherwise - look up a handler function based on the type of *obj* (see next - paragraph), call :meth:`~email.message.EmailMessage.clear_content` on the - *msg*, and call the handler function, passing through all arguments. The - expectation is that the handler will transform and store *obj* into - *msg*, possibly making other changes to *msg* as well, such as adding - various MIME headers to encode information needed to interpret the stored - data. - - To find the handler, obtain the type of *obj* (``typ = type(obj)``), and - look for the following keys in the registry, stopping with the first one - found: - - * the type itself (``typ``) - * the type's fully qualified name (``typ.__module__ + '.' + - typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). - - If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key - yields a handler, check for a handler for the key ``None``. If there is - no handler for ``None``, raise a :exc:`KeyError` for the fully - qualified name of the type. - - Also add a :mailheader:`MIME-Version` header if one is not present (see - also :class:`.MIMEPart`). - - - .. method:: add_get_handler(key, handler) - - Record the function *handler* as the handler for *key*. For the possible - values of *key*, see :meth:`get_content`. - - - .. method:: add_set_handler(typekey, handler) - - Record *handler* as the function to call when an object of a type - matching *typekey* is passed to :meth:`set_content`. For the possible - values of *typekey*, see :meth:`set_content`. - - -Content Manager Instances -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Currently the email package provides only one concrete content manager, -:data:`raw_data_manager`, although more may be added in the future. -:data:`raw_data_manager` is the -:attr:`~email.policy.EmailPolicy.content_manager` provided by -:attr:`~email.policy.EmailPolicy` and its derivatives. - - -.. data:: raw_data_manager - - This content manager provides only a minimum interface beyond that provided - by :class:`~email.message.Message` itself: it deals only with text, raw - byte strings, and :class:`~email.message.Message` objects. Nevertheless, it - provides significant advantages compared to the base API: ``get_content`` on - a text part will return a unicode string without the application needing to - manually decode it, ``set_content`` provides a rich set of options for - controlling the headers added to a part and controlling the content transfer - encoding, and it enables the use of the various ``add_`` methods, thereby - simplifying the creation of multipart messages. - - .. method:: get_content(msg, errors='replace') - - Return the payload of the part as either a string (for ``text`` parts), a - :class:`~email.message.EmailMessage` object (for ``message/rfc822`` - parts), or a ``bytes`` object (for all other non-multipart types). Raise - a :exc:`KeyError` if called on a ``multipart``. If the part is a - ``text`` part and *errors* is specified, use it as the error handler when - decoding the payload to unicode. The default error handler is - ``replace``. - - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ - cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'bytes'>, maintype, subtype, cte="base64", \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'Message'>, cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'list'>, subtype='mixed', \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - - Add headers and payload to *msg*: - - Add a :mailheader:`Content-Type` header with a ``maintype/subtype`` - value. - - * For ``str``, set the MIME ``maintype`` to ``text``, and set the - subtype to *subtype* if it is specified, or ``plain`` if it is not. - * For ``bytes``, use the specified *maintype* and *subtype*, or - raise a :exc:`TypeError` if they are not specified. - * For :class:`~email.message.Message` objects, set the maintype to - ``message``, and set the subtype to *subtype* if it is specified - or ``rfc822`` if it is not. If *subtype* is ``partial``, raise an - error (``bytes`` objects must be used to construct - ``message/partial`` parts). - * For *<'list'>*, which should be a list of - :class:`~email.message.Message` objects, set the ``maintype`` to - ``multipart``, and the ``subtype`` to *subtype* if it is - specified, and ``mixed`` if it is not. If the message parts in - the *<'list'>* have :mailheader:`MIME-Version` headers, remove - them. - - If *charset* is provided (which is valid only for ``str``), encode the - string to bytes using the specified character set. The default is - ``utf-8``. If the specified *charset* is a known alias for a standard - MIME charset name, use the standard charset instead. - - If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to - that value. For ``str`` objects, if it is not set use heuristics to - determine the most compact encoding. Possible values for *cte* are - ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``. - If the input cannot be encoded in the specified encoding (eg: ``7bit``), - raise a :exc:`ValueError`. For :class:`~email.message.Message`, per - :rfc:`2046`, raise an error if a *cte* of ``quoted-printable`` or - ``base64`` is requested for *subtype* ``rfc822``, and for any *cte* - other than ``7bit`` for *subtype* ``external-body``. For - ``message/rfc822``, use ``8bit`` if *cte* is not specified. For all - other values of *subtype*, use ``7bit``. - - .. note:: A *cte* of ``binary`` does not actually work correctly yet. - The ``Message`` object as modified by ``set_content`` is correct, but - :class:`~email.generator.BytesGenerator` does not serialize it - correctly. - - If *disposition* is set, use it as the value of the - :mailheader:`Content-Disposition` header. If not specified, and - *filename* is specified, add the header with the value ``attachment``. - If it is not specified and *filename* is also not specified, do not add - the header. The only valid values for *disposition* are ``attachment`` - and ``inline``. - - If *filename* is specified, use it as the value of the ``filename`` - parameter of the :mailheader:`Content-Disposition` header. There is no - default. - - If *cid* is specified, add a :mailheader:`Content-ID` header with - *cid* as its value. - - If *params* is specified, iterate its ``items`` method and use the - resulting ``(key, value)`` pairs to set additional parameters on the - :mailheader:`Content-Type` header. - - If *headers* is specified and is a list of strings of the form - ``headername: headervalue`` or a list of ``header`` objects - (distinguised from strings by having a ``name`` attribute), add the - headers to *msg*. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.encoders.rst --- a/Doc/library/email.encoders.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.encoders.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.encoders`: Encoders -------------------------------- +:mod:`email`: Encoders +---------------------- .. module:: email.encoders :synopsis: Encoders for email message payloads. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.errors.rst --- a/Doc/library/email.errors.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.errors.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- +:mod:`email`: Exception and Defect classes +------------------------------------------ .. module:: email.errors :synopsis: The exception classes used by the email package. @@ -25,8 +25,7 @@ Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse ` or - :meth:`Parser.parsestr ` methods. + from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. Situations where it can be raised include finding an envelope header after the first :rfc:`2822` header of the message, finding a continuation line before the @@ -38,8 +37,7 @@ Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse ` or - :meth:`Parser.parsestr ` methods. + from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. Situations where it can be raised include not being able to find the starting or terminating boundary in a :mimetype:`multipart/\*` message when strict parsing @@ -48,20 +46,19 @@ .. exception:: MultipartConversionError() - Raised when a payload is added to a :class:`~email.message.Message` object - using :meth:`add_payload`, but the payload is already a scalar and the - message's :mailheader:`Content-Type` main type is not either - :mimetype:`multipart` or missing. :exc:`MultipartConversionError` multiply - inherits from :exc:`MessageError` and the built-in :exc:`TypeError`. + Raised when a payload is added to a :class:`Message` object using + :meth:`add_payload`, but the payload is already a scalar and the message's + :mailheader:`Content-Type` main type is not either :mimetype:`multipart` or + missing. :exc:`MultipartConversionError` multiply inherits from + :exc:`MessageError` and the built-in :exc:`TypeError`. - Since :meth:`Message.add_payload` is deprecated, this exception is rarely - raised in practice. However the exception may also be raised if the - :meth:`~email.message.Message.attach` + Since :meth:`Message.add_payload` is deprecated, this exception is rarely raised + in practice. However the exception may also be raised if the :meth:`attach` method is called on an instance of a class derived from :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). -Here's the list of the defects that the :class:`~email.parser.FeedParser` +Here's the list of the defects that the :class:`~email.mime.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message where the problem was found, so for example, if a message nested inside a :mimetype:`multipart/alternative` had a malformed header, that nested message @@ -76,38 +73,17 @@ * :class:`StartBoundaryNotFoundDefect` -- The start boundary claimed in the :mailheader:`Content-Type` header was never found. -* :class:`CloseBoundaryNotFoundDefect` -- A start boundary was found, but - no corresponding close boundary was ever found. - - .. versionadded:: 3.3 - * :class:`FirstHeaderLineIsContinuationDefect` -- The message had a continuation line as its first header line. * :class:`MisplacedEnvelopeHeaderDefect` - A "Unix From" header was found in the middle of a header block. -* :class:`MissingHeaderBodySeparatorDefect` - A line was found while parsing - headers that had no leading white space but contained no ':'. Parsing - continues assuming that the line represents the first line of the body. - - .. versionadded:: 3.3 - * :class:`MalformedHeaderDefect` -- A header was found that was missing a colon, or was otherwise malformed. - .. deprecated:: 3.3 - This defect has not been used for several Python versions. +* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a + :mimetype:`multipart`, but no subparts were found. Note that when a message has + this defect, its :meth:`is_multipart` method may return false even though its + content type claims to be :mimetype:`multipart`. -* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a - :mimetype:`multipart`, but no subparts were found. Note that when a message - has this defect, its :meth:`~email.message.Message.is_multipart` method may - return false even though its content type claims to be :mimetype:`multipart`. - -* :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - enocded bytes, the padding was not correct. Enough padding is added to - perform the decode, but the resulting decoded bytes may be invalid. - -* :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - enocded bytes, characters outside the base64 alphebet were encountered. - The characters are ignored, but the resulting decoded bytes may be invalid. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.generator.rst --- a/Doc/library/email.generator.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.generator.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- +:mod:`email`: Generating MIME documents +--------------------------------------- .. module:: email.generator :synopsis: Generate flat text email messages from a message structure. @@ -17,10 +17,10 @@ standards-compliant way, should handle MIME and non-MIME email messages just fine, and is designed so that the transformation from flat text, to a message structure via the :class:`~email.parser.Parser` class, and back to flat text, -is idempotent (the input is identical to the output) [#]_. On the other hand, -using the Generator on a :class:`~email.message.Message` constructed by program -may result in changes to the :class:`~email.message.Message` object as defaults -are filled in. +is idempotent (the input is identical to the output). On the other hand, using +the Generator on a :class:`~email.message.Message` constructed by program may +result in changes to the :class:`~email.message.Message` object as defaults are +filled in. :class:`bytes` output can be generated using the :class:`BytesGenerator` class. If the message object structure contains non-ASCII bytes, this generator's @@ -32,7 +32,8 @@ :mod:`email.generator` module: -.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78, *, policy=None) +.. class:: Generator(outfp, mangle_from_=True, maxheaderlen=78, *, \ + policy=policy.default) The constructor for the :class:`Generator` class takes a :term:`file-like object` called *outfp* for an argument. *outfp* must support the :meth:`write` method @@ -54,9 +55,8 @@ The default is 78, as recommended (but not required) by :rfc:`2822`. The *policy* keyword specifies a :mod:`~email.policy` object that controls a - number of aspects of the generator's operation. If no *policy* is specified, - then the *policy* attached to the message object passed to :attr:`flatten` - is used. + number of aspects of the generator's operation. The default policy + maintains backward compatibility. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -80,19 +80,19 @@ Optional *linesep* specifies the line separator character used to terminate lines in the output. If specified it overrides the value - specified by the *msg*\'s or ``Generator``\'s ``policy``. + specified by the ``Generator``\'s ``policy``. - Because strings cannot represent non-ASCII bytes, if the policy that - applies when ``flatten`` is run has :attr:`~email.policy.Policy.cte_type` - set to ``8bit``, ``Generator`` will operate as if it were set to - ``7bit``. This means that messages parsed with a Bytes parser that have - a :mailheader:`Content-Transfer-Encoding` of ``8bit`` will be converted - to a use a ``7bit`` Content-Transfer-Encoding. Non-ASCII bytes in the - headers will be :rfc:`2047` encoded with a charset of ``unknown-8bit``. + Because strings cannot represent non-ASCII bytes, ``Generator`` ignores + the value of the :attr:`~email.policy.Policy.must_be_7bit` + :mod:`~email.policy` setting and operates as if it were set ``True``. + This means that messages parsed with a Bytes parser that have a + :mailheader:`Content-Transfer-Encoding` of 8bit will be converted to a + use a 7bit Content-Transfer-Encoding. Non-ASCII bytes in the headers + will be :rfc:`2047` encoded with a charset of `unknown-8bit`. .. versionchanged:: 3.2 - Added support for re-encoding ``8bit`` message bodies, and the - *linesep* argument. + Added support for re-encoding 8bit message bodies, and the *linesep* + argument. .. method:: clone(fp) @@ -112,7 +112,7 @@ :mod:`email.message`. .. class:: BytesGenerator(outfp, mangle_from_=True, maxheaderlen=78, *, \ - policy=None) + policy=policy.default) The constructor for the :class:`BytesGenerator` class takes a binary :term:`file-like object` called *outfp* for an argument. *outfp* must @@ -134,11 +134,9 @@ wrapping. The default is 78, as recommended (but not required) by :rfc:`2822`. - The *policy* keyword specifies a :mod:`~email.policy` object that controls a - number of aspects of the generator's operation. If no *policy* is specified, - then the *policy* attached to the message object passed to :attr:`flatten` - is used. + number of aspects of the generator's operation. The default policy + maintains backward compatibility. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -151,13 +149,13 @@ at *msg* to the output file specified when the :class:`BytesGenerator` instance was created. Subparts are visited depth-first and the resulting text will be properly MIME encoded. If the :mod:`~email.policy` option - :attr:`~email.policy.Policy.cte_type` is ``8bit`` (the default), + :attr:`~email.policy.Policy.must_be_7bit` is ``False`` (the default), then any bytes with the high bit set in the original parsed message that have not been modified will be copied faithfully to the output. If - ``cte_type`` is ``7bit``, the bytes will be converted as needed - using an ASCII-compatible Content-Transfer-Encoding. In particular, - RFC-invalid non-ASCII bytes in headers will be encoded using the MIME - ``unknown-8bit`` character set, thus rendering them RFC-compliant. + ``must_be_7bit`` is true, the bytes will be converted as needed using an + ASCII content-transfer-encoding. In particular, RFC-invalid non-ASCII + bytes in headers will be encoded using the MIME ``unknown-8bit`` + character set, thus rendering them RFC-compliant. .. XXX: There should be a complementary option that just does the RFC compliance transformation but leaves CTE 8bit parts alone. @@ -176,7 +174,7 @@ Optional *linesep* specifies the line separator character used to terminate lines in the output. If specified it overrides the value - specified by the ``Generator``\ or *msg*\ 's ``policy``. + specified by the ``Generator``\ 's ``policy``. .. method:: clone(fp) @@ -199,7 +197,7 @@ representing the part. -.. class:: DecodedGenerator(outfp, mangle_from_=True, maxheaderlen=78, fmt=None) +.. class:: DecodedGenerator(outfp[, mangle_from_=True, maxheaderlen=78, fmt=None) This class, derived from :class:`Generator` walks through all the subparts of a message. If the subpart is of main type :mimetype:`text`, then it prints the @@ -225,12 +223,3 @@ The default value for *fmt* is ``None``, meaning :: [Non-text (%(type)s) part of message omitted, filename %(filename)s] - - -.. rubric:: Footnotes - -.. [#] This statement assumes that you use the appropriate setting for the - ``unixfrom`` argument, and that you set maxheaderlen=0 (which will - preserve whatever the input line lengths were). It is also not strictly - true, since in many cases runs of whitespace in headers are collapsed - into single blanks. The latter is a bug that will eventually be fixed. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.header.rst --- a/Doc/library/email.header.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.header.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- +:mod:`email`: Internationalized headers +--------------------------------------- .. module:: email.header :synopsis: Representing non-ASCII headers @@ -31,8 +31,8 @@ >>> msg = Message() >>> h = Header('p\xf6stal', 'iso-8859-1') >>> msg['Subject'] = h - >>> msg.as_string() - 'Subject: =?iso-8859-1?q?p=F6stal?=\n\n' + >>> print(msg.as_string()) + Subject: =?iso-8859-1?q?p=F6stal?= @@ -176,7 +176,7 @@ >>> from email.header import decode_header >>> decode_header('=?iso-8859-1?q?p=F6stal?=') - [(b'p\xf6stal', 'iso-8859-1')] + [('p\xf6stal', 'iso-8859-1')] .. function:: make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' ') diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.headerregistry.rst --- a/Doc/library/email.headerregistry.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,453 +0,0 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- - -.. module:: email.headerregistry - :synopsis: Automatic Parsing of headers based on the field name - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - - -.. note:: - - The headerregistry module has been included in the standard library on a - :term:`provisional basis `. Backwards incompatible - changes (up to and including removal of the module) may occur if deemed - necessary by the core developers. - -.. versionadded:: 3.3 - as a :term:`provisional module `. - -Headers are represented by customized subclasses of :class:`str`. The -particular class used to represent a given header is determined by the -:attr:`~email.policy.EmailPolicy.header_factory` of the :mod:`~email.policy` in -effect when the headers are created. This section documents the particular -``header_factory`` implemented by the email package for handling :RFC:`5322` -compliant email messages, which not only provides customized header objects for -various header types, but also provides an extension mechanism for applications -to add their own custom header types. - -When using any of the policy objects derived from -:data:`~email.policy.EmailPolicy`, all headers are produced by -:class:`.HeaderRegistry` and have :class:`.BaseHeader` as their last base -class. Each header class has an additional base class that is determined by -the type of the header. For example, many headers have the class -:class:`.UnstructuredHeader` as their other base class. The specialized second -class for a header is determined by the name of the header, using a lookup -table stored in the :class:`.HeaderRegistry`. All of this is managed -transparently for the typical application program, but interfaces are provided -for modifying the default behavior for use by more complex applications. - -The sections below first document the header base classes and their attributes, -followed by the API for modifying the behavior of :class:`.HeaderRegistry`, and -finally the support classes used to represent the data parsed from structured -headers. - - -.. class:: BaseHeader(name, value) - - *name* and *value* are passed to ``BaseHeader`` from the - :attr:`~email.policy.EmailPolicy.header_factory` call. The string value of - any header object is the *value* fully decoded to unicode. - - This base class defines the following read-only properties: - - - .. attribute:: name - - The name of the header (the portion of the field before the ':'). This - is exactly the value passed in the - :attr:`~email.policy.EmailPolicy.header_factory` call for *name*; that - is, case is preserved. - - - .. attribute:: defects - - A tuple of :exc:`~email.errors.HeaderDefect` instances reporting any - RFC compliance problems found during parsing. The email package tries to - be complete about detecting compliance issues. See the :mod:`~email.errors` - module for a discussion of the types of defects that may be reported. - - - .. attribute:: max_count - - The maximum number of headers of this type that can have the same - ``name``. A value of ``None`` means unlimited. The ``BaseHeader`` value - for this attribute is ``None``; it is expected that specialized header - classes will override this value as needed. - - ``BaseHeader`` also provides the following method, which is called by the - email library code and should not in general be called by application - programs: - - .. method:: fold(*, policy) - - Return a string containing :attr:`~email.policy.Policy.linesep` - characters as required to correctly fold the header according - to *policy*. A :attr:`~email.policy.Policy.cte_type` of - ``8bit`` will be treated as if it were ``7bit``, since strings - may not contain binary data. - - - ``BaseHeader`` by itself cannot be used to create a header object. It - defines a protocol that each specialized header cooperates with in order to - produce the header object. Specifically, ``BaseHeader`` requires that - the specialized class provide a :func:`classmethod` named ``parse``. This - method is called as follows:: - - parse(string, kwds) - - ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. - ``defects`` is an empty list. The parse method should append any detected - defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may - contain transport encoded parts, but should correctly handle all valid - unicode characters as well so that it can parse un-encoded header values. - - ``BaseHeader``'s ``__new__`` then creates the header instance, and calls its - ``init`` method. The specialized class only needs to provide an ``init`` - method if it wishes to set additional attributes beyond those provided by - ``BaseHeader`` itself. Such an ``init`` method should look like this:: - - def init(self, *args, **kw): - self._myattr = kw.pop('myattr') - super().init(*args, **kw) - - That is, anything extra that the specialized class puts in to the ``kwds`` - dictionary should be removed and handled, and the remaining contents of - ``kw`` (and ``args``) passed to the ``BaseHeader`` ``init`` method. - - -.. class:: UnstructuredHeader - - An "unstructured" header is the default type of header in :rfc:`5322`. - Any header that does not have a specified syntax is treated as - unstructured. The classic example of an unstructured header is the - :mailheader:`Subject` header. - - In :rfc:`5322`, an unstructured header is a run of arbitrary text in the - ASCII character set. :rfc:`2047`, however, has an :rfc:`5322` compatible - mechanism for encoding non-ASCII text as ASCII characters within a header - value. When a *value* containing encoded words is passed to the - constructor, the ``UnstructuredHeader`` parser converts such encoded words - back in to the original unicode, following the :rfc:`2047` rules for - unstructured text. The parser uses heuristics to attempt to decode certain - non-compliant encoded words. Defects are registered in such cases, as well - as defects for issues such as invalid characters within the encoded words or - the non-encoded text. - - This header type provides no additional attributes. - - -.. class:: DateHeader - - :rfc:`5322` specifies a very specific format for dates within email headers. - The ``DateHeader`` parser recognizes that date format, as well as - recognizing a number of variant forms that are sometimes found "in the - wild". - - This header type provides the following additional attributes: - - .. attribute:: datetime - - If the header value can be recognized as a valid date of one form or - another, this attribute will contain a :class:`~datetime.datetime` - instance representing that date. If the timezone of the input date is - specified as ``-0000`` (indicating it is in UTC but contains no - information about the source timezone), then :attr:`.datetime` will be a - naive :class:`~datetime.datetime`. If a specific timezone offset is - found (including `+0000`), then :attr:`.datetime` will contain an aware - ``datetime`` that uses :class:`datetime.timezone` to record the timezone - offset. - - The ``decoded`` value of the header is determined by formatting the - ``datetime`` according to the :rfc:`5322` rules; that is, it is set to:: - - email.utils.format_datetime(self.datetime) - - When creating a ``DateHeader``, *value* may be - :class:`~datetime.datetime` instance. This means, for example, that - the following code is valid and does what one would expect:: - - msg['Date'] = datetime(2011, 7, 15, 21) - - Because this is a naive ``datetime`` it will be interpreted as a UTC - timestamp, and the resulting value will have a timezone of ``-0000``. Much - more useful is to use the :func:`~email.utils.localtime` function from the - :mod:`~email.utils` module:: - - msg['Date'] = utils.localtime() - - This example sets the date header to the current time and date using - the current timezone offset. - - -.. class:: AddressHeader - - Address headers are one of the most complex structured header types. - The ``AddressHeader`` class provides a generic interface to any address - header. - - This header type provides the following additional attributes: - - - .. attribute:: groups - - A tuple of :class:`.Group` objects encoding the - addresses and groups found in the header value. Addresses that are - not part of a group are represented in this list as single-address - ``Groups`` whose :attr:`~.Group.display_name` is ``None``. - - - .. attribute:: addresses - - A tuple of :class:`.Address` objects encoding all - of the individual addresses from the header value. If the header value - contains any groups, the individual addresses from the group are included - in the list at the point where the group occurs in the value (that is, - the list of addresses is "flattened" into a one dimensional list). - - The ``decoded`` value of the header will have all encoded words decoded to - unicode. :class:`~encodings.idna` encoded domain names are also decoded to unicode. The - ``decoded`` value is set by :attr:`~str.join`\ ing the :class:`str` value of - the elements of the ``groups`` attribute with ``', '``. - - A list of :class:`.Address` and :class:`.Group` objects in any combination - may be used to set the value of an address header. ``Group`` objects whose - ``display_name`` is ``None`` will be interpreted as single addresses, which - allows an address list to be copied with groups intact by using the list - obtained ``groups`` attribute of the source header. - - -.. class:: SingleAddressHeader - - A subclass of :class:`.AddressHeader` that adds one - additional attribute: - - - .. attribute:: address - - The single address encoded by the header value. If the header value - actually contains more than one address (which would be a violation of - the RFC under the default :mod:`~email.policy`), accessing this attribute - will result in a :exc:`ValueError`. - - -Many of the above classes also have a ``Unique`` variant (for example, -``UniqueUnstructuredHeader``). The only difference is that in the ``Unique`` -variant, :attr:`~.BaseHeader.max_count` is set to 1. - - -.. class:: MIMEVersionHeader - - There is really only one valid value for the :mailheader:`MIME-Version` - header, and that is ``1.0``. For future proofing, this header class - supports other valid version numbers. If a version number has a valid value - per :rfc:`2045`, then the header object will have non-``None`` values for - the following attributes: - - .. attribute:: version - - The version number as a string, with any whitespace and/or comments - removed. - - .. attribute:: major - - The major version number as an integer - - .. attribute:: minor - - The minor version number as an integer - - -.. class:: ParameterizedMIMEHeader - - MOME headers all start with the prefix 'Content-'. Each specific header has - a certain value, described under the class for that header. Some can - also take a list of supplemental parameters, which have a common format. - This class serves as a base for all the MIME headers that take parameters. - - .. attribute:: params - - A dictionary mapping parameter names to parameter values. - - -.. class:: ContentTypeHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Type` header. - - .. attribute:: content_type - - The content type string, in the form ``maintype/subtype``. - - .. attribute:: maintype - - .. attribute:: subtype - - -.. class:: ContentDispositionHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Disposition` header. - - .. attribute:: content-disposition - - ``inline`` and ``attachment`` are the only valid values in common use. - - -.. class:: ContentTransferEncoding - - Handles the :mailheader:`Content-Transfer-Encoding` header. - - .. attribute:: cte - - Valid values are ``7bit``, ``8bit``, ``base64``, and - ``quoted-printable``. See :rfc:`2045` for more information. - - - -.. class:: HeaderRegistry(base_class=BaseHeader, \ - default_class=UnstructuredHeader, \ - use_default_map=True) - - This is the factory used by :class:`~email.policy.EmailPolicy` by default. - ``HeaderRegistry`` builds the class used to create a header instance - dynamically, using *base_class* and a specialized class retrieved from a - registry that it holds. When a given header name does not appear in the - registry, the class specified by *default_class* is used as the specialized - class. When *use_default_map* is ``True`` (the default), the standard - mapping of header names to classes is copied in to the registry during - initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. - - The default mappings are: - - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader - - ``HeaderRegistry`` has the following methods: - - - .. method:: map_to_type(self, name, cls) - - *name* is the name of the header to be mapped. It will be converted to - lower case in the registry. *cls* is the specialized class to be used, - along with *base_class*, to create the class used to instantiate headers - that match *name*. - - - .. method:: __getitem__(name) - - Construct and return a class to handle creating a *name* header. - - - .. method:: __call__(name, value) - - Retrieves the specialized header associated with *name* from the - registry (using *default_class* if *name* does not appear in the - registry) and composes it with *base_class* to produce a class, - calls the constructed class's constructor, passing it the same - argument list, and finally returns the class instance created thereby. - - -The following classes are the classes used to represent data parsed from -structured headers and can, in general, be used by an application program to -construct structured values to assign to specific headers. - - -.. class:: Address(display_name='', username='', domain='', addr_spec=None) - - The class used to represent an email address. The general form of an - address is:: - - [display_name] - - or:: - - username@domain - - where each part must conform to specific syntax rules spelled out in - :rfc:`5322`. - - As a convenience *addr_spec* can be specified instead of *username* and - *domain*, in which case *username* and *domain* will be parsed from the - *addr_spec*. An *addr_spec* must be a properly RFC quoted string; if it is - not ``Address`` will raise an error. Unicode characters are allowed and - will be property encoded when serialized. However, per the RFCs, unicode is - *not* allowed in the username portion of the address. - - .. attribute:: display_name - - The display name portion of the address, if any, with all quoting - removed. If the address does not have a display name, this attribute - will be an empty string. - - .. attribute:: username - - The ``username`` portion of the address, with all quoting removed. - - .. attribute:: domain - - The ``domain`` portion of the address. - - .. attribute:: addr_spec - - The ``username@domain`` portion of the address, correctly quoted - for use as a bare address (the second form shown above). This - attribute is not mutable. - - .. method:: __str__() - - The ``str`` value of the object is the address quoted according to - :rfc:`5322` rules, but with no Content Transfer Encoding of any non-ASCII - characters. - - To support SMTP (:rfc:`5321`), ``Address`` handles one special case: if - ``username`` and ``domain`` are both the empty string (or ``None``), then - the string value of the ``Address`` is ``<>``. - - -.. class:: Group(display_name=None, addresses=None) - - The class used to represent an address group. The general form of an - address group is:: - - display_name: [address-list]; - - As a convenience for processing lists of addresses that consist of a mixture - of groups and single addresses, a ``Group`` may also be used to represent - single addresses that are not part of a group by setting *display_name* to - ``None`` and providing a list of the single address as *addresses*. - - .. attribute:: display_name - - The ``display_name`` of the group. If it is ``None`` and there is - exactly one ``Address`` in ``addresses``, then the ``Group`` represents a - single address that is not in a group. - - .. attribute:: addresses - - A possibly empty tuple of :class:`.Address` objects representing the - addresses in the group. - - .. method:: __str__() - - The ``str`` value of a ``Group`` is formatted according to :rfc:`5322`, - but with no Content Transfer Encoding of any non-ASCII characters. If - ``display_name`` is none and there is a single ``Address`` in the - ``addresses`` list, the ``str`` value will be the same as the ``str`` of - that single ``Address``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.iterators.rst --- a/Doc/library/email.iterators.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.iterators.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,14 +1,13 @@ -:mod:`email.iterators`: Iterators ---------------------------------- +:mod:`email`: Iterators +----------------------- .. module:: email.iterators :synopsis: Iterate over a message object tree. Iterating over a message object tree is fairly easy with the -:meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over -message object trees. +:meth:`Message.walk` method. The :mod:`email.iterators` module provides some +useful higher level iterations over message object trees. .. function:: body_line_iterator(msg, decode=False) @@ -17,11 +16,9 @@ string payloads line-by-line. It skips over all the subpart headers, and it skips over any subpart with a payload that isn't a Python string. This is somewhat equivalent to reading the flat text representation of the message from - a file using :meth:`~io.TextIOBase.readline`, skipping over all the - intervening headers. + a file using :meth:`readline`, skipping over all the intervening headers. - Optional *decode* is passed through to :meth:`Message.get_payload - `. + Optional *decode* is passed through to :meth:`Message.get_payload`. .. function:: typed_subpart_iterator(msg, maintype='text', subtype=None) @@ -36,22 +33,14 @@ Thus, by default :func:`typed_subpart_iterator` returns each subpart that has a MIME type of :mimetype:`text/\*`. - The following function has been added as a useful debugging tool. It should *not* be considered part of the supported public interface for the package. + .. function:: _structure(msg, fp=None, level=0, include_default=False) Prints an indented representation of the content types of the message object - structure. For example: - - .. testsetup:: - - >>> import email - >>> from email.iterators import _structure - >>> somefile = open('Lib/test/test_email/data/msg_02.txt') - - .. doctest:: + structure. For example:: >>> msg = email.message_from_file(somefile) >>> _structure(msg) @@ -71,10 +60,6 @@ text/plain text/plain - .. testsetup:: - - >>> somefile.close() - Optional *fp* is a file-like object to print the output to. It must be suitable for Python's :func:`print` function. *level* is used internally. *include_default*, if true, prints the default type as well. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.message.rst --- a/Doc/library/email.message.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.message.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- +:mod:`email`: Representing an email message +------------------------------------------- .. module:: email.message :synopsis: The base class representing email messages. @@ -31,40 +31,26 @@ Here are the methods of the :class:`Message` class: -.. class:: Message(policy=compat32) +.. class:: Message() - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update and serialize the representation - of the message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. + The constructor takes no arguments. - .. versionchanged:: 3.3 The *policy* keyword argument was added. - - .. method:: as_string(unixfrom=False, maxheaderlen=0, policy=None) + .. method:: as_string(unixfrom=False, maxheaderlen=0) Return the entire message flattened as a string. When optional *unixfrom* - is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compabitility reasons, - *maxheaderlen* defaults to ``0``, so if you want a different value you - must override it explicitly (the value specified for *max_line_length* in - the policy will be ignored by this method). The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the ``Generator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). + is ``True``, the envelope header is included in the returned string. + *unixfrom* defaults to ``False``. Flattening the message may trigger + changes to the :class:`Message` if defaults need to be filled in to + complete the transformation to a string (for example, MIME boundaries may + be generated or modified). Note that this method is provided as a convenience and may not always format the message the way you want. For example, by default it does not do the mangling of lines that begin with ``From`` that is required by the unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.Generator` instance and use its - :meth:`~email.generator.Generator.flatten` method directly. For example:: + :class:`~email.generator.Generator` instance and use its :meth:`flatten` + method directly. For example:: from io import StringIO from email.generator import Generator @@ -73,69 +59,17 @@ g.flatten(msg) text = fp.getvalue() - If the message object contains binary data that is not encoded according - to RFC standards, the non-compliant data will be replaced by unicode - "unknown character" code points. (See also :meth:`.as_bytes` and - :class:`~email.generator.BytesGenerator`.) - - .. versionchanged:: 3.4 the *policy* keyword argument was added. - .. method:: __str__() - Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a - string containing the formatted message. - - - .. method:: as_bytes(unixfrom=False, policy=None) - - Return the entire message flattened as a bytes object. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the - ``BytesGenerator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not always - format the message the way you want. For example, by default it does - not do the mangling of lines that begin with ``From`` that is - required by the unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.BytesGenerator` instance and use its - :meth:`~email.generator.BytesGenerator.flatten` method directly. - For example:: - - from io import BytesIO - from email.generator import BytesGenerator - fp = BytesIO() - g = BytesGenerator(fp, mangle_from_=True, maxheaderlen=60) - g.flatten(msg) - text = fp.getvalue() - - .. versionadded:: 3.4 - - - .. method:: __bytes__() - - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a - bytes object containing the formatted message. - - .. versionadded:: 3.4 + Equivalent to ``as_string(unixfrom=True)``. .. method:: is_multipart() Return ``True`` if the message's payload is a list of sub-\ :class:`Message` objects, otherwise return ``False``. When - :meth:`is_multipart` returns ``False``, the payload should be a string - object. (Note that :meth:`is_multipart` returning ``True`` does not - necessarily mean that "msg.get_content_maintype() == 'multipart'" will - return the ``True``. For example, ``is_multipart`` will return ``True`` - when the :class:`Message` is of type ``message/rfc822``.) + :meth:`is_multipart` returns False, the payload should be a string object. .. method:: set_unixfrom(unixfrom) @@ -177,14 +111,10 @@ header. When ``True`` and the message is not a multipart, the payload will be decoded if this header's value is ``quoted-printable`` or ``base64``. If some other encoding is used, or :mailheader:`Content-Transfer-Encoding` - header is missing, the payload is + header is missing, or if the payload has bogus base64 data, the payload is returned as-is (undecoded). In all cases the returned value is binary data. If the message is a multipart and the *decode* flag is ``True``, - then ``None`` is returned. If the payload is base64 and it was not - perfectly formed (missing padding, characters outside the base64 - alphabet), then an appropriate defect will be added to the message's - defect property (:class:`~email.errors.InvalidBase64PaddingDefect` or - :class:`~email.errors.InvalidBase64CharactersDefect`, respectively). + then ``None`` is returned. When *decode* is ``False`` (the default) the body is returned as a string without decoding the :mailheader:`Content-Transfer-Encoding`. However, @@ -470,8 +400,7 @@ to ``False``. - .. method:: set_param(param, value, header='Content-Type', requote=True, \ - charset=None, language='', replace=False) + .. method:: set_param(param, value, header='Content-Type', requote=True, charset=None, language='') Set a parameter in the :mailheader:`Content-Type` header. If the parameter already exists in the header, its value will be replaced with @@ -488,12 +417,6 @@ language, defaulting to the empty string. Both *charset* and *language* should be strings. - If *replace* is ``False`` (the default) the header is moved to the - end of the list of headers. If *replace* is ``True``, the header - will be updated in place. - - .. versionchanged:: 3.4 ``replace`` keyword was added. - .. method:: del_param(param, header='content-type', requote=True) @@ -543,8 +466,8 @@ Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`~email.errors.HeaderParseError` is raised if the - message object has no :mailheader:`Content-Type` header. + necessary. A :exc:`HeaderParseError` is raised if the message object has + no :mailheader:`Content-Type` header. Note that using this method is subtly different than deleting the old :mailheader:`Content-Type` header and adding a new one with the new @@ -578,15 +501,6 @@ will be *failobj*. - .. method:: get_content_disposition() - - Return the lowercased value (without parameters) of the message's - :mailheader:`Content-Disposition` header if it has one, or ``None``. The - possible values for this method are *inline*, *attachment* or ``None`` - if the message follows :rfc:`2183`. - - .. versionadded:: 3.5 - .. method:: walk() The :meth:`walk` method is an all-purpose generator which can be used to @@ -595,58 +509,16 @@ iterator in a ``for`` loop; each iteration returns the next subpart. Here's an example that prints the MIME type of every part of a multipart - message structure: + message structure:: - .. testsetup:: - - >>> from email import message_from_binary_file - >>> with open('Lib/test/test_email/data/msg_16.txt', 'rb') as f: - ... msg = message_from_binary_file(f) - >>> from email.iterators import _structure - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_type()) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - ``walk`` iterates over the subparts of any part where - :meth:`is_multipart` returns ``True``, even though - ``msg.get_content_maintype() == 'multipart'`` may return ``False``. We - can see this in our example by making use of the ``_structure`` debug - helper function: - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_maintype() == 'multipart'), - ... part.is_multipart()) - True True - False False - False True - False False - False False - False True - False False - >>> _structure(msg) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - Here the ``message`` parts are not ``multiparts``, but they do contain - subparts. ``is_multipart()`` returns ``True`` and ``walk`` descends - into the subparts. - + >>> for part in msg.walk(): + ... print(part.get_content_type()) + multipart/report + text/plain + message/delivery-status + text/plain + text/plain + message/rfc822 :class:`Message` objects can also optionally contain two instance attributes, which can be used when generating the plain text of a MIME message. @@ -682,8 +554,7 @@ the end of the message. You do not need to set the epilogue to the empty string in order for the - :class:`~email.generator.Generator` to print a newline at the end of the - file. + :class:`Generator` to print a newline at the end of the file. .. attribute:: defects diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.mime.rst --- a/Doc/library/email.mime.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.mime.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- +:mod:`email`: Creating email and MIME objects from scratch +---------------------------------------------------------- .. module:: email.mime :synopsis: Build MIME messages. @@ -35,8 +35,7 @@ *_maintype* is the :mailheader:`Content-Type` major type (e.g. :mimetype:`text` or :mimetype:`image`), and *_subtype* is the :mailheader:`Content-Type` minor type (e.g. :mimetype:`plain` or :mimetype:`gif`). *_params* is a parameter - key/value dictionary and is passed directly to :meth:`Message.add_header - `. + key/value dictionary and is passed directly to :meth:`Message.add_header`. The :class:`MIMEBase` class always adds a :mailheader:`Content-Type` header (based on *_maintype*, *_subtype*, and *_params*), and a @@ -51,9 +50,8 @@ A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base class for MIME messages that are not :mimetype:`multipart`. The primary - purpose of this class is to prevent the use of the - :meth:`~email.message.Message.attach` method, which only makes sense for - :mimetype:`multipart` messages. If :meth:`~email.message.Message.attach` + purpose of this class is to prevent the use of the :meth:`attach` method, + which only makes sense for :mimetype:`multipart` messages. If :meth:`attach` is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. @@ -76,8 +74,7 @@ *_subparts* is a sequence of initial subparts for the payload. It must be possible to convert this sequence to a list. You can always attach new subparts - to the message by using the :meth:`Message.attach - ` method. + to the message by using the :meth:`Message.attach` method. Additional parameters for the :mailheader:`Content-Type` header are taken from the keyword arguments, or passed into the *_params* argument, which is a keyword @@ -98,10 +95,8 @@ Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the data for transport. This callable takes one argument, which is - the :class:`MIMEApplication` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add + the :class:`MIMEApplication` instance. It should use :meth:`get_payload` and + :meth:`set_payload` to change the payload to encoded form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -121,15 +116,13 @@ this data can be decoded by the standard Python module :mod:`sndhdr`, then the subtype will be automatically included in the :mailheader:`Content-Type` header. Otherwise you can explicitly specify the audio subtype via the *_subtype* - argument. If the minor type could not be guessed and *_subtype* was not given, + parameter. If the minor type could not be guessed and *_subtype* was not given, then :exc:`TypeError` is raised. Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the audio data for transport. This callable takes one argument, - which is the :class:`MIMEAudio` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add + which is the :class:`MIMEAudio` instance. It should use :meth:`get_payload` and + :meth:`set_payload` to change the payload to encoded form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -149,15 +142,13 @@ this data can be decoded by the standard Python module :mod:`imghdr`, then the subtype will be automatically included in the :mailheader:`Content-Type` header. Otherwise you can explicitly specify the image subtype via the *_subtype* - argument. If the minor type could not be guessed and *_subtype* was not given, + parameter. If the minor type could not be guessed and *_subtype* was not given, then :exc:`TypeError` is raised. Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the image data for transport. This callable takes one argument, - which is the :class:`MIMEImage` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add + which is the :class:`MIMEImage` instance. It should use :meth:`get_payload` and + :meth:`set_payload` to change the payload to encoded form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -192,21 +183,7 @@ :class:`MIMEText` class is used to create MIME objects of major type :mimetype:`text`. *_text* is the string for the payload. *_subtype* is the minor type and defaults to :mimetype:`plain`. *_charset* is the character - set of the text and is passed as an argument to the + set of the text and is passed as a parameter to the :class:`~email.mime.nonmultipart.MIMENonMultipart` constructor; it defaults - to ``us-ascii`` if the string contains only ``ascii`` code points, and - ``utf-8`` otherwise. The *_charset* parameter accepts either a string or a - :class:`~email.charset.Charset` instance. - - Unless the *_charset* argument is explicitly set to ``None``, the - MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Endcoding` - header. This means that a subsequent ``set_payload`` call will not result - in an encoded payload, even if a charset is passed in the ``set_payload`` - command. You can "reset" this behavior by deleting the - ``Content-Transfer-Encoding`` header, after which a ``set_payload`` call - will automatically encode the new payload (and add a new - :mailheader:`Content-Transfer-Encoding` header). - - .. versionchanged:: 3.5 - *_charset* also accepts :class:`~email.charset.Charset` instances. + to ``us-ascii`` if the string contains only ``ascii`` codepoints, and + ``utf-8`` otherwise. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.parser.rst --- a/Doc/library/email.parser.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.parser.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- +:mod:`email`: Parsing email messages +------------------------------------ .. module:: email.parser :synopsis: Parse flat text email messages to produce a message object structure. @@ -7,8 +7,7 @@ Message object structures can be created in one of two ways: they can be created from whole cloth by instantiating :class:`~email.message.Message` objects and -stringing them together via :meth:`~email.message.Message.attach` and -:meth:`~email.message.Message.set_payload` calls, or they +stringing them together via :meth:`attach` and :meth:`set_payload` calls, or they can be created by parsing a flat text representation of the email message. The :mod:`email` package provides a standard parser that understands most email @@ -17,9 +16,8 @@ :class:`~email.message.Message` instance of the object structure. For simple, non-MIME messages the payload of this root object will likely be a string containing the text of the message. For MIME messages, the root object will -return ``True`` from its :meth:`~email.message.Message.is_multipart` method, and -the subparts can be accessed via the :meth:`~email.message.Message.get_payload` -and :meth:`~email.message.Message.walk` methods. +return ``True`` from its :meth:`is_multipart` method, and the subparts can be +accessed via the :meth:`get_payload` and :meth:`walk` methods. There are actually two parser interfaces available for use, the classic :class:`Parser` API and the incremental :class:`FeedParser` API. The classic @@ -60,18 +58,15 @@ Here is the API for the :class:`FeedParser`: -.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.compat32) +.. class:: FeedParser(_factory=email.message.Message, *, policy=policy.default) Create a :class:`FeedParser` instance. Optional *_factory* is a no-argument callable that will be called whenever a new message object is needed. It defaults to the :class:`email.message.Message` class. - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update the representation of the - message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. + The *policy* keyword specifies a :mod:`~email.policy` object that controls a + number of aspects of the parser's operation. The default policy maintains + backward compatibility. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -112,11 +107,10 @@ message body, instead setting the payload to the raw body as a string. They have the same API as the :class:`Parser` and :class:`BytesParser` classes. -.. versionadded:: 3.3 - The BytesHeaderParser class. +.. versionadded:: 3.3 BytesHeaderParser -.. class:: Parser(_class=email.message.Message, *, policy=policy.compat32) +.. class:: Parser(_class=email.message.Message, *, policy=policy.default) The constructor for the :class:`Parser` class takes an optional argument *_class*. This must be a callable factory (such as a function or a class), and @@ -124,12 +118,9 @@ :class:`~email.message.Message` (see :mod:`email.message`). The factory will be called without arguments. - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update the representation of the - message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. + The *policy* keyword specifies a :mod:`~email.policy` object that controls a + number of aspects of the parser's operation. The default policy maintains + backward compatibility. .. versionchanged:: 3.3 Removed the *strict* argument that was deprecated in 2.4. Added the @@ -142,11 +133,10 @@ Read all the data from the file-like object *fp*, parse the resulting text, and return the root message object. *fp* must support both the - :meth:`~io.TextIOBase.readline` and the :meth:`~io.TextIOBase.read` - methods on file-like objects. + :meth:`readline` and the :meth:`read` methods on file-like objects. The text contained in *fp* must be formatted as a block of :rfc:`2822` - style headers and header continuation lines, optionally preceded by an + style headers and header continuation lines, optionally preceded by a envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the message (which may contain MIME-encoded subparts). @@ -165,31 +155,28 @@ Optional *headersonly* is as with the :meth:`parse` method. -.. class:: BytesParser(_class=email.message.Message, *, policy=policy.compat32) +.. class:: BytesParser(_class=email.message.Message, *, policy=policy.default) This class is exactly parallel to :class:`Parser`, but handles bytes input. The *_class* and *strict* arguments are interpreted in the same way as for the :class:`Parser` constructor. - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update the representation of the - message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. + The *policy* keyword specifies a :mod:`~email.policy` object that + controls a number of aspects of the parser's operation. The default + policy maintains backward compatibility. .. versionchanged:: 3.3 Removed the *strict* argument. Added the *policy* keyword. - .. method:: parse(fp, headersonly=False) + .. method:: parse(fp, headeronly=False) Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` - methods on file-like objects. + both the :meth:`readline` and the :meth:`read` methods on file-like + objects. The bytes contained in *fp* must be formatted as a block of :rfc:`2822` - style headers and header continuation lines, optionally preceded by an + style headers and header continuation lines, optionally preceded by a envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the message (which may contain MIME-encoded subparts, including subparts @@ -218,45 +205,43 @@ .. currentmodule:: email .. function:: message_from_string(s, _class=email.message.Message, *, \ - policy=policy.compat32) + policy=policy.default) Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as - with the :class:`~email.parser.Parser` class constructor. + with the :class:`Parser` class constructor. .. versionchanged:: 3.3 Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_bytes(s, _class=email.message.Message, *, \ - policy=policy.compat32) + policy=policy.default) Return a message object structure from a byte string. This is exactly equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and - *strict* are interpreted as with the :class:`~email.parser.Parser` class - constructor. + *strict* are interpreted as with the :class:`Parser` class constructor. .. versionadded:: 3.2 .. versionchanged:: 3.3 Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_file(fp, _class=email.message.Message, *, \ - policy=policy.compat32) + policy=policy.default) Return a message object structure tree from an open :term:`file object`. This is exactly equivalent to ``Parser().parse(fp)``. *_class* - and *policy* are interpreted as with the :class:`~email.parser.Parser` class - constructor. + and *policy* are interpreted as with the :class:`Parser` class constructor. .. versionchanged:: Removed the *strict* argument. Added the *policy* keyword. .. function:: message_from_binary_file(fp, _class=email.message.Message, *, \ - policy=policy.compat32) + policy=policy.default) Return a message object structure tree from an open binary :term:`file object`. This is exactly equivalent to ``BytesParser().parse(fp)``. - *_class* and *policy* are interpreted as with the - :class:`~email.parser.Parser` class constructor. + *_class* and *policy* are interpreted as with the :class:`Parser` + class constructor. .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -265,7 +250,7 @@ Here's an example of how you might use this at an interactive Python prompt:: >>> import email - >>> msg = email.message_from_string(myString) # doctest: +SKIP + >>> msg = email.message_from_string(myString) Additional notes @@ -275,35 +260,32 @@ * Most non-\ :mimetype:`multipart` type messages are parsed as a single message object with a string payload. These objects will return ``False`` for - :meth:`~email.message.Message.is_multipart`. Their - :meth:`~email.message.Message.get_payload` method will return a string object. + :meth:`is_multipart`. Their :meth:`get_payload` method will return a string + object. * All :mimetype:`multipart` type messages will be parsed as a container message object with a list of sub-message objects for their payload. The outer - container message will return ``True`` for - :meth:`~email.message.Message.is_multipart` and their - :meth:`~email.message.Message.get_payload` method will return the list of - :class:`~email.message.Message` subparts. + container message will return ``True`` for :meth:`is_multipart` and their + :meth:`get_payload` method will return the list of :class:`~email.message.Message` + subparts. * Most messages with a content type of :mimetype:`message/\*` (e.g. :mimetype:`message/delivery-status` and :mimetype:`message/rfc822`) will also be parsed as container object containing a list payload of length 1. Their - :meth:`~email.message.Message.is_multipart` method will return ``True``. - The single element in the list payload will be a sub-message object. + :meth:`is_multipart` method will return ``True``. The single element in the + list payload will be a sub-message object. * Some non-standards compliant messages may not be internally consistent about their :mimetype:`multipart`\ -edness. Such messages may have a :mailheader:`Content-Type` header of type :mimetype:`multipart`, but their - :meth:`~email.message.Message.is_multipart` method may return ``False``. - If such messages were parsed with the :class:`~email.parser.FeedParser`, - they will have an instance of the - :class:`~email.errors.MultipartInvariantViolationDefect` class in their - *defects* attribute list. See :mod:`email.errors` for details. + :meth:`is_multipart` method may return ``False``. If such messages were parsed + with the :class:`FeedParser`, they will have an instance of the + :class:`MultipartInvariantViolationDefect` class in their *defects* attribute + list. See :mod:`email.errors` for details. .. rubric:: Footnotes .. [#] As of email package version 3.0, introduced in Python 2.4, the classic - :class:`~email.parser.Parser` was re-implemented in terms of the - :class:`~email.parser.FeedParser`, so the semantics and results are - identical between the two parsers. + :class:`Parser` was re-implemented in terms of the :class:`FeedParser`, so the + semantics and results are identical between the two parsers. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.policy.rst --- a/Doc/library/email.policy.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.policy.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,12 +1,9 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ +:mod:`email`: Policy Objects +---------------------------- .. module:: email.policy :synopsis: Controlling the parsing and generating of messages -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - .. versionadded:: 3.3 @@ -26,124 +23,81 @@ control the behavior of various components of the email package during use. :class:`Policy` instances can be passed to various classes and methods in the email package to alter the default behavior. The settable values and their -defaults are described below. +defaults are described below. The :mod:`policy` module also provides some +pre-created :class:`Policy` instances. In addition to a :const:`default` +instance, there are instances tailored for certain applications. For example +there is an :const:`SMTP` :class:`Policy` with defaults appropriate for +generating output to be sent to an SMTP server. These are listed `below +`. -There is a default policy used by all classes in the email package. This -policy is named :class:`Compat32`, with a corresponding pre-defined instance -named :const:`compat32`. It provides for complete backward compatibility (in -some cases, including bug compatibility) with the pre-Python3.3 version of the -email package. - -The first part of this documentation covers the features of :class:`Policy`, an -:term:`abstract base class` that defines the features that are common to all -policy objects, including :const:`compat32`. This includes certain hook -methods that are called internally by the email package, which a custom policy -could override to obtain different behavior. - -When a :class:`~email.message.Message` object is created, it acquires a policy. -By default this will be :const:`compat32`, but a different policy can be -specified. If the ``Message`` is created by a :mod:`~email.parser`, a policy -passed to the parser will be the policy used by the ``Message`` it creates. If -the ``Message`` is created by the program, then the policy can be specified -when it is created. When a ``Message`` is passed to a :mod:`~email.generator`, -the generator uses the policy from the ``Message`` by default, but you can also -pass a specific policy to the generator that will override the one stored on -the ``Message`` object. - -:class:`Policy` instances are immutable, but they can be cloned, accepting the -same keyword arguments as the class constructor and returning a new -:class:`Policy` instance that is a copy of the original but with the specified -attributes values changed. +In general an application will only need to deal with setting the policy at the +input and output boundaries. Once parsed, a message is represented by a +:class:`~email.message.Message` object, which is designed to be independent of +the format that the message has "on the wire" when it is received, transmitted, +or displayed. Thus, a :class:`Policy` can be specified when parsing a message +to create a :class:`~email.message.Message`, and again when turning the +:class:`~email.message.Message` into some other representation. While often a +program will use the same :class:`Policy` for both input and output, the two +can be different. As an example, the following code could be used to read an email message from a -file on disk and pass it to the system ``sendmail`` program on a Unix system: +file on disk and pass it to the system ``sendmail`` program on a Unix system:: -.. testsetup:: - - >>> from unittest import mock - >>> mocker = mock.patch('subprocess.Popen') - >>> m = mocker.start() - >>> proc = mock.MagicMock() - >>> m.return_value = proc - >>> proc.stdin.close.return_value = None - >>> mymsg = open('mymsg.txt', 'w') - >>> mymsg.write('To: abc@xyz.com\n\n') - 17 - >>> mymsg.flush() - -.. doctest:: - - >>> from email import message_from_binary_file + >>> from email import msg_from_binary_file >>> from email.generator import BytesGenerator - >>> from email import policy + >>> import email.policy >>> from subprocess import Popen, PIPE - >>> with open('mymsg.txt', 'rb') as f: - ... msg = message_from_binary_file(f, policy=policy.default) - >>> p = Popen(['sendmail', msg['To'].addresses[0]], stdin=PIPE) - >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n')) + >>> with open('mymsg.txt', 'b') as f: + ... msg = msg_from_binary_file(f, policy=email.policy.mbox) + >>> p = Popen(['sendmail', msg['To'][0].address], stdin=PIPE) + >>> g = BytesGenerator(p.stdin, policy=email.policy.SMTP) >>> g.flatten(msg) >>> p.stdin.close() >>> rc = p.wait() -.. testsetup:: - - >>> mymsg.close() - >>> mocker.stop() - >>> import os - >>> os.remove('mymsg.txt') - -Here we are telling :class:`~email.generator.BytesGenerator` to use the RFC -correct line separator characters when creating the binary string to feed into -``sendmail's`` ``stdin``, where the default policy would use ``\n`` line -separators. +.. XXX email.policy.mbox/MBOX does not exist yet Some email package methods accept a *policy* keyword argument, allowing the policy to be overridden for that method. For example, the following code uses -the :meth:`~email.message.Message.as_bytes` method of the *msg* object from -the previous example and writes the message to a file using the native line -separators for the platform on which it is running:: +the :meth:`~email.message.Message.as_string` method of the *msg* object from the +previous example and re-write it to a file using the native line separators for +the platform on which it is running:: >>> import os + >>> mypolicy = email.policy.Policy(linesep=os.linesep) >>> with open('converted.txt', 'wb') as f: - ... f.write(msg.as_bytes(policy=msg.policy.clone(linesep=os.linesep))) - 17 + ... f.write(msg.as_string(policy=mypolicy)) + +Policy instances are immutable, but they can be cloned, accepting the same +keyword arguments as the class constructor and returning a new :class:`Policy` +instance that is a copy of the original but with the specified attributes +values changed. For example, the following creates an SMTP policy that will +raise any defects detected as errors:: + + >>> strict_SMTP = email.policy.SMTP.clone(raise_on_defect=True) Policy objects can also be combined using the addition operator, producing a policy object whose settings are a combination of the non-default values of the summed objects:: - >>> compat_SMTP = policy.compat32.clone(linesep='\r\n') - >>> compat_strict = policy.compat32.clone(raise_on_defect=True) - >>> compat_strict_SMTP = compat_SMTP + compat_strict + >>> strict_SMTP = email.policy.SMTP + email.policy.strict This operation is not commutative; that is, the order in which the objects are added matters. To illustrate:: - >>> policy100 = policy.compat32.clone(max_line_length=100) - >>> policy80 = policy.compat32.clone(max_line_length=80) - >>> apolicy = policy100 + policy80 + >>> Policy = email.policy.Policy + >>> apolicy = Policy(max_line_length=100) + Policy(max_line_length=80) >>> apolicy.max_line_length 80 - >>> apolicy = policy80 + policy100 + >>> apolicy = Policy(max_line_length=80) + Policy(max_line_length=100) >>> apolicy.max_line_length 100 .. class:: Policy(**kw) - This is the :term:`abstract base class` for all policy classes. It provides - default implementations for a couple of trivial methods, as well as the - implementation of the immutability property, the :meth:`clone` method, and - the constructor semantics. - - The constructor of a policy class can be passed various keyword arguments. - The arguments that may be specified are any non-method properties on this - class, plus any additional non-method properties on the concrete class. A - value specified in the constructor will override the default value for the - corresponding attribute. - - This class defines the following properties, and thus values for the - following may be passed in the constructor of any policy class: + The valid constructor keyword arguments are any of the attributes listed + below. .. attribute:: max_line_length @@ -156,30 +110,18 @@ The string to be used to terminate lines in serialized output. The default is ``\n`` because that's the internal end-of-line discipline used - by Python, though ``\r\n`` is required by the RFCs. + by Python, though ``\r\n`` is required by the RFCs. See `Policy + Instances`_ for policies that use an RFC conformant linesep. Setting it + to :attr:`os.linesep` may also be useful. - .. attribute:: cte_type + .. attribute:: must_be_7bit - Controls the type of Content Transfer Encodings that may be or are - required to be used. The possible values are: - - .. tabularcolumns:: |l|L| - - ======== =============================================================== - ``7bit`` all data must be "7 bit clean" (ASCII-only). This means that - where necessary data will be encoded using either - quoted-printable or base64 encoding. - - ``8bit`` data is not constrained to be 7 bit clean. Data in headers is - still required to be ASCII-only and so will be encoded (see - 'binary_fold' below for an exception), but body parts may use - the ``8bit`` CTE. - ======== =============================================================== - - A ``cte_type`` value of ``8bit`` only works with ``BytesGenerator``, not - ``Generator``, because strings cannot contain binary data. If a - ``Generator`` is operating under a policy that specifies - ``cte_type=8bit``, it will act as if ``cte_type`` is ``7bit``. + If ``True``, data output by a bytes generator is limited to ASCII + characters. If :const:`False` (the default), then bytes with the high + bit set are preserved and/or allowed in certain contexts (for example, + where possible a content transfer encoding of ``8bit`` will be used). + String generators act as if ``must_be_7bit`` is ``True`` regardless of + the policy in effect, since a string cannot represent non-ASCII bytes. .. attribute:: raise_on_defect @@ -187,395 +129,56 @@ :const:`False` (the default), defects will be passed to the :meth:`register_defect` method. + :mod:`Policy` object also have the following methods: + .. method:: handle_defect(obj, defect) - .. attribute:: mangle_from\_ + *obj* is the object on which to register the defect. *defect* should be + an instance of a subclass of :class:`~email.errors.Defect`. + If :attr:`raise_on_defect` + is ``True`` the defect is raised as an exception. Otherwise *obj* and + *defect* are passed to :meth:`register_defect`. This method is intended + to be called by parsers when they encounter defects, and will not be + called by code that uses the email library unless that code is + implementing an alternate parser. - If :const:`True`, lines starting with *"From "* in the body are - escaped by putting a ``>`` in front of them. This parameter is used when - the message is being serialized by a generator. - Default: :const:`False`. + .. method:: register_defect(obj, defect) - .. versionadded:: 3.5 - The *mangle_from_* parameter. + *obj* is the object on which to register the defect. *defect* should be + a subclass of :class:`~email.errors.Defect`. This method is part of the + public API so that custom ``Policy`` subclasses can implement alternate + handling of defects. The default implementation calls the ``append`` + method of the ``defects`` attribute of *obj*. - The following :class:`Policy` method is intended to be called by code using - the email library to create policy instances with custom settings: - - .. method:: clone(**kw) + .. method:: clone(obj, *kw) Return a new :class:`Policy` instance whose attributes have the same values as the current instance, except where those attributes are given new values by the keyword arguments. - The remaining :class:`Policy` methods are called by the email package code, - and are not intended to be called by an application using the email package. - A custom policy must implement all of these methods. - .. method:: handle_defect(obj, defect) +Policy Instances +^^^^^^^^^^^^^^^^ - Handle a *defect* found on *obj*. When the email package calls this - method, *defect* will always be a subclass of - :class:`~email.errors.Defect`. - - The default implementation checks the :attr:`raise_on_defect` flag. If - it is ``True``, *defect* is raised as an exception. If it is ``False`` - (the default), *obj* and *defect* are passed to :meth:`register_defect`. - - .. method:: register_defect(obj, defect) - - Register a *defect* on *obj*. In the email package, *defect* will always - be a subclass of :class:`~email.errors.Defect`. - - The default implementation calls the ``append`` method of the ``defects`` - attribute of *obj*. When the email package calls :attr:`handle_defect`, - *obj* will normally have a ``defects`` attribute that has an ``append`` - method. Custom object types used with the email package (for example, - custom ``Message`` objects) should also provide such an attribute, - otherwise defects in parsed messages will raise unexpected errors. - - .. method:: header_max_count(name) - - Return the maximum allowed number of headers named *name*. - - Called when a header is added to a :class:`~email.message.Message` - object. If the returned value is not ``0`` or ``None``, and there are - already a number of headers with the name *name* equal to the value - returned, a :exc:`ValueError` is raised. - - Because the default behavior of ``Message.__setitem__`` is to append the - value to the list of headers, it is easy to create duplicate headers - without realizing it. This method allows certain headers to be limited - in the number of instances of that header that may be added to a - ``Message`` programmatically. (The limit is not observed by the parser, - which will faithfully produce as many headers as exist in the message - being parsed.) - - The default implementation returns ``None`` for all header names. - - .. method:: header_source_parse(sourcelines) - - The email package calls this method with a list of strings, each string - ending with the line separation characters found in the source being - parsed. The first line includes the field header name and separator. - All whitespace in the source is preserved. The method should return the - ``(name, value)`` tuple that is to be stored in the ``Message`` to - represent the parsed header. - - If an implementation wishes to retain compatibility with the existing - email package policies, *name* should be the case preserved name (all - characters up to the '``:``' separator), while *value* should be the - unfolded value (all line separator characters removed, but whitespace - kept intact), stripped of leading whitespace. - - *sourcelines* may contain surrogateescaped binary data. - - There is no default implementation - - .. method:: header_store_parse(name, value) - - The email package calls this method with the name and value provided by - the application program when the application program is modifying a - ``Message`` programmatically (as opposed to a ``Message`` created by a - parser). The method should return the ``(name, value)`` tuple that is to - be stored in the ``Message`` to represent the header. - - If an implementation wishes to retain compatibility with the existing - email package policies, the *name* and *value* should be strings or - string subclasses that do not change the content of the passed in - arguments. - - There is no default implementation - - .. method:: header_fetch_parse(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` when that header is requested by the - application program, and whatever the method returns is what is passed - back to the application as the value of the header being retrieved. - Note that there may be more than one header with the same name stored in - the ``Message``; the method is passed the specific name and value of the - header destined to be returned to the application. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the value returned by the method. - - There is no default implementation - - .. method:: fold(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` for a given header. The method should return a - string that represents that header "folded" correctly (according to the - policy settings) by composing the *name* with the *value* and inserting - :attr:`linesep` characters at the appropriate places. See :rfc:`5322` - for a discussion of the rules for folding email headers. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the string returned by the method. - - .. method:: fold_binary(name, value) - - The same as :meth:`fold`, except that the returned value should be a - bytes object rather than a string. - - *value* may contain surrogateescaped binary data. These could be - converted back into binary data in the returned bytes object. - - -.. class:: Compat32(**kw) - - This concrete :class:`Policy` is the backward compatibility policy. It - replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, - :const:`compat32`, that is used as the default policy. Thus the default - behavior of the email package is to maintain compatibility with Python 3.2. - - The following attributes have values that are different from the - :class:`Policy` default: - - .. attribute:: mangle_from_ - - The default is ``True``. - - The class provides the following concrete implementations of the - abstract methods of :class:`Policy`: - - .. method:: header_source_parse(sourcelines) - - The name is parsed as everything up to the '``:``' and returned - unmodified. The value is determined by stripping leading whitespace off - the remainder of the first line, joining all subsequent lines together, - and stripping any trailing carriage return or linefeed characters. - - .. method:: header_store_parse(name, value) - - The name and value are returned unmodified. - - .. method:: header_fetch_parse(name, value) - - If the value contains binary data, it is converted into a - :class:`~email.header.Header` object using the ``unknown-8bit`` charset. - Otherwise it is returned unmodified. - - .. method:: fold(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. Non-ASCII binary data are - CTE encoded using the ``unknown-8bit`` charset. - - .. method:: fold_binary(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. If ``cte_type`` is - ``7bit``, non-ascii binary data is CTE encoded using the ``unknown-8bit`` - charset. Otherwise the original source header is used, with its existing - line breaks and any (RFC invalid) binary data it may contain. - - -An instance of :class:`Compat32` is provided as a module constant: - -.. data:: compat32 - - An instance of :class:`Compat32`, providing backward compatibility with the - behavior of the email package in Python 3.2. - - -.. note:: - - The documentation below describes new policies that are included in the - standard library on a :term:`provisional basis `. - Backwards incompatible changes (up to and including removal of the feature) - may occur if deemed necessary by the core developers. - - -.. class:: EmailPolicy(**kw) - - This concrete :class:`Policy` provides behavior that is intended to be fully - compliant with the current email RFCs. These include (but are not limited - to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs. - - This policy adds new header parsing and folding algorithms. Instead of - simple strings, headers are ``str`` subclasses with attributes that depend - on the type of the field. The parsing and folding algorithm fully implement - :rfc:`2047` and :rfc:`5322`. - - In addition to the settable attributes listed above that apply to all - policies, this policy adds the following additional attributes: - - .. attribute:: utf8 - - If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages - formatted in this way may be passed to SMTP servers that support - the ``SMTPUTF8`` extension (:rfc:`6531`). - - .. attribute:: refold_source - - If the value for a header in the ``Message`` object originated from a - :mod:`~email.parser` (as opposed to being set by a program), this - attribute indicates whether or not a generator should refold that value - when transforming the message back into stream form. The possible values - are: - - ======== =============================================================== - ``none`` all source values use original folding - - ``long`` source values that have any line that is longer than - ``max_line_length`` will be refolded - - ``all`` all values are refolded. - ======== =============================================================== - - The default is ``long``. - - .. attribute:: header_factory - - A callable that takes two arguments, ``name`` and ``value``, where - ``name`` is a header field name and ``value`` is an unfolded header field - value, and returns a string subclass that represents that header. A - default ``header_factory`` (see :mod:`~email.headerregistry`) is provided - that understands some of the :RFC:`5322` header field types. (Currently - address fields and date fields have special treatment, while all other - fields are treated as unstructured. This list will be completed before - the extension is marked stable.) - - .. attribute:: content_manager - - An object with at least two methods: get_content and set_content. When - the :meth:`~email.message.Message.get_content` or - :meth:`~email.message.Message.set_content` method of a - :class:`~email.message.Message` object is called, it calls the - corresponding method of this object, passing it the message object as its - first argument, and any arguments or keywords that were passed to it as - additional arguments. By default ``content_manager`` is set to - :data:`~email.contentmanager.raw_data_manager`. - - .. versionadded:: 3.4 - - - The class provides the following concrete implementations of the abstract - methods of :class:`Policy`: - - .. method:: header_max_count(name) - - Returns the value of the - :attr:`~email.headerregistry.BaseHeader.max_count` attribute of the - specialized class used to represent the header with the given name. - - .. method:: header_source_parse(sourcelines) - - The implementation of this method is the same as that for the - :class:`Compat32` policy. - - .. method:: header_store_parse(name, value) - - The name is returned unchanged. If the input value has a ``name`` - attribute and it matches *name* ignoring case, the value is returned - unchanged. Otherwise the *name* and *value* are passed to - ``header_factory``, and the resulting header object is returned as - the value. In this case a ``ValueError`` is raised if the input value - contains CR or LF characters. - - .. method:: header_fetch_parse(name, value) - - If the value has a ``name`` attribute, it is returned to unmodified. - Otherwise the *name*, and the *value* with any CR or LF characters - removed, are passed to the ``header_factory``, and the resulting - header object is returned. Any surrogateescaped bytes get turned into - the unicode unknown-character glyph. - - .. method:: fold(name, value) - - Header folding is controlled by the :attr:`refold_source` policy setting. - A value is considered to be a 'source value' if and only if it does not - have a ``name`` attribute (having a ``name`` attribute means it is a - header object of some sort). If a source value needs to be refolded - according to the policy, it is converted into a header object by - passing the *name* and the *value* with any CR and LF characters removed - to the ``header_factory``. Folding of a header object is done by - calling its ``fold`` method with the current policy. - - Source values are split into lines using :meth:`~str.splitlines`. If - the value is not to be refolded, the lines are rejoined using the - ``linesep`` from the policy and returned. The exception is lines - containing non-ascii binary data. In that case the value is refolded - regardless of the ``refold_source`` setting, which causes the binary data - to be CTE encoded using the ``unknown-8bit`` charset. - - .. method:: fold_binary(name, value) - - The same as :meth:`fold` if :attr:`~Policy.cte_type` is ``7bit``, except - that the returned value is bytes. - - If :attr:`~Policy.cte_type` is ``8bit``, non-ASCII binary data is - converted back - into bytes. Headers with binary data are not refolded, regardless of the - ``refold_header`` setting, since there is no way to know whether the - binary data consists of single byte characters or multibyte characters. - -The following instances of :class:`EmailPolicy` provide defaults suitable for -specific application domains. Note that in the future the behavior of these -instances (in particular the ``HTTP`` instance) may be adjusted to conform even -more closely to the RFCs relevant to their domains. +The following instances of :class:`Policy` provide defaults suitable for +specific common application domains. .. data:: default - An instance of ``EmailPolicy`` with all defaults unchanged. This policy - uses the standard Python ``\n`` line endings rather than the RFC-correct - ``\r\n``. + An instance of :class:`Policy` with all defaults unchanged. .. data:: SMTP - Suitable for serializing messages in conformance with the email RFCs. - Like ``default``, but with ``linesep`` set to ``\r\n``, which is RFC - compliant. - -.. data:: SMTPUTF8 - - The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. - Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP trasmission if the - sender or recipient addresses have non-ASCII characters (the - :meth:`smtplib.SMTP.send_message` method handles this automatically). + Output serialized from a message will conform to the email and SMTP + RFCs. The only changed attribute is :attr:`linesep`, which is set to + ``\r\n``. .. data:: HTTP - Suitable for serializing headers with for use in HTTP traffic. Like - ``SMTP`` except that ``max_line_length`` is set to ``None`` (unlimited). + Suitable for use when serializing headers for use in HTTP traffic. + :attr:`linesep` is set to ``\r\n``, and :attr:`max_line_length` is set to + :const:`None` (unlimited). .. data:: strict - Convenience instance. The same as ``default`` except that - ``raise_on_defect`` is set to ``True``. This allows any policy to be made - strict by writing:: - - somepolicy + policy.strict - -With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of -the email package is changed from the Python 3.2 API in the following ways: - - * Setting a header on a :class:`~email.message.Message` results in that - header being parsed and a header object created. - - * Fetching a header value from a :class:`~email.message.Message` results - in that header being parsed and a header object created and - returned. - - * Any header object, or any header that is refolded due to the - policy settings, is folded using an algorithm that fully implements the - RFC folding algorithms, including knowing where encoded words are required - and allowed. - -From the application view, this means that any header obtained through the -:class:`~email.message.Message` is a header object with extra -attributes, whose string value is the fully decoded unicode value of the -header. Likewise, a header may be assigned a new value, or a new header -created, using a unicode string, and the policy will take care of converting -the unicode string into the correct RFC encoded form. - -The header objects and their attributes are described in -:mod:`~email.headerregistry`. + :attr:`raise_on_defect` is set to :const:`True`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.rst --- a/Doc/library/email.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.rst Mon Jan 25 17:05:13 2016 +0100 @@ -52,8 +52,6 @@ email.parser.rst email.generator.rst email.policy.rst - email.headerregistry.rst - email.contentmanager.rst email.mime.rst email.header.rst email.charset.rst @@ -91,19 +89,15 @@ +---------------+------------------------------+-----------------------+ | :const:`2.5` | Python 2.2.2+ and Python 2.3 | Python 2.1 to 2.5 | +---------------+------------------------------+-----------------------+ -| :const:`3.0` | Python 2.4 and Python 2.5 | Python 2.3 to 2.6 | +| :const:`3.0` | Python 2.4 | Python 2.3 to 2.5 | +---------------+------------------------------+-----------------------+ -| :const:`4.0` | Python 2.5 to Python 2.7 | Python 2.3 to 2.7 | +| :const:`4.0` | Python 2.5 | Python 2.3 to 2.5 | +---------------+------------------------------+-----------------------+ | :const:`5.0` | Python 3.0 and Python 3.1 | Python 3.0 to 3.2 | +---------------+------------------------------+-----------------------+ -| :const:`5.1` | Python 3.2 | Python 3.2 | +| :const:`5.1` | Python 3.2 | Python 3.0 to 3.2 | +---------------+------------------------------+-----------------------+ -After Version 5.1 (Python 3.2), the email package no longer has a version that -is separate from the Python version. (See the :ref:`whatsnew-index` documents -for the respective Python versions for details on changes.) - Here are the major differences between :mod:`email` version 5.1 and version 5.0: @@ -152,15 +146,14 @@ *Note that the version 3 names will continue to work until Python 2.6*. * The :mod:`email.mime.application` module was added, which contains the - :class:`~email.mime.application.MIMEApplication` class. + :class:`MIMEApplication` class. * Methods that were deprecated in version 3 have been removed. These include :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`. * Fixes have been added for :rfc:`2231` support which can change some of the - return types for :func:`Message.get_param ` - and friends. Under some + return types for :func:`Message.get_param` and friends. Under some circumstances, values which used to return a 3-tuple now return simple strings (specifically, if all extended parameter segments were unencoded, there is no language and charset designation expected, so the return type is now a simple @@ -169,24 +162,23 @@ Here are the major differences between :mod:`email` version 3 and version 2: -* The :class:`~email.parser.FeedParser` class was introduced, and the - :class:`~email.parser.Parser` class was implemented in terms of the - :class:`~email.parser.FeedParser`. All parsing therefore is +* The :class:`FeedParser` class was introduced, and the :class:`Parser` class + was implemented in terms of the :class:`FeedParser`. All parsing therefore is non-strict, and parsing will make a best effort never to raise an exception. Problems found while parsing messages are stored in the message's *defect* attribute. * All aspects of the API which raised :exc:`DeprecationWarning`\ s in version 2 have been removed. These include the *_encoder* argument to the - :class:`~email.mime.text.MIMEText` constructor, the - :meth:`Message.add_payload` method, the :func:`Utils.dump_address_pair` - function, and the functions :func:`Utils.decode` and :func:`Utils.encode`. + :class:`MIMEText` constructor, the :meth:`Message.add_payload` method, the + :func:`Utils.dump_address_pair` function, and the functions :func:`Utils.decode` + and :func:`Utils.encode`. * New :exc:`DeprecationWarning`\ s have been added to: :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`, and the *strict* - argument to the :class:`~email.parser.Parser` class. These are expected to - be removed in future versions. + argument to the :class:`Parser` class. These are expected to be removed in + future versions. * Support for Pythons earlier than 2.3 has been removed. @@ -194,61 +186,53 @@ * The :mod:`email.Header` and :mod:`email.Charset` modules have been added. -* The pickle format for :class:`~email.message.Message` instances has changed. - Since this was never (and still isn't) formally defined, this isn't - considered a backward incompatibility. However if your application pickles - and unpickles :class:`~email.message.Message` instances, be aware that in - :mod:`email` version 2, :class:`~email.message.Message` instances now have - private variables *_charset* and *_default_type*. +* The pickle format for :class:`Message` instances has changed. Since this was + never (and still isn't) formally defined, this isn't considered a backward + incompatibility. However if your application pickles and unpickles + :class:`Message` instances, be aware that in :mod:`email` version 2, + :class:`Message` instances now have private variables *_charset* and + *_default_type*. -* Several methods in the :class:`~email.message.Message` class have been - deprecated, or their signatures changed. Also, many new methods have been - added. See the documentation for the :class:`~email.message.Message` class - for details. The changes should be completely backward compatible. +* Several methods in the :class:`Message` class have been deprecated, or their + signatures changed. Also, many new methods have been added. See the + documentation for the :class:`Message` class for details. The changes should be + completely backward compatible. * The object structure has changed in the face of :mimetype:`message/rfc822` - content types. In :mod:`email` version 1, such a type would be represented - by a scalar payload, i.e. the container message's - :meth:`~email.message.Message.is_multipart` returned false, - :meth:`~email.message.Message.get_payload` was not a list object, but a - single :class:`~email.message.Message` instance. + content types. In :mod:`email` version 1, such a type would be represented by a + scalar payload, i.e. the container message's :meth:`is_multipart` returned + false, :meth:`get_payload` was not a list object, but a single :class:`Message` + instance. This structure was inconsistent with the rest of the package, so the object representation for :mimetype:`message/rfc822` content types was changed. In :mod:`email` version 2, the container *does* return ``True`` from - :meth:`~email.message.Message.is_multipart`, and - :meth:`~email.message.Message.get_payload` returns a list containing a single - :class:`~email.message.Message` item. + :meth:`is_multipart`, and :meth:`get_payload` returns a list containing a single + :class:`Message` item. - Note that this is one place that backward compatibility could not be - completely maintained. However, if you're already testing the return type of - :meth:`~email.message.Message.get_payload`, you should be fine. You just need - to make sure your code doesn't do a :meth:`~email.message.Message.set_payload` - with a :class:`~email.message.Message` instance on a container with a content - type of :mimetype:`message/rfc822`. + Note that this is one place that backward compatibility could not be completely + maintained. However, if you're already testing the return type of + :meth:`get_payload`, you should be fine. You just need to make sure your code + doesn't do a :meth:`set_payload` with a :class:`Message` instance on a container + with a content type of :mimetype:`message/rfc822`. -* The :class:`~email.parser.Parser` constructor's *strict* argument was added, - and its :meth:`~email.parser.Parser.parse` and - :meth:`~email.parser.Parser.parsestr` methods grew a *headersonly* argument. - The *strict* flag was also added to functions :func:`email.message_from_file` - and :func:`email.message_from_string`. +* The :class:`Parser` constructor's *strict* argument was added, and its + :meth:`parse` and :meth:`parsestr` methods grew a *headersonly* argument. The + *strict* flag was also added to functions :func:`email.message_from_file` and + :func:`email.message_from_string`. -* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten - ` instead. The - :class:`~email.generator.Generator` class has also grown the - :meth:`~email.generator.Generator.clone` method. +* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten` + instead. The :class:`Generator` class has also grown the :meth:`clone` method. -* The :class:`~email.generator.DecodedGenerator` class in the - :mod:`email.generator` module was added. +* The :class:`DecodedGenerator` class in the :mod:`email.Generator` module was + added. -* The intermediate base classes - :class:`~email.mime.nonmultipart.MIMENonMultipart` and - :class:`~email.mime.multipart.MIMEMultipart` have been added, and interposed - in the class hierarchy for most of the other MIME-related derived classes. +* The intermediate base classes :class:`MIMENonMultipart` and + :class:`MIMEMultipart` have been added, and interposed in the class hierarchy + for most of the other MIME-related derived classes. -* The *_encoder* argument to the :class:`~email.mime.text.MIMEText` constructor - has been deprecated. Encoding now happens implicitly based on the - *_charset* argument. +* The *_encoder* argument to the :class:`MIMEText` constructor has been + deprecated. Encoding now happens implicitly based on the *_charset* argument. * The following functions in the :mod:`email.Utils` module have been deprecated: :func:`dump_address_pairs`, :func:`decode`, and :func:`encode`. The following @@ -262,7 +246,7 @@ ------------------------------- The :mod:`email` package was originally prototyped as a separate library called -`mimelib `_. Changes have been made so that method names +`mimelib `_. Changes have been made so that method names are more consistent, and some methods or modules have either been added or removed. The semantics of some of the methods have also changed. For the most part, any functionality available in :mod:`mimelib` is still available in the @@ -281,22 +265,17 @@ * :func:`messageFromFile` has been renamed to :func:`message_from_file`. -The :class:`~email.message.Message` class has the following differences: +The :class:`Message` class has the following differences: -* The method :meth:`asString` was renamed to - :meth:`~email.message.Message.as_string`. +* The method :meth:`asString` was renamed to :meth:`as_string`. -* The method :meth:`ismultipart` was renamed to - :meth:`~email.message.Message.is_multipart`. +* The method :meth:`ismultipart` was renamed to :meth:`is_multipart`. -* The :meth:`~email.message.Message.get_payload` method has grown a *decode* - optional argument. +* The :meth:`get_payload` method has grown a *decode* optional argument. -* The method :meth:`getall` was renamed to - :meth:`~email.message.Message.get_all`. +* The method :meth:`getall` was renamed to :meth:`get_all`. -* The method :meth:`addheader` was renamed to - :meth:`~email.message.Message.add_header`. +* The method :meth:`addheader` was renamed to :meth:`add_header`. * The method :meth:`gettype` was renamed to :meth:`get_type`. @@ -304,57 +283,48 @@ * The method :meth:`getsubtype` was renamed to :meth:`get_subtype`. -* The method :meth:`getparams` was renamed to - :meth:`~email.message.Message.get_params`. Also, whereas :meth:`getparams` - returned a list of strings, :meth:`~email.message.Message.get_params` returns - a list of 2-tuples, effectively the key/value pairs of the parameters, split - on the ``'='`` sign. +* The method :meth:`getparams` was renamed to :meth:`get_params`. Also, whereas + :meth:`getparams` returned a list of strings, :meth:`get_params` returns a list + of 2-tuples, effectively the key/value pairs of the parameters, split on the + ``'='`` sign. -* The method :meth:`getparam` was renamed to - :meth:`~email.message.Message.get_param`. +* The method :meth:`getparam` was renamed to :meth:`get_param`. -* The method :meth:`getcharsets` was renamed to - :meth:`~email.message.Message.get_charsets`. +* The method :meth:`getcharsets` was renamed to :meth:`get_charsets`. -* The method :meth:`getfilename` was renamed to - :meth:`~email.message.Message.get_filename`. +* The method :meth:`getfilename` was renamed to :meth:`get_filename`. -* The method :meth:`getboundary` was renamed to - :meth:`~email.message.Message.get_boundary`. +* The method :meth:`getboundary` was renamed to :meth:`get_boundary`. -* The method :meth:`setboundary` was renamed to - :meth:`~email.message.Message.set_boundary`. +* The method :meth:`setboundary` was renamed to :meth:`set_boundary`. * The method :meth:`getdecodedpayload` was removed. To get similar - functionality, pass the value 1 to the *decode* flag of the - :meth:`~email.message.Message.get_payload` method. + functionality, pass the value 1 to the *decode* flag of the get_payload() + method. * The method :meth:`getpayloadastext` was removed. Similar functionality is - supported by the :class:`~email.generator.DecodedGenerator` class in the - :mod:`email.generator` module. + supported by the :class:`DecodedGenerator` class in the :mod:`email.generator` + module. * The method :meth:`getbodyastext` was removed. You can get similar - functionality by creating an iterator with - :func:`~email.iterators.typed_subpart_iterator` in the :mod:`email.iterators` - module. + functionality by creating an iterator with :func:`typed_subpart_iterator` in the + :mod:`email.iterators` module. -The :class:`~email.parser.Parser` class has no differences in its public -interface. It does have some additional smarts to recognize -:mimetype:`message/delivery-status` type messages, which it represents as a -:class:`~email.message.Message` instance containing separate -:class:`~email.message.Message` subparts for each header block in the delivery -status notification [#]_. +The :class:`Parser` class has no differences in its public interface. It does +have some additional smarts to recognize :mimetype:`message/delivery-status` +type messages, which it represents as a :class:`Message` instance containing +separate :class:`Message` subparts for each header block in the delivery status +notification [#]_. -The :class:`~email.generator.Generator` class has no differences in its public -interface. There is a new class in the :mod:`email.generator` module though, -called :class:`~email.generator.DecodedGenerator` which provides most of the -functionality previously available in the :meth:`Message.getpayloadastext` -method. +The :class:`Generator` class has no differences in its public interface. There +is a new class in the :mod:`email.generator` module though, called +:class:`DecodedGenerator` which provides most of the functionality previously +available in the :meth:`Message.getpayloadastext` method. The following modules and classes have been changed: -* The :class:`~email.mime.base.MIMEBase` class constructor arguments *_major* - and *_minor* have changed to *_maintype* and *_subtype* respectively. +* The :class:`MIMEBase` class constructor arguments *_major* and *_minor* have + changed to *_maintype* and *_subtype* respectively. * The ``Image`` class/module has been renamed to ``MIMEImage``. The *_minor* argument has been renamed to *_subtype*. @@ -367,8 +337,7 @@ but that clashed with the Python standard library module :mod:`rfc822` on some case-insensitive file systems. - Also, the :class:`~email.mime.message.MIMEMessage` class now represents any - kind of MIME message + Also, the :class:`MIMEMessage` class now represents any kind of MIME message with main type :mimetype:`message`. It takes an optional argument *_subtype* which is used to set the MIME subtype. *_subtype* defaults to :mimetype:`rfc822`. @@ -378,8 +347,8 @@ :mod:`email.utils` module. The ``MsgReader`` class/module has been removed. Its functionality is most -closely supported in the :func:`~email.iterators.body_line_iterator` function -in the :mod:`email.iterators` module. +closely supported in the :func:`body_line_iterator` function in the +:mod:`email.iterators` module. .. rubric:: Footnotes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/email.util.rst --- a/Doc/library/email.util.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/email.util.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- +:mod:`email`: Miscellaneous utilities +------------------------------------- .. module:: email.utils :synopsis: Miscellaneous email package utilities. @@ -41,16 +41,15 @@ characters. Can be an instance of :class:`str` or a :class:`~email.charset.Charset`. Defaults to ``utf-8``. - .. versionchanged:: 3.3 - Added the *charset* option. + .. versionchanged: 3.3 added the *charset* option .. function:: getaddresses(fieldvalues) This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: + :meth:`Message.get_all`. Here's a simple example that gets all the recipients + of a message:: from email.utils import getaddresses @@ -98,9 +97,12 @@ .. function:: mktime_tz(tuple) - Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC - timestamp (seconds since the Epoch). If the timezone item in the - tuple is ``None``, assume local time. + Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC timestamp. It + the timezone item in the tuple is ``None``, assume local time. Minor + deficiency: :func:`mktime_tz` interprets the first 8 elements of *tuple* as a + local time and then compensates for the timezone difference. This may yield a + slight error around changes in daylight savings time, though not worth worrying + about for common use. .. function:: formatdate(timeval=None, localtime=False, usegmt=False) @@ -138,22 +140,6 @@ .. versionadded:: 3.3 -.. function:: localtime(dt=None) - - Return local time as an aware datetime object. If called without - arguments, return current time. Otherwise *dt* argument should be a - :class:`~datetime.datetime` instance, and it is converted to the local time - zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. In this - case, a positive or zero value for *isdst* causes ``localtime`` to presume - initially that summer time (for example, Daylight Saving Time) is or is not - (respectively) in effect for the specified time. A negative value for - *isdst* causes the ``localtime`` to attempt to divine whether summer time - is in effect for the specified time. - - .. versionadded:: 3.3 - - .. function:: make_msgid(idstring=None, domain=None) Returns a string suitable for an :rfc:`2822`\ -compliant @@ -164,8 +150,7 @@ may be useful certain cases, such as a constructing distributed system that uses a consistent domain name across multiple hosts. - .. versionchanged:: 3.2 - Added the *domain* keyword. + .. versionchanged:: 3.2 domain keyword added .. function:: decode_rfc2231(s) @@ -184,11 +169,10 @@ .. function:: collapse_rfc2231_value(value, errors='replace', fallback_charset='us-ascii') When a header parameter is encoded in :rfc:`2231` format, - :meth:`Message.get_param ` may return a - 3-tuple containing the character set, + :meth:`Message.get_param` may return a 3-tuple containing the character set, language, and value. :func:`collapse_rfc2231_value` turns this into a unicode string. Optional *errors* is passed to the *errors* argument of :class:`str`'s - :func:`~str.encode` method; it defaults to ``'replace'``. Optional + :func:`encode` method; it defaults to ``'replace'``. Optional *fallback_charset* specifies the character set to use if the one in the :rfc:`2231` header is not known by Python; it defaults to ``'us-ascii'``. @@ -207,3 +191,4 @@ .. [#] Note that the sign of the timezone offset is the opposite of the sign of the ``time.timezone`` variable for the same timezone; the latter variable follows the POSIX standard while this module follows :rfc:`2822`. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ensurepip.rst --- a/Doc/library/ensurepip.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,131 +0,0 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== - -.. module:: ensurepip - :synopsis: Bootstrapping the "pip" installer into an existing Python - installation or virtual environment. - -.. versionadded:: 3.4 - -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` -installer into an existing Python installation or virtual environment. This -bootstrapping approach reflects the fact that ``pip`` is an independent -project with its own release cycle, and the latest available stable version -is bundled with maintenance and feature releases of the CPython reference -interpreter. - -In most cases, end users of Python shouldn't need to invoke this module -directly (as ``pip`` should be bootstrapped by default), but it may be -needed if installing ``pip`` was skipped when installing Python (or -when creating a virtual environment) or after explicitly uninstalling -``pip``. - -.. note:: - - This module *does not* access the internet. All of the components - needed to bootstrap ``pip`` are included as internal parts of the - package. - -.. seealso:: - - :ref:`installing-index` - The end user guide for installing Python packages - - :pep:`453`: Explicit bootstrapping of pip in Python installations - The original rationale and specification for this module. - - -Command line interface ----------------------- - -The command line interface is invoked using the interpreter's ``-m`` switch. - -The simplest possible invocation is:: - - python -m ensurepip - -This invocation will install ``pip`` if it is not already installed, -but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one bundled with ``ensurepip``, pass the -``--upgrade`` option:: - - python -m ensurepip --upgrade - -By default, ``pip`` is installed into the current virtual environment -(if one is active) or into the system site packages (if there is no -active virtual environment). The installation location can be controlled -through two additional command line options: - -* ``--root ``: Installs ``pip`` relative to the given root directory - rather than the root of the currently active virtual environment (if any) - or the default root for the current Python installation. -* ``--user``: Installs ``pip`` into the user site packages directory rather - than globally for the current Python installation (this option is not - permitted inside an active virtual environment). - -By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where -X.Y stands for the version of Python used to invoke ``ensurepip``). The -scripts installed can be controlled through two additional command line -options: - -* ``--altinstall``: if an alternate installation is requested, the ``pipX`` - script will *not* be installed. - -* ``--default-pip``: if a "default pip" installation is requested, the - ``pip`` script will be installed in addition to the two regular scripts. - -Providing both of the script selection options will trigger an exception. - - -Module API ----------- - -:mod:`ensurepip` exposes two functions for programmatic use: - -.. function:: version() - - Returns a string specifying the bundled version of pip that will be - installed when bootstrapping an environment. - -.. function:: bootstrap(root=None, upgrade=False, user=False, \ - altinstall=False, default_pip=False, \ - verbosity=0) - - Bootstraps ``pip`` into the current or designated environment. - - *root* specifies an alternative root directory to install relative to. - If *root* is None, then installation uses the default install location - for the current environment. - - *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the bundled version. - - *user* indicates whether to use the user scheme rather than installing - globally. - - By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where - X.Y stands for the current version of Python). - - If *altinstall* is set, then ``pipX`` will *not* be installed. - - If *default_pip* is set, then ``pip`` will be installed in addition to - the two regular scripts. - - Setting both *altinstall* and *default_pip* will trigger - :exc:`ValueError`. - - *verbosity* controls the level of output to :data:`sys.stdout` from the - bootstrapping operation. - - .. note:: - - The bootstrapping process has side effects on both ``sys.path`` and - ``os.environ``. Invoking the command line interface in a subprocess - instead allows these side effects to be avoided. - - .. note:: - - The bootstrapping process may install additional modules required by - ``pip``, but other software should not assume those dependencies will - always be present by default (as the dependencies may be removed in a - future version of ``pip``). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/enum.rst --- a/Doc/library/enum.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,777 +0,0 @@ -:mod:`enum` --- Support for enumerations -======================================== - -.. module:: enum - :synopsis: Implementation of an enumeration class. - -.. :moduleauthor:: Ethan Furman -.. :sectionauthor:: Barry Warsaw , -.. :sectionauthor:: Eli Bendersky , -.. :sectionauthor:: Ethan Furman - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/enum.py` - ----------------- - -An enumeration is a set of symbolic names (members) bound to unique, -constant values. Within an enumeration, the members can be compared -by identity, and the enumeration itself can be iterated over. - - -Module Contents ---------------- - -This module defines two enumeration classes that can be used to define unique -sets of names and values: :class:`Enum` and :class:`IntEnum`. It also defines -one decorator, :func:`unique`. - -.. class:: Enum - - Base class for creating enumerated constants. See section - `Functional API`_ for an alternate construction syntax. - -.. class:: IntEnum - - Base class for creating enumerated constants that are also - subclasses of :class:`int`. - -.. function:: unique - - Enum class decorator that ensures only one name is bound to any one value. - - -Creating an Enum ----------------- - -Enumerations are created using the :keyword:`class` syntax, which makes them -easy to read and write. An alternative creation method is described in -`Functional API`_. To define an enumeration, subclass :class:`Enum` as -follows:: - - >>> from enum import Enum - >>> class Color(Enum): - ... red = 1 - ... green = 2 - ... blue = 3 - ... - -.. note:: Nomenclature - - - The class :class:`Color` is an *enumeration* (or *enum*) - - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are - *enumeration members* (or *enum members*). - - The enum members have *names* and *values* (the name of - :attr:`Color.red` is ``red``, the value of :attr:`Color.blue` is - ``3``, etc.) - -.. note:: - - Even though we use the :keyword:`class` syntax to create Enums, Enums - are not normal Python classes. See `How are Enums different?`_ for - more details. - -Enumeration members have human readable string representations:: - - >>> print(Color.red) - Color.red - -...while their ``repr`` has more information:: - - >>> print(repr(Color.red)) - - -The *type* of an enumeration member is the enumeration it belongs to:: - - >>> type(Color.red) - - >>> isinstance(Color.green, Color) - True - >>> - -Enum members also have a property that contains just their item name:: - - >>> print(Color.red.name) - red - -Enumerations support iteration, in definition order:: - - >>> class Shake(Enum): - ... vanilla = 7 - ... chocolate = 4 - ... cookies = 9 - ... mint = 3 - ... - >>> for shake in Shake: - ... print(shake) - ... - Shake.vanilla - Shake.chocolate - Shake.cookies - Shake.mint - -Enumeration members are hashable, so they can be used in dictionaries and sets:: - - >>> apples = {} - >>> apples[Color.red] = 'red delicious' - >>> apples[Color.green] = 'granny smith' - >>> apples == {Color.red: 'red delicious', Color.green: 'granny smith'} - True - - -Programmatic access to enumeration members and their attributes ---------------------------------------------------------------- - -Sometimes it's useful to access members in enumerations programmatically (i.e. -situations where ``Color.red`` won't do because the exact color is not known -at program-writing time). ``Enum`` allows such access:: - - >>> Color(1) - - >>> Color(3) - - -If you want to access enum members by *name*, use item access:: - - >>> Color['red'] - - >>> Color['green'] - - -If you have an enum member and need its :attr:`name` or :attr:`value`:: - - >>> member = Color.red - >>> member.name - 'red' - >>> member.value - 1 - - -Duplicating enum members and values ------------------------------------ - -Having two enum members with the same name is invalid:: - - >>> class Shape(Enum): - ... square = 2 - ... square = 3 - ... - Traceback (most recent call last): - ... - TypeError: Attempted to reuse key: 'square' - -However, two enum members are allowed to have the same value. Given two members -A and B with the same value (and A defined first), B is an alias to A. By-value -lookup of the value of A and B will return A. By-name lookup of B will also -return A:: - - >>> class Shape(Enum): - ... square = 2 - ... diamond = 1 - ... circle = 3 - ... alias_for_square = 2 - ... - >>> Shape.square - - >>> Shape.alias_for_square - - >>> Shape(2) - - -.. note:: - - Attempting to create a member with the same name as an already - defined attribute (another member, a method, etc.) or attempting to create - an attribute with the same name as a member is not allowed. - - -Ensuring unique enumeration values ----------------------------------- - -By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, the following decorator can be used to -ensure each value is used only once in the enumeration: - -.. decorator:: unique - -A :keyword:`class` decorator specifically for enumerations. It searches an -enumeration's :attr:`__members__` gathering any aliases it finds; if any are -found :exc:`ValueError` is raised with the details:: - - >>> from enum import Enum, unique - >>> @unique - ... class Mistake(Enum): - ... one = 1 - ... two = 2 - ... three = 3 - ... four = 3 - ... - Traceback (most recent call last): - ... - ValueError: duplicate values found in : four -> three - - -Iteration ---------- - -Iterating over the members of an enum does not provide the aliases:: - - >>> list(Shape) - [, , ] - -The special attribute ``__members__`` is an ordered dictionary mapping names -to members. It includes all names defined in the enumeration, including the -aliases:: - - >>> for name, member in Shape.__members__.items(): - ... name, member - ... - ('square', ) - ('diamond', ) - ('circle', ) - ('alias_for_square', ) - -The ``__members__`` attribute can be used for detailed programmatic access to -the enumeration members. For example, finding all the aliases:: - - >>> [name for name, member in Shape.__members__.items() if member.name != name] - ['alias_for_square'] - - -Comparisons ------------ - -Enumeration members are compared by identity:: - - >>> Color.red is Color.red - True - >>> Color.red is Color.blue - False - >>> Color.red is not Color.blue - True - -Ordered comparisons between enumeration values are *not* supported. Enum -members are not integers (but see `IntEnum`_ below):: - - >>> Color.red < Color.blue - Traceback (most recent call last): - File "", line 1, in - TypeError: unorderable types: Color() < Color() - -Equality comparisons are defined though:: - - >>> Color.blue == Color.red - False - >>> Color.blue != Color.red - True - >>> Color.blue == Color.blue - True - -Comparisons against non-enumeration values will always compare not equal -(again, :class:`IntEnum` was explicitly designed to behave differently, see -below):: - - >>> Color.blue == 2 - False - - -Allowed members and attributes of enumerations ----------------------------------------------- - -The examples above use integers for enumeration values. Using integers is -short and handy (and provided by default by the `Functional API`_), but not -strictly enforced. In the vast majority of use-cases, one doesn't care what -the actual value of an enumeration is. But if the value *is* important, -enumerations can have arbitrary values. - -Enumerations are Python classes, and can have methods and special methods as -usual. If we have this enumeration:: - - >>> class Mood(Enum): - ... funky = 1 - ... happy = 3 - ... - ... def describe(self): - ... # self is the member here - ... return self.name, self.value - ... - ... def __str__(self): - ... return 'my custom str! {0}'.format(self.value) - ... - ... @classmethod - ... def favorite_mood(cls): - ... # cls here is the enumeration - ... return cls.happy - ... - -Then:: - - >>> Mood.favorite_mood() - - >>> Mood.happy.describe() - ('happy', 3) - >>> str(Mood.funky) - 'my custom str! 1' - -The rules for what is allowed are as follows: names that start and end with a -with a single underscore are reserved by enum and cannot be used; all other -attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.) and descriptors (methods are also descriptors). - -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then -whatever value(s) were given to the enum member will be passed into those -methods. See `Planet`_ for an example. - - -Restricted subclassing of enumerations --------------------------------------- - -Subclassing an enumeration is allowed only if the enumeration does not define -any members. So this is forbidden:: - - >>> class MoreColor(Color): - ... pink = 17 - ... - Traceback (most recent call last): - ... - TypeError: Cannot extend enumerations - -But this is allowed:: - - >>> class Foo(Enum): - ... def some_behavior(self): - ... pass - ... - >>> class Bar(Foo): - ... happy = 1 - ... sad = 2 - ... - -Allowing subclassing of enums that define members would lead to a violation of -some important invariants of types and instances. On the other hand, it makes -sense to allow sharing some common behavior between a group of enumerations. -(See `OrderedEnum`_ for an example.) - - -Pickling --------- - -Enumerations can be pickled and unpickled:: - - >>> from test.test_enum import Fruit - >>> from pickle import dumps, loads - >>> Fruit.tomato is loads(dumps(Fruit.tomato)) - True - -The usual restrictions for pickling apply: picklable enums must be defined in -the top level of a module, since unpickling requires them to be importable -from that module. - -.. note:: - - With pickle protocol version 4 it is possible to easily pickle enums - nested in other classes. - -It is possible to modify how Enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. - - -Functional API --------------- - -The :class:`Enum` class is callable, providing the following functional API:: - - >>> Animal = Enum('Animal', 'ant bee cat dog') - >>> Animal - - >>> Animal.ant - - >>> Animal.ant.value - 1 - >>> list(Animal) - [, , , ] - -The semantics of this API resemble :class:`~collections.namedtuple`. The first -argument of the call to :class:`Enum` is the name of the enumeration. - -The second argument is the *source* of enumeration member names. It can be a -whitespace-separated string of names, a sequence of names, a sequence of -2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to -values. The last two options enable assigning arbitrary values to -enumerations; the others auto-assign increasing integers starting with 1 (use -the ``start`` parameter to specify a different starting value). A -new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: - - >>> class Animal(Enum): - ... ant = 1 - ... bee = 2 - ... cat = 3 - ... dog = 4 - ... - -The reason for defaulting to ``1`` as the starting number and not ``0`` is -that ``0`` is ``False`` in a boolean sense, but enum members all evaluate -to ``True``. - -Pickling enums created with the functional API can be tricky as frame stack -implementation details are used to try and figure out which module the -enumeration is being created in (e.g. it will fail if you use a utility -function in separate module, and also may not work on IronPython or Jython). -The solution is to specify the module name explicitly as follows:: - - >>> Animal = Enum('Animal', 'ant bee cat dog', module=__name__) - -.. warning:: - - If ``module`` is not supplied, and Enum cannot determine what it is, - the new Enum members will not be unpicklable; to keep errors closer to - the source, pickling will be disabled. - -The new pickle protocol 4 also, in some circumstances, relies on -:attr:`__qualname__` being set to the location where pickle will be able -to find the class. For example, if the class was made available in class -SomeData in the global scope:: - - >>> Animal = Enum('Animal', 'ant bee cat dog', qualname='SomeData.Animal') - -The complete signature is:: - - Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=, start=1) - -:value: What the new Enum class will record as its name. - -:names: The Enum members. This can be a whitespace or comma separated string - (values will start at 1 unless otherwise specified):: - - 'red green blue' | 'red,green,blue' | 'red, green, blue' - - or an iterator of names:: - - ['red', 'green', 'blue'] - - or an iterator of (name, value) pairs:: - - [('cyan', 4), ('magenta', 5), ('yellow', 6)] - - or a mapping:: - - {'chartreuse': 7, 'sea_green': 11, 'rosemary': 42} - -:module: name of module where new Enum class can be found. - -:qualname: where in module new Enum class can be found. - -:type: type to mix in to new Enum class. - -:start: number to start counting at if only names are passed in. - -.. versionchanged:: 3.5 - The *start* parameter was added. - - -Derived Enumerations --------------------- - -IntEnum -^^^^^^^ - -A variation of :class:`Enum` is provided which is also a subclass of -:class:`int`. Members of an :class:`IntEnum` can be compared to integers; -by extension, integer enumerations of different types can also be compared -to each other:: - - >>> from enum import IntEnum - >>> class Shape(IntEnum): - ... circle = 1 - ... square = 2 - ... - >>> class Request(IntEnum): - ... post = 1 - ... get = 2 - ... - >>> Shape == 1 - False - >>> Shape.circle == 1 - True - >>> Shape.circle == Request.post - True - -However, they still can't be compared to standard :class:`Enum` enumerations:: - - >>> class Shape(IntEnum): - ... circle = 1 - ... square = 2 - ... - >>> class Color(Enum): - ... red = 1 - ... green = 2 - ... - >>> Shape.circle == Color.red - False - -:class:`IntEnum` values behave like integers in other ways you'd expect:: - - >>> int(Shape.circle) - 1 - >>> ['a', 'b', 'c'][Shape.circle] - 'b' - >>> [i for i in range(Shape.square)] - [0, 1] - -For the vast majority of code, :class:`Enum` is strongly recommended, -since :class:`IntEnum` breaks some semantic promises of an enumeration (by -being comparable to integers, and thus by transitivity to other -unrelated enumerations). It should be used only in special cases where -there's no other choice; for example, when integer constants are -replaced with enumerations and backwards compatibility is required with code -that still expects integers. - - -Others -^^^^^^ - -While :class:`IntEnum` is part of the :mod:`enum` module, it would be very -simple to implement independently:: - - class IntEnum(int, Enum): - pass - -This demonstrates how similar derived enumerations can be defined; for example -a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`. - -Some rules: - -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` - example above. -2. While :class:`Enum` can have members of any type, once you mix in an - additional type, all the members must have values of that type, e.g. - :class:`int` above. This restriction does not apply to mix-ins which only - add methods and don't specify another data type such as :class:`int` or - :class:`str`. -3. When another data type is mixed in, the :attr:`value` attribute is *not the - same* as the enum member itself, although it is equivalent and will compare - equal. -4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and - :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for - IntEnum) treat the enum member as its mixed-in type. -5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in - type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or - :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes. - - -Interesting examples --------------------- - -While :class:`Enum` and :class:`IntEnum` are expected to cover the majority of -use-cases, they cannot cover them all. Here are recipes for some different -types of enumerations that can be used directly, or as examples for creating -one's own. - - -AutoNumber -^^^^^^^^^^ - -Avoids having to specify the value for each enumeration member:: - - >>> class AutoNumber(Enum): - ... def __new__(cls): - ... value = len(cls.__members__) + 1 - ... obj = object.__new__(cls) - ... obj._value_ = value - ... return obj - ... - >>> class Color(AutoNumber): - ... red = () - ... green = () - ... blue = () - ... - >>> Color.green.value == 2 - True - -.. note:: - - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after - class creation for lookup of existing members. - - -OrderedEnum -^^^^^^^^^^^ - -An ordered enumeration that is not based on :class:`IntEnum` and so maintains -the normal :class:`Enum` invariants (such as not being comparable to other -enumerations):: - - >>> class OrderedEnum(Enum): - ... def __ge__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value >= other.value - ... return NotImplemented - ... def __gt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value > other.value - ... return NotImplemented - ... def __le__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value <= other.value - ... return NotImplemented - ... def __lt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value < other.value - ... return NotImplemented - ... - >>> class Grade(OrderedEnum): - ... A = 5 - ... B = 4 - ... C = 3 - ... D = 2 - ... F = 1 - ... - >>> Grade.C < Grade.A - True - - -DuplicateFreeEnum -^^^^^^^^^^^^^^^^^ - -Raises an error if a duplicate member name is found instead of creating an -alias:: - - >>> class DuplicateFreeEnum(Enum): - ... def __init__(self, *args): - ... cls = self.__class__ - ... if any(self.value == e.value for e in cls): - ... a = self.name - ... e = cls(self.value).name - ... raise ValueError( - ... "aliases not allowed in DuplicateFreeEnum: %r --> %r" - ... % (a, e)) - ... - >>> class Color(DuplicateFreeEnum): - ... red = 1 - ... green = 2 - ... blue = 3 - ... grene = 2 - ... - Traceback (most recent call last): - ... - ValueError: aliases not allowed in DuplicateFreeEnum: 'grene' --> 'green' - -.. note:: - - This is a useful example for subclassing Enum to add or change other - behaviors as well as disallowing aliases. If the only desired change is - disallowing aliases, the :func:`unique` decorator can be used instead. - - -Planet -^^^^^^ - -If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member -will be passed to those methods:: - - >>> class Planet(Enum): - ... MERCURY = (3.303e+23, 2.4397e6) - ... VENUS = (4.869e+24, 6.0518e6) - ... EARTH = (5.976e+24, 6.37814e6) - ... MARS = (6.421e+23, 3.3972e6) - ... JUPITER = (1.9e+27, 7.1492e7) - ... SATURN = (5.688e+26, 6.0268e7) - ... URANUS = (8.686e+25, 2.5559e7) - ... NEPTUNE = (1.024e+26, 2.4746e7) - ... def __init__(self, mass, radius): - ... self.mass = mass # in kilograms - ... self.radius = radius # in meters - ... @property - ... def surface_gravity(self): - ... # universal gravitational constant (m3 kg-1 s-2) - ... G = 6.67300E-11 - ... return G * self.mass / (self.radius * self.radius) - ... - >>> Planet.EARTH.value - (5.976e+24, 6378140.0) - >>> Planet.EARTH.surface_gravity - 9.802652743337129 - - -How are Enums different? ------------------------- - -Enums have a custom metaclass that affects many aspects of both derived Enum -classes and their instances (members). - - -Enum Classes -^^^^^^^^^^^^ - -The :class:`EnumMeta` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that -allow one to do things with an :class:`Enum` class that fail on a typical -class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is -responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). - - -Enum Members (aka instances) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The most interesting thing about Enum members is that they are singletons. -:class:`EnumMeta` creates them all while it is creating the :class:`Enum` -class itself, and then puts a custom :meth:`__new__` in place to ensure -that no new ones are ever instantiated by returning only the existing -member instances. - - -Finer Points -^^^^^^^^^^^^ - -:class:`Enum` members are instances of an :class:`Enum` class, and even -though they are accessible as `EnumClass.member`, they should not be accessed -directly from the member as that lookup may fail or, worse, return something -besides the :class:`Enum` member you looking for:: - - >>> class FieldTypes(Enum): - ... name = 0 - ... value = 1 - ... size = 2 - ... - >>> FieldTypes.value.size - - >>> FieldTypes.size.value - 2 - -.. versionchanged:: 3.5 - -Boolean evaluation: Enum classes that are mixed with non-Enum types (such as -:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in -type's rules; otherwise, all members evaluate as ``True``. To make your own -Enum's boolean evaluation depend on the member's value add the following to -your class:: - - def __bool__(self): - return bool(self.value) - -The :attr:`__members__` attribute is only available on the class. - -If you give your :class:`Enum` subclass extra methods, like the `Planet`_ -class above, those methods will show up in a :func:`dir` of the member, -but not of the class:: - - >>> dir(Planet) - ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] - >>> dir(Planet.EARTH) - ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] - -The :meth:`__new__` method will only be used for the creation of the -:class:`Enum` members -- after that it is replaced. Any custom :meth:`__new__` -method must create the object and set the :attr:`_value_` attribute -appropriately. - -If you wish to change how :class:`Enum` members are looked up you should either -write a helper function or a :func:`classmethod` for the :class:`Enum` -subclass. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/errno.rst --- a/Doc/library/errno.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/errno.rst Mon Jan 25 17:05:13 2016 +0100 @@ -41,10 +41,7 @@ .. data:: EINTR - Interrupted system call. - - .. seealso:: - This error is mapped to the exception :exc:`InterruptedError`. + Interrupted system call .. data:: EIO diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/exceptions.rst --- a/Doc/library/exceptions.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/exceptions.rst Mon Jan 25 17:05:13 2016 +0100 @@ -28,43 +28,29 @@ interpreter raises the same exception; but beware that there is nothing to prevent user code from raising an inappropriate error. -The built-in exception classes can be subclassed to define new exceptions; -programmers are encouraged to derive new exceptions from the :exc:`Exception` -class or one of its subclasses, and not from :exc:`BaseException`. More -information on defining exceptions is available in the Python Tutorial under +The built-in exception classes can be sub-classed to define new exceptions; +programmers are encouraged to at least derive new exceptions from the +:exc:`Exception` class and not :exc:`BaseException`. More information on +defining exceptions is available in the Python Tutorial under :ref:`tut-userexceptions`. -When raising (or re-raising) an exception in an :keyword:`except` or -:keyword:`finally` clause +When raising (or re-raising) an exception in an :keyword:`except` clause :attr:`__context__` is automatically set to the last exception caught; if the new exception is not handled the traceback that is eventually displayed will include the originating exception(s) and the final exception. -When raising a new exception (rather than using a bare ``raise`` to re-raise -the exception currently being handled), the implicit exception context can be -supplemented with an explicit cause by using :keyword:`from` with -:keyword:`raise`:: +This implicit exception chain can be made explicit by using :keyword:`from` +with :keyword:`raise`. The single argument to :keyword:`from` must be an +exception or :const:`None`, and it will be set as :attr:`__cause__` on the +raised exception. If :attr:`__cause__` is an exception it will be displayed +instead of :attr:`__context__`; if :attr:`__cause__` is None, +:attr:`__context__` will not be displayed by the default exception handling +code. (Note: the default value for :attr:`__context__` is :const:`None`, +while the default value for :attr:`__cause__` is :const:`Ellipsis`.) - raise new_exc from original_exc - -The expression following :keyword:`from` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`, while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. - -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. - -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. +In either case, the default exception handling code will not display +any of the remaining links in the :attr:`__context__` chain if +:attr:`__cause__` has been set. Base classes @@ -83,7 +69,7 @@ .. attribute:: args The tuple of arguments given to the exception constructor. Some built-in - exceptions (like :exc:`OSError`) expect a certain number of arguments and + exceptions (like :exc:`IOError`) expect a certain number of arguments and assign a special meaning to the elements of this tuple, while others are usually called only with a single string giving an error message. @@ -147,9 +133,10 @@ .. exception:: EOFError - Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and - :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) + Raised when one of the built-in functions (:func:`input` or :func:`raw_input`) + hits an end-of-file condition (EOF) without reading any data. (N.B.: the + :meth:`file.read` and :meth:`file.readline` methods return an empty string + when they hit EOF.) .. exception:: FloatingPointError @@ -162,8 +149,7 @@ .. exception:: GeneratorExit - Raised when a :term:`generator` or :term:`coroutine` is closed; - see :meth:`generator.close` and :meth:`coroutine.close`. It + Raise when a :term:`generator`\'s :meth:`close` method is called. It directly inherits from :exc:`BaseException` instead of :exc:`Exception` since it is technically not an error. @@ -173,14 +159,6 @@ Raised when an :keyword:`import` statement fails to find the module definition or when a ``from ... import`` fails to find a name that is to be imported. - The :attr:`name` and :attr:`path` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the module - that was attempted to be imported and the path to any file which triggered - the exception, respectively. - - .. versionchanged:: 3.3 - Added the :attr:`name` and :attr:`path` attributes. - .. exception:: IndexError @@ -232,91 +210,43 @@ classes to override the method. -.. exception:: OSError([arg]) - OSError(errno, strerror[, filename[, winerror[, filename2]]]) +.. exception:: OSError .. index:: module: errno This exception is raised when a system function returns a system-related error, including I/O failures such as "file not found" or "disk full" - (not for illegal argument types or other incidental errors). + (not for illegal argument types or other incidental errors). Often a + subclass of :exc:`OSError` will actually be raised as described in + `OS exceptions`_ below. The :attr:`errno` attribute is a numeric error + code from the C variable :c:data:`errno`. - The second form of the constructor sets the corresponding attributes, - described below. The attributes default to :const:`None` if not - specified. For backwards compatibility, if three arguments are passed, - the :attr:`~BaseException.args` attribute contains only a 2-tuple - of the first two constructor arguments. + Under Windows, the :attr:`winerror` attribute gives you the native + Windows error code. The :attr:`errno` attribute is then an approximate + translation, in POSIX terms, of that native error code. - The constructor often actually returns a subclass of :exc:`OSError`, as - described in `OS exceptions`_ below. The particular subclass depends on - the final :attr:`.errno` value. This behaviour only occurs when - constructing :exc:`OSError` directly or via an alias, and is not - inherited when subclassing. + Under all platforms, the :attr:`strerror` attribute is the corresponding + error message as provided by the operating system (as formatted by the C + functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` + Windows). - .. attribute:: errno - - A numeric error code from the C variable :c:data:`errno`. - - .. attribute:: winerror - - Under Windows, this gives you the native - Windows error code. The :attr:`.errno` attribute is then an approximate - translation, in POSIX terms, of that native error code. - - Under Windows, if the *winerror* constructor argument is an integer, - the :attr:`.errno` attribute is determined from the Windows error code, - and the *errno* argument is ignored. On other platforms, the - *winerror* argument is ignored, and the :attr:`winerror` attribute - does not exist. - - .. attribute:: strerror - - The corresponding error message, as provided by - the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` - under Windows. - - .. attribute:: filename - filename2 - - For exceptions that involve a file system path (such as :func:`open` or - :func:`os.unlink`), :attr:`filename` is the file name passed to the function. - For functions that involve two file system paths (such as - :func:`os.rename`), :attr:`filename2` corresponds to the second - file name passed to the function. - + For exceptions that involve a file system path (such as :func:`open` or + :func:`os.unlink`), the exception instance will contain an additional + attribute, :attr:`filename`, which is the file name passed to the function. .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`VMSError`, :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the - constructor may return a subclass. - - .. versionchanged:: 3.4 - The :attr:`filename` attribute is now the original file name passed to - the function, instead of the name encoded to or decoded from the - filesystem encoding. Also, the *filename2* constructor argument and - attribute was added. + :exc:`mmap.error` have been merged into :exc:`OSError`. .. exception:: OverflowError Raised when the result of an arithmetic operation is too large to be represented. This cannot occur for integers (which would rather raise - :exc:`MemoryError` than give up). However, for historical reasons, - OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. - - -.. exception:: RecursionError - - This exception is derived from :exc:`RuntimeError`. It is raised when the - interpreter detects that the maximum recursion depth (see - :func:`sys.getrecursionlimit`) is exceeded. - - .. versionadded:: 3.5 - Previously, a plain :exc:`RuntimeError` was raised. + :exc:`MemoryError` than give up). Because of the lack of standardization of + floating point exception handling in C, most floating point operations also + aren't checked. .. exception:: ReferenceError @@ -331,43 +261,28 @@ Raised when an error is detected that doesn't fall in any of the other categories. The associated value is a string indicating what precisely went - wrong. + wrong. (This exception is mostly a relic from a previous version of the + interpreter; it is not used very much any more.) .. exception:: StopIteration Raised by built-in function :func:`next` and an :term:`iterator`\'s - :meth:`~iterator.__next__` method to signal that there are no further - items produced by the iterator. + :meth:`__next__` method to signal that there are no further items to be + produced by the iterator. The exception object has a single attribute :attr:`value`, which is given as an argument when constructing the exception, and defaults to :const:`None`. - When a :term:`generator` or :term:`coroutine` function - returns, a new :exc:`StopIteration` instance is + When a generator function returns, a new :exc:`StopIteration` instance is raised, and the value returned by the function is used as the :attr:`value` parameter to the constructor of the exception. - If a generator function defined in the presence of a ``from __future__ - import generator_stop`` directive raises :exc:`StopIteration`, it will be - converted into a :exc:`RuntimeError` (retaining the :exc:`StopIteration` - as the new exception's cause). - .. versionchanged:: 3.3 Added ``value`` attribute and the ability for generator functions to use it to return a value. - .. versionchanged:: 3.5 - Introduced the RuntimeError transformation. - -.. exception:: StopAsyncIteration - - Must be raised by :meth:`__anext__` method of an - :term:`asynchronous iterator` object to stop the iteration. - - .. versionadded:: 3.5 - .. exception:: SyntaxError Raised when the parser encounters a syntax error. This may occur in an @@ -407,28 +322,28 @@ .. exception:: SystemExit - This exception is raised by the :func:`sys.exit` function. It inherits from - :exc:`BaseException` instead of :exc:`Exception` so that it is not accidentally - caught by code that catches :exc:`Exception`. This allows the exception to - properly propagate up and cause the interpreter to exit. When it is not - handled, the Python interpreter exits; no stack traceback is printed. The - constructor accepts the same optional argument passed to :func:`sys.exit`. - If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if - it has another type (such as a string), the object's value is printed and + This exception is raised by the :func:`sys.exit` function. When it is not + handled, the Python interpreter exits; no stack traceback is printed. If the + associated value is an integer, it specifies the system exit status (passed + to C's :c:func:`exit` function); if it is ``None``, the exit status is zero; + if it has another type (such as a string), the object's value is printed and the exit status is one. + Instances have an attribute :attr:`code` which is set to the proposed exit + status or error message (defaulting to ``None``). Also, this exception derives + directly from :exc:`BaseException` and not :exc:`Exception`, since it is not + technically an error. + A call to :func:`sys.exit` is translated into an exception so that clean-up handlers (:keyword:`finally` clauses of :keyword:`try` statements) can be executed, and so that a debugger can execute a script without running the risk of losing control. The :func:`os._exit` function can be used if it is absolutely positively necessary to exit immediately (for example, in the child - process after a call to :func:`os.fork`). + process after a call to :func:`fork`). - .. attribute:: code - - The exit status or error message that is passed to the constructor. - (Defaults to ``None``.) + The exception inherits from :exc:`BaseException` instead of :exc:`Exception` so + that it is not accidentally caught by code that catches :exc:`Exception`. This + allows the exception to properly propagate up and cause the interpreter to exit. .. exception:: TypeError @@ -449,30 +364,6 @@ Raised when a Unicode-related encoding or decoding error occurs. It is a subclass of :exc:`ValueError`. - :exc:`UnicodeError` has attributes that describe the encoding or decoding - error. For example, ``err.object[err.start:err.end]`` gives the particular - invalid input that the codec failed on. - - .. attribute:: encoding - - The name of the encoding that raised the error. - - .. attribute:: reason - - A string describing the specific codec error. - - .. attribute:: object - - The object the codec was attempting to encode or decode. - - .. attribute:: start - - The first index of invalid data in :attr:`object`. - - .. attribute:: end - - The index after the last invalid data in :attr:`object`. - .. exception:: UnicodeEncodeError @@ -513,6 +404,10 @@ .. exception:: IOError +.. exception:: VMSError + + Only available on VMS. + .. exception:: WindowsError Only available on Windows. @@ -547,35 +442,34 @@ .. exception:: ConnectionError - A base class for connection-related issues. - - Subclasses are :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`, + A base class for connection-related issues. Subclasses are + :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`, :exc:`ConnectionRefusedError` and :exc:`ConnectionResetError`. -.. exception:: BrokenPipeError + .. exception:: BrokenPipeError - A subclass of :exc:`ConnectionError`, raised when trying to write on a - pipe while the other end has been closed, or trying to write on a socket - which has been shutdown for writing. - Corresponds to :c:data:`errno` ``EPIPE`` and ``ESHUTDOWN``. + A subclass of :exc:`ConnectionError`, raised when trying to write on a + pipe while the other end has been closed, or trying to write on a socket + which has been shutdown for writing. + Corresponds to :c:data:`errno` ``EPIPE`` and ``ESHUTDOWN``. -.. exception:: ConnectionAbortedError + .. exception:: ConnectionAbortedError - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is aborted by the peer. - Corresponds to :c:data:`errno` ``ECONNABORTED``. + A subclass of :exc:`ConnectionError`, raised when a connection attempt + is aborted by the peer. + Corresponds to :c:data:`errno` ``ECONNABORTED``. -.. exception:: ConnectionRefusedError + .. exception:: ConnectionRefusedError - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is refused by the peer. - Corresponds to :c:data:`errno` ``ECONNREFUSED``. + A subclass of :exc:`ConnectionError`, raised when a connection attempt + is refused by the peer. + Corresponds to :c:data:`errno` ``ECONNREFUSED``. -.. exception:: ConnectionResetError + .. exception:: ConnectionResetError - A subclass of :exc:`ConnectionError`, raised when a connection is - reset by the peer. - Corresponds to :c:data:`errno` ``ECONNRESET``. + A subclass of :exc:`ConnectionError`, raised when a connection is + reset by the peer. + Corresponds to :c:data:`errno` ``ECONNRESET``. .. exception:: FileExistsError @@ -590,12 +484,7 @@ .. exception:: InterruptedError Raised when a system call is interrupted by an incoming signal. - Corresponds to :c:data:`errno` :py:data:`~errno.EINTR`. - - .. versionchanged:: 3.5 - Python now retries system calls when a syscall is interrupted by a - signal, except if the signal handler raises an exception (see :pep:`475` - for the rationale), instead of raising :exc:`InterruptedError`. + Corresponds to :c:data:`errno` ``EEINTR``. .. exception:: IsADirectoryError @@ -632,6 +521,7 @@ .. seealso:: :pep:`3151` - Reworking the OS and IO exception hierarchy + PEP written and implemented by Antoine Pitrou. Warnings @@ -662,7 +552,7 @@ .. exception:: SyntaxWarning - Base class for warnings about dubious syntax. + Base class for warnings about dubious syntax .. exception:: RuntimeWarning @@ -688,7 +578,7 @@ .. exception:: BytesWarning - Base class for warnings related to :class:`bytes` and :class:`bytearray`. + Base class for warnings related to :class:`bytes` and :class:`buffer`. .. exception:: ResourceWarning diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/faulthandler.rst --- a/Doc/library/faulthandler.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/faulthandler.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,14 +4,12 @@ .. module:: faulthandler :synopsis: Dump the Python traceback. -.. versionadded:: 3.3 - This module contains functions to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. Call :func:`faulthandler.enable` to install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment -variable or by using the :option:`-X` ``faulthandler`` command line option. +variable or by using :option:`-X` ``faulthandler`` command line option. The fault handler is compatible with system fault handlers like Apport or the Windows fault handler. The module uses an alternative stack for signal handlers @@ -25,11 +23,10 @@ * Only ASCII is supported. The ``backslashreplace`` error handler is used on encoding. -* Each string is limited to 500 characters. +* Each string is limited to 100 characters. * Only the filename, the function name and the line number are displayed. (no source code) * It is limited to 100 frames and 100 threads. -* The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see tracebacks, applications must be run in the terminal. A log file can @@ -38,18 +35,17 @@ The module is implemented in C, so tracebacks can be dumped on a crash or when Python is deadlocked. +.. versionadded:: 3.3 -Dumping the traceback ---------------------- + +Dump the traceback +------------------ .. function:: dump_traceback(file=sys.stderr, all_threads=True) Dump the tracebacks of all threads into *file*. If *all_threads* is ``False``, dump only the current thread. - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - Fault handler state ------------------- @@ -62,12 +58,6 @@ produce tracebacks for every running thread. Otherwise, dump only the current thread. - The *file* must be kept open until the fault handler is disabled: see - :ref:`issue with file descriptors `. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -78,10 +68,10 @@ Check if the fault handler is enabled. -Dumping the tracebacks after a timeout --------------------------------------- +Dump the tracebacks after a timeout +----------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_tracebacks_later(timeout, repeat=False, file=sys.stderr, exit=False) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -91,23 +81,16 @@ call replaces previous parameters and resets the timeout. The timer has a sub-second resolution. - The *file* must be kept open until the traceback is dumped or - :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file - descriptors `. - This function is implemented using a watchdog thread and therefore is not available if Python is compiled with threads disabled. - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. +.. function:: cancel_dump_tracebacks_later() -.. function:: cancel_dump_traceback_later() + Cancel the last call to :func:`dump_tracebacks_later`. - Cancel the last call to :func:`dump_traceback_later`. - -Dumping the traceback on a user signal --------------------------------------- +Dump the traceback on a user signal +----------------------------------- .. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) @@ -115,14 +98,8 @@ the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. - The *file* must be kept open until the signal is unregistered by - :func:`unregister`: see :ref:`issue with file descriptors `. - Not available on Windows. - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal @@ -132,12 +109,10 @@ Not available on Windows. -.. _faulthandler-fd: +File descriptor issue +--------------------- -Issue with file descriptors ---------------------------- - -:func:`enable`, :func:`dump_traceback_later` and :func:`register` keep the +:func:`enable`, :func:`dump_tracebacks_later` and :func:`register` keep the file descriptor of their *file* argument. If the file is closed and its file descriptor is reused by a new file, or if :func:`os.dup2` is used to replace the file descriptor, the traceback will be written into a different file. Call @@ -147,20 +122,14 @@ Example ------- -.. highlight:: sh +Example of a segmentation fault on Linux: :: -Example of a segmentation fault on Linux with and without enabling the fault -handler:: - - $ python3 -c "import ctypes; ctypes.string_at(0)" - Segmentation fault - - $ python3 -q -X faulthandler + $ python -q -X faulthandler >>> import ctypes >>> ctypes.string_at(0) Fatal Python error: Segmentation fault - Current thread 0x00007fb899f39700 (most recent call first): + Current thread 0x00007fb899f39700: File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at File "", line 1 in Segmentation fault diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/fcntl.rst --- a/Doc/library/fcntl.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/fcntl.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= +:mod:`fcntl` --- The :func:`fcntl` and :func:`ioctl` system calls +================================================================= .. module:: fcntl :platform: Unix @@ -16,63 +16,57 @@ All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by -``sys.stdin.fileno()``, or an :class:`io.IOBase` object, such as ``sys.stdin`` -itself, which provides a :meth:`~io.IOBase.fileno` that returns a genuine file -descriptor. +``sys.stdin.fileno()``, or a :class:`io.IOBase` object, such as ``sys.stdin`` +itself, which provides a :meth:`fileno` that returns a genuine file descriptor. .. versionchanged:: 3.3 - Operations in this module used to raise an :exc:`IOError` where they now - raise an :exc:`OSError`. + Operations in this module used to raise a :exc:`IOError` where they now + raise a :exc:`OSError`. The module defines the following functions: -.. function:: fcntl(fd, cmd, arg=0) +.. function:: fcntl(fd, op[, arg]) - Perform the operation *cmd* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). The values used - for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C - header files. The argument *arg* can either be an integer value, or a - :class:`bytes` object. With an integer value, the return value of this - function is the integer return value of the C :c:func:`fcntl` call. When - the argument is bytes it represents a binary structure, e.g. created by - :func:`struct.pack`. The binary data is copied to a buffer whose address is - passed to the C :c:func:`fcntl` call. The return value after a successful - call is the contents of the buffer, converted to a :class:`bytes` object. - The length of the returned object will be the same as the length of the - *arg* argument. This is limited to 1024 bytes. If the information returned - in the buffer by the operating system is larger than 1024 bytes, this is - most likely to result in a segmentation violation or a more subtle data - corruption. + Perform the requested operation on file descriptor *fd* (file objects providing + a :meth:`fileno` method are accepted as well). The operation is defined by *op* + and is operating system dependent. These codes are also found in the + :mod:`fcntl` module. The argument *arg* is optional, and defaults to the integer + value ``0``. When present, it can either be an integer value, or a string. + With the argument missing or an integer value, the return value of this function + is the integer return value of the C :c:func:`fcntl` call. When the argument is + a string it represents a binary structure, e.g. created by :func:`struct.pack`. + The binary data is copied to a buffer whose address is passed to the C + :c:func:`fcntl` call. The return value after a successful call is the contents + of the buffer, converted to a string object. The length of the returned string + will be the same as the length of the *arg* argument. This is limited to 1024 + bytes. If the information returned in the buffer by the operating system is + larger than 1024 bytes, this is most likely to result in a segmentation + violation or a more subtle data corruption. If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. -.. function:: ioctl(fd, request, arg=0, mutate_flag=True) +.. function:: ioctl(fd, op[, arg[, mutate_flag]]) - This function is identical to the :func:`~fcntl.fcntl` function, except - that the argument handling is even more complicated. + This function is identical to the :func:`fcntl` function, except that the + argument handling is even more complicated. - The *request* parameter is limited to values that can fit in 32-bits. - Additional constants of interest for use as the *request* argument can be - found in the :mod:`termios` module, under the same names as used in - the relevant C header files. + The op parameter is limited to values that can fit in 32-bits. - The parameter *arg* can be one of an integer, an object supporting the - read-only buffer interface (like :class:`bytes`) or an object supporting - the read-write buffer interface (like :class:`bytearray`). + The parameter *arg* can be one of an integer, absent (treated identically to the + integer ``0``), an object supporting the read-only buffer interface (most likely + a plain Python string) or an object supporting the read-write buffer interface. - In all but the last case, behaviour is as for the :func:`~fcntl.fcntl` - function. + In all but the last case, behaviour is as for the :func:`fcntl` function. If a mutable buffer is passed, then the behaviour is determined by the value of the *mutate_flag* parameter. If it is false, the buffer's mutability is ignored and behaviour is as for a read-only buffer, except that the 1024 byte limit mentioned above is avoided -- - so long as the buffer you pass is at least as long as what the operating system + so long as the buffer you pass is as least as long as what the operating system wants to put there, things should work. If *mutate_flag* is true (the default), then the buffer is (in effect) passed @@ -83,8 +77,6 @@ buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back into the supplied buffer. - If the :c:func:`ioctl` fails, an :exc:`IOError` exception is raised. - An example:: >>> import array, fcntl, struct, termios, os @@ -99,27 +91,25 @@ array('h', [13341]) -.. function:: flock(fd, operation) +.. function:: flock(fd, op) - Perform the lock operation *operation* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual + Perform the lock operation *op* on file descriptor *fd* (file objects providing + a :meth:`fileno` method are accepted as well). See the Unix manual :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) - If the :c:func:`flock` fails, an :exc:`IOError` exception is raised. +.. function:: lockf(fd, operation, [length, [start, [whence]]]) -.. function:: lockf(fd, cmd, len=0, start=0, whence=0) - - This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. - *fd* is the file descriptor of the file to lock or unlock, and *cmd* - is one of the following values: + This is essentially a wrapper around the :func:`fcntl` locking calls. *fd* is + the file descriptor of the file to lock or unlock, and *operation* is one of the + following values: * :const:`LOCK_UN` -- unlock * :const:`LOCK_SH` -- acquire a shared lock * :const:`LOCK_EX` -- acquire an exclusive lock - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be + When *operation* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. If :const:`LOCK_NB` is used and the lock cannot be acquired, an :exc:`OSError` will be raised and the exception will have an *errno* @@ -128,16 +118,16 @@ systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. - *len* is the number of bytes to lock, *start* is the byte offset at - which the lock starts, relative to *whence*, and *whence* is as with - :func:`io.IOBase.seek`, specifically: + *length* is the number of bytes to lock, *start* is the byte offset at which the + lock starts, relative to *whence*, and *whence* is as with :func:`fileobj.seek`, + specifically: - * :const:`0` -- relative to the start of the file (:data:`os.SEEK_SET`) - * :const:`1` -- relative to the current buffer position (:data:`os.SEEK_CUR`) - * :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`) + * :const:`0` -- relative to the start of the file (:const:`SEEK_SET`) + * :const:`1` -- relative to the current buffer position (:const:`SEEK_CUR`) + * :const:`2` -- relative to the end of the file (:const:`SEEK_END`) The default for *start* is 0, which means to start at the beginning of the file. - The default for *len* is 0 which means to lock to the end of the file. The + The default for *length* is 0 which means to lock to the end of the file. The default for *whence* is also 0. Examples (all on a SVR4 compliant system):: @@ -151,16 +141,15 @@ rv = fcntl.fcntl(f, fcntl.F_SETLKW, lockdata) Note that in the first example the return value variable *rv* will hold an -integer value; in the second example it will hold a :class:`bytes` object. The -structure lay-out for the *lockdata* variable is system dependent --- therefore -using the :func:`flock` call may be better. +integer value; in the second example it will hold a string value. The structure +lay-out for the *lockdata* variable is system dependent --- therefore using the +:func:`flock` call may be better. .. seealso:: Module :mod:`os` - If the locking flags :data:`~os.O_SHLOCK` and :data:`~os.O_EXLOCK` are - present in the :mod:`os` module (on BSD only), the :func:`os.open` - function provides an alternative to the :func:`lockf` and :func:`flock` - functions. + If the locking flags :const:`O_SHLOCK` and :const:`O_EXLOCK` are present + in the :mod:`os` module (on BSD only), the :func:`os.open` function + provides an alternative to the :func:`lockf` and :func:`flock` functions. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/filecmp.rst --- a/Doc/library/filecmp.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/filecmp.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,16 +21,15 @@ Compare the files named *f1* and *f2*, returning ``True`` if they seem equal, ``False`` otherwise. - If *shallow* is true, files with identical :func:`os.stat` signatures are - taken to be equal. Otherwise, the contents of the files are compared. + Unless *shallow* is given and is false, files with identical :func:`os.stat` + signatures are taken to be equal. + + Files that were compared using this function will not be compared again unless + their :func:`os.stat` signature changes. Note that no external programs are called from this function, giving it portability and efficiency. - This function uses a cache for past comparisons and the results, - with cache entries invalidated if the :func:`os.stat` information for the - file changes. The entire cache may be cleared using :func:`clear_cache`. - .. function:: cmpfiles(dir1, dir2, common, shallow=True) @@ -52,13 +51,13 @@ one of the three returned lists. -.. function:: clear_cache() +Example:: - Clear the filecmp cache. This may be useful if a file is compared so quickly - after it is modified that it is within the mtime resolution of - the underlying filesystem. - - .. versionadded:: 3.4 + >>> import filecmp + >>> filecmp.cmp('undoc.rst', 'undoc.rst') + True + >>> filecmp.cmp('undoc.rst', 'index.rst') + False .. _dircmp-objects: @@ -66,33 +65,36 @@ The :class:`dircmp` class ------------------------- +:class:`dircmp` instances are built using this constructor: + + .. class:: dircmp(a, b, ignore=None, hide=None) - Construct a new directory comparison object, to compare the directories *a* - and *b*. *ignore* is a list of names to ignore, and defaults to - :attr:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and - defaults to ``[os.curdir, os.pardir]``. - - The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. + Construct a new directory comparison object, to compare the directories *a* and + *b*. *ignore* is a list of names to ignore, and defaults to ``['RCS', 'CVS', + 'tags']``. *hide* is a list of names to hide, and defaults to ``[os.curdir, + os.pardir]``. The :class:`dircmp` class provides the following methods: + .. method:: report() - Print (to :data:`sys.stdout`) a comparison between *a* and *b*. + Print (to ``sys.stdout``) a comparison between *a* and *b*. + .. method:: report_partial_closure() Print a comparison between *a* and *b* and common immediate subdirectories. + .. method:: report_full_closure() Print a comparison between *a* and *b* and common subdirectories (recursively). - The :class:`dircmp` class offers a number of interesting attributes that may be + The :class:`dircmp` offers a number of interesting attributes that may be used to get various bits of information about the directory trees being compared. @@ -101,16 +103,6 @@ to compute are used. - .. attribute:: left - - The directory *a*. - - - .. attribute:: right - - The directory *b*. - - .. attribute:: left_list Files and subdirectories in *a*, filtered by *hide* and *ignore*. @@ -143,7 +135,7 @@ .. attribute:: common_files - Files in both *a* and *b*. + Files in both *a* and *b* .. attribute:: common_funny @@ -154,14 +146,12 @@ .. attribute:: same_files - Files which are identical in both *a* and *b*, using the class's - file comparison operator. + Files which are identical in both *a* and *b*. .. attribute:: diff_files - Files which are in both *a* and *b*, whose contents differ according - to the class's file comparison operator. + Files which are in both *a* and *b*, whose contents differ. .. attribute:: funny_files @@ -174,24 +164,3 @@ A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp` objects. -.. attribute:: DEFAULT_IGNORES - - .. versionadded:: 3.4 - - List of directories ignored by :class:`dircmp` by default. - - -Here is a simplified example of using the ``subdirs`` attribute to search -recursively through two directories to show common different files:: - - >>> from filecmp import dircmp - >>> def print_diff_files(dcmp): - ... for name in dcmp.diff_files: - ... print("diff_file %s found in %s and %s" % (name, dcmp.left, - ... dcmp.right)) - ... for sub_dcmp in dcmp.subdirs.values(): - ... print_diff_files(sub_dcmp) - ... - >>> dcmp = dircmp('dir1', 'dir2') # doctest: +SKIP - >>> print_diff_files(dcmp) # doctest: +SKIP - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/fileinput.rst --- a/Doc/library/fileinput.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/fileinput.rst Mon Jan 25 17:05:13 2016 +0100 @@ -136,12 +136,11 @@ Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, - :meth:`isstdin`, :meth:`nextfile` and :meth:`close` correspond to the - functions of the same name in the module. In addition it has a - :meth:`~io.TextIOBase.readline` method which returns the next input line, - and a :meth:`__getitem__` method which implements the sequence behavior. - The sequence must be accessed in strictly sequential order; random access - and :meth:`~io.TextIOBase.readline` cannot be mixed. + :meth:`isstdin`, :meth:`nextfile` and :meth:`close` correspond to the functions + of the same name in the module. In addition it has a :meth:`readline` method + which returns the next input line, and a :meth:`__getitem__` method which + implements the sequence behavior. The sequence must be accessed in strictly + sequential order; random access and :meth:`readline` cannot be mixed. With *mode* you can specify which file mode will be passed to :func:`open`. It must be one of ``'r'``, ``'rU'``, ``'U'`` and ``'rb'``. @@ -160,9 +159,6 @@ .. versionchanged:: 3.2 Can be used as a context manager. - .. deprecated:: 3.4 - The ``'rU'`` and ``'U'`` modes. - **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/filesys.rst --- a/Doc/library/filesys.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/filesys.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,7 +12,6 @@ .. toctree:: - pathlib.rst os.path.rst fileinput.rst stat.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/fnmatch.rst --- a/Doc/library/fnmatch.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/fnmatch.rst Mon Jan 25 17:05:13 2016 +0100 @@ -29,9 +29,6 @@ | ``[!seq]`` | matches any character not in *seq* | +------------+------------------------------------+ -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - .. index:: module: glob Note that the filename separator (``'/'`` on Unix) is *not* special to this @@ -77,16 +74,18 @@ Return the shell-style *pattern* converted to a regular expression. + Be aware there is no way to quote meta-characters. + Example: >>> import fnmatch, re >>> >>> regex = fnmatch.translate('*.txt') >>> regex - '.*\\.txt\\Z(?ms)' + '.*\\.txt$' >>> reobj = re.compile(regex) >>> reobj.match('foobar.txt') - <_sre.SRE_Match object; span=(0, 10), match='foobar.txt'> + <_sre.SRE_Match object at 0x...> .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/formatter.rst --- a/Doc/library/formatter.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/formatter.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,10 +3,6 @@ .. module:: formatter :synopsis: Generic output formatter and device interface. - :deprecated: - -.. deprecated:: 3.4 - Due to lack of usage, the formatter module has been deprecated. This module supports two interface definitions, each with multiple diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/fractions.rst --- a/Doc/library/fractions.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/fractions.rst Mon Jan 25 17:05:13 2016 +0100 @@ -56,6 +56,7 @@ Fraction(0, 1) >>> Fraction('3/7') Fraction(3, 7) + [40794 refs] >>> Fraction(' -3/7 ') Fraction(-3, 7) >>> Fraction('1.414213 \t\n') @@ -77,31 +78,20 @@ :class:`numbers.Rational`, and implements all of the methods and operations from that class. :class:`Fraction` instances are hashable, and should be treated as immutable. In addition, - :class:`Fraction` has the following properties and methods: + :class:`Fraction` has the following methods: .. versionchanged:: 3.2 The :class:`Fraction` constructor now accepts :class:`float` and :class:`decimal.Decimal` instances. - .. attribute:: numerator - - Numerator of the Fraction in lowest term. - - .. attribute:: denominator - - Denominator of the Fraction in lowest term. - - .. method:: from_float(flt) This class method constructs a :class:`Fraction` representing the exact value of *flt*, which must be a :class:`float`. Beware that - ``Fraction.from_float(0.3)`` is not the same value as ``Fraction(3, 10)``. + ``Fraction.from_float(0.3)`` is not the same value as ``Fraction(3, 10)`` - .. note:: - - From Python 3.2 onwards, you can also construct a + .. note:: From Python 3.2 onwards, you can also construct a :class:`Fraction` instance directly from a :class:`float`. @@ -110,9 +100,7 @@ This class method constructs a :class:`Fraction` representing the exact value of *dec*, which must be a :class:`decimal.Decimal` instance. - .. note:: - - From Python 3.2 onwards, you can also construct a + .. note:: From Python 3.2 onwards, you can also construct a :class:`Fraction` instance directly from a :class:`decimal.Decimal` instance. @@ -172,9 +160,6 @@ sign as *b* if *b* is nonzero; otherwise it takes the sign of *a*. ``gcd(0, 0)`` returns ``0``. - .. deprecated:: 3.5 - Use :func:`math.gcd` instead. - .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ftplib.rst --- a/Doc/library/ftplib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/ftplib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -23,17 +23,16 @@ Here's a sample session using the :mod:`ftplib` module:: >>> from ftplib import FTP - >>> ftp = FTP('ftp.debian.org') # connect to host, default port - >>> ftp.login() # user anonymous, passwd anonymous@ - '230 Login successful.' - >>> ftp.cwd('debian') # change into "debian" directory - >>> ftp.retrlines('LIST') # list directory contents - -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README - ... - drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool - drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project - drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools - '226 Directory send OK.' + >>> ftp = FTP('ftp.cwi.nl') # connect to host, default port + >>> ftp.login() # user anonymous, passwd anonymous@ + >>> ftp.retrlines('LIST') # list directory contents + total 24418 + drwxrwsr-x 5 ftp-usr pdmaint 1536 Mar 20 09:48 . + dr-xr-srwt 105 ftp-usr pdmaint 1536 Mar 21 14:32 .. + -rw-r--r-- 1 ftp-usr pdmaint 5305 Mar 20 09:48 INDEX + . + . + . >>> ftp.retrbinary('RETR README', open('README', 'wb').write) '226 Transfer complete.' >>> ftp.quit() @@ -52,7 +51,8 @@ will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. - The :class:`FTP` class supports the :keyword:`with` statement, e.g.: + :class:`FTP` class supports the :keyword:`with` statement. Here is a sample + on how using it: >>> from ftplib import FTP >>> with FTP("ftp1.at.proftpd.org") as ftp: @@ -79,34 +79,40 @@ :rfc:`4217`. Connect as usual to port 21 implicitly securing the FTP control connection before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* -- they - can point to PEM-formatted private key and certificate chain files - (respectively) for the SSL connection. + explicitly ask for it by calling the :meth:`prot_p` method. + *keyfile* and *certfile* are optional -- they can contain a PEM formatted + private key and certificate chain file name for the SSL connection. + *context* parameter is a :class:`ssl.SSLContext` object which allows + bundling SSL configuration options, certificates and private keys into a + single (potentially long-lived) structure. *source_address* is a 2-tuple + ``(host, port)`` for the socket to bind to as its source address before + connecting. .. versionadded:: 3.2 .. versionchanged:: 3.3 *source_address* parameter was added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). + Here's a sample session using the :class:`FTP_TLS` class: - Here's a sample session using the :class:`FTP_TLS` class:: - - >>> ftps = FTP_TLS('ftp.pureftpd.org') - >>> ftps.login() - '230 Anonymous user logged in' - >>> ftps.prot_p() - '200 Data protection level set to "private"' - >>> ftps.nlst() - ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] + >>> from ftplib import FTP_TLS + >>> ftps = FTP_TLS('ftp.python.org') + >>> ftps.login() # login anonymously before securing control channel + >>> ftps.prot_p() # switch to secure data connection + >>> ftps.retrlines('LIST') # list directory content securely + total 9 + drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . + drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. + drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin + drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc + d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming + drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib + drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub + drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr + -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg + '226 Transfer complete.' + >>> ftps.quit() + >>> .. exception:: error_reply @@ -261,34 +267,34 @@ Passive mode is on by default. -.. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) +.. method:: FTP.storbinary(cmd, file, blocksize=8192, callback=None, rest=None) Store a file in binary transfer mode. *cmd* should be an appropriate - ``STOR`` command: ``"STOR filename"``. *fp* is a :term:`file object` - (opened in binary mode) which is read until EOF using its :meth:`~io.IOBase.read` - method in blocks of size *blocksize* to provide the data to be stored. - The *blocksize* argument defaults to 8192. *callback* is an optional single - parameter callable that is called on each block of data after it is sent. - *rest* means the same thing as in the :meth:`transfercmd` method. + ``STOR`` command: ``"STOR filename"``. *file* is an open :term:`file object` + which is read until EOF using its :meth:`read` method in blocks of size + *blocksize* to provide the data to be stored. The *blocksize* argument + defaults to 8192. *callback* is an optional single parameter callable that + is called on each block of data after it is sent. *rest* means the same thing + as in the :meth:`transfercmd` method. .. versionchanged:: 3.2 *rest* parameter added. -.. method:: FTP.storlines(cmd, fp, callback=None) +.. method:: FTP.storlines(cmd, file, callback=None) Store a file in ASCII transfer mode. *cmd* should be an appropriate ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the - :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` - method to provide the data to be stored. *callback* is an optional single - parameter callable that is called on each line after it is sent. + open :term:`file object` *file* using its :meth:`readline` method to provide + the data to be stored. *callback* is an optional single parameter callable + that is called on each line after it is sent. .. method:: FTP.transfercmd(cmd, rest=None) - Initiate a transfer over the data connection. If the transfer is active, send an + Initiate a transfer over the data connection. If the transfer is active, send a ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and - accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` + accept the connection. If the server is passive, send a ``EPSV`` or ``PASV`` command, connect to it, and start the transfer command. Either way, return the socket for the connection. @@ -314,7 +320,7 @@ .. method:: FTP.mlsd(path="", facts=[]) - List a directory in a standardized format by using ``MLSD`` command + List a directory in a standardized format by using MLSD command (:rfc:`3659`). If *path* is omitted the current directory is assumed. *facts* is a list of strings representing the type of information desired (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a @@ -333,7 +339,7 @@ directory). Multiple arguments can be used to pass non-standard options to the ``NLST`` command. - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + .. deprecated:: 3.3 use :meth:`mlsd` instead. .. method:: FTP.dir(argument[, ...]) @@ -345,7 +351,7 @@ as a *callback* function as for :meth:`retrlines`; the default prints to ``sys.stdout``. This method returns ``None``. - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. + .. deprecated:: 3.3 use :meth:`mlsd` instead. .. method:: FTP.rename(fromname, toname) @@ -400,10 +406,10 @@ .. method:: FTP.close() Close the connection unilaterally. This should not be applied to an already - closed connection such as after a successful call to :meth:`~FTP.quit`. - After this call the :class:`FTP` instance should not be used any more (after - a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the - connection by issuing another :meth:`login` method). + closed connection such as after a successful call to :meth:`quit`. After this + call the :class:`FTP` instance should not be used any more (after a call to + :meth:`close` or :meth:`quit` you cannot reopen the connection by issuing + another :meth:`login` method). FTP_TLS Objects @@ -413,17 +419,11 @@ .. attribute:: FTP_TLS.ssl_version - The SSL version to use (defaults to :attr:`ssl.PROTOCOL_SSLv23`). + The SSL version to use (defaults to *TLSv1*). .. method:: FTP_TLS.auth() - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). + Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute. .. method:: FTP_TLS.ccc() @@ -440,3 +440,5 @@ .. method:: FTP_TLS.prot_c() Set up clear text data connection. + + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/functions.rst --- a/Doc/library/functions.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/functions.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,12 +14,12 @@ :func:`all` :func:`dir` :func:`hex` :func:`next` :func:`slice` :func:`any` :func:`divmod` :func:`id` :func:`object` :func:`sorted` :func:`ascii` :func:`enumerate` :func:`input` :func:`oct` :func:`staticmethod` -:func:`bin` :func:`eval` :func:`int` :func:`open` |func-str|_ +:func:`bin` :func:`eval` :func:`int` :func:`open` :func:`str` :func:`bool` :func:`exec` :func:`isinstance` :func:`ord` :func:`sum` :func:`bytearray` :func:`filter` :func:`issubclass` :func:`pow` :func:`super` -:func:`bytes` :func:`float` :func:`iter` :func:`print` |func-tuple|_ +:func:`bytes` :func:`float` :func:`iter` :func:`print` :func:`tuple` :func:`callable` :func:`format` :func:`len` :func:`property` :func:`type` -:func:`chr` |func-frozenset|_ |func-list|_ |func-range|_ :func:`vars` +:func:`chr` |func-frozenset|_ :func:`list` :func:`range` :func:`vars` :func:`classmethod` :func:`getattr` :func:`locals` :func:`repr` :func:`zip` :func:`compile` :func:`globals` :func:`map` :func:`reversed` :func:`__import__` :func:`complex` :func:`hasattr` :func:`max` :func:`round` @@ -33,10 +33,6 @@ .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` -.. |func-list| replace:: ``list()`` -.. |func-str| replace:: ``str()`` -.. |func-tuple| replace:: ``tuple()`` -.. |func-range| replace:: ``range()`` .. function:: abs(x) @@ -48,7 +44,7 @@ .. function:: all(iterable) - Return ``True`` if all elements of the *iterable* are true (or if the iterable + Return True if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: def all(iterable): @@ -60,8 +56,8 @@ .. function:: any(iterable) - Return ``True`` if any element of the *iterable* is true. If the iterable - is empty, return ``False``. Equivalent to:: + Return True if any element of the *iterable* is true. If the iterable + is empty, return False. Equivalent to:: def any(iterable): for element in iterable: @@ -85,22 +81,21 @@ :meth:`__index__` method that returns an integer. -.. class:: bool([x]) +.. function:: bool([x]) - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false - or omitted, this returns ``False``; otherwise it returns ``True``. The - :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). - It cannot be subclassed further. Its only instances are ``False`` and + Convert a value to a Boolean, using the standard :ref:`truth testing + procedure `. If *x* is false or omitted, this returns ``False``; + otherwise it returns ``True``. :class:`bool` is also a class, which is a + subclass of :class:`int` (see :ref:`typesnumeric`). Class :class:`bool` + cannot be subclassed further. Its only instances are ``False`` and ``True`` (see :ref:`bltin-boolean-values`). .. index:: pair: Boolean; type -.. _func-bytearray: -.. class:: bytearray([source[, encoding[, errors]]]) +.. function:: bytearray([source[, encoding[, errors]]]) - Return a new array of bytes. The :class:`bytearray` class is a mutable + Return a new array of bytes. The :class:`bytearray` type is a mutable sequence of integers in the range 0 <= x < 256. It has most of the usual methods of mutable sequences, described in :ref:`typesseq-mutable`, as well as most methods that the :class:`bytes` type has, see :ref:`bytes-methods`. @@ -123,11 +118,8 @@ Without an argument, an array of size 0 is created. - See also :ref:`binaryseq` and :ref:`typebytearray`. - -.. _func-bytes: -.. class:: bytes([source[, encoding[, errors]]]) +.. function:: bytes([source[, encoding[, errors]]]) Return a new "bytes" object, which is an immutable sequence of integers in the range ``0 <= x < 256``. :class:`bytes` is an immutable version of @@ -138,8 +130,6 @@ Bytes objects can also be created with literals, see :ref:`strings`. - See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. - .. function:: callable(object) @@ -156,12 +146,11 @@ .. function:: chr(i) - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(957)`` returns the string ``'ν'``. This is the inverse of :func:`ord`. - - The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. + Return the string representing a character whose Unicode codepoint is the integer + *i*. For example, ``chr(97)`` returns the string ``'a'``. This is the + inverse of :func:`ord`. The valid range for the argument is from 0 through + 1,114,111 (0x10FFFF in base 16). :exc:`ValueError` will be raised if *i* is + outside that range. .. function:: classmethod(function) @@ -194,9 +183,9 @@ .. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) Compile the *source* into a code or AST object. Code objects can be executed - by :func:`exec` or :func:`eval`. *source* can either be a normal string, a - byte string, or an AST object. Refer to the :mod:`ast` module documentation - for information on how to work with AST objects. + by :func:`exec` or :func:`eval`. *source* can either be a string or an AST + object. Refer to the :mod:`ast` module documentation for information on how + to work with AST objects. The *filename* argument should give the file from which the code was read; pass some recognizable value if it wasn't read from a file (``''`` is @@ -211,7 +200,7 @@ The optional arguments *flags* and *dont_inherit* control which future statements (see :pep:`236`) affect the compilation of *source*. If neither is present (or both are zero) the code is compiled with those future - statements that are in effect in the code that is calling :func:`compile`. If the + statements that are in effect in the code that is calling compile. If the *flags* argument is given and *dont_inherit* is not (or is zero) then the future statements specified by the *flags* argument are used in addition to those that would be used anyway. If *dont_inherit* is a non-zero integer then @@ -220,8 +209,8 @@ Future statements are specified by bits which can be bitwise ORed together to specify multiple statements. The bitfield required to specify a given feature - can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on - the :class:`~__future__._Feature` instance in the :mod:`__future__` module. + can be found as the :attr:`compiler_flag` attribute on the :class:`_Feature` + instance in the :mod:`__future__` module. The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as @@ -232,9 +221,6 @@ This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`TypeError` if the source contains null bytes. - If you want to parse Python code into its AST representation, see - :func:`ast.parse`. - .. note:: When compiling a string with multi-line code in ``'single'`` or @@ -247,16 +233,15 @@ does not have to end in a newline anymore. Added the *optimize* parameter. -.. class:: complex([real[, imag]]) +.. function:: complex([real[, imag]]) - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. + Create a complex number with the value *real* + *imag*\*j or convert a string or + number to a complex number. If the first parameter is a string, it will be + interpreted as a complex number and the function must be called without a second + parameter. The second parameter can never be a string. Each argument may be any + numeric type (including complex). If *imag* is omitted, it defaults to zero and + the function serves as a numeric conversion function like :func:`int` + and :func:`float`. If both arguments are omitted, returns ``0j``. .. note:: @@ -277,16 +262,14 @@ .. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) +.. function:: dict([arg]) :noindex: - Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + Create a new data dictionary, optionally with items taken from *arg*. + The dictionary type is described in :ref:`typesmapping`. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built in :class:`list`, :class:`set`, and + :class:`tuple` classes, and the :mod:`collections` module. .. function:: dir([object]) @@ -322,18 +305,17 @@ >>> import struct >>> dir() # show the names in the module namespace - ['__builtins__', '__name__', 'struct'] - >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', - '__initializing__', '__loader__', '__name__', '__package__', - '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', + ['__builtins__', '__doc__', '__name__', 'struct'] + >>> dir(struct) # show the names in the struct module + ['Struct', '__builtins__', '__doc__', '__file__', '__name__', + '__package__', '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', 'unpack', 'unpack_from'] - >>> class Shape: - ... def __dir__(self): - ... return ['area', 'perimeter', 'location'] + >>> class Shape(object): + def __dir__(self): + return ['area', 'perimeter', 'location'] >>> s = Shape() >>> dir(s) - ['area', 'location', 'perimeter'] + ['area', 'perimeter', 'location'] .. note:: @@ -360,10 +342,10 @@ .. function:: enumerate(iterable, start=0) Return an enumerate object. *iterable* must be a sequence, an - :term:`iterator`, or some other object which supports iteration. - The :meth:`~iterator.__next__` method of the iterator returned by - :func:`enumerate` returns a tuple containing a count (from *start* which - defaults to 0) and the values obtained from iterating over *iterable*. + :term:`iterator`, or some other object which supports iteration. The + :meth:`__next__` method of the iterator returned by :func:`enumerate` returns a + tuple containing a count (from *start* which defaults to 0) and the + values obtained from iterating over *iterable*. >>> seasons = ['Spring', 'Summer', 'Fall', 'Winter'] >>> list(enumerate(seasons)) @@ -414,7 +396,6 @@ See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. -.. index:: builtin: exec .. function:: exec(object[, globals[, locals]]) @@ -432,10 +413,7 @@ current scope. If only *globals* is provided, it must be a dictionary, which will be used for both the global and the local variables. If *globals* and *locals* are given, they are used for the global and local variables, - respectively. If provided, *locals* can be any mapping object. Remember - that at module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. + respectively. If provided, *locals* can be any mapping object. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module @@ -474,13 +452,13 @@ elements of *iterable* for which *function* returns false. -.. class:: float([x]) +.. function:: float([x]) .. index:: single: NaN single: Infinity - Return a floating point number constructed from a number or string *x*. + Convert a string or a number to floating point. If the argument is a string, it should contain a decimal number, optionally preceded by a sign, and optionally embedded in whitespace. The optional @@ -527,44 +505,37 @@ The float type is described in :ref:`typesnumeric`. - .. index:: - single: __format__ - single: string; format() (built-in function) - .. function:: format(value[, format_spec]) + .. index:: + pair: str; format + single: __format__ + Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type of the *value* argument, however there is a standard formatting syntax that is used by most built-in types: :ref:`formatspec`. The default *format_spec* is an empty string which usually gives the same - effect as calling :func:`str(value) `. + effect as calling ``str(value)``. A call to ``format(value, format_spec)`` is translated to - ``type(value).__format__(value, format_spec)`` which bypasses the instance + ``type(value).__format__(format_spec)`` which bypasses the instance dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches - :mod:`object` and the *format_spec* is non-empty, or if either the - *format_spec* or the return value are not strings. - - .. versionchanged:: 3.4 - ``object().__format__(format_spec)`` raises :exc:`TypeError` - if *format_spec* is not an empty string. + :exc:`TypeError` exception is raised if the method is not found or if either + the *format_spec* or the return value are not strings. .. _func-frozenset: -.. class:: frozenset([iterable]) +.. function:: frozenset([iterable]) :noindex: - Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and - :ref:`types-set` for documentation about this class. + Return a frozenset object, optionally with elements taken from *iterable*. + The frozenset type is described in :ref:`types-set`. - For other containers see the built-in :class:`set`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. + For other containers see the built in :class:`dict`, :class:`list`, and + :class:`tuple` classes, and the :mod:`collections` module. .. function:: getattr(object, name[, default]) @@ -593,16 +564,11 @@ .. function:: hash(object) - Return the hash value of the object (if it has one). Hash values are - integers. They are used to quickly compare dictionary keys during a - dictionary lookup. Numeric values that compare equal have the same hash - value (even if they are of different types, as is the case for 1 and 1.0). + Return the hash value of the object (if it has one). Hash values are integers. + They are used to quickly compare dictionary keys during a dictionary lookup. + Numeric values that compare equal have the same hash value (even if they are of + different types, as is the case for 1 and 1.0). - .. note:: - - For object's with custom :meth:`__hash__` methods, note that :func:`hash` - truncates the return value based on the bit width of the host machine. - See :meth:`__hash__` for details. .. function:: help([object]) @@ -615,26 +581,12 @@ This function is added to the built-in namespace by the :mod:`site` module. - .. versionchanged:: 3.4 - Changes to :mod:`pydoc` and :mod:`inspect` mean that the reported - signatures for callables are now more comprehensive and consistent. - .. function:: hex(x) - Convert an integer number to a lowercase hexadecimal string - prefixed with "0x", for example: - - >>> hex(255) - '0xff' - >>> hex(-42) - '-0x2a' - - If x is not a Python :class:`int` object, it has to define an __index__() - method that returns an integer. - - See also :func:`int` for converting a hexadecimal string to an - integer using a base of 16. + Convert an integer number to a hexadecimal string. The result is a valid Python + expression. If *x* is not a Python :class:`int` object, it has to define an + :meth:`__index__` method that returns an integer. .. note:: @@ -659,29 +611,23 @@ to a string (stripping a trailing newline), and returns that. When EOF is read, :exc:`EOFError` is raised. Example:: - >>> s = input('--> ') # doctest: +SKIP + >>> s = input('--> ') --> Monty Python's Flying Circus - >>> s # doctest: +SKIP + >>> s "Monty Python's Flying Circus" If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. -.. class:: int(x=0) - int(x, base=10) +.. function:: int([number | string[, base]]) - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() `. For floating point numbers, this - truncates towards zero. - - If *x* is not a number or if *base* is given, then *x* must be a string, - :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer - literal ` in radix *base*. Optionally, the literal can be - preceded by ``+`` or ``-`` (with no space in between) and surrounded by - whitespace. A base-n literal consists of the digits 0 to n-1, with ``a`` - to ``z`` (or ``A`` to ``Z``) having + Convert a number or string to an integer. If no arguments are given, return + ``0``. If a number is given, return ``number.__int__()``. Conversion of + floating point numbers to integers truncates towards zero. A string must be + a base-radix integer literal optionally preceded by '+' or '-' (with no space + in between) and optionally surrounded by whitespace. A base-n literal + consists of the digits 0 to n-1, with 'a' to 'z' (or 'A' to 'Z') having values 10 to 35. The default *base* is 10. The allowed values are 0 and 2-36. Base-2, -8, and -16 literals can be optionally prefixed with ``0b``/``0B``, ``0o``/``0O``, or ``0x``/``0X``, as with integer literals in code. Base 0 @@ -691,22 +637,16 @@ The integer type is described in :ref:`typesnumeric`. - .. versionchanged:: 3.4 - If *base* is not an instance of :class:`int` and the *base* object has a - :meth:`base.__index__ ` method, that method is called - to obtain an integer for the base. Previous versions used - :meth:`base.__int__ ` instead of :meth:`base.__index__ - `. .. function:: isinstance(object, classinfo) Return true if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect or :term:`virtual `) subclass thereof. If *object* is not - an object of the given type, the function always returns false. - If *classinfo* is a tuple of type objects (or recursively, other such - tuples), return true if *object* is an instance of any of the types. - If *classinfo* is not a type or tuple of types and such tuples, + an object of the given type, the function always returns false. If + *classinfo* is not a class (type object), it may be a tuple of type objects, + or may recursively contain other such tuples (other sequence types are not + accepted). If *classinfo* is not a type or tuple of types and such tuples, a :exc:`TypeError` exception is raised. @@ -729,16 +669,13 @@ starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its - :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will - be returned. - - See also :ref:`typeiter`. + will call *object* with no arguments for each call to its :meth:`__next__` + method; if the value returned is equal to *sentinel*, :exc:`StopIteration` + will be raised, otherwise the value will be returned. One useful application of the second form of :func:`iter` is to read lines of a file until a certain line is reached. The following example reads a file - until the :meth:`~io.TextIOBase.readline` method returns an empty string:: + until the :meth:`readline` method returns an empty string:: with open('mydata.txt') as fp: for line in iter(fp.readline, ''): @@ -748,16 +685,19 @@ .. function:: len(s) Return the length (the number of items) of an object. The argument may be a - sequence (such as a string, bytes, tuple, list, or range) or a collection - (such as a dictionary, set, or frozen set). + sequence (string, tuple or list) or a mapping (dictionary). -.. _func-list: -.. class:: list([iterable]) - :noindex: +.. function:: list([iterable]) - Rather than being a function, :class:`list` is actually a mutable - sequence type, as documented in :ref:`typesseq-list` and :ref:`typesseq`. + Return a list whose items are the same and in the same order as *iterable*'s + items. *iterable* may be either a sequence, a container that supports + iteration, or an iterator object. If *iterable* is already a list, a copy is + made and returned, similar to ``iterable[:]``. For instance, ``list('abc')`` + returns ``['a', 'b', 'c']`` and ``list( (1, 2, 3) )`` returns ``[1, 2, 3]``. + If no argument is given, returns a new empty list, ``[]``. + + :class:`list` is a mutable sequence type, as documented in :ref:`typesseq`. .. function:: locals() @@ -780,31 +720,20 @@ already arranged into argument tuples, see :func:`itertools.starmap`\. -.. function:: max(iterable, *[, key, default]) - max(arg1, arg2, *args[, key]) +.. function:: max(iterable[, args...], *[, key]) - Return the largest item in an iterable or the largest of two or more - arguments. + With a single argument *iterable*, return the largest item of a non-empty + iterable (such as a string, tuple or list). With more than one argument, return + the largest of the arguments. - If one positional argument is provided, it should be an :term:`iterable`. - The largest item in the iterable is returned. If two or more positional - arguments are provided, the largest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. + The optional keyword-only *key* argument specifies a one-argument ordering + function like that used for :meth:`list.sort`. If multiple items are maximal, the function returns the first one encountered. This is consistent with other sort-stability preserving tools such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and ``heapq.nlargest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. - .. _func-memoryview: .. function:: memoryview(obj) @@ -814,40 +743,28 @@ :ref:`typememoryview` for more information. -.. function:: min(iterable, *[, key, default]) - min(arg1, arg2, *args[, key]) +.. function:: min(iterable[, args...], *[, key]) - Return the smallest item in an iterable or the smallest of two or more - arguments. + With a single argument *iterable*, return the smallest item of a non-empty + iterable (such as a string, tuple or list). With more than one argument, return + the smallest of the arguments. - If one positional argument is provided, it should be an :term:`iterable`. - The smallest item in the iterable is returned. If two or more positional - arguments are provided, the smallest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. + The optional keyword-only *key* argument specifies a one-argument ordering + function like that used for :meth:`list.sort`. If multiple items are minimal, the function returns the first one encountered. This is consistent with other sort-stability preserving tools such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, iterable, key=keyfunc)``. - .. versionadded:: 3.4 - The *default* keyword-only argument. - - .. function:: next(iterator[, default]) - Retrieve the next item from the *iterator* by calling its - :meth:`~iterator.__next__` method. If *default* is given, it is returned - if the iterator is exhausted, otherwise :exc:`StopIteration` is raised. + Retrieve the next item from the *iterator* by calling its :meth:`__next__` + method. If *default* is given, it is returned if the iterator is exhausted, + otherwise :exc:`StopIteration` is raised. -.. class:: object() +.. function:: object() Return a new featureless object. :class:`object` is a base for all classes. It has the methods that are common to all instances of Python classes. This @@ -855,8 +772,8 @@ .. note:: - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. + :class:`object` does *not* have a :attr:`__dict__`, so you can't assign + arbitrary attributes to an instance of the :class:`object` class. .. function:: oct(x) @@ -866,13 +783,10 @@ :meth:`__index__` method that returns an integer. - .. index:: - single: file object; open() built-in function - .. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) - Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. + Open *file* and return a corresponding stream. If the file cannot be opened, + an :exc:`OSError` is raised. *file* is either a string or bytes object giving the pathname (absolute or relative to the current working directory) of the file to be opened or @@ -883,25 +797,23 @@ *mode* is an optional string that specifies the mode in which the file is opened. It defaults to ``'r'`` which means open for reading in text mode. Other common values are ``'w'`` for writing (truncating the file if it - already exists), ``'x'`` for exclusive creation and ``'a'`` for appending - (which on *some* Unix systems, means that *all* writes append to the end of - the file regardless of the current seek position). In text mode, if - *encoding* is not specified the encoding used is platform dependent: - ``locale.getpreferredencoding(False)`` is called to get the current locale - encoding. (For reading and writing raw bytes use binary mode and leave - *encoding* unspecified.) The available modes are: + already exists), and ``'a'`` for appending (which on *some* Unix systems, + means that *all* writes append to the end of the file regardless of the + current seek position). In text mode, if *encoding* is not specified the + encoding used is platform dependent. (For reading and writing raw bytes use + binary mode and leave *encoding* unspecified.) The available modes are: ========= =============================================================== Character Meaning - ========= =============================================================== + --------- --------------------------------------------------------------- ``'r'`` open for reading (default) ``'w'`` open for writing, truncating the file first - ``'x'`` open for exclusive creation, failing if the file already exists ``'a'`` open for writing, appending to the end of the file if it exists ``'b'`` binary mode ``'t'`` text mode (default) ``'+'`` open a disk file for updating (reading and writing) - ``'U'`` :term:`universal newlines` mode (deprecated) + ``'U'`` universal newline mode (for backwards compatibility; should + not be used in new code) ========= =============================================================== The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). @@ -925,85 +837,58 @@ *buffering* is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size - in bytes of a fixed-size chunk buffer. When no *buffering* argument is - given, the default buffering policy works as follows: + of a fixed-size chunk buffer. When no *buffering* argument is given, the + default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer is chosen using a heuristic trying to determine the underlying device's "block size" and falling back on :attr:`io.DEFAULT_BUFFER_SIZE`. On many systems, the buffer will typically be 4096 or 8192 bytes long. - * "Interactive" text files (files for which :meth:`~io.IOBase.isatty` - returns ``True``) use line buffering. Other text files use the policy - described above for binary files. + * "Interactive" text files (files for which :meth:`isatty` returns True) use + line buffering. Other text files use the policy described above for binary + files. *encoding* is the name of the encoding used to decode or encode the file. This should only be used in text mode. The default encoding is platform dependent (whatever :func:`locale.getpreferredencoding` returns), but any - :term:`text encoding` supported by Python - can be used. See the :mod:`codecs` module for + encoding supported by Python can be used. See the :mod:`codecs` module for the list of supported encodings. *errors* is an optional string that specifies how encoding and decoding - errors are to be handled--this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with - :func:`codecs.register_error` is also valid. The standard names - include: + errors are to be handled--this cannot be used in binary mode. Pass + ``'strict'`` to raise a :exc:`ValueError` exception if there is an encoding + error (the default of ``None`` has the same effect), or pass ``'ignore'`` to + ignore errors. (Note that ignoring encoding errors can lead to data loss.) + ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted + where there is malformed data. When writing, ``'xmlcharrefreplace'`` + (replace with the appropriate XML character reference) or + ``'backslashreplace'`` (replace with backslashed escape sequences) can be + used. Any other error handling name that has been registered with + :func:`codecs.register_error` is also valid. - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. + *newline* controls how universal newlines works (it only applies to text + mode). It can be ``None``, ``''``, ``'\n'``, ``'\r'``, and ``'\r\n'``. It + works as follows: - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. + * On input, if *newline* is ``None``, universal newlines mode is enabled. + Lines in the input can end in ``'\n'``, ``'\r'``, or ``'\r\n'``, and these + are translated into ``'\n'`` before being returned to the caller. If it is + ``''``, universal newline mode is enabled, but line endings are returned to + the caller untranslated. If it has any of the other legal values, input + lines are only terminated by the given string, and the line ending is + returned to the caller untranslated. - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as code - points in the Unicode Private Use Area ranging from U+DC80 to - U+DCFF. These private code points will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference ``&#nnn;``. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. - - .. index:: - single: universal newlines; open() built-in function - - *newline* controls how :term:`universal newlines` mode works (it only - applies to text mode). It can be ``None``, ``''``, ``'\n'``, ``'\r'``, and - ``'\r\n'``. It works as follows: - - * When reading input from the stream, if *newline* is ``None``, universal - newlines mode is enabled. Lines in the input can end in ``'\n'``, - ``'\r'``, or ``'\r\n'``, and these are translated into ``'\n'`` before - being returned to the caller. If it is ``''``, universal newlines mode is - enabled, but line endings are returned to the caller untranslated. If it - has any of the other legal values, input lines are only terminated by the - given string, and the line ending is returned to the caller untranslated. - - * When writing output to the stream, if *newline* is ``None``, any ``'\n'`` - characters written are translated to the system default line separator, - :data:`os.linesep`. If *newline* is ``''`` or ``'\n'``, no translation - takes place. If *newline* is any of the other legal values, any ``'\n'`` - characters written are translated to the given string. + * On output, if *newline* is ``None``, any ``'\n'`` characters written are + translated to the system default line separator, :data:`os.linesep`. If + *newline* is ``''``, no translation takes place. If *newline* is any of + the other legal values, any ``'\n'`` characters written are translated to + the given string. If *closefd* is ``False`` and a file descriptor rather than a filename was given, the underlying file descriptor will be kept open when the file is - closed. If a filename is given *closefd* must be ``True`` (the default) - otherwise an error will be raised. + closed. If a filename is given *closefd* has no effect and must be ``True`` + (the default). A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -1011,30 +896,18 @@ :mod:`os.open` as *opener* results in functionality similar to passing ``None``). - The newly created file is :ref:`non-inheritable `. + .. versionchanged:: 3.3 + The *opener* parameter was added. - The following example uses the :ref:`dir_fd ` parameter of the - :func:`os.open` function to open a file relative to a given directory:: - - >>> import os - >>> dir_fd = os.open('somedir', os.O_RDONLY) - >>> def opener(path, flags): - ... return os.open(path, flags, dir_fd=dir_fd) - ... - >>> with open('spamspam.txt', 'w', opener=opener) as f: - ... print('This will be written to somedir/spamspam.txt', file=f) - ... - >>> os.close(dir_fd) # don't leak a file descriptor - - The type of :term:`file object` returned by the :func:`open` function - depends on the mode. When :func:`open` is used to open a file in a text - mode (``'w'``, ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of + The type of file object returned by the :func:`open` function depends on the + mode. When :func:`open` is used to open a file in a text mode (``'w'``, + ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of :class:`io.TextIOBase` (specifically :class:`io.TextIOWrapper`). When used to open a file in a binary mode with buffering, the returned class is a subclass of :class:`io.BufferedIOBase`. The exact class varies: in read - binary mode, it returns an :class:`io.BufferedReader`; in write binary and - append binary modes, it returns an :class:`io.BufferedWriter`, and in - read/write mode, it returns an :class:`io.BufferedRandom`. When buffering is + binary mode, it returns a :class:`io.BufferedReader`; in write binary and + append binary modes, it returns a :class:`io.BufferedWriter`, and in + read/write mode, it returns a :class:`io.BufferedRandom`. When buffering is disabled, the raw stream, a subclass of :class:`io.RawIOBase`, :class:`io.FileIO`, is returned. @@ -1052,31 +925,16 @@ and :mod:`shutil`. .. versionchanged:: 3.3 - The *opener* parameter was added. - The ``'x'`` mode was added. :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - :exc:`FileExistsError` is now raised if the file opened in exclusive - creation mode (``'x'``) already exists. - .. versionchanged:: 3.4 - The file is now non-inheritable. - .. deprecated-removed:: 3.4 4.0 - - The ``'U'`` mode. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - +.. XXX works for bytes too, but should it? .. function:: ord(c) Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, - ``ord('a')`` returns the integer ``97`` and ``ord('ν')`` returns ``957``. - This is the inverse of :func:`chr`. + representing the Unicode code + point of that character. For example, ``ord('a')`` returns the integer ``97`` + and ``ord('\u2020')`` returns ``8224``. This is the inverse of :func:`chr`. .. function:: pow(x, y[, z]) @@ -1095,39 +953,34 @@ must be of integer types, and *y* must be non-negative. -.. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) +.. function:: print([object, ...], *, sep=' ', end='\\n', file=sys.stdout, flush=False) - Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end* and *file*, if present, must be given as keyword + Print *object*\(s) to the stream *file*, separated by *sep* and followed by + *end*. *sep*, *end* and *file*, if present, must be given as keyword arguments. All non-keyword arguments are converted to strings like :func:`str` does and written to the stream, separated by *sep* and followed by *end*. Both *sep* and *end* must be strings; they can also be ``None``, which means to use the - default values. If no *objects* are given, :func:`print` will just write + default values. If no *object* is given, :func:`print` will just write *end*. The *file* argument must be an object with a ``write(string)`` method; if it - is not present or ``None``, :data:`sys.stdout` will be used. Since printed - arguments are converted to text strings, :func:`print` cannot be used with - binary mode file objects. For these, use ``file.write(...)`` instead. - - Whether output is buffered is usually determined by *file*, but if the - *flush* keyword argument is true, the stream is forcibly flushed. + is not present or ``None``, :data:`sys.stdout` will be used. Whether output + is buffered is usually determined by *file*, but if the *flush* keyword + argument is true, the stream is forcibly flushed. .. versionchanged:: 3.3 Added the *flush* keyword argument. -.. class:: property(fget=None, fset=None, fdel=None, doc=None) +.. function:: property(fget=None, fset=None, fdel=None, doc=None) Return a property attribute. - *fget* is a function for getting an attribute value. *fset* is a function - for setting an attribute value. *fdel* is a function for deleting an attribute - value. And *doc* creates a docstring for the attribute. - - A typical use is to define a managed attribute ``x``:: + *fget* is a function for getting an attribute value, likewise *fset* is a + function for setting, and *fdel* a function for del'ing, an attribute. Typical + use is to define a managed attribute ``x``:: class C: def __init__(self): @@ -1135,16 +988,13 @@ def getx(self): return self._x - def setx(self, value): self._x = value - def delx(self): del self._x - x = property(getx, setx, delx, "I'm the 'x' property.") - If *c* is an instance of *C*, ``c.x`` will invoke the getter, + If then *c* is an instance of *C*, ``c.x`` will invoke the getter, ``c.x = value`` will invoke the setter and ``del c.x`` the deleter. If given, *doc* will be the docstring of the property attribute. Otherwise, the @@ -1160,14 +1010,13 @@ """Get the current voltage.""" return self._voltage - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" - for a read-only attribute with the same name, and it sets the docstring for - *voltage* to "Get the current voltage." + turns the :meth:`voltage` method into a "getter" for a read-only attribute + with the same name. - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: + A property object has :attr:`getter`, :attr:`setter`, and :attr:`deleter` + methods usable as decorators that create a copy of the property with the + corresponding accessor function set to the decorated function. This is + best explained with an example:: class C: def __init__(self): @@ -1190,20 +1039,83 @@ additional functions the same name as the original property (``x`` in this case.) - The returned property object also has the attributes ``fget``, ``fset``, and + The returned property also has the attributes ``fget``, ``fset``, and ``fdel`` corresponding to the constructor arguments. - .. versionchanged:: 3.5 - The docstrings of property objects are now writeable. +.. XXX does accept objects with __index__ too +.. function:: range([start,] stop[, step]) -.. _func-range: -.. function:: range(stop) - range(start, stop[, step]) - :noindex: + This is a versatile function to create iterables yielding arithmetic + progressions. It is most often used in :keyword:`for` loops. The arguments + must be integers. If the *step* argument is omitted, it defaults to ``1``. + If the *start* argument is omitted, it defaults to ``0``. The full form + returns an iterable of integers ``[start, start + step, start + 2 * step, + ...]``. If *step* is positive, the last element is the largest ``start + i * + step`` less than *stop*; if *step* is negative, the last element is the + smallest ``start + i * step`` greater than *stop*. *step* must not be zero + (or else :exc:`ValueError` is raised). Range objects have read-only data + attributes :attr:`start`, :attr:`stop` and :attr:`step` which return the + argument values (or their default). Example: - Rather than being a function, :class:`range` is actually an immutable - sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. + >>> list(range(10)) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(range(1, 11)) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + >>> list(range(0, 30, 5)) + [0, 5, 10, 15, 20, 25] + >>> list(range(0, 10, 3)) + [0, 3, 6, 9] + >>> list(range(0, -10, -1)) + [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] + >>> list(range(0)) + [] + >>> list(range(1, 0)) + [] + + Range objects implement the :class:`collections.Sequence` ABC, and provide + features such as containment tests, element index lookup, slicing and + support for negative indices (see :ref:`typesseq`): + + >>> r = range(0, 20, 2) + >>> r + range(0, 20, 2) + >>> 11 in r + False + >>> 10 in r + True + >>> r.index(10) + 5 + >>> r[5] + 10 + >>> r[:5] + range(0, 10, 2) + >>> r[-1] + 18 + + Testing range objects for equality with ``==`` and ``!=`` compares + them as sequences. That is, two range objects are considered equal if + they represent the same sequence of values. (Note that two range + objects that compare equal might have different :attr:`start`, + :attr:`stop` and :attr:`step` attributes, for example ``range(0) == + range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.) + + Ranges containing absolute values larger than :data:`sys.maxsize` are permitted + but some features (such as :func:`len`) will raise :exc:`OverflowError`. + + .. versionchanged:: 3.2 + Implement the Sequence ABC. + Support slicing and negative indices. + Test integers for membership in constant time instead of iterating + through all items. + + .. versionchanged:: 3.3 + Define '==' and '!=' to compare range objects based on the + sequence of values they define (instead of comparing based on + object identity). + + .. versionadded:: 3.3 + The :attr:`start`, :attr:`stop` and :attr:`step` attributes. .. function:: repr(object) @@ -1225,18 +1137,18 @@ arguments starting at ``0``). -.. function:: round(number[, ndigits]) +.. function:: round(x[, n]) - Return the floating point value *number* rounded to *ndigits* digits after - the decimal point. If *ndigits* is omitted, it returns the nearest integer - to its input. Delegates to ``number.__round__(ndigits)``. + Return the floating point value *x* rounded to *n* digits after the decimal + point. If *n* is omitted, it defaults to zero. Delegates to + ``x.__round__(n)``. For the built-in types supporting :func:`round`, values are rounded to the - closest multiple of 10 to the power minus *ndigits*; if two multiples are - equally close, rounding is done toward the even choice (so, for example, - both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). The return value is an integer if called with one argument, - otherwise of the same type as *number*. + closest multiple of 10 to the power minus *n*; if two multiples are equally + close, rounding is done toward the even choice (so, for example, both + ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is ``2``). + The return value is an integer if called with one argument, otherwise of the + same type as *x*. .. note:: @@ -1248,16 +1160,11 @@ .. _func-set: -.. class:: set([iterable]) +.. function:: set([iterable]) :noindex: - Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and - :ref:`types-set` for documentation about this class. - - For other containers see the built-in :class:`frozenset`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. + Return a new set, optionally with elements taken from *iterable*. + The set type is described in :ref:`types-set`. .. function:: setattr(object, name, value) @@ -1269,20 +1176,19 @@ ``x.foobar = 123``. -.. class:: slice(stop) - slice(start, stop[, step]) +.. function:: slice([start,] stop[, step]) .. index:: single: Numerical Python Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop` and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however they are used by Numerical Python and other third party extensions. - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. + ``None``. Slice objects have read-only data attributes :attr:`start`, + :attr:`stop` and :attr:`step` which merely return the argument values (or their + default). They have no other explicit functionality; however they are used by + Numerical Python and other third party extensions. Slice objects are also + generated when extended indexing syntax is used. For example: + ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` + for an alternate version that returns an iterator. .. function:: sorted(iterable[, key][, reverse]) @@ -1301,12 +1207,8 @@ Use :func:`functools.cmp_to_key` to convert an old-style *cmp* function to a *key* function. - The built-in :func:`sorted` function is guaranteed to be stable. A sort is - stable if it guarantees not to change the relative order of elements that - compare equal --- this is helpful for sorting in multiple passes (for - example, sort by department, then by salary grade). - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. + For sorting examples and a brief sorting tutorial, see `Sorting HowTo + `_\. .. function:: staticmethod(function) @@ -1332,19 +1234,37 @@ For more information on static methods, consult the documentation on the standard type hierarchy in :ref:`types`. - .. index:: - single: string; str() (built-in function) +.. function:: str([object[, encoding[, errors]]]) -.. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - :noindex: + Return a string version of an object, using one of the following modes: - Return a :class:`str` version of *object*. See :func:`str` for details. + If *encoding* and/or *errors* are given, :func:`str` will decode the + *object* which can either be a byte string or a character buffer using + the codec for *encoding*. The *encoding* parameter is a string giving + the name of an encoding; if the encoding is not known, :exc:`LookupError` + is raised. Error handling is done according to *errors*; this specifies the + treatment of characters which are invalid in the input encoding. If + *errors* is ``'strict'`` (the default), a :exc:`ValueError` is raised on + errors, while a value of ``'ignore'`` causes errors to be silently ignored, + and a value of ``'replace'`` causes the official Unicode replacement character, + U+FFFD, to be used to replace input characters which cannot be decoded. + See also the :mod:`codecs` module. - ``str`` is the built-in string :term:`class`. For general information - about strings, see :ref:`textseq`. + When only *object* is given, this returns its nicely printable representation. + For strings, this is the string itself. The difference with ``repr(object)`` + is that ``str(object)`` does not always attempt to return a string that is + acceptable to :func:`eval`; its goal is to return a printable string. + With no arguments, this returns the empty string. + + Objects can specify what ``str(object)`` returns by defining a :meth:`__str__` + special method. + + For more information on strings see :ref:`typesseq` which describes sequence + functionality (strings are sequences), and also the string-specific methods + described in the :ref:`string-methods` section. To output formatted strings, + see the :ref:`string-formatting` section. In addition see the + :ref:`stringservices` section. .. function:: sum(iterable[, start]) @@ -1366,10 +1286,9 @@ been overridden in a class. The search order is same as that used by :func:`getattr` except that the *type* itself is skipped. - The :attr:`~class.__mro__` attribute of the *type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. + The :attr:`__mro__` attribute of the *type* lists the method resolution + search order used by both :func:`getattr` and :func:`super`. The attribute + is dynamic and can change whenever the inheritance hierarchy is updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If @@ -1405,69 +1324,69 @@ Accordingly, :func:`super` is undefined for implicit lookups using statements or operators such as ``super()[name]``. - Also note that, aside from the zero argument form, :func:`super` is not - limited to use inside methods. The two argument form specifies the - arguments exactly and makes the appropriate references. The zero - argument form only works inside a class definition, as the compiler fills - in the necessary details to correctly retrieve the class being defined, - as well as accessing the current instance for ordinary methods. + Also note that :func:`super` is not limited to use inside methods. The two + argument form specifies the arguments exactly and makes the appropriate + references. The zero argument form automatically searches the stack frame + for the class (``__class__``) and the first argument. For practical suggestions on how to design cooperative classes using :func:`super`, see `guide to using super() `_. -.. _func-tuple: .. function:: tuple([iterable]) - :noindex: - Rather than being a function, :class:`tuple` is actually an immutable - sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. + Return a tuple whose items are the same and in the same order as *iterable*'s + items. *iterable* may be a sequence, a container that supports iteration, or an + iterator object. If *iterable* is already a tuple, it is returned unchanged. + For instance, ``tuple('abc')`` returns ``('a', 'b', 'c')`` and ``tuple([1, 2, + 3])`` returns ``(1, 2, 3)``. If no argument is given, returns a new empty + tuple, ``()``. + :class:`tuple` is an immutable sequence type, as documented in :ref:`typesseq`. -.. class:: type(object) - type(name, bases, dict) + +.. function:: type(object) .. index:: object: type - With one argument, return the type of an *object*. The return value is a - type object and generally the same object as returned by - :attr:`object.__class__ `. + Return the type of an *object*. The return value is a type object and + generally the same object as returned by ``object.__class__``. The :func:`isinstance` built-in function is recommended for testing the type of an object, because it takes subclasses into account. + With three arguments, :func:`type` functions as a constructor as detailed + below. - With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~class.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and becomes the :attr:`~object.__dict__` attribute. For - example, the following two statements create identical :class:`type` objects: + +.. function:: type(name, bases, dict) + :noindex: + + Return a new type object. This is essentially a dynamic form of the + :keyword:`class` statement. The *name* string is the class name and becomes the + :attr:`__name__` attribute; the *bases* tuple itemizes the base classes and + becomes the :attr:`__bases__` attribute; and the *dict* dictionary is the + namespace containing definitions for class body and becomes the :attr:`__dict__` + attribute. For example, the following two statements create identical + :class:`type` objects: >>> class X: ... a = 1 ... >>> X = type('X', (object,), dict(a=1)) - See also :ref:`bltin-type-objects`. - .. function:: vars([object]) - Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`__dict__` attribute. + Without an argument, act like :func:`locals`. - Objects such as modules and instances have an updateable :attr:`__dict__` - attribute; however, other objects may have write restrictions on their - :attr:`__dict__` attributes (for example, classes use a - dictproxy to prevent direct dictionary updates). + With a module, class or class instance object as argument (or anything else that + has a :attr:`__dict__` attribute), return that attribute. - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. - + .. note:: + The returned dictionary should not be modified: + the effects on the corresponding symbol table are undefined. [#]_ .. function:: zip(*iterables) @@ -1494,9 +1413,7 @@ The left-to-right evaluation order of the iterables is guaranteed. This makes possible an idiom for clustering a data series into n-length groups - using ``zip(*[iter(s)]*n)``. This repeats the *same* iterator ``n`` times - so that each output tuple has the result of ``n`` calls to the iterator. - This has the effect of dividing the input into n-length chunks. + using ``zip(*[iter(s)]*n)``. :func:`zip` should only be used with unequal length inputs when you don't care about trailing, unmatched values from the longer iterables. If those @@ -1515,7 +1432,7 @@ True -.. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) +.. function:: __import__(name, globals={}, locals={}, fromlist=[], level=0) .. index:: statement: import @@ -1529,11 +1446,9 @@ This function is invoked by the :keyword:`import` statement. It can be replaced (by importing the :mod:`builtins` module and assigning to ``builtins.__import__``) in order to change semantics of the - :keyword:`import` statement, but doing so is **strongly** discouraged as it - is usually simpler to use import hooks (see :pep:`302`) to attain the same - goals and does not cause issues with code which assumes the default import - implementation is in use. Direct use of :func:`__import__` is also - discouraged in favor of :func:`importlib.import_module`. + :keyword:`import` statement, but nowadays it is usually simpler to use import + hooks (see :pep:`302`). Direct use of :func:`__import__` is rare, except in + cases where you want to import a module whose name is only known at runtime. The function imports the module *name*, potentially using the given *globals* and *locals* to determine how to interpret the name in a package context. @@ -1545,8 +1460,7 @@ *level* specifies whether to use absolute or relative imports. ``0`` (the default) means only perform absolute imports. Positive values for *level* indicate the number of parent directories to search relative to the - directory of the module calling :func:`__import__` (see :pep:`328` for the - details). + directory of the module calling :func:`__import__`. When the *name* variable is of the form ``package.module``, normally, the top-level package (the name up till the first dot) is returned, *not* the @@ -1579,13 +1493,13 @@ If you simply want to import a module (potentially within a package) by name, use :func:`importlib.import_module`. - .. versionchanged:: 3.3 - Negative values for *level* are no longer supported (which also changes - the default value to 0). - .. rubric:: Footnotes .. [#] Note that the parser only accepts the Unix-style end of line convention. If you are reading the code from a file, make sure to use newline conversion mode to convert Windows or Mac-style newlines. + +.. [#] In the current implementation, local variable bindings cannot normally be + affected this way, but variables retrieved from other scopes (such as modules) + can be. This may change. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/functools.rst --- a/Doc/library/functools.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/functools.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,6 @@ .. moduleauthor:: Peter Harris .. moduleauthor:: Raymond Hettinger .. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa .. sectionauthor:: Peter Harris **Source code:** :source:`Lib/functools.py` @@ -21,8 +20,8 @@ .. function:: cmp_to_key(func) - Transform an old-style comparison function to a :term:`key function`. Used - with tools that accept key functions (such as :func:`sorted`, :func:`min`, + Transform an old-style comparison function to a key function. Used with + tools that accept key functions (such as :func:`sorted`, :func:`min`, :func:`max`, :func:`heapq.nlargest`, :func:`heapq.nsmallest`, :func:`itertools.groupby`). This function is primarily used as a transition tool for programs being converted from Python 2 which supported the use of @@ -31,18 +30,17 @@ A comparison function is any callable that accept two arguments, compares them, and returns a negative number for less-than, zero for equality, or a positive number for greater-than. A key function is a callable that accepts one - argument and returns another value to be used as the sort key. + argument and returns another value indicating the position in the desired + collation sequence. Example:: sorted(iterable, key=cmp_to_key(locale.strcoll)) # locale-aware sort order - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - .. versionadded:: 3.2 -.. decorator:: lru_cache(maxsize=128, typed=False) +.. decorator:: lru_cache(maxsize=100, typed=False) Decorator to wrap a function with a memoizing callable that saves up to the *maxsize* most recent calls. It can save time when an expensive or I/O bound @@ -51,9 +49,8 @@ Since a dictionary is used to cache results, the positional and keyword arguments to the function must be hashable. - If *maxsize* is set to None, the LRU feature is disabled and the cache can - grow without bound. The LRU feature performs best when *maxsize* is a - power-of-two. + If *maxsize* is set to None, the LRU feature is disabled and the cache + can grow without bound. If *typed* is set to True, function arguments of different types will be cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated @@ -73,7 +70,7 @@ bypassing the cache, or for rewrapping the function with a different cache. An `LRU (least recently used) cache - `_ works + `_ works best when the most recent calls are the best predictors of upcoming calls (for example, the most popular articles on a news server tend to change each day). The cache's size limit assures that the cache does not grow without bound on @@ -81,7 +78,7 @@ Example of an LRU cache for static web content:: - @lru_cache(maxsize=32) + @lru_cache(maxsize=20) def get_pep(num): 'Retrieve text of a Python Enhancement Proposal' resource = 'http://www.python.org/dev/peps/pep-%04d/' % num @@ -95,8 +92,8 @@ ... pep = get_pep(n) ... print(n, len(pep)) - >>> get_pep.cache_info() - CacheInfo(hits=3, misses=8, maxsize=32, currsize=8) + >>> print(get_pep.cache_info()) + CacheInfo(hits=3, misses=8, maxsize=20, currsize=8) Example of efficiently computing `Fibonacci numbers `_ @@ -110,10 +107,10 @@ return n return fib(n-1) + fib(n-2) - >>> [fib(n) for n in range(16)] + >>> print([fib(n) for n in range(16)]) [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610] - >>> fib.cache_info() + >>> print(fib.cache_info()) CacheInfo(hits=28, misses=16, maxsize=None, currsize=16) .. versionadded:: 3.2 @@ -135,34 +132,15 @@ @total_ordering class Student: - def _is_valid_operand(self, other): - return (hasattr(other, "lastname") and - hasattr(other, "firstname")) def __eq__(self, other): - if not self._is_valid_operand(other): - return NotImplemented return ((self.lastname.lower(), self.firstname.lower()) == (other.lastname.lower(), other.firstname.lower())) def __lt__(self, other): - if not self._is_valid_operand(other): - return NotImplemented return ((self.lastname.lower(), self.firstname.lower()) < (other.lastname.lower(), other.firstname.lower())) - .. note:: - - While this decorator makes it easy to create well behaved totally - ordered types, it *does* come at the cost of slower execution and - more complex stack traces for the derived comparison methods. If - performance benchmarking indicates this is a bottleneck for a given - application, implementing all six rich comparison methods instead is - likely to provide an easy speed boost. - .. versionadded:: 3.2 - .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for - unrecognised types is now supported. .. function:: partial(func, *args, **keywords) @@ -195,50 +173,6 @@ 18 -.. class:: partialmethod(func, *args, **keywords) - - Return a new :class:`partialmethod` descriptor which behaves - like :class:`partial` except that it is designed to be used as a method - definition rather than being directly callable. - - *func* must be a :term:`descriptor` or a callable (objects which are both, - like normal functions, are handled as descriptors). - - When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or - another instance of :class:`partialmethod`), calls to ``__get__`` are - delegated to the underlying descriptor, and an appropriate - :class:`partial` object returned as the result. - - When *func* is a non-descriptor callable, an appropriate bound method is - created dynamically. This behaves like a normal Python function when - used as a method: the *self* argument will be inserted as the first - positional argument, even before the *args* and *keywords* supplied to - the :class:`partialmethod` constructor. - - Example:: - - >>> class Cell(object): - ... def __init__(self): - ... self._alive = False - ... @property - ... def alive(self): - ... return self._alive - ... def set_state(self, state): - ... self._alive = bool(state) - ... set_alive = partialmethod(set_state, True) - ... set_dead = partialmethod(set_state, False) - ... - >>> c = Cell() - >>> c.alive - False - >>> c.set_alive() - >>> c.alive - True - - .. versionadded:: 3.4 - - .. function:: reduce(function, iterable[, initializer]) Apply *function* of two arguments cumulatively to the items of *sequence*, from @@ -250,123 +184,6 @@ a default when the sequence is empty. If *initializer* is not given and *sequence* contains only one item, the first item is returned. - Roughly equivalent to:: - - def reduce(function, iterable, initializer=None): - it = iter(iterable) - if initializer is None: - value = next(it) - else: - value = initializer - for element in it: - value = function(value, element) - return value - - -.. decorator:: singledispatch(default) - - Transforms a function into a :term:`single-dispatch ` :term:`generic function`. - - To define a generic function, decorate it with the ``@singledispatch`` - decorator. Note that the dispatch happens on the type of the first argument, - create your function accordingly:: - - >>> from functools import singledispatch - >>> @singledispatch - ... def fun(arg, verbose=False): - ... if verbose: - ... print("Let me just say,", end=" ") - ... print(arg) - - To add overloaded implementations to the function, use the :func:`register` - attribute of the generic function. It is a decorator, taking a type - parameter and decorating a function implementing the operation for that - type:: - - >>> @fun.register(int) - ... def _(arg, verbose=False): - ... if verbose: - ... print("Strength in numbers, eh?", end=" ") - ... print(arg) - ... - >>> @fun.register(list) - ... def _(arg, verbose=False): - ... if verbose: - ... print("Enumerate this:") - ... for i, elem in enumerate(arg): - ... print(i, elem) - - To enable registering lambdas and pre-existing functions, the - :func:`register` attribute can be used in a functional form:: - - >>> def nothing(arg, verbose=False): - ... print("Nothing.") - ... - >>> fun.register(type(None), nothing) - - The :func:`register` attribute returns the undecorated function which - enables decorator stacking, pickling, as well as creating unit tests for - each variant independently:: - - >>> @fun.register(float) - ... @fun.register(Decimal) - ... def fun_num(arg, verbose=False): - ... if verbose: - ... print("Half of your number:", end=" ") - ... print(arg / 2) - ... - >>> fun_num is fun - False - - When called, the generic function dispatches on the type of the first - argument:: - - >>> fun("Hello, world.") - Hello, world. - >>> fun("test.", verbose=True) - Let me just say, test. - >>> fun(42, verbose=True) - Strength in numbers, eh? 42 - >>> fun(['spam', 'spam', 'eggs', 'spam'], verbose=True) - Enumerate this: - 0 spam - 1 spam - 2 eggs - 3 spam - >>> fun(None) - Nothing. - >>> fun(1.23) - 0.615 - - Where there is no registered implementation for a specific type, its - method resolution order is used to find a more generic implementation. - The original function decorated with ``@singledispatch`` is registered - for the base ``object`` type, which means it is used if no better - implementation is found. - - To check which implementation will the generic function choose for - a given type, use the ``dispatch()`` attribute:: - - >>> fun.dispatch(float) - - >>> fun.dispatch(dict) # note: default implementation - - - To access all registered implementations, use the read-only ``registry`` - attribute:: - - >>> fun.registry.keys() - dict_keys([, , , - , , - ]) - >>> fun.registry[float] - - >>> fun.registry[object] - - - .. versionadded:: 3.4 - .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) @@ -382,8 +199,8 @@ To allow access to the original function for introspection and other purposes (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function - automatically adds a ``__wrapped__`` attribute to the wrapper that refers to - the function being wrapped. + automatically adds a __wrapped__ attribute to the wrapper that refers to + the original function. The main intended use for this function is in :term:`decorator` functions which wrap the decorated function and return the wrapper. If the wrapper function is @@ -406,18 +223,12 @@ .. versionchanged:: 3.2 Missing attributes no longer trigger an :exc:`AttributeError`. - .. versionchanged:: 3.4 - The ``__wrapped__`` attribute now always refers to the wrapped - function, even if that function defined a ``__wrapped__`` attribute. - (see :issue:`17482`) - .. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) - This is a convenience function for invoking :func:`update_wrapper` as a - function decorator when defining a wrapper function. It is equivalent to - ``partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated)``. - For example:: + This is a convenience function for invoking ``partial(update_wrapper, + wrapped=wrapped, assigned=assigned, updated=updated)`` as a function decorator + when defining a wrapper function. For example: >>> from functools import wraps >>> def my_decorator(f): @@ -477,3 +288,4 @@ are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods during instance attribute look-up. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/gc.rst --- a/Doc/library/gc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/gc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -67,25 +67,6 @@ returned. -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - .. function:: set_threshold(threshold0[, threshold1[, threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting @@ -140,8 +121,8 @@ Return a list of objects directly referred to by any of the arguments. The referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects + :attr:`tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :attr:`tp_traverse` methods are supported only by objects that support garbage collection, and are only required to visit objects that may be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. @@ -149,8 +130,8 @@ .. function:: is_tracked(obj) - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't + Returns True if the object is currently tracked by the garbage collector, + False otherwise. As a general rule, instances of atomic types aren't tracked and instances of non-atomic types (containers, user-defined objects...) are. However, some type-specific optimizations can be present in order to suppress the garbage collector footprint of simple instances @@ -172,64 +153,36 @@ .. versionadded:: 3.1 -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): +The following variable is provided for read-only access (you can mutate its +value but should not rebind it): .. data:: garbage - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-NULL ``tp_del`` slot. + A list of objects which the collector found to be unreachable but could not be + freed (uncollectable objects). By default, this list contains only objects with + :meth:`__del__` methods. Objects that have :meth:`__del__` methods and are + part of a reference cycle cause the entire reference cycle to be uncollectable, + including objects not necessarily in the cycle but reachable only from it. + Python doesn't collect such cycles automatically because, in general, it isn't + possible for Python to guess a safe order in which to run the :meth:`__del__` + methods. If you know a safe order, you can force the issue by examining the + *garbage* list, and explicitly breaking cycles due to your objects within the + list. Note that these objects are kept alive even so by virtue of being in the + *garbage* list, so they should be removed from *garbage* too. For example, + after breaking cycles, do ``del gc.garbage[:]`` to empty the list. It's + generally better to avoid the issue by not creating cycles containing objects + with :meth:`__del__` methods, and *garbage* can be examined in that case to + verify that no such cycles are being created. - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. + If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be added + to this list rather than freed. .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a + If this list is non-empty at interpreter shutdown, a :exc:`ResourceWarning` is emitted, which is silent by default. If :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects are printed. - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :attr:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - The following constants are provided for use with :func:`set_debug`: @@ -252,8 +205,8 @@ to the ``garbage`` list. .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. + Also print the contents of the :data:`garbage` list at interpreter + shutdown, if it isn't empty. .. data:: DEBUG_SAVEALL diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/getopt.rst --- a/Doc/library/getopt.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/getopt.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,6 @@ -------------- .. note:: - The :mod:`getopt` module is a parser for command line options whose API is designed to be familiar to users of the C :c:func:`getopt` function. Users who are unfamiliar with the C :c:func:`getopt` function or who would like to write diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/getpass.rst --- a/Doc/library/getpass.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/getpass.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,11 +13,10 @@ .. function:: getpass(prompt='Password: ', stream=None) Prompt the user for a password without echoing. The user is prompted using - the string *prompt*, which defaults to ``'Password: '``. On Unix, the - prompt is written to the file-like object *stream* using the replace error - handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this - argument is ignored on Windows). + the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt + is written to the file-like object *stream*. *stream* defaults to the + controlling terminal (:file:`/dev/tty`) or if that is unavailable to + ``sys.stderr`` (this argument is ignored on Windows). If echo free input is unavailable getpass() falls back to printing a warning message to *stream* and reading from ``sys.stdin`` and diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/gettext.rst --- a/Doc/library/gettext.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/gettext.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,8 +3,8 @@ .. module:: gettext :synopsis: Multilingual internationalization services. -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw +.. moduleauthor:: Barry A. Warsaw +.. sectionauthor:: Barry A. Warsaw **Source code:** :source:`Lib/gettext.py` @@ -94,10 +94,9 @@ The Plural formula is taken from the catalog header. It is a C or Python expression that has a free variable *n*; the expression evaluates to the index - of the plural in the catalog. See - `the GNU gettext documentation `__ - for the precise syntax to be used in :file:`.po` files and the - formulas for a variety of languages. + of the plural in the catalog. See the GNU gettext documentation for the precise + syntax to be used in :file:`.po` files and the formulas for a variety of + languages. .. function:: lngettext(singular, plural, n) @@ -344,9 +343,9 @@ The entire set of key/value pairs are placed into a dictionary and set as the "protected" :attr:`_info` instance variable. -If the :file:`.mo` file's magic number is invalid, the major version number is -unexpected, or if other problems occur while reading the file, instantiating a -:class:`GNUTranslations` class can raise :exc:`OSError`. +If the :file:`.mo` file's magic number is invalid, or if other problems occur +while reading the file, instantiating a :class:`GNUTranslations` class can raise +:exc:`OSError`. The following methods are overridden from the base class implementation: @@ -452,42 +451,35 @@ In this example, the string ``'writing a log message'`` is marked as a candidate for translation, while the strings ``'mylog.txt'`` and ``'w'`` are not. -There are a few tools to extract the strings meant for translation. -The original GNU :program:`gettext` only supported C or C++ source -code but its extended version :program:`xgettext` scans code written -in a number of languages, including Python, to find strings marked as -translatable. `Babel `__ is a Python -internationalization library that includes a :file:`pybabel` script to -extract and compile message catalogs. François Pinard's program -called :program:`xpot` does a similar job and is available as part of -his `po-utils package `__. +The Python distribution comes with two tools which help you generate the message +catalogs once you've prepared your source code. These may or may not be +available from a binary distribution, but they can be found in a source +distribution, in the :file:`Tools/i18n` directory. -(Python also includes pure-Python versions of these programs, called -:program:`pygettext.py` and :program:`msgfmt.py`; some Python distributions -will install them for you. :program:`pygettext.py` is similar to -:program:`xgettext`, but only understands Python source code and -cannot handle other programming languages such as C or C++. -:program:`pygettext.py` supports a command-line interface similar to -:program:`xgettext`; for details on its use, run ``pygettext.py ---help``. :program:`msgfmt.py` is binary compatible with GNU -:program:`msgfmt`. With these two programs, you may not need the GNU -:program:`gettext` package to internationalize your Python -applications.) +The :program:`pygettext` [#]_ program scans all your Python source code looking +for the strings you previously marked as translatable. It is similar to the GNU +:program:`gettext` program except that it understands all the intricacies of +Python source code, but knows nothing about C or C++ source code. You don't +need GNU ``gettext`` unless you're also going to be translating C code (such as +C extension modules). -:program:`xgettext`, :program:`pygettext`, and similar tools generate -:file:`.po` files that are message catalogs. They are structured -human-readable files that contain every marked string in the source -code, along with a placeholder for the translated versions of these -strings. +:program:`pygettext` generates textual Uniforum-style human readable message +catalog :file:`.pot` files, essentially structured human readable files which +contain every marked string in the source code, along with a placeholder for the +translation strings. :program:`pygettext` is a command line script that supports +a similar command line interface as :program:`xgettext`; for details on its use, +run:: -Copies of these :file:`.po` files are then handed over to the -individual human translators who write translations for every -supported natural language. They send back the completed -language-specific versions as a :file:`.po` file that's -compiled into a machine-readable :file:`.mo` binary catalog file using -the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at -run-time. + pygettext.py --help + +Copies of these :file:`.pot` files are then handed over to the individual human +translators who write language-specific versions for every supported natural +language. They send you back the filled in language-specific versions as a +:file:`.po` file. Using the :program:`msgfmt.py` [#]_ program (in the +:file:`Tools/i18n` directory), you take the :file:`.po` files from your +translators and generate the machine-readable :file:`.mo` binary catalog files. +The :file:`.mo` files are what the :mod:`gettext` module uses for the actual +translation processing during run-time. How you use the :mod:`gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two @@ -525,7 +517,7 @@ import gettext gettext.install('myapplication') -If you need to set the locale directory, you can pass it into the +If you need to set the locale directory, you can pass these into the :func:`install` function:: import gettext @@ -598,8 +590,7 @@ namespace. Note that the second use of :func:`_` will not identify "a" as being -translatable to the :program:`gettext` program, because the parameter -is not a string literal. +translatable to the :program:`pygettext` program, since it is not a string. Another way to handle this is with the following example:: @@ -615,14 +606,11 @@ for a in animals: print(_(a)) -In this case, you are marking translatable strings with the function -:func:`N_`, which won't conflict with any definition of :func:`_`. -However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`N_`. :program:`xgettext`, -:program:`pygettext`, ``pybabel extract``, and :program:`xpot` all -support this through the use of the :option:`-k` command-line switch. -The choice of :func:`N_` here is totally arbitrary; it could have just -as easily been :func:`MarkThisStringForTranslation`. +In this case, you are marking translatable strings with the function :func:`N_`, +[#]_ which won't conflict with any definition of :func:`_`. However, you will +need to teach your message extraction program to look for translatable strings +marked with :func:`N_`. :program:`pygettext` and :program:`xpot` both support +this through the use of command line switches. Acknowledgements @@ -657,3 +645,16 @@ absolute path at the start of your application. .. [#] See the footnote for :func:`bindtextdomain` above. + +.. [#] François Pinard has written a program called :program:`xpot` which does a + similar job. It is available as part of his `po-utils package + `_. + +.. [#] :program:`msgfmt.py` is binary compatible with GNU :program:`msgfmt` except that + it provides a simpler, all-Python implementation. With this and + :program:`pygettext.py`, you generally won't need to install the GNU + :program:`gettext` package to internationalize your Python applications. + +.. [#] The choice of :func:`N_` here is totally arbitrary; it could have just as easily + been :func:`MarkThisStringForTranslation`. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/glob.rst --- a/Doc/library/glob.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/glob.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,24 +12,15 @@ -------------- The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character -ranges expressed with ``[]`` will be correctly matched. This is done by using -the :func:`os.listdir` and :func:`fnmatch.fnmatch` functions in concert, and -not by actually invoking a subshell. Note that unlike :func:`fnmatch.fnmatch`, -:mod:`glob` treats filenames beginning with a dot (``.``) as special cases. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) +according to the rules used by the Unix shell. No tilde expansion is done, but +``*``, ``?``, and character ranges expressed with ``[]`` will be correctly +matched. This is done by using the :func:`os.listdir` and +:func:`fnmatch.fnmatch` functions in concert, and not by actually invoking a +subshell. (For tilde and shell variable expansion, use +:func:`os.path.expanduser` and :func:`os.path.expandvars`.) -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - -.. function:: glob(pathname, *, recursive=False) +.. function:: glob(pathname) Return a possibly-empty list of path names that match *pathname*, which must be a string containing a path specification. *pathname* can be either absolute @@ -37,38 +28,15 @@ :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken symlinks are included in the results (as in the shell). - If *recursive* is true, the pattern "``**``" will match any files and zero or - more directories and subdirectories. If the pattern is followed by an - ``os.sep``, only directories and subdirectories match. - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. versionchanged:: 3.5 - Support for recursive globs using "``**``". - - -.. function:: iglob(pathname, recursive=False) +.. function:: iglob(pathname) Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. -.. function:: escape(pathname) - - Escape all special characters (``'?'``, ``'*'`` and ``'['``). - This is useful if you want to match an arbitrary literal string that may - have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows - ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. - - .. versionadded:: 3.4 - - -For example, consider a directory containing the following files: -:file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub` -which contains only the file :file:`3.txt`. :func:`glob` will produce +For example, consider a directory containing only the following files: +:file:`1.gif`, :file:`2.txt`, and :file:`card.gif`. :func:`glob` will produce the following results. Notice how any leading components of the path are preserved. :: @@ -79,20 +47,7 @@ ['1.gif', 'card.gif'] >>> glob.glob('?.gif') ['1.gif'] - >>> glob.glob('**/*.txt', recursive=True) - ['2.txt', 'sub/3.txt'] - >>> glob.glob('./**/', recursive=True) - ['./', './sub/'] -If the directory contains files starting with ``.`` they won't be matched by -default. For example, consider a directory containing :file:`card.gif` and -:file:`.card.gif`:: - - >>> import glob - >>> glob.glob('*.gif') - ['card.gif'] - >>> glob.glob('.c*') - ['.card.gif'] .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/grp.rst --- a/Doc/library/grp.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/grp.rst Mon Jan 25 17:05:13 2016 +0100 @@ -42,9 +42,6 @@ Return the group database entry for the given numeric group ID. :exc:`KeyError` is raised if the entry asked for cannot be found. - .. deprecated:: 3.6 - Since Python 3.6 the support of non-integer arguments like floats or - strings in :func:`getgrgid` is deprecated. .. function:: getgrnam(name) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/gzip.rst --- a/Doc/library/gzip.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/gzip.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,11 +13,9 @@ The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the -:func:`.open`, :func:`compress` and :func:`decompress` convenience functions. -The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, -automatically compressing or decompressing the data so that it looks like an -ordinary :term:`file object`. +The :mod:`gzip` module provides the :class:`GzipFile` class. The :class:`GzipFile` +class reads and writes :program:`gzip`\ -format files, automatically compressing +or decompressing the data so that it looks like an ordinary :term:`file object`. Note that additional file formats which can be decompressed by the :program:`gzip` and :program:`gunzip` programs, such as those produced by @@ -26,37 +24,6 @@ The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) - - Open a gzip-compressed file in binary or text mode, returning a :term:`file - object`. - - The *filename* argument can be an actual filename (a :class:`str` or - :class:`bytes` object), or an existing file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, - ``'w'``, ``'wb'``, ``'x'`` or ``'xb'`` for binary mode, or ``'rt'``, - ``'at'``, ``'wt'``, or ``'xt'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 0 to 9, as for the - :class:`GzipFile` constructor. - - For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`GzipFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.3 - Added support for *filename* being a file object, support for text mode, - and the *encoding*, *errors* and *newline* arguments. - - .. versionchanged:: 3.4 - Added support for the ``'x'``, ``'xb'`` and ``'xt'`` modes. - - .. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) Constructor for the :class:`GzipFile` class, which simulates most of the @@ -64,39 +31,41 @@ method. At least one of *fileobj* and *filename* must be given a non-trivial value. - The new class instance is based on *fileobj*, which can be a regular file, an - :class:`io.BytesIO` object, or any other object which simulates a file. It + The new class instance is based on *fileobj*, which can be a regular file, a + :class:`StringIO` object, or any other object which simulates a file. It defaults to ``None``, in which case *filename* is opened to provide a file object. When *fileobj* is not ``None``, the *filename* argument is only used to be - included in the :program:`gzip` file header, which may include the original + included in the :program:`gzip` file header, which may includes the original filename of the uncompressed file. It defaults to the filename of *fileobj*, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``, - ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or - written. The default is the mode of *fileobj* if discernible; otherwise, the - default is ``'rb'``. + or ``'wb'``, depending on whether the file will be read or written. The default + is the mode of *fileobj* if discernible; otherwise, the default is ``'rb'``. - Note that the file is always opened in binary mode. To open a compressed file - in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an - :class:`io.TextIOWrapper`). + Note that the file is always opened in binary mode; text mode is not + supported. If you need to read a compressed file in text mode, wrap your + :class:`GzipFile` with an :class:`io.TextIOWrapper`. - The *compresslevel* argument is an integer from ``0`` to ``9`` controlling - the level of compression; ``1`` is fastest and produces the least - compression, and ``9`` is slowest and produces the most compression. ``0`` - is no compression. The default is ``9``. + The *compresslevel* argument is an integer from ``1`` to ``9`` controlling the + level of compression; ``1`` is fastest and produces the least compression, and + ``9`` is slowest and produces the most compression. The default is ``9``. The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. + the stream when compressing. All :program:`gzip` compressed streams are + required to contain a timestamp. If omitted or ``None``, the current + time is used. This module ignores the timestamp when decompressing; + however, some programs, such as :program:`gunzip`\ , make use of it. + The format of the timestamp is the same as that of the return value of + ``time.time()`` and of the ``st_mtime`` attribute of the object returned + by ``os.stat()``. Calling a :class:`GzipFile` object's :meth:`close` method does not close *fileobj*, since you might wish to append more material after the compressed - data. This also allows you to pass an :class:`io.BytesIO` object opened for + data. This also allows you to pass a :class:`io.BytesIO` object opened for writing as *fileobj*, and retrieve the resulting memory buffer using the :class:`io.BytesIO` object's :meth:`~io.BytesIO.getvalue` method. @@ -104,53 +73,35 @@ including iteration and the :keyword:`with` statement. Only the :meth:`truncate` method isn't implemented. - :class:`GzipFile` also provides the following method and attribute: + :class:`GzipFile` also provides the following method: - .. method:: peek(n) + .. method:: peek([n]) Read *n* uncompressed bytes without advancing the file position. At most one single read on the compressed stream is done to satisfy the call. The number of bytes returned may be more or less than requested. - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`GzipFile`, it may change the position of the underlying - file object (e.g. if the :class:`GzipFile` was constructed with the - *fileobj* parameter). - .. versionadded:: 3.2 - .. attribute:: mtime - - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. - - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. - .. versionchanged:: 3.1 - Support for the :keyword:`with` statement was added, along with the - *mtime* constructor argument and :attr:`mtime` attribute. + Support for the :keyword:`with` statement was added. .. versionchanged:: 3.2 - Support for zero-padded and unseekable files was added. + Support for zero-padded files was added. + + .. versionchanged:: 3.2 + Support for unseekable files was added. .. versionchanged:: 3.3 The :meth:`io.BufferedIOBase.read1` method is now implemented. - .. versionchanged:: 3.4 - Added support for the ``'x'`` and ``'xb'`` modes. - .. versionchanged:: 3.5 - Added support for writing arbitrary - :term:`bytes-like objects `. - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. +.. function:: open(filename, mode='rb', compresslevel=9) + This is a shorthand for ``GzipFile(filename,`` ``mode,`` ``compresslevel)``. + The *filename* argument is required; *mode* defaults to ``'rb'`` and + *compresslevel* defaults to ``9``. .. function:: compress(data, compresslevel=9) @@ -189,10 +140,9 @@ Example of how to GZIP compress an existing file:: import gzip - import shutil with open('/home/joe/file.txt', 'rb') as f_in: with gzip.open('/home/joe/file.txt.gz', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) + f_out.writelines(f_in) Example of how to GZIP compress a binary string:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/hashlib.rst --- a/Doc/library/hashlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/hashlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -23,45 +23,37 @@ digests. The modern term is secure hash. .. note:: - - If you want the adler32 or crc32 hash functions, they are available in + If you want the adler32 or crc32 hash functions they are available in the :mod:`zlib` module. .. warning:: - Some algorithms have known hash collision weaknesses, refer to the "See - also" section at the end. - - -.. _hash-algorithms: - -Hash algorithms ---------------- + Some algorithms have known hash collision weaknesses, see the FAQ at the end. There is one constructor method named for each type of :dfn:`hash`. All return a hash object with the same simple interface. For example: use :func:`sha1` to -create a SHA1 hash object. You can now feed this object with :term:`bytes-like -object`\ s (normally :class:`bytes`) using the :meth:`update` method. -At any point you can ask it for the :dfn:`digest` of the +create a SHA1 hash object. You can now feed this object with objects conforming +to the buffer interface (normally :class:`bytes` objects) using the +:meth:`update` method. At any point you can ask it for the :dfn:`digest` of the concatenation of the data fed to it so far using the :meth:`digest` or :meth:`hexdigest` methods. .. note:: - For better multithreading performance, the Python :term:`GIL` is released for - data larger than 2047 bytes at object creation or on update. + For better multithreading performance, the Python GIL is released for + strings of more than 2047 bytes at object creation or on update. .. note:: - Feeding string objects into :meth:`update` is not supported, as hashes work + Feeding string objects is to :meth:`update` is not supported, as hashes work on bytes, not on characters. .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, -and :func:`sha512`. Additional algorithms may also be available depending upon -the OpenSSL library that Python uses on your platform. +:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, and +:func:`sha512`. Additional algorithms may also be available depending upon the +OpenSSL library that Python uses on your platform. For example, to obtain the digest of the byte string ``b'Nobody inspects the spammish repetition'``:: @@ -101,18 +93,18 @@ .. data:: algorithms_guaranteed - A set containing the names of the hash algorithms guaranteed to be supported + Contains the names of the hash algorithms guaranteed to be supported by this module on all platforms. .. versionadded:: 3.2 .. data:: algorithms_available - A set containing the names of the hash algorithms that are available in the - running Python interpreter. These names will be recognized when passed to - :func:`new`. :attr:`algorithms_guaranteed` will always be a subset. The - same algorithm may appear multiple times in this set under different names - (thanks to OpenSSL). + Contains the names of the hash algorithms that are available + in the running Python interpreter. These names will be recognized + when passed to :func:`new`. :attr:`algorithms_guaranteed` + will always be a subset. Duplicate algorithms with different + name formats may appear in this set (thanks to OpenSSL). .. versionadded:: 3.2 @@ -128,18 +120,6 @@ The internal block size of the hash algorithm in bytes. -A hash object has the following attributes: - -.. attribute:: hash.name - - The canonical name of this hash, always lowercase and always suitable as a - parameter to :func:`new` to create another hash of this type. - - .. versionchanged:: 3.4 - The name attribute has been present in CPython since its inception, but - until Python 3.4 was not formally specified, so may not exist on some - platforms. - A hash object has the following methods: @@ -152,7 +132,7 @@ .. versionchanged:: 3.1 The Python GIL is released to allow other threads to run while hash - updates on data larger than 2047 bytes is taking place when using hash + updates on data larger than 2048 bytes is taking place when using hash algorithms supplied by OpenSSL. @@ -176,46 +156,6 @@ compute the digests of data sharing a common initial substring. -Key derivation --------------- - -Key derivation and key stretching algorithms are designed for secure password -hashing. Naive algorithms such as ``sha1(password)`` are not resistant against -brute-force attacks. A good password hashing function must be tunable, slow, and -include a `salt `_. - - -.. function:: pbkdf2_hmac(name, password, salt, rounds, dklen=None) - - The function provides PKCS#5 password-based key derivation function 2. It - uses HMAC as pseudorandom function. - - The string *name* is the desired name of the hash digest algorithm for - HMAC, e.g. 'sha1' or 'sha256'. *password* and *salt* are interpreted as - buffers of bytes. Applications and libraries should limit *password* to - a sensible value (e.g. 1024). *salt* should be about 16 or more bytes from - a proper source, e.g. :func:`os.urandom`. - - The number of *rounds* should be chosen based on the hash algorithm and - computing power. As of 2013, at least 100,000 rounds of SHA-256 is suggested. - - *dklen* is the length of the derived key. If *dklen* is ``None`` then the - digest size of the hash algorithm *name* is used, e.g. 64 for SHA-512. - - >>> import hashlib, binascii - >>> dk = hashlib.pbkdf2_hmac('sha256', b'password', b'salt', 100000) - >>> binascii.hexlify(dk) - b'0394a2ede332c9a13eb82e9b24631604c31df978b4e2f0fbd2c549944f9d79a5' - - .. versionadded:: 3.4 - - .. note:: - - A fast implementation of *pbkdf2_hmac* is available with OpenSSL. The - Python implementation uses an inline version of :mod:`hmac`. It is about - three times slower and doesn't release the GIL. - - .. seealso:: Module :mod:`hmac` @@ -227,9 +167,7 @@ http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf The FIPS 180-2 publication on Secure Hash Algorithms. - https://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms + http://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms Wikipedia article with information on which algorithms have known issues and what that means regarding their use. - http://www.ietf.org/rfc/rfc2898.txt - PKCS #5: Password-Based Cryptography Specification Version 2.0 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/heapq.rst --- a/Doc/library/heapq.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/heapq.rst Mon Jan 25 17:05:13 2016 +0100 @@ -47,8 +47,7 @@ .. function:: heappop(heap) Pop and return the smallest item from the *heap*, maintaining the heap - invariant. If the heap is empty, :exc:`IndexError` is raised. To access the - smallest item without popping it, use ``heap[0]``. + invariant. If the heap is empty, :exc:`IndexError` is raised. .. function:: heappushpop(heap, item) @@ -82,7 +81,7 @@ The module also offers three general purpose functions based on heaps. -.. function:: merge(*iterables, key=None, reverse=False) +.. function:: merge(*iterables) Merge multiple sorted inputs into a single sorted output (for example, merge timestamped entries from multiple log files). Returns an :term:`iterator` @@ -92,18 +91,6 @@ not pull the data into memory all at once, and assumes that each of the input streams is already sorted (smallest to largest). - Has two optional arguments which must be specified as keyword arguments. - - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). - - *reverse* is a boolean value. If set to ``True``, then the input elements - are merged as if each comparison were reversed. - - .. versionchanged:: 3.5 - Added the optional *key* and *reverse* parameters. - .. function:: nlargest(n, iterable, key=None) @@ -125,8 +112,7 @@ The latter two functions perform best for smaller values of *n*. For larger values, it is more efficient to use the :func:`sorted` function. Also, when ``n==1``, it is more efficient to use the built-in :func:`min` and :func:`max` -functions. If repeated usage of these functions is required, consider turning -the iterable into an actual heap. +functions. Basic Examples @@ -137,6 +123,7 @@ time:: >>> def heapsort(iterable): + ... 'Equivalent to sorted(iterable)' ... h = [] ... for value in iterable: ... heappush(h, value) @@ -145,9 +132,6 @@ >>> heapsort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0]) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] -This is similar to ``sorted(iterable)``, but unlike :func:`sorted`, this -implementation is not stable. - Heap elements can be tuples. This is useful for assigning comparison values (such as task priorities) alongside the main record being tracked:: @@ -262,7 +246,7 @@ the sort is going on, provided that the inserted items are not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition -means the smallest scheduled time. When an event schedules other events for +means the smallest scheduled time. When an event schedule other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). @@ -274,11 +258,11 @@ the worst cases might be terrible. Heaps are also very useful in big disk sorts. You most probably all know that a -big sort implies producing "runs" (which are pre-sorted sequences, whose size is +big sort implies producing "runs" (which are pre-sorted sequences, which size is usually related to the amount of CPU memory), followed by a merging passes for these runs, which merging is often very cleverly organised [#]_. It is very important that the initial sort produces the longest runs possible. Tournaments -are a good way to achieve that. If, using all the memory available to hold a +are a good way to that. If, using all the memory available to hold a tournament, you replace and percolate items that happen to fit the current run, you'll produce runs which are twice the size of the memory for random input, and much better for input fuzzily ordered. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/hmac.rst --- a/Doc/library/hmac.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/hmac.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,6 +3,7 @@ .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation + for Python. .. moduleauthor:: Gerhard Häring .. sectionauthor:: Gerhard Häring @@ -15,32 +16,20 @@ .. function:: new(key, msg=None, digestmod=None) - Return a new hmac object. *key* is a bytes or bytearray object giving the - secret key. If *msg* is present, the method call ``update(msg)`` is made. - *digestmod* is the digest name, digest constructor or module for the HMAC - object to use. It supports any name suitable to :func:`hashlib.new` and - defaults to the :data:`hashlib.md5` constructor. - - .. versionchanged:: 3.4 - Parameter *key* can be a bytes or bytearray object. - Parameter *msg* can be of any type supported by :mod:`hashlib`. - Parameter *digestmod* can be the name of a hash algorithm. - - .. deprecated:: 3.4 - MD5 as implicit default digest for *digestmod* is deprecated. + Return a new hmac object. *key* is a bytes object giving the secret key. If + *msg* is present, the method call ``update(msg)`` is made. *digestmod* is + the digest constructor or module for the HMAC object to use. It defaults to + the :func:`hashlib.md5` constructor. An HMAC object has the following methods: .. method:: HMAC.update(msg) - Update the hmac object with *msg*. Repeated calls are equivalent to a - single call with the concatenation of all the arguments: + Update the hmac object with the bytes object *msg*. Repeated calls are + equivalent to a single call with the concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``. - .. versionchanged:: 3.4 - Parameter *msg* can be of any type supported by :mod:`hashlib`. - .. method:: HMAC.digest() @@ -49,13 +38,6 @@ given to the constructor. It may contain non-ASCII bytes, including NUL bytes. - .. warning:: - - When comparing the output of :meth:`digest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - .. method:: HMAC.hexdigest() @@ -63,13 +45,6 @@ length containing only hexadecimal digits. This may be used to exchange the value safely in email or other non-binary environments. - .. warning:: - - When comparing the output of :meth:`hexdigest` to an externally-supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - .. method:: HMAC.copy() @@ -77,45 +52,6 @@ compute the digests of strings that share a common initial substring. -A hash object has the following attributes: - -.. attribute:: HMAC.digest_size - - The size of the resulting HMAC digest in bytes. - -.. attribute:: HMAC.block_size - - The internal block size of the hash algorithm in bytes. - - .. versionadded:: 3.4 - -.. attribute:: HMAC.name - - The canonical name of this HMAC, always lowercase, e.g. ``hmac-md5``. - - .. versionadded:: 3.4 - - -This module also provides the following helper function: - -.. function:: compare_digest(a, b) - - Return ``a == b``. This function uses an approach designed to prevent - timing analysis by avoiding content-based short circuiting behaviour, - making it appropriate for cryptography. *a* and *b* must both be of the - same type: either :class:`str` (ASCII only, as e.g. returned by - :meth:`HMAC.hexdigest`), or a :term:`bytes-like object`. - - .. note:: - - If *a* and *b* are of different lengths, or if an error occurs, - a timing attack could theoretically reveal information about the - types and lengths of *a* and *b*--but not their values. - - - .. versionadded:: 3.3 - - .. seealso:: Module :mod:`hashlib` diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/html.entities.rst --- a/Doc/library/html.entities.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/html.entities.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,20 +9,11 @@ -------------- -This module defines four dictionaries, :data:`html5`, -:data:`name2codepoint`, :data:`codepoint2name`, and :data:`entitydefs`. - - -.. data:: html5 - - A dictionary that maps HTML5 named character references [#]_ to the - equivalent Unicode character(s), e.g. ``html5['gt;'] == '>'``. - Note that the trailing semicolon is included in the name (e.g. ``'gt;'``), - however some of the names are accepted by the standard even without the - semicolon: in this case the name is present with and without the ``';'``. - See also :func:`html.unescape`. - - .. versionadded:: 3.3 +This module defines three dictionaries, ``name2codepoint``, ``codepoint2name``, +and ``entitydefs``. ``entitydefs`` is used to provide the :attr:`entitydefs` +attribute of the :class:`html.parser.HTMLParser` class. The definition provided +here contains all the entities defined by XHTML 1.0 that can be handled using +simple textual substitution in the Latin-1 character set (ISO-8859-1). .. data:: entitydefs @@ -33,14 +24,9 @@ .. data:: name2codepoint - A dictionary that maps HTML entity names to the Unicode code points. + A dictionary that maps HTML entity names to the Unicode codepoints. .. data:: codepoint2name - A dictionary that maps Unicode code points to HTML entity names. - - -.. rubric:: Footnotes - -.. [#] See http://www.w3.org/TR/html5/syntax.html#named-character-references + A dictionary that maps Unicode codepoints to HTML entity names. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/html.parser.rst --- a/Doc/library/html.parser.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/html.parser.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,13 +16,13 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(*, convert_charrefs=True) +.. class:: HTMLParser(strict=True) - Create a parser instance able to parse invalid markup. - - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are - automatically converted to the corresponding Unicode characters. + Create a parser instance. If *strict* is ``True`` (the default), invalid + HTML results in :exc:`~html.parser.HTMLParseError` exceptions [#]_. If + *strict* is ``False``, the parser uses heuristics to make a best guess at + the intention of any invalid HTML it encounters, similar to the way most + browsers do. Using ``strict=False`` is advised. An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are @@ -32,11 +32,19 @@ This parser does not check that end tags match start tags or call the end-tag handler for elements which are closed implicitly by closing an outer element. - .. versionchanged:: 3.4 - *convert_charrefs* keyword argument added. + .. versionchanged:: 3.2 *strict* keyword added - .. versionchanged:: 3.5 - The default value for argument *convert_charrefs* is now ``True``. +An exception is defined as well: + + +.. exception:: HTMLParseError + + Exception raised by the :class:`HTMLParser` class when it encounters an error + while parsing and *strict* is ``True``. This exception provides three + attributes: :attr:`msg` is a brief message explaining the error, + :attr:`lineno` is the number of the line on which the broken construct was + detected, and :attr:`offset` is the number of characters into the line at + which the construct starts. Example HTML Parser Application @@ -56,7 +64,7 @@ def handle_data(self, data): print("Encountered some data :", data) - parser = MyHTMLParser() + parser = MyHTMLParser(strict=False) parser.feed('Test' '

Parse me!

') @@ -163,8 +171,7 @@ This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. + (e.g. ``'gt'``). .. method:: HTMLParser.handle_charref(name) @@ -172,8 +179,7 @@ This method is called to process decimal and hexadecimal numeric character references of the form ``&#NNN;`` and ``&#xNNN;``. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. + in this case the method will receive ``'62'`` or ``'x3E'``. .. method:: HTMLParser.handle_comment(data) @@ -185,7 +191,7 @@ The content of Internet Explorer conditional comments (condcoms) will also be sent to this method, so, for ````, - this method will receive ``'[if IE 9]>IE9-specific contentIE-specific content`` markup. It is sometimes useful to be overridden by a - derived class. The base class implementation does nothing. + derived class. The base class implementation raises an :exc:`HTMLParseError` + when *strict* is ``True``. .. _htmlparser-examples: @@ -255,7 +262,7 @@ def handle_decl(self, data): print("Decl :", data) - parser = MyHTMLParser() + parser = MyHTMLParser(strict=False) Parsing a doctype:: @@ -307,8 +314,7 @@ Num ent : > Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but -:meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: +:meth:`~HTMLParser.handle_data` might be called more than once:: >>> for chunk in ['buff', 'ered ', 'text']: ... parser.feed(chunk) @@ -329,3 +335,9 @@ Data : tag soup End tag : p End tag : a + +.. rubric:: Footnotes + +.. [#] For backward compatibility reasons *strict* mode does not raise + exceptions for all non-compliant HTML. That is, some invalid HTML + is tolerated even in *strict* mode. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/html.rst --- a/Doc/library/html.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/html.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,8 @@ .. module:: html :synopsis: Helpers for manipulating HTML. +.. versionadded:: 3.2 + **Source code:** :source:`Lib/html/__init__.py` -------------- @@ -17,23 +19,3 @@ characters in HTML. If the optional flag *quote* is true, the characters (``"``) and (``'``) are also translated; this helps for inclusion in an HTML attribute value delimited by quotes, as in ````. - - .. versionadded:: 3.2 - - -.. function:: unescape(s) - - Convert all named and numeric character references (e.g. ``>``, - ``>``, ``&x3e;``) in the string *s* to the corresponding unicode - characters. This function uses the rules defined by the HTML 5 standard - for both valid and invalid character references, and the :data:`list of - HTML 5 named character references `. - - .. versionadded:: 3.4 - --------------- - -Submodules in the ``html`` package are: - -* :mod:`html.parser` -- HTML/XHTML parser with lenient parsing mode -* :mod:`html.entities` -- HTML entity definitions diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/http.client.rst --- a/Doc/library/http.client.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/http.client.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,11 +19,6 @@ HTTPS protocols. It is normally not used directly --- the module :mod:`urllib.request` uses it to handle URLs that use HTTP and HTTPS. -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level http client interface. - .. note:: HTTPS support is only available if Python was compiled with SSL support @@ -32,8 +27,7 @@ The module provides the following classes: -.. class:: HTTPConnection(host, port=None[, timeout], \ - source_address=None) +.. class:: HTTPConnection(host, port=None[, strict[, timeout[, source_address]]]) An :class:`HTTPConnection` instance represents one transaction with an HTTP server. It should be instantiated passing it a host and optional port @@ -48,37 +42,41 @@ For example, the following calls all create instances that connect to the server at the same host and port:: - >>> h1 = http.client.HTTPConnection('www.python.org') - >>> h2 = http.client.HTTPConnection('www.python.org:80') - >>> h3 = http.client.HTTPConnection('www.python.org', 80) - >>> h4 = http.client.HTTPConnection('www.python.org', 80, timeout=10) + >>> h1 = http.client.HTTPConnection('www.cwi.nl') + >>> h2 = http.client.HTTPConnection('www.cwi.nl:80') + >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80) + >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80, timeout=10) .. versionchanged:: 3.2 *source_address* was added. - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - not longer supported. + .. versionchanged:: 3.2 + The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" + are not supported anymore. -.. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, timeout], \ - source_address=None, *, context=None, \ - check_hostname=None) +.. class:: HTTPSConnection(host, port=None, key_file=None, cert_file=None[, strict[, timeout[, source_address]]], *, context=None, check_hostname=None) A subclass of :class:`HTTPConnection` that uses SSL for communication with secure servers. Default port is ``443``. If *context* is specified, it must be a :class:`ssl.SSLContext` instance describing the various SSL - options. + options. If *context* is specified and has a :attr:`~ssl.SSLContext.verify_mode` + of either :data:`~ssl.CERT_OPTIONAL` or :data:`~ssl.CERT_REQUIRED`, then + by default *host* is matched against the host name(s) allowed by the + server's certificate. If you want to change that behaviour, you can + explicitly set *check_hostname* to False. *key_file* and *cert_file* are deprecated, please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should be used - instead. + :meth:`ssl.SSLContext.load_cert_chain` instead. - Please read :ref:`ssl-security` for more information on best practices. + If you access arbitrary hosts on the Internet, it is recommended to + require certificate checking and feed the *context* with a set of + trusted CA certificates:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations('/etc/pki/tls/certs/ca-bundle.crt') + h = client.HTTPSConnection('svn.python.org', 443, context=context) .. versionchanged:: 3.2 *source_address*, *context* and *check_hostname* were added. @@ -87,25 +85,19 @@ This class now supports HTTPS virtual hosts if possible (that is, if :data:`ssl.HAS_SNI` is true). - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - no longer supported. + .. versionchanged:: 3.2 + The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" + are not supported anymore. - .. versionchanged:: 3.4.3 - This class now performs all the necessary certificate and hostname checks - by default. To revert to the previous, unverified, behavior - :func:`ssl._create_unverified_context` can be passed to the *context* - parameter. - -.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) +.. class:: HTTPResponse(sock, debuglevel=0[, strict], method=None, url=None) Class whose instances are returned upon successful connection. Not instantiated directly by user. - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9 style "Simple Responses" are - no longer supported. + .. versionchanged:: 3.2 + The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses" + are not supported anymore. The following exceptions are raised as appropriate: @@ -173,25 +165,8 @@ A subclass of :exc:`HTTPException`. Raised if a server responds with a HTTP status code that we don't understand. +The constants defined in this module are: -.. exception:: LineTooLong - - A subclass of :exc:`HTTPException`. Raised if an excessively long line - is received in the HTTP protocol from the server. - - -.. exception:: RemoteDisconnected - - A subclass of :exc:`ConnectionResetError` and :exc:`BadStatusLine`. Raised - by :meth:`HTTPConnection.getresponse` when the attempt to read the response - results in no data read from the connection, indicating that the remote end - has closed the connection. - - .. versionadded:: 3.5 - Previously, :exc:`BadStatusLine`\ ``('')`` was raised. - - -The constants defined in this module are: .. data:: HTTP_PORT @@ -202,15 +177,206 @@ The default port for the HTTPS protocol (always ``443``). +and also the following constants for integer status codes: + ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| Constant | Value | Definition | ++==========================================+=========+=======================================================================+ +| :const:`CONTINUE` | ``100`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.1.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`SWITCHING_PROTOCOLS` | ``101`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.1.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`PROCESSING` | ``102`` | WEBDAV, `RFC 2518, Section 10.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`OK` | ``200`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`CREATED` | ``201`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`ACCEPTED` | ``202`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.3 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NON_AUTHORITATIVE_INFORMATION` | ``203`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.4 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NO_CONTENT` | ``204`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.5 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`RESET_CONTENT` | ``205`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.6 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`PARTIAL_CONTENT` | ``206`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.2.7 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`MULTI_STATUS` | ``207`` | WEBDAV `RFC 2518, Section 10.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`IM_USED` | ``226`` | Delta encoding in HTTP, | +| | | :rfc:`3229`, Section 10.4.1 | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`MULTIPLE_CHOICES` | ``300`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`MOVED_PERMANENTLY` | ``301`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`FOUND` | ``302`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.3 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`SEE_OTHER` | ``303`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.4 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NOT_MODIFIED` | ``304`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.5 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`USE_PROXY` | ``305`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.6 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`TEMPORARY_REDIRECT` | ``307`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.3.8 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`BAD_REQUEST` | ``400`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`UNAUTHORIZED` | ``401`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`PAYMENT_REQUIRED` | ``402`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.3 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`FORBIDDEN` | ``403`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.4 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NOT_FOUND` | ``404`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.5 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`METHOD_NOT_ALLOWED` | ``405`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.6 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NOT_ACCEPTABLE` | ``406`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.7 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`PROXY_AUTHENTICATION_REQUIRED` | ``407`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.8 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`REQUEST_TIMEOUT` | ``408`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.9 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`CONFLICT` | ``409`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.10 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`GONE` | ``410`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.11 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`LENGTH_REQUIRED` | ``411`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.12 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`PRECONDITION_FAILED` | ``412`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.13 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`REQUEST_ENTITY_TOO_LARGE` | ``413`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.14 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`REQUEST_URI_TOO_LONG` | ``414`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.15 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`UNSUPPORTED_MEDIA_TYPE` | ``415`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.16 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`REQUESTED_RANGE_NOT_SATISFIABLE` | ``416`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.17 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`EXPECTATION_FAILED` | ``417`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.4.18 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`UNPROCESSABLE_ENTITY` | ``422`` | WEBDAV, `RFC 2518, Section 10.3 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`LOCKED` | ``423`` | WEBDAV `RFC 2518, Section 10.4 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`FAILED_DEPENDENCY` | ``424`` | WEBDAV, `RFC 2518, Section 10.5 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`UPGRADE_REQUIRED` | ``426`` | HTTP Upgrade to TLS, | +| | | :rfc:`2817`, Section 6 | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`INTERNAL_SERVER_ERROR` | ``500`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.5.1 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NOT_IMPLEMENTED` | ``501`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.5.2 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`BAD_GATEWAY` | ``502`` | HTTP/1.1 `RFC 2616, Section | +| | | 10.5.3 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`SERVICE_UNAVAILABLE` | ``503`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.5.4 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`GATEWAY_TIMEOUT` | ``504`` | HTTP/1.1 `RFC 2616, Section | +| | | 10.5.5 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`HTTP_VERSION_NOT_SUPPORTED` | ``505`` | HTTP/1.1, `RFC 2616, Section | +| | | 10.5.6 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`INSUFFICIENT_STORAGE` | ``507`` | WEBDAV, `RFC 2518, Section 10.6 | +| | | `_ | ++------------------------------------------+---------+-----------------------------------------------------------------------+ +| :const:`NOT_EXTENDED` | ``510`` | An HTTP Extension Framework, | +| | | :rfc:`2774`, Section 7 | ++------------------------------------------+---------+-----------------------------------------------------------------------+ + + .. data:: responses This dictionary maps the HTTP 1.1 status codes to the W3C names. Example: ``http.client.responses[http.client.NOT_FOUND]`` is ``'Not Found'``. -See :ref:`http-status-codes` for a list of HTTP status codes that are -available in this module as constants. - .. _httpconnection-objects: @@ -223,33 +389,23 @@ .. method:: HTTPConnection.request(method, url, body=None, headers={}) This will send a request to the server using the HTTP request - method *method* and the selector *url*. + method *method* and the selector *url*. If the *body* argument is + present, it should be string or bytes object of data to send after + the headers are finished. Strings are encoded as ISO-8859-1, the + default charset for HTTP. To use other encodings, pass a bytes + object. The Content-Length header is set to the length of the + string. - If *body* is specified, the specified data is sent after the headers are - finished. It may be a string, a :term:`bytes-like object`, an open - :term:`file object`, or an iterable of :term:`bytes-like object`\s. If - *body* is a string, it is encoded as ISO-8851-1, the default for HTTP. If - it is a bytes-like object the bytes are sent as is. If it is a :term:`file - object`, the contents of the file is sent; this file object should support - at least the ``read()`` method. If the file object has a ``mode`` - attribute, the data returned by the ``read()`` method will be encoded as - ISO-8851-1 unless the ``mode`` attribute contains the substring ``b``, - otherwise the data returned by ``read()`` is sent as is. If *body* is an - iterable, the elements of the iterable are sent as is until the iterable is - exhausted. + The *body* may also be an open :term:`file object`, in which case the + contents of the file is sent; this file object should support ``fileno()`` + and ``read()`` methods. The header Content-Length is automatically set to + the length of the file as reported by stat. The *body* argument may also be + an iterable and Content-Length header should be explicitly provided when the + body is an iterable. The *headers* argument should be a mapping of extra HTTP headers to send with the request. - If *headers* does not contain a Content-Length item, one is added - automatically if possible. If *body* is ``None``, the Content-Length header - is set to ``0`` for methods that expect a body (``PUT``, ``POST``, and - ``PATCH``). If *body* is a string or bytes object, the Content-Length - header is set to its length. If *body* is a :term:`file object` and it - works to call :func:`~os.fstat` on the result of its ``fileno()`` method, - then the Content-Length header is set to the ``st_size`` reported by the - ``fstat`` call. Otherwise no Content-Length header is added. - .. versionadded:: 3.2 *body* can now be an iterable. @@ -263,11 +419,6 @@ Note that you must have read the whole response before you can send a new request to the server. - .. versionchanged:: 3.5 - If a :exc:`ConnectionError` or subclass is raised, the - :class:`HTTPConnection` object will be ready to reconnect when - a new request is sent. - .. method:: HTTPConnection.set_debuglevel(level) @@ -281,34 +432,18 @@ .. method:: HTTPConnection.set_tunnel(host, port=None, headers=None) - Set the host and the port for HTTP Connect Tunnelling. This allows running - the connection through a proxy server. + Set the host and the port for HTTP Connect Tunnelling. Normally used when it + is required to a HTTPS Connection through a proxy server. - The host and port arguments specify the endpoint of the tunneled connection - (i.e. the address included in the CONNECT request, *not* the address of the - proxy server). - - The headers argument should be a mapping of extra HTTP headers to send with - the CONNECT request. - - For example, to tunnel through a HTTPS proxy server running locally on port - 8080, we would pass the address of the proxy to the :class:`HTTPSConnection` - constructor, and the address of the host that we eventually want to reach to - the :meth:`~HTTPConnection.set_tunnel` method:: - - >>> import http.client - >>> conn = http.client.HTTPSConnection("localhost", 8080) - >>> conn.set_tunnel("www.python.org") - >>> conn.request("HEAD","/index.html") + The headers argument should be a mapping of extra HTTP headers to send + with the CONNECT request. .. versionadded:: 3.2 .. method:: HTTPConnection.connect() - Connect to the server specified when the object was created. By default, - this is called automatically when making a request if the client does not - already have a connection. + Connect to the server specified when the object was created. .. method:: HTTPConnection.close() @@ -419,7 +554,7 @@ .. attribute:: HTTPResponse.closed - Is ``True`` if the stream is closed. + Is True if the stream is closed. Examples -------- @@ -427,18 +562,18 @@ Here is an example session that uses the ``GET`` method:: >>> import http.client - >>> conn = http.client.HTTPSConnection("www.python.org") - >>> conn.request("GET", "/") + >>> conn = http.client.HTTPConnection("www.python.org") + >>> conn.request("GET", "/index.html") >>> r1 = conn.getresponse() >>> print(r1.status, r1.reason) 200 OK >>> data1 = r1.read() # This will return entire content. >>> # The following example demonstrates reading data in chunks. - >>> conn.request("GET", "/") + >>> conn.request("GET", "/index.html") >>> r1 = conn.getresponse() >>> while not r1.closed: ... print(r1.read(200)) # 200 bytes - b'\n 1 3 6 10 15`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, keyfunc] sub-iterators grouped by value of keyfunc(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` -============================ ============================ ================================================= ============================================================= +==================== ============================ ================================================= ============================================================= +Iterator Arguments Results Example +==================== ============================ ================================================= ============================================================= +:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` +:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` +:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` +:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` +:func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` +:func:`groupby` iterable[, keyfunc] sub-iterators grouped by value of keyfunc(v) +:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` +:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` +:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` +:func:`tee` it, n it1, it2 , ... itn splits one iterator into n +:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` +==================== ============================ ================================================= ============================================================= **Combinatoric generators:** @@ -87,15 +86,10 @@ .. function:: accumulate(iterable[, func]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) If the input iterable is empty, the - output iterable will also be empty. + Make an iterator that returns accumulated sums. Elements may be any addable + type including :class:`Decimal` or :class:`Fraction`. If the optional + *func* argument is supplied, it should be a function of two arguments + and it will be used instead of addition. Equivalent to:: @@ -104,10 +98,7 @@ # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 it = iter(iterable) - try: - total = next(it) - except StopIteration: - return + total = next(it) yield total for element in it: total = func(total, element) @@ -139,13 +130,10 @@ >>> inputs = repeat(x0, 36) # only the initial value is used >>> [format(x, '.2f') for x in accumulate(inputs, logistic_map)] ['0.40', '0.91', '0.30', '0.81', '0.60', '0.92', '0.29', '0.79', '0.63', - '0.88', '0.39', '0.90', '0.33', '0.84', '0.52', '0.95', '0.18', '0.57', + '0.88' ,'0.39', '0.90', '0.33', '0.84', '0.52', '0.95', '0.18', '0.57', '0.93', '0.25', '0.71', '0.79', '0.63', '0.88', '0.39', '0.91', '0.32', '0.83', '0.54', '0.95', '0.20', '0.60', '0.91', '0.30', '0.80', '0.60'] - See :func:`functools.reduce` for a similar function that returns only the - final accumulated value. - .. versionadded:: 3.2 .. versionchanged:: 3.3 @@ -168,8 +156,9 @@ .. classmethod:: chain.from_iterable(iterable) Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Roughly equivalent to:: + single iterable argument that is evaluated lazily. Equivalent to:: + @classmethod def from_iterable(iterables): # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F for it in iterables: @@ -289,7 +278,7 @@ .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often + Make an iterator that returns evenly spaced values starting with *n*. Often used as an argument to :func:`map` to generate consecutive data points. Also, used with :func:`zip` to add sequence numbers. Equivalent to:: @@ -408,15 +397,11 @@ def _grouper(self, tgtkey): while self.currkey == tgtkey: yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: - return + self.currvalue = next(self.it) # Exit on StopIteration self.currkey = self.keyfunc(self.currvalue) -.. function:: islice(iterable, stop) - islice(iterable, start, stop[, step]) +.. function:: islice(iterable, [start,] stop [, step]) Make an iterator that returns selected elements from the iterable. If *start* is non-zero, then elements from the iterable are skipped until start is reached. @@ -435,10 +420,7 @@ # islice('ABCDEFG', 0, None, 2) --> A C E G s = slice(*args) it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) - try: - nexti = next(it) - except StopIteration: - return + nexti = next(it) for i, element in enumerate(iterable): if i == nexti: yield element @@ -596,10 +578,7 @@ def gen(mydeque): while True: if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return + newval = next(it) # fetch a new value and for d in deques: # load it to all the deques d.append(newval) yield mydeque.popleft() @@ -674,11 +653,6 @@ "Return function(0), function(1), ..." return map(function, count(start)) - def tail(n, iterable): - "Return an iterator over the last n items" - # tail(3, 'ABCDEFG') --> E F G - return iter(collections.deque(iterable, maxlen=n)) - def consume(iterator, n): "Advance the iterator n-steps ahead. If n is none, consume entirely." # Use functions that consume iterators at C speed. @@ -730,9 +704,8 @@ next(b, None) return zip(a, b) - def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" + def grouper(n, iterable, fillvalue=None): + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" args = [iter(iterable)] * n return zip_longest(*args, fillvalue=fillvalue) @@ -787,7 +760,7 @@ """ Call a function repeatedly until an exception is raised. Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead + Like __builtin__.iter(func, sentinel) but uses an exception instead of a sentinel to end the loop. Examples: @@ -806,19 +779,6 @@ except exception: pass - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item - for which pred(item) is true. - - """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) - def random_product(*args, repeat=1): "Random selection from itertools.product(*args, **kwds)" pools = [tuple(pool) for pool in args] * repeat diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/json.rst --- a/Doc/library/json.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/json.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,12 +6,8 @@ .. moduleauthor:: Bob Ippolito .. sectionauthor:: Bob Ippolito -`JSON (JavaScript Object Notation) `_, specified by -:rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, -is a lightweight data interchange format inspired by -`JavaScript `_ object literal syntax -(although it is not a strict subset of JavaScript [#rfc-errata]_ ). +`JSON (JavaScript Object Notation) `_ is a subset of JavaScript +syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`json` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. @@ -84,7 +80,6 @@ ... def default(self, obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... # Let the base class default method raise the TypeError ... return json.JSONEncoder.default(self, obj) ... >>> json.dumps(2 + 1j, cls=ComplexEncoder) @@ -95,40 +90,32 @@ ['[2.0', ', 1.0', ']'] -.. highlight:: bash +.. highlight:: none Using json.tool from the shell to validate and pretty-print:: - $ echo '{"json":"obj"}' | python -m json.tool + $ echo '{"json":"obj"}' | python -mjson.tool { "json": "obj" } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) + $ echo '{ 1.2:3.4}' | python -mjson.tool + Expecting property name: line 1 column 2 (char 2) -See :ref:`json-commandline` for detailed documentation. - -.. highlight:: python3 +.. highlight:: python .. note:: - JSON is a subset of `YAML `_ 1.2. The JSON produced by - this module's default settings (in particular, the default *separators* - value) is also a subset of YAML 1.0 and 1.1. This module can thus also be - used as a YAML serializer. + The JSON produced by this module's default settings is a subset of + YAML, so it may be used as a serializer for that as well. Basic Usage ----------- -.. function:: dump(obj, fp, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) +.. function:: dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, **kw) Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting - :term:`file-like object`) using this :ref:`conversion table - `. + file-like object). If *skipkeys* is ``True`` (default: ``False``), then dict keys that are not of a basic type (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, @@ -155,39 +142,25 @@ object members will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), + indents that many spaces per level. If *indent* is a string (such at '\t'), that string is used to indent each level. - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. - - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. + If *separators* is an ``(item_separator, dict_separator)`` tuple, then it + will be used instead of the default ``(', ', ': ')`` separators. ``(',', + ':')`` is the most compact JSON representation. *default(obj)* is a function that should return a serializable version of *obj* or raise :exc:`TypeError`. The default simply raises :exc:`TypeError`. - If *sort_keys* is ``True`` (default: ``False``), then the output of - dictionaries will be sorted by key. - To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the :meth:`default` method to serialize additional types), specify it with the *cls* kwarg; otherwise :class:`JSONEncoder` is used. -.. function:: dumps(obj, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) +.. function:: dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, **kw) - Serialize *obj* to a JSON formatted :class:`str` using this :ref:`conversion - table `. The arguments have the same meaning as in - :func:`dump`. + Serialize *obj* to a JSON formatted :class:`str`. The arguments have the + same meaning as in :func:`dump`. .. note:: @@ -199,22 +172,20 @@ Keys in key/value pairs of JSON are always of the type :class:`str`. When a dictionary is converted into JSON, all the keys of the dictionary are - coerced to strings. As a result of this, if a dictionary is converted + coerced to strings. As a result of this, if a dictionary is convered into JSON and then back into a dictionary, the dictionary may not equal the original one. That is, ``loads(dumps(x)) != x`` if x has non-string keys. .. function:: load(fp, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - Deserialize *fp* (a ``.read()``-supporting :term:`file-like object` - containing a JSON document) to a Python object using this :ref:`conversion - table `. + Deserialize *fp* (a ``.read()``-supporting file-like object containing a JSON + document) to a Python object. *object_hook* is an optional function that will be called with the result of any object literal decoded (a :class:`dict`). The return value of *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). + to implement custom decoders (e.g. JSON-RPC class hinting). *object_pairs_hook* is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The @@ -238,32 +209,25 @@ (e.g. :class:`float`). *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers + strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``, ``'null'``, ``'true'``, + ``'false'``. This can be used to raise an exception if invalid JSON numbers are encountered. - .. versionchanged:: 3.1 - *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls`` kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments will be passed to the constructor of the class. - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. .. function:: loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) Deserialize *s* (a :class:`str` instance containing a JSON document) to a - Python object using this :ref:`conversion table `. + Python object. The other arguments have the same meaning as in :func:`load`, except *encoding* which is ignored and deprecated. - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. -Encoders and Decoders +Encoders and decoders --------------------- .. class:: JSONDecoder(object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) @@ -272,8 +236,6 @@ Performs the following translations in decoding by default: - .. _json-to-py-table: - +---------------+-------------------+ | JSON | Python | +===============+===================+ @@ -333,16 +295,11 @@ those with character codes in the 0-31 range, including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``. - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. .. method:: decode(s) Return the Python representation of *s* (a :class:`str` instance - containing a JSON document). - - :exc:`JSONDecodeError` will be raised if the given JSON document is not - valid. + containing a JSON document) .. method:: raw_decode(s) @@ -360,28 +317,23 @@ Supports the following objects and types by default: - .. _py-to-json-table: - - +----------------------------------------+---------------+ - | Python | JSON | - +========================================+===============+ - | dict | object | - +----------------------------------------+---------------+ - | list, tuple | array | - +----------------------------------------+---------------+ - | str | string | - +----------------------------------------+---------------+ - | int, float, int- & float-derived Enums | number | - +----------------------------------------+---------------+ - | True | true | - +----------------------------------------+---------------+ - | False | false | - +----------------------------------------+---------------+ - | None | null | - +----------------------------------------+---------------+ - - .. versionchanged:: 3.4 - Added support for int- and float-derived Enum classes. + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str | string | + +-------------------+---------------+ + | int, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ To extend this to recognize other objects, subclass and implement a :meth:`default` method with another method that returns a serializable object @@ -411,23 +363,14 @@ will be sorted by key; this is useful for regression tests to ensure that JSON serializations can be compared on a day-to-day basis. - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. + If *indent* is a non-negative integer (it is ``None`` by default), then JSON + array elements and object members will be pretty-printed with that indent + level. An indent level of 0 will only insert newlines. ``None`` is the most + compact representation. If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. + tuple. The default is ``(', ', ': ')``. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate whitespace. If specified, *default* is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable version of the @@ -450,7 +393,6 @@ pass else: return list(iterable) - # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, o) @@ -470,226 +412,3 @@ for chunk in json.JSONEncoder().iterencode(bigobject): mysocket.write(chunk) - - -Exceptions ----------- - -.. exception:: JSONDecodeError(msg, doc, pos, end=None) - - Subclass of :exc:`ValueError` with the following additional attributes: - - .. attribute:: msg - - The unformatted error message. - - .. attribute:: doc - - The JSON document being parsed. - - .. attribute:: pos - - The start index of *doc* where parsing failed. - - .. attribute:: lineno - - The line corresponding to *pos*. - - .. attribute:: colno - - The column corresponding to *pos*. - - .. versionadded:: 3.5 - - -Standard Compliance and Interoperability ----------------------------------------- - -The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. -This section details this module's level of compliance with the RFC. -For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and -parameters other than those explicitly mentioned, are not considered. - -This module does not comply with the RFC in a strict fashion, implementing some -extensions that are valid JavaScript but not valid JSON. In particular: - -- Infinite and NaN number values are accepted and output; -- Repeated names within an object are accepted, and only the value of the last - name-value pair is used. - -Since the RFC permits RFC-compliant parsers to accept input texts that are not -RFC-compliant, this module's deserializer is technically RFC-compliant under -default settings. - -Character Encodings -^^^^^^^^^^^^^^^^^^^ - -The RFC requires that JSON be represented using either UTF-8, UTF-16, or -UTF-32, with UTF-8 being the recommended default for maximum interoperability. - -As permitted, though not required, by the RFC, this module's serializer sets -*ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. - -Other than the *ensure_ascii* parameter, this module is defined strictly in -terms of conversion between Python objects and -:class:`Unicode strings `, and thus does not otherwise directly address -the issue of character encodings. - -The RFC prohibits adding a byte order mark (BOM) to the start of a JSON text, -and this module's serializer does not add a BOM to its output. -The RFC permits, but does not require, JSON deserializers to ignore an initial -BOM in their input. This module's deserializer raises a :exc:`ValueError` -when an initial BOM is present. - -The RFC does not explicitly forbid JSON strings which contain byte sequences -that don't correspond to valid Unicode characters (e.g. unpaired UTF-16 -surrogates), but it does note that they may cause interoperability problems. -By default, this module accepts and outputs (when present in the original -:class:`str`) code points for such sequences. - - -Infinite and NaN Number Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC does not permit the representation of infinite or NaN number values. -Despite that, by default, this module accepts and outputs ``Infinity``, -``-Infinity``, and ``NaN`` as if they were valid JSON number literal values:: - - >>> # Neither of these calls raises an exception, but the results are not valid JSON - >>> json.dumps(float('-inf')) - '-Infinity' - >>> json.dumps(float('nan')) - 'NaN' - >>> # Same when deserializing - >>> json.loads('-Infinity') - -inf - >>> json.loads('NaN') - nan - -In the serializer, the *allow_nan* parameter can be used to alter this -behavior. In the deserializer, the *parse_constant* parameter can be used to -alter this behavior. - - -Repeated Names Within an Object -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC specifies that the names within a JSON object should be unique, but -does not mandate how repeated names in JSON objects should be handled. By -default, this module does not raise an exception; instead, it ignores all but -the last name-value pair for a given name:: - - >>> weird_json = '{"x": 1, "x": 2, "x": 3}' - >>> json.loads(weird_json) - {'x': 3} - -The *object_pairs_hook* parameter can be used to alter this behavior. - - -Top-level Non-Object, Non-Array Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The old version of JSON specified by the obsolete :rfc:`4627` required that -the top-level value of a JSON text must be either a JSON object or array -(Python :class:`dict` or :class:`list`), and could not be a JSON null, -boolean, number, or string value. :rfc:`7159` removed that restriction, and -this module does not and has never implemented that restriction in either its -serializer or its deserializer. - -Regardless, for maximum interoperability, you may wish to voluntarily adhere -to the restriction yourself. - - -Implementation Limitations -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some JSON deserializer implementations may set limits on: - -* the size of accepted JSON texts -* the maximum level of nesting of JSON objects and arrays -* the range and precision of JSON numbers -* the content and maximum length of JSON strings - -This module does not impose any such limits beyond those of the relevant -Python datatypes themselves or the Python interpreter itself. - -When serializing to JSON, beware any such limitations in applications that may -consume your JSON. In particular, it is common for JSON numbers to be -deserialized into IEEE 754 double precision numbers and thus subject to that -representation's range and precision limitations. This is especially relevant -when serializing Python :class:`int` values of extremely large magnitude, or -when serializing instances of "exotic" numerical types such as -:class:`decimal.Decimal`. - -.. highlight:: bash -.. module:: json.tool - -.. _json-commandline: - -Command Line Interface ----------------------- - -The :mod:`json.tool` module provides a simple command line interface to validate -and pretty-print JSON objects. - -If the optional ``infile`` and ``outfile`` arguments are not -specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively:: - - $ echo '{"json": "obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - -.. versionchanged:: 3.5 - The output is now in the same order as the input. Use the - :option:`--sort-keys` option to sort the output of dictionaries - alphabetically by key. - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. cmdoption:: infile - - The JSON file to be validated or pretty-printed:: - - $ python -m json.tool mp_films.json - [ - { - "title": "And Now for Something Completely Different", - "year": 1971 - }, - { - "title": "Monty Python and the Holy Grail", - "year": 1975 - } - ] - - If *infile* is not specified, read from :attr:`sys.stdin`. - -.. cmdoption:: outfile - - Write the output of the *infile* to the given *outfile*. Otherwise, write it - to :attr:`sys.stdout`. - -.. cmdoption:: --sort-keys - - Sort the output of dictionaries alphabetically by key. - - .. versionadded:: 3.5 - -.. cmdoption:: -h, --help - - Show the help message. - - -.. rubric:: Footnotes - -.. [#rfc-errata] As noted in `the errata for RFC 7159 - `_, - JSON permits literal U+2028 (LINE SEPARATOR) and - U+2029 (PARAGRAPH SEPARATOR) characters in strings, whereas JavaScript - (as of ECMAScript Edition 5.1) does not. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/linecache.rst --- a/Doc/library/linecache.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/linecache.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,15 +9,11 @@ -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`linecache` module allows one to get any line from any file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. -The :func:`tokenize.open` function is used to open files. This -function uses :func:`tokenize.detect_encoding` to get the encoding of the -file; in the absence of an encoding token, the file encoding defaults to UTF-8. - The :mod:`linecache` module defines the following functions: @@ -47,17 +43,10 @@ changed on disk, and you require the updated version. If *filename* is omitted, it will check all the entries in the cache. -.. function:: lazycache(filename, module_globals) - - Capture enough detail about a non-file-based module to permit getting its - lines later via :func:`getline` even if *module_globals* is None in the later - call. This avoids doing I/O until a line is actually needed, without having - to carry the module globals around indefinitely. - - .. versionadded:: 3.5 Example:: >>> import linecache - >>> linecache.getline(linecache.__file__, 8) - 'import sys\n' + >>> linecache.getline('/etc/passwd', 4) + 'sys:x:3:3:sys:/dev:/bin/sh\n' + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/locale.rst --- a/Doc/library/locale.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/locale.rst Mon Jan 25 17:05:13 2016 +0100 @@ -55,8 +55,6 @@ Returns the database of the local conventions as a dictionary. This dictionary has the following strings as keys: - .. tabularcolumns:: |l|l|L| - +----------------------+-------------------------------------+--------------------------------+ | Category | Key | Meaning | +======================+=====================================+================================+ @@ -162,22 +160,22 @@ .. data:: D_T_FMT - Get a string that can be used as a format string for :func:`time.strftime` to + Get a string that can be used as a format string for :func:`strftime` to represent date and time in a locale-specific way. .. data:: D_FMT - Get a string that can be used as a format string for :func:`time.strftime` to + Get a string that can be used as a format string for :func:`strftime` to represent a date in a locale-specific way. .. data:: T_FMT - Get a string that can be used as a format string for :func:`time.strftime` to + Get a string that can be used as a format string for :func:`strftime` to represent a time in a locale-specific way. .. data:: T_FMT_AMPM - Get a format string for :func:`time.strftime` to represent time in the am/pm + Get a format string for :func:`strftime` to represent time in the am/pm format. .. data:: DAY_1 ... DAY_7 @@ -204,7 +202,7 @@ .. data:: RADIXCHAR - Get the radix character (decimal dot, decimal comma, etc.). + Get the radix character (decimal dot, decimal comma, etc.) .. data:: THOUSEP @@ -241,24 +239,24 @@ then-emperor's reign. Normally it should not be necessary to use this value directly. Specifying - the ``E`` modifier in their format strings causes the :func:`time.strftime` + the ``E`` modifier in their format strings causes the :func:`strftime` function to use this information. The format of the returned string is not specified, and therefore you should not assume knowledge of it on different systems. .. data:: ERA_D_T_FMT - Get a format string for :func:`time.strftime` to represent date and time in a + Get a format string for :func:`strftime` to represent date and time in a locale-specific era-based way. .. data:: ERA_D_FMT - Get a format string for :func:`time.strftime` to represent a date in a + Get a format string for :func:`strftime` to represent a date in a locale-specific era-based way. .. data:: ERA_T_FMT - Get a format string for :func:`time.strftime` to represent a time in a + Get a format string for :func:`strftime` to represent a time in a locale-specific era-based way. .. data:: ALT_DIGITS @@ -387,14 +385,6 @@ ``str(float)``, but takes the decimal point into account. -.. function:: delocalize(string) - - Converts a string into a normalized number string, following the - :const:`LC_NUMERIC` settings. - - .. versionadded:: 3.5 - - .. function:: atof(string) Converts a string to a floating point number, following the :const:`LC_NUMERIC` @@ -485,11 +475,8 @@ locale somewhat painful to use correctly. Initially, when a program is started, the locale is the ``C`` locale, no matter -what the user's preferred locale is. There is one exception: the -:data:`LC_CTYPE` category is changed at startup to set the current locale -encoding to the user's preferred locale encoding. The program must explicitly -say that it wants the user's preferred locale settings for other categories by -calling ``setlocale(LC_ALL, '')``. +what the user's preferred locale is. The program must explicitly say that it +wants the user's preferred locale settings by calling ``setlocale(LC_ALL, '')``. It is generally a bad idea to call :func:`setlocale` in some library routine, since as a side effect it affects the entire program. Saving and restoring it diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/logging.config.rst --- a/Doc/library/logging.config.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/logging.config.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,10 +17,6 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` -**Source code:** :source:`Lib/logging/config.py` - --------------- - This section describes the API for configuring the logging module. .. _logging-config-api: @@ -80,94 +76,39 @@ .. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True) - Reads the logging configuration from a :mod:`configparser`\-format file. The - format of the file should be as described in - :ref:`logging-config-fileformat`. - This function can be called several times from an application, allowing an - end user to select from various pre-canned configurations (if the developer - provides a mechanism to present the choices and load the chosen - configuration). - - :param fname: A filename, or a file-like object, or an instance derived - from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is - instantiated, and the configuration read by it from the - object passed in ``fname``. If that has a :meth:`readline` - method, it is assumed to be a file-like object and read using - :meth:`~configparser.ConfigParser.read_file`; otherwise, - it is assumed to be a filename and passed to - :meth:`~configparser.ConfigParser.read`. - + Reads the logging configuration from a :mod:`configparser`\-format file + named *fname*. This function can be called several times from an + application, allowing an end user to select from various pre-canned + configurations (if the developer provides a mechanism to present the choices + and load the chosen configuration). :param defaults: Defaults to be passed to the ConfigParser can be specified in this argument. :param disable_existing_loggers: If specified as ``False``, loggers which exist when this call is made are left - enabled. The default is ``True`` because this + alone. The default is ``True`` because this enables old behaviour in a backward- compatible way. This behaviour is to disable any existing loggers unless they or their ancestors are explicitly named in the logging configuration. - .. versionchanged:: 3.4 - An instance of a subclass of :class:`~configparser.RawConfigParser` is - now accepted as a value for ``fname``. This facilitates: - * Use of a configuration file where logging configuration is just part - of the overall application configuration. - * Use of a configuration read from a file, and then modified by the using - application (e.g. based on command-line parameters or other aspects - of the runtime environment) before being passed to ``fileConfig``. - -.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) +.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT) Starts up a socket server on the specified port, and listens for new configurations. If no port is specified, the module's default :const:`DEFAULT_LOGGING_CONFIG_PORT` is used. Logging configurations will be sent as a file suitable for processing by :func:`fileConfig`. Returns a - :class:`~threading.Thread` instance on which you can call - :meth:`~threading.Thread.start` to start the server, and which you can - :meth:`~threading.Thread.join` when appropriate. To stop the server, + :class:`Thread` instance on which you can call :meth:`start` to start the + server, and which you can :meth:`join` when appropriate. To stop the server, call :func:`stopListening`. - The ``verify`` argument, if specified, should be a callable which should - verify whether bytes received across the socket are valid and should be - processed. This could be done by encrypting and/or signing what is sent - across the socket, such that the ``verify`` callable can perform - signature verification and/or decryption. The ``verify`` callable is called - with a single argument - the bytes received across the socket - and should - return the bytes to be processed, or None to indicate that the bytes should - be discarded. The returned bytes could be the same as the passed in bytes - (e.g. when only verification is done), or they could be completely different - (perhaps if decryption were performed). - To send a configuration to the socket, read in the configuration file and send it to the socket as a string of bytes preceded by a four-byte length string packed in binary using ``struct.pack('>L', n)``. - .. note:: - - Because portions of the configuration are passed through - :func:`eval`, use of this function may open its users to a security risk. - While the function only binds to a socket on ``localhost``, and so does - not accept connections from remote machines, there are scenarios where - untrusted code could be run under the account of the process which calls - :func:`listen`. Specifically, if the process calling :func:`listen` runs - on a multi-user machine where users cannot trust each other, then a - malicious user could arrange to run essentially arbitrary code in a - victim user's process, simply by connecting to the victim's - :func:`listen` socket and sending a configuration which runs whatever - code the attacker wants to have executed in the victim's process. This is - especially easy to do if the default port is used, but not hard even if a - different port is used). To avoid the risk of this happening, use the - ``verify`` argument to :func:`listen` to prevent unrecognised - configurations from being applied. - - .. versionchanged:: 3.4. - The ``verify`` argument was added. .. function:: stopListening() @@ -212,11 +153,11 @@ * *formatters* - the corresponding value will be a dict in which each key is a formatter id and each value is a dict describing how to - configure the corresponding :class:`~logging.Formatter` instance. + configure the corresponding Formatter instance. The configuring dict is searched for keys ``format`` and ``datefmt`` (with defaults of ``None``) and these are used to construct a - :class:`~logging.Formatter` instance. + :class:`logging.Formatter` instance. * *filters* - the corresponding value will be a dict in which each key is a filter id and each value is a dict describing how to configure @@ -615,18 +556,6 @@ specified in a section called ``[formatter_form01]``. The root logger configuration must be specified in a section called ``[logger_root]``. -.. note:: - - The :func:`fileConfig` API is older than the :func:`dictConfig` API and does - not provide functionality to cover certain aspects of logging. For example, - you cannot configure :class:`~logging.Filter` objects, which provide for - filtering of messages beyond simple integer levels, using :func:`fileConfig`. - If you need to have instances of :class:`~logging.Filter` in your logging - configuration, you will need to use :func:`dictConfig`. Note that future - enhancements to configuration functionality will be added to - :func:`dictConfig`, so it's worth considering transitioning to this newer - API when it's convenient to do so. - Examples of these sections in the file are given below. :: [loggers] @@ -762,17 +691,8 @@ The ``class`` entry is optional. It indicates the name of the formatter's class (as a dotted module and class name.) This option is useful for instantiating a -:class:`~logging.Formatter` subclass. Subclasses of -:class:`~logging.Formatter` can present exception tracebacks in an expanded or -condensed format. - -.. note:: - - Due to the use of :func:`eval` as described above, there are - potential security risks which result from using the :func:`listen` to send - and receive configurations via sockets. The risks are limited to where - multiple users with no mutual trust run code on the same machine; see the - :func:`listen` documentation for more information. +:class:`Formatter` subclass. Subclasses of :class:`Formatter` can present +exception tracebacks in an expanded or condensed format. .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/logging.handlers.rst --- a/Doc/library/logging.handlers.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/logging.handlers.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,10 +17,6 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` -**Source code:** :source:`Lib/logging/handlers.py` - --------------- - .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of @@ -57,8 +53,8 @@ .. method:: flush() Flushes the stream by calling its :meth:`flush` method. Note that the - :meth:`close` method is inherited from :class:`~logging.Handler` and so - does no output, so an explicit :meth:`flush` call may be needed at times. + :meth:`close` method is inherited from :class:`Handler` and so does + no output, so an explicit :meth:`flush` call may be needed at times. .. versionchanged:: 3.2 The ``StreamHandler`` class now has a ``terminator`` attribute, default @@ -149,8 +145,8 @@ This handler is not appropriate for use under Windows, because under Windows open log files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need for such a handler. Furthermore, -*ST_INO* is not supported under Windows; :func:`~os.stat` always returns zero -for this value. +*ST_INO* is not supported under Windows; :func:`stat` always returns zero for +this value. .. class:: WatchedFileHandler(filename[,mode[, encoding[, delay]]]) @@ -162,19 +158,11 @@ first call to :meth:`emit`. By default, the file grows indefinitely. - .. method:: reopenIfNeeded() - - Checks to see if the file has changed. If it has, the existing stream is - flushed and closed and the file opened again, typically as a precursor to - outputting the record to the file. - - .. versionadded:: 3.6 - - .. method:: emit(record) - Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to - reopen the file if it has changed. + Outputs the record to the file, but first checks to see if the file has + changed. If it has, the existing stream is flushed and closed and the + file opened again, before outputting the record to the file. .. _base-rotating-handler: @@ -238,7 +226,7 @@ renamed to the destination. :param source: The source filename. This is normally the base - filename, e.g. 'test.log'. + filename, e.g. 'test.log' :param dest: The destination filename. This is normally what the source is rotated to, e.g. 'test.log.1'. @@ -277,16 +265,15 @@ You can use the *maxBytes* and *backupCount* values to allow the file to :dfn:`rollover` at a predetermined size. When the size is about to be exceeded, the file is closed and a new file is silently opened for output. Rollover occurs - whenever the current log file is nearly *maxBytes* in length; if either of - *maxBytes* or *backupCount* is zero, rollover never occurs. If *backupCount* - is non-zero, the system will save old log files by appending the extensions - '.1', '.2' etc., to the filename. For example, with a *backupCount* of 5 and - a base file name of :file:`app.log`, you would get :file:`app.log`, - :file:`app.log.1`, :file:`app.log.2`, up to :file:`app.log.5`. The file being - written to is always :file:`app.log`. When this file is filled, it is closed - and renamed to :file:`app.log.1`, and if files :file:`app.log.1`, - :file:`app.log.2`, etc. exist, then they are renamed to :file:`app.log.2`, - :file:`app.log.3` etc. respectively. + whenever the current log file is nearly *maxBytes* in length; if *maxBytes* is + zero, rollover never occurs. If *backupCount* is non-zero, the system will save + old log files by appending the extensions '.1', '.2' etc., to the filename. For + example, with a *backupCount* of 5 and a base file name of :file:`app.log`, you + would get :file:`app.log`, :file:`app.log.1`, :file:`app.log.2`, up to + :file:`app.log.5`. The file being written to is always :file:`app.log`. When + this file is filled, it is closed and renamed to :file:`app.log.1`, and if files + :file:`app.log.1`, :file:`app.log.2`, etc. exist, then they are renamed to + :file:`app.log.2`, :file:`app.log.3` etc. respectively. .. method:: doRollover() @@ -309,7 +296,7 @@ timed intervals. -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None) +.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False) Returns a new instance of the :class:`TimedRotatingFileHandler` class. The specified file is opened and used as the stream for logging. On rotating it also @@ -330,15 +317,11 @@ +----------------+-----------------------+ | ``'D'`` | Days | +----------------+-----------------------+ - | ``'W0'-'W6'`` | Weekday (0=Monday) | + | ``'W'`` | Week day (0=Monday) | +----------------+-----------------------+ | ``'midnight'`` | Roll over at midnight | +----------------+-----------------------+ - When using weekday-based rotation, specify 'W0' for Monday, 'W1' for - Tuesday, and so on up to 'W6' for Sunday. In this case, the value passed for - *interval* isn't used. - The system will save old log files by appending extensions to the filename. The extensions are date-and-time based, using the strftime format ``%Y-%m-%d_%H-%M-%S`` or a leading portion thereof, depending on the @@ -359,12 +342,6 @@ If *delay* is true, then file opening is deferred until the first call to :meth:`emit`. - If *atTime* is not ``None``, it must be a ``datetime.time`` instance which - specifies the time of day when rollover occurs, for the cases where rollover - is set to happen "at midnight" or "on a particular weekday". - - .. versionchanged:: 3.4 - *atTime* parameter was added. .. method:: doRollover() @@ -390,9 +367,6 @@ Returns a new instance of the :class:`SocketHandler` class intended to communicate with a remote machine whose address is given by *host* and *port*. - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. .. method:: close() @@ -405,8 +379,7 @@ binary format. If there is an error with the socket, silently drops the packet. If the connection was previously lost, re-establishes the connection. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. + :class:`LogRecord`, use the :func:`makeLogRecord` function. .. method:: handleError() @@ -444,7 +417,7 @@ .. method:: createSocket() Tries to create a socket; on failure, uses an exponential back-off - algorithm. On initial failure, the handler will drop the message it was + algorithm. On intial failure, the handler will drop the message it was trying to send. When subsequent messages are handled by the same instance, it will not try connecting until some time has passed. The default parameters are such that the initial delay is one second, and if @@ -478,17 +451,13 @@ Returns a new instance of the :class:`DatagramHandler` class intended to communicate with a remote machine whose address is given by *host* and *port*. - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. .. method:: emit() Pickles the record's attribute dictionary and writes it to the socket in binary format. If there is an error with the socket, silently drops the packet. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. + :class:`LogRecord`, use the :func:`makeLogRecord` function. .. method:: makeSocket() @@ -735,7 +704,7 @@ supports sending logging messages to an email address via SMTP. -.. class:: SMTPHandler(mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None, timeout=1.0) +.. class:: SMTPHandler(mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None) Returns a new instance of the :class:`SMTPHandler` class. The instance is initialized with the from and to addresses and subject line of the email. The @@ -751,12 +720,6 @@ and certificate file. (This tuple is passed to the :meth:`smtplib.SMTP.starttls` method.) - A timeout can be specified for communication with the SMTP server using the - *timeout* argument. - - .. versionadded:: 3.3 - The *timeout* argument was added. - .. method:: emit(record) Formats the record and sends it to the specified addressees. @@ -781,7 +744,7 @@ :class:`BufferingHandler`, which is an abstract class. This buffers logging records in memory. Whenever each record is added to the buffer, a check is made by calling :meth:`shouldFlush` to see if the buffer should be flushed. If it -should, then :meth:`flush` is expected to do the flushing. +should, then :meth:`flush` is expected to do the needful. .. class:: BufferingHandler(capacity) @@ -848,43 +811,21 @@ ``POST`` semantics. -.. class:: HTTPHandler(host, url, method='GET', secure=False, credentials=None, context=None) +.. class:: HTTPHandler(host, url, method='GET', secure=False, credentials=None) Returns a new instance of the :class:`HTTPHandler` class. The *host* can be - of the form ``host:port``, should you need to use a specific port number. If - no *method* is specified, ``GET`` is used. If *secure* is true, a HTTPS - connection will be used. The *context* parameter may be set to a - :class:`ssl.SSLContext` instance to configure the SSL settings used for the - HTTPS connection. If *credentials* is specified, it should be a 2-tuple - consisting of userid and password, which will be placed in a HTTP + of the form ``host:port``, should you need to use a specific port number. + If no *method* is specified, ``GET`` is used. If *secure* is True, an HTTPS + connection will be used. If *credentials* is specified, it should be a + 2-tuple consisting of userid and password, which will be placed in an HTTP 'Authorization' header using Basic authentication. If you specify credentials, you should also specify secure=True so that your userid and password are not passed in cleartext across the wire. - .. versionchanged:: 3.5 - The *context* parameter was added. - - .. method:: mapLogRecord(record) - - Provides a dictionary, based on ``record``, which is to be URL-encoded - and sent to the web server. The default implementation just returns - ``record.__dict__``. This method can be overridden if e.g. only a - subset of :class:`~logging.LogRecord` is to be sent to the web server, or - if more specific customization of what's sent to the server is required. .. method:: emit(record) - Sends the record to the Web server as an URL-encoded dictionary. The - :meth:`mapLogRecord` method is used to convert the record to the - dictionary to be sent. - - .. note:: Since preparing a record for sending it to a Web server is not - the same as a generic formatting operation, using - :meth:`~logging.Handler.setFormatter` to specify a - :class:`~logging.Formatter` for a :class:`HTTPHandler` has no effect. - Instead of calling :meth:`~logging.Handler.format`, this handler calls - :meth:`mapLogRecord` and then :func:`urllib.parse.urlencode` to encode the - dictionary in a form suitable for sending to a Web server. + Sends the record to the Web server as a percent-encoded dictionary. .. _queue-handler: @@ -935,7 +876,7 @@ Enqueues the record on the queue using ``put_nowait()``; you may want to override this if you want to use blocking behaviour, or a - timeout, or a customized queue implementation. + timeout, or a customised queue implementation. @@ -961,20 +902,13 @@ possible, while any potentially slow operations (such as sending an email via :class:`SMTPHandler`) are done on a separate thread. -.. class:: QueueListener(queue, *handlers, respect_handler_level=False) +.. class:: QueueListener(queue, *handlers) Returns a new instance of the :class:`QueueListener` class. The instance is initialized with the queue to send messages to and a list of handlers which will handle entries placed on the queue. The queue can be any queue- like object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. If ``respect_handler_level`` is ``True``, - a handler's level is respected (compared with the level for the message) when - deciding whether to pass messages to that handler; otherwise, the behaviour - is as in previous Python versions - to always pass each message to each - handler. - - .. versionchanged:: 3.5 - The ``respect_handler_levels`` argument was added. + to know how to get messages from it. .. method:: dequeue(block) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/logging.rst --- a/Doc/library/logging.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/logging.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,10 +21,6 @@ * :ref:`Logging Cookbook ` -**Source code:** :source:`Lib/logging/__init__.py` - --------------- - This module defines functions and classes which implement a flexible event logging system for applications and libraries. @@ -53,46 +49,24 @@ Logger Objects -------------- -Loggers have the following attributes and methods. Note that Loggers are never +Loggers have the following attributes and methods. Note that Loggers are never instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - +``logging.getLogger(name)``. .. class:: Logger .. attribute:: Logger.propagate - If this evaluates to true, events logged to this logger will be passed to the - handlers of higher level (ancestor) loggers, in addition to any handlers - attached to this logger. Messages are passed directly to the ancestor - loggers' handlers - neither the level nor filters of the ancestor loggers in - question are considered. + If this evaluates to true, logging messages are passed by this logger and by + its child loggers to the handlers of higher level (ancestor) loggers. + Messages are passed directly to the ancestor loggers' handlers - neither the + level nor filters of the ancestor loggers in question are considered. If this evaluates to false, logging messages are not passed to the handlers of ancestor loggers. The constructor sets this attribute to ``True``. - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. .. method:: Logger.setLevel(lvl) @@ -113,14 +87,10 @@ If the root is reached, and it has a level of NOTSET, then all messages will be processed. Otherwise, the root's level will be used as the effective level. - See :ref:`levels` for a list of levels. - .. versionchanged:: 3.2 The *lvl* parameter now accepts a string representation of the level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. + such as :const:`INFO`. .. method:: Logger.isEnabledFor(lvl) @@ -136,9 +106,7 @@ Indicates the effective level for this logger. If a value other than :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. + :const:`NOTSET` is found, and that value is returned. .. method:: Logger.getChild(suffix) @@ -159,16 +127,14 @@ *msg* using the string formatting operator. (Note that this means that you can use keywords in the format string, together with a single dictionary argument.) - There are three keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be + There are three keyword arguments in *kwargs* which are inspected: *exc_info* + which, if it does not evaluate as false, causes exception information to be added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. + :func:`sys.exc_info`) is provided, it is used; otherwise, :func:`sys.exc_info` + is called to get the exception information. The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging + False. If specified as True, stack information is added to the logging message, including the actual logging call. Note that this is not the same stack information as that displayed through specifying *exc_info*: The former is stack frames from the bottom of the stack up to the logging call @@ -222,9 +188,6 @@ .. versionadded:: 3.2 The *stack_info* parameter was added. - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - .. method:: Logger.info(msg, *args, **kwargs) @@ -259,7 +222,7 @@ interpreted as for :meth:`debug`. -.. method:: Logger.exception(msg, *args, **kwargs) +.. method:: Logger.exception(msg, *args) Logs a message with level :const:`ERROR` on this logger. The arguments are interpreted as for :meth:`debug`. Exception info is added to the logging @@ -279,10 +242,7 @@ .. method:: Logger.filter(record) Applies this logger's filters to the record and returns a true value if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. + record is to be processed. .. method:: Logger.addHandler(hdlr) @@ -319,7 +279,7 @@ Checks to see if this logger has any handlers configured. This is done by looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching + Returns True if a handler was found, else False. The method stops searching up the hierarchy whenever a logger with the 'propagate' attribute set to False is found - that will be the last logger which is checked for the existence of handlers. @@ -327,34 +287,6 @@ .. versionadded:: 3.2 -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - - .. _handler: Handler Objects @@ -395,8 +327,6 @@ severe than *lvl* will be ignored. When a handler is created, the level is set to :const:`NOTSET` (which causes all messages to be processed). - See :ref:`levels` for a list of levels. - .. versionchanged:: 3.2 The *lvl* parameter now accepts a string representation of the level such as 'INFO' as an alternative to the integer constants @@ -421,10 +351,7 @@ .. method:: Handler.filter(record) Applies this handler's filters to the record and returns a true value if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. + record is to be processed. .. method:: Handler.flush() @@ -486,9 +413,7 @@ responsible for converting a :class:`LogRecord` to (usually) a string which can be interpreted by either a human or an external system. The base :class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. +supplied, the default value of ``'%(message)s'`` is used. A Formatter can be initialized with a format string which makes use of knowledge of the :class:`LogRecord` attributes - such as the default value mentioned above @@ -511,8 +436,7 @@ The *style* parameter can be one of '%', '{' or '$' and determines how the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. See :ref:`formatting-styles` - for more information on using {- and $-formatting for log messages. + :meth:`str.format` or :class:`string.Template`. .. versionchanged:: 3.2 The *style* parameter was added. @@ -610,12 +534,12 @@ yes. If deemed appropriate, the record may be modified in-place by this method. -Note that filters attached to handlers are consulted before an event is +Note that filters attached to handlers are consulted whenever an event is emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. +whenever an event is logged to the handler (using :meth:`debug`, :meth:`info`, +etc.) This means that events which have been generated by descendant loggers +will not be filtered by a logger's filter setting, unless the filter has also +been applied to those descendant loggers. You don't actually need to subclass ``Filter``: you can pass any instance which has a ``filter`` method with the same semantics. @@ -659,9 +583,7 @@ record. :param name: The name of the logger used to log the event represented by - this LogRecord. Note that this name will always have this - value, even though it may be emitted by a handler attached to - a different (ancestor) logger. + this LogRecord. :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) Note that this is converted to *two* attributes of the LogRecord: ``levelno`` for the numeric value and ``levelname`` for the @@ -743,9 +665,7 @@ | Attribute name | Format | Description | +================+=========================+===============================================+ | args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | +| | format this yourself. | produce ``message``. | +----------------+-------------------------+-----------------------------------------------+ | asctime | ``%(asctime)s`` | Human-readable time when the | | | | :class:`LogRecord` was created. By default | @@ -813,9 +733,6 @@ | threadName | ``%(threadName)s`` | Thread name (if available). | +----------------+-------------------------+-----------------------------------------------+ -.. versionchanged:: 3.1 - *processName* was added. - .. _logger-adapter: @@ -823,7 +740,7 @@ --------------------- :class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on +information into logging calls. For a usage example , see the section on :ref:`adding contextual information to your logging output `. .. class:: LoggerAdapter(logger, extra) @@ -840,18 +757,17 @@ (possibly modified) versions of the arguments passed in. In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their +methods of :class:`Logger`, i.e. :meth:`debug`, :meth:`info`, :meth:`warning`, +:meth:`error`, :meth:`exception`, :meth:`critical`, :meth:`log`, +:meth:`isEnabledFor`, :meth:`getEffectiveLevel`, :meth:`setLevel`, +:meth:`hasHandlers`. These methods have the same signatures as their counterparts in :class:`Logger`, so you can use the two types of instances interchangeably. .. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + The :meth:`isEnabledFor`, :meth:`getEffectiveLevel`, :meth:`setLevel` and + :meth:`hasHandlers` methods were added to :class:`LoggerAdapter`. These + methods delegate to the underlying logger. Thread Safety @@ -891,8 +807,8 @@ Return either the standard :class:`Logger` class, or the last class passed to :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: + definition, to ensure that installing a customised :class:`Logger` class will + not undo customisations already applied by other code. For example:: class MyLogger(logging.getLoggerClass()): # ... override behaviour here @@ -924,7 +840,7 @@ is called to get the exception information. The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging + False. If specified as True, stack information is added to the logging message, including the actual logging call. Note that this is not the same stack information as that displayed through specifying *exc_info*: The former is stack frames from the bottom of the stack up to the logging call @@ -1005,7 +921,7 @@ are interpreted as for :func:`debug`. -.. function:: exception(msg, *args, **kwargs) +.. function:: exception(msg, *args) Logs a message with level :const:`ERROR` on the root logger. The arguments are interpreted as for :func:`debug`. Exception info is added to the logging @@ -1016,15 +932,14 @@ Logs a message with level *level* on the root logger. The other arguments are interpreted as for :func:`debug`. - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. + PLEASE NOTE: The above module-level functions which delegate to the root + logger should *not* be used in threads, in versions of Python earlier than + 2.7.1 and 3.2, unless at least one handler has been added to the root + logger *before* the threads are started. These convenience functions call + :func:`basicConfig` to ensure that at least one handler is available; in + earlier versions of Python, this can (under rare circumstances) lead to + handlers being added multiple times to the root logger, which can in turn + lead to multiple messages for the same event. .. function:: disable(lvl) @@ -1034,10 +949,7 @@ effect is to disable all logging calls of severity *lvl* and below, so that if you call it with a value of INFO, then all INFO and DEBUG events would be discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. + according to the logger's effective level. .. function:: addLevelName(lvl, levelName) @@ -1049,8 +961,8 @@ registered using this function, levels should be positive integers and they should increase in increasing order of severity. - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. + NOTE: If you are thinking of defining your own levels, please see the section + on :ref:`custom-levels`. .. function:: getLevelName(lvl) @@ -1062,16 +974,6 @@ of the defined levels is passed in, the corresponding string representation is returned. Otherwise, the string 'Level %s' % lvl is returned. - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. .. function:: makeLogRecord(attrdict) @@ -1092,17 +994,15 @@ This function does nothing if the root logger already has handlers configured for it. - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. + PLEASE NOTE: This function should be called from the main thread + before other threads are started. In versions of Python prior to + 2.7.1 and 3.2, if this function is called from multiple threads, + it is possible (in rare circumstances) that a handler will be added + to the root logger more than once, leading to unexpected results + such as messages being duplicated in the log. The following keyword arguments are supported. - .. tabularcolumns:: |l|L| - +--------------+---------------------------------------------+ | Format | Description | +==============+=============================================+ @@ -1198,21 +1098,6 @@ :kwargs: Additional keyword arguments. -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - Integration with the warnings module ------------------------------------ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/lzma.rst --- a/Doc/library/lzma.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/lzma.rst Mon Jan 25 17:05:13 2016 +0100 @@ -29,54 +29,17 @@ Reading and writing compressed files ------------------------------------ -.. function:: open(filename, mode="rb", \*, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) +.. class:: LZMAFile(filename=None, mode="r", \*, fileobj=None, format=None, check=-1, preset=None, filters=None) - Open an LZMA-compressed file in binary or text mode, returning a :term:`file - object`. + Open an LZMA-compressed file. - The *filename* argument can be either an actual file name (given as a - :class:`str` or :class:`bytes` object), in which case the named file is - opened, or it can be an existing file object to read from or write to. - - The *mode* argument can be any of ``"r"``, ``"rb"``, ``"w"``, ``"wb"``, - ``"x"``, ``"xb"``, ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, - ``"wt"``, ``"xt"``, or ``"at"`` for text mode. The default is ``"rb"``. - - When opening a file for reading, the *format* and *filters* arguments have - the same meanings as for :class:`LZMADecompressor`. In this case, the *check* - and *preset* arguments should not be used. - - When opening a file for writing, the *format*, *check*, *preset* and - *filters* arguments have the same meanings as for :class:`LZMACompressor`. - - For binary mode, this function is equivalent to the :class:`LZMAFile` - constructor: ``LZMAFile(filename, mode, ...)``. In this case, the *encoding*, - *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`LZMAFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.4 - Added support for the ``"x"``, ``"xb"`` and ``"xt"`` modes. - - -.. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None) - - Open an LZMA-compressed file in binary mode. - - An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate - directly on a named file. The *filename* argument specifies either the file - object to wrap, or the name of the file to open (as a :class:`str` or - :class:`bytes` object). When wrapping an existing file object, the wrapped - file will not be closed when the :class:`LZMAFile` is closed. + An :class:`LZMAFile` can wrap an existing :term:`file object` (given by + *fileobj*), or operate directly on a named file (named by *filename*). + Exactly one of these two parameters should be provided. If *fileobj* is + provided, it is not closed when the :class:`LZMAFile` is closed. The *mode* argument can be either ``"r"`` for reading (default), ``"w"`` for - overwriting, ``"x"`` for exclusive creation, or ``"a"`` for appending. These - can equivalently be given as ``"rb"``, ``"wb"``, ``"xb"`` and ``"ab"`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of + overwriting, or ``"a"`` for appending. If *fileobj* is provided, a mode of ``"w"`` does not truncate the file, and is instead equivalent to ``"a"``. When opening a file for reading, the input file may be the concatenation of @@ -102,18 +65,6 @@ byte of data will be returned, unless EOF has been reached. The exact number of bytes returned is unspecified (the *size* argument is ignored). - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`LZMAFile`, it may change the position of the underlying - file object (e.g. if the :class:`LZMAFile` was constructed by passing a - file object for *filename*). - - .. versionchanged:: 3.4 - Added support for the ``"x"`` and ``"xb"`` modes. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - Compressing and decompressing data in memory -------------------------------------------- @@ -174,7 +125,7 @@ In addition to being more CPU-intensive, compression with higher presets also requires much more memory (and produces output that needs more memory to decompress). With preset ``9`` for example, the overhead for an - :class:`LZMACompressor` object can be as high as 800 MiB. For this reason, + :class:`LZMACompressor` object can be as high as 800MiB. For this reason, it is generally best to stick with the default preset. The *filters* argument (if provided) should be a filter chain specifier. @@ -225,32 +176,13 @@ decompress a multi-stream input with :class:`LZMADecompressor`, you must create a new decompressor for each stream. - .. method:: decompress(data, max_length=-1) + .. method:: decompress(data) - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. - - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. - - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an `EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. + Decompress *data* (a :class:`bytes` object), returning a :class:`bytes` + object containing the decompressed data for at least part of the input. + Some of *data* may be buffered internally, for use in later calls to + :meth:`decompress`. The returned data should be concatenated with the + output of any previous calls to :meth:`decompress`. .. attribute:: check @@ -260,7 +192,7 @@ .. attribute:: eof - ``True`` if the end-of-stream marker has been reached. + True if the end-of-stream marker has been reached. .. attribute:: unused_data @@ -268,12 +200,6 @@ Before the end of the stream is reached, this will be ``b""``. - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 .. function:: compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None) @@ -299,7 +225,7 @@ Miscellaneous ------------- -.. function:: is_check_supported(check) +.. function:: check_is_supported(check) Returns true if the given integrity check is supported on this system. @@ -343,8 +269,8 @@ * ``preset``: A compression preset to use as a source of default values for options that are not specified explicitly. - * ``dict_size``: Dictionary size in bytes. This should be between 4 KiB and - 1.5 GiB (inclusive). + * ``dict_size``: Dictionary size in bytes. This should be between 4KiB and + 1.5GiB (inclusive). * ``lc``: Number of literal context bits. * ``lp``: Number of literal position bits. The sum ``lc + lp`` must be at most 4. @@ -376,15 +302,15 @@ Reading in a compressed file:: import lzma - with lzma.open("file.xz") as f: - file_content = f.read() + with lzma.LZMAFile("file.xz") as f: + file_content = f.read() Creating a compressed file:: import lzma data = b"Insert Data Here" - with lzma.open("file.xz", "w") as f: - f.write(data) + with lzma.LZMAFile("file.xz", "w") as f: + f.write(data) Compressing data in memory:: @@ -408,7 +334,7 @@ import lzma with open("file.xz", "wb") as f: f.write(b"This data will not be compressed\n") - with lzma.open(f, "w") as lzf: + with lzma.LZMAFile(fileobj=f, mode="w") as lzf: lzf.write(b"This *will* be compressed\n") f.write(b"Not compressed\n") @@ -419,5 +345,5 @@ {"id": lzma.FILTER_DELTA, "dist": 5}, {"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_EXTREME}, ] - with lzma.open("file.xz", "w", filters=my_filters) as f: + with lzma.LZMAFile("file.xz", "w", filters=my_filters) as f: f.write(b"blah blah blah") diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/mailbox.rst --- a/Doc/library/mailbox.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/mailbox.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,9 +10,9 @@ This module defines two classes, :class:`Mailbox` and :class:`Message`, for accessing and manipulating on-disk mailboxes and the messages they contain. :class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's -:class:`~email.message.Message` class with format-specific state and behavior. -Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. +:class:`Message` extends the :mod:`email.Message` module's :class:`Message` +class with format-specific state and behavior. Supported mailbox formats are +Maildir, mbox, MH, Babyl, and MMDF. .. seealso:: @@ -81,7 +81,7 @@ it. Parameter *message* may be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, a byte string, or a + :class:`email.Message.Message` instance, a string, a byte string, or a file-like object (which should be open in binary mode). If *message* is an instance of the appropriate format-specific :class:`Message` subclass (e.g., if it's an @@ -89,8 +89,7 @@ format-specific information is used. Otherwise, reasonable defaults for format-specific information are used. - .. versionchanged:: 3.2 - Support for binary input was added. + .. versionchanged:: 3.2 support for binary input .. method:: remove(key) @@ -112,7 +111,7 @@ :exc:`KeyError` exception if no message already corresponds to *key*. As with :meth:`add`, parameter *message* may be a :class:`Message` - instance, an :class:`email.message.Message` instance, a string, a byte + instance, an :class:`email.Message.Message` instance, a string, a byte string, or a file-like object (which should be open in binary mode). If *message* is an instance of the appropriate format-specific :class:`Message` subclass @@ -202,7 +201,7 @@ .. versionchanged:: 3.2 The file object really is a binary file; previously it was incorrectly returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to + context manager protocol: you can use a :keyword:`with` statement to automatically close it. .. note:: @@ -487,7 +486,7 @@ `Configuring Netscape Mail on Unix: Why The Content-Length Format is Bad `_ An argument for using the original mbox format rather than a variation. - `"mbox" is a family of several mutually incompatible mailbox formats `_ + `"mbox" is a family of several mutually incompatible mailbox formats `_ A history of mbox variations. @@ -674,8 +673,8 @@ In Babyl mailboxes, the headers of a message are not stored contiguously with the body of the message. To generate a file-like representation, the - headers and body are copied together into an :class:`io.BytesIO` instance, - which has an API identical to that of a + headers and body are copied together into a :class:`StringIO` instance + (from the :mod:`StringIO` module), which has an API identical to that of a file. As a result, the file-like object is truly independent of the underlying mailbox but does not save memory compared to a string representation. @@ -757,12 +756,11 @@ .. class:: Message(message=None) - A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add - mailbox-format-specific state and behavior. + A subclass of the :mod:`email.Message` module's :class:`Message`. Subclasses of + :class:`mailbox.Message` add mailbox-format-specific state and behavior. If *message* is omitted, the new instance is created in a default, empty state. - If *message* is an :class:`email.message.Message` instance, its contents are + If *message* is an :class:`email.Message.Message` instance, its contents are copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`Message` instance. If *message* is a string, a byte string, @@ -1009,7 +1007,7 @@ Set the "From " line to *from_*, which should be specified without a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a + *time_* is specified, it should be a :class:`struct_time` instance, a tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use :meth:`time.gmtime`). @@ -1268,7 +1266,7 @@ Set the message's visible headers to be the same as the headers in *message*. Parameter *visible* should be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, or a file-like object + :class:`email.Message.Message` instance, a string, or a file-like object (which should be open in text mode). @@ -1380,7 +1378,7 @@ Set the "From " line to *from_*, which should be specified without a leading "From " or trailing newline. For convenience, *time_* may be specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a + *time_* is specified, it should be a :class:`struct_time` instance, a tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use :meth:`time.gmtime`). @@ -1550,7 +1548,7 @@ due to malformed messages in the mailbox:: import mailbox - import email.errors + import email.Errors list_names = ('python-list', 'python-dev', 'python-bugs') @@ -1560,7 +1558,7 @@ for key in inbox.iterkeys(): try: message = inbox[key] - except email.errors.MessageParseError: + except email.Errors.MessageParseError: continue # The message is malformed. Just leave it. for name in list_names: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/mailcap.rst --- a/Doc/library/mailcap.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/mailcap.rst Mon Jan 25 17:05:13 2016 +0100 @@ -71,6 +71,6 @@ >>> import mailcap >>> d=mailcap.getcaps() - >>> mailcap.findmatch(d, 'video/mpeg', filename='tmp1223') - ('xmpeg tmp1223', {'view': 'xmpeg %s'}) + >>> mailcap.findmatch(d, 'video/mpeg', filename='/tmp/tmp1223') + ('xmpeg /tmp/tmp1223', {'view': 'xmpeg %s'}) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/markup.rst --- a/Doc/library/markup.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/markup.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,13 +9,20 @@ Language (SGML) and the Hypertext Markup Language (HTML), and several interfaces for working with the Extensible Markup Language (XML). +It is important to note that modules in the :mod:`xml` package require that +there be at least one SAX-compliant XML parser available. The Expat parser is +included with Python, so the :mod:`xml.parsers.expat` module will always be +available. + +The documentation for the :mod:`xml.dom` and :mod:`xml.sax` packages are the +definition of the Python bindings for the DOM and SAX interfaces. + .. toctree:: html.rst html.parser.rst html.entities.rst - xml.rst xml.etree.elementtree.rst xml.dom.rst xml.dom.minidom.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/marshal.rst --- a/Doc/library/marshal.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/marshal.rst Mon Jan 25 17:05:13 2016 +0100 @@ -40,11 +40,10 @@ point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, frozensets, dictionaries, and code objects, where it should be understood that tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported. +the values contained therein are themselves supported; and recursive lists, sets +and dictionaries should not be written (they will cause infinite loops). The singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be marshalled and unmarshalled. -For format *version* lower than 3, recursive lists, sets and dictionaries cannot -be written (see below). There are functions that read/write files as well as functions operating on strings. @@ -104,9 +103,7 @@ Indicates the format that the module uses. Version 0 is the historical format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. - Version 3 adds support for object instancing and recursion. - The current version is 4. + for floating point numbers. The current version is 2. .. rubric:: Footnotes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/math.rst --- a/Doc/library/math.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/math.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,9 +4,6 @@ .. module:: math :synopsis: Mathematical functions (sin() etc.). -.. testsetup:: - - from math import fsum This module is always available. It provides access to the mathematical functions defined by the C standard. @@ -31,14 +28,14 @@ Return the ceiling of *x*, the smallest integer greater than or equal to *x*. If *x* is not a float, delegates to ``x.__ceil__()``, which should return an - :class:`~numbers.Integral` value. + :class:`Integral` value. .. function:: copysign(x, y) - Return a float with the magnitude (absolute value) of *x* but the sign of - *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` - returns *-1.0*. + Return *x* with the sign of *y*. On a platform that supports + signed zeros, ``copysign(1.0, -0.0)`` returns *-1.0*. + .. function:: fabs(x) @@ -53,7 +50,7 @@ Return the floor of *x*, the largest integer less than or equal to *x*. If *x* is not a float, delegates to ``x.__floor__()``, which should return an - :class:`~numbers.Integral` value. + :class:`Integral` value. .. function:: fmod(x, y) @@ -100,48 +97,6 @@ `_\. -.. function:: gcd(a, b) - - Return the greatest common divisor of the integers *a* and *b*. If either - *a* or *b* is nonzero, then the value of ``gcd(a, b)`` is the largest - positive integer that divides both *a* and *b*. ``gcd(0, 0)`` returns - ``0``. - - .. versionadded:: 3.5 - - -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - .. function:: isfinite(x) Return ``True`` if *x* is neither an infinity nor a NaN, and @@ -175,9 +130,8 @@ .. function:: trunc(x) - Return the :class:`~numbers.Real` value *x* truncated to an - :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + Return the :class:`Real` value *x* truncated to an :class:`Integral` (usually + an integer). Delegates to ``x.__trunc__()``. Note that :func:`frexp` and :func:`modf` have a different call/return pattern @@ -258,10 +212,6 @@ ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` is undefined, and raises :exc:`ValueError`. - Unlike the built-in ``**`` operator, :func:`math.pow` converts both - its arguments to type :class:`float`. Use ``**`` or the built-in - :func:`pow` function for computing exact integer powers. - .. function:: sqrt(x) @@ -322,12 +272,12 @@ .. function:: degrees(x) - Convert angle *x* from radians to degrees. + Converts angle *x* from radians to degrees. .. function:: radians(x) - Convert angle *x* from degrees to radians. + Converts angle *x* from degrees to radians. Hyperbolic functions -------------------- @@ -425,22 +375,6 @@ The mathematical constant e = 2.718281..., to available precision. -.. data:: inf - - A floating-point positive infinity. (For negative infinity, use - ``-math.inf``.) Equivalent to the output of ``float('inf')``. - - .. versionadded:: 3.5 - - -.. data:: nan - - A floating-point "not a number" (NaN) value. Equivalent to the output of - ``float('nan')``. - - .. versionadded:: 3.5 - - .. impl-detail:: The :mod:`math` module consists mostly of thin wrappers around the platform C diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/mimetypes.rst --- a/Doc/library/mimetypes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/mimetypes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -44,7 +44,7 @@ The optional *strict* argument is a flag specifying whether the list of known MIME types is limited to only the official types `registered with IANA - `_. + `_. When *strict* is ``True`` (the default), only the IANA types are supported; when *strict* is ``False``, some additional non-standard but commonly used MIME types are also recognized. @@ -85,9 +85,6 @@ :const:`knownfiles` takes precedence over those named before it. Calling :func:`init` repeatedly is allowed. - Specifying an empty list for *files* will prevent the system defaults from - being applied: only the well-known values will be present from a built-in list. - .. versionchanged:: 3.2 Previously, Windows registry settings were ignored. @@ -106,8 +103,8 @@ extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. - When *strict* is ``True`` (the default), the mapping will be added to the - official MIME types, otherwise to the non-standard ones. + When *strict* is ``True`` (the default), the mapping will added to the official MIME + types, otherwise to the non-standard ones. .. data:: inited diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/mmap.rst --- a/Doc/library/mmap.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/mmap.rst Mon Jan 25 17:05:13 2016 +0100 @@ -106,19 +106,19 @@ with open("hello.txt", "r+b") as f: # memory-map the file, size 0 means whole file - mm = mmap.mmap(f.fileno(), 0) + map = mmap.mmap(f.fileno(), 0) # read content via standard file methods - print(mm.readline()) # prints b"Hello Python!\n" + print(map.readline()) # prints b"Hello Python!\n" # read content via slice notation - print(mm[:5]) # prints b"Hello" + print(map[:5]) # prints b"Hello" # update content using slice notation; # note that new content must have same size - mm[6:] = b" world!\n" + map[6:] = b" world!\n" # ... and read again using standard file methods - mm.seek(0) - print(mm.readline()) # prints b"Hello world!\n" + map.seek(0) + print(map.readline()) # prints b"Hello world!\n" # close the map - mm.close() + map.close() :class:`mmap` can also be used as a context manager in a :keyword:`with` @@ -126,8 +126,8 @@ import mmap - with mmap.mmap(-1, 13) as mm: - mm.write("Hello world!") + with mmap.mmap(-1, 13) as map: + map.write("Hello world!") .. versionadded:: 3.2 Context manager support. @@ -139,30 +139,29 @@ import mmap import os - mm = mmap.mmap(-1, 13) - mm.write(b"Hello world!") + map = mmap.mmap(-1, 13) + map.write(b"Hello world!") pid = os.fork() if pid == 0: # In a child process - mm.seek(0) - print(mm.readline()) + map.seek(0) + print(map.readline()) - mm.close() + map.close() Memory-mapped file objects support the following methods: .. method:: close() - Closes the mmap. Subsequent calls to other methods of the object will - result in a ValueError exception being raised. This will not close - the open file. + Close the file. Subsequent calls to other methods of the object will + result in an exception being raised. .. attribute:: closed - ``True`` if the file is closed. + True if the file is closed. .. versionadded:: 3.2 @@ -174,9 +173,6 @@ Optional arguments *start* and *end* are interpreted as in slice notation. Returns ``-1`` on failure. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) @@ -237,9 +233,6 @@ Optional arguments *start* and *end* are interpreted as in slice notation. Returns ``-1`` on failure. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. method:: seek(pos[, whence]) @@ -267,9 +260,6 @@ were written. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. method:: write_byte(byte) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/modulefinder.rst --- a/Doc/library/modulefinder.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/modulefinder.rst Mon Jan 25 17:05:13 2016 +0100 @@ -53,7 +53,7 @@ .. attribute:: modules A dictionary mapping module names to modules. See - :ref:`modulefinder-example`. + :ref:`modulefinder-example` .. _modulefinder-example: @@ -104,7 +104,7 @@ re: __module__,finditer,_expand itertools: __main__: re,itertools,baconhameggs - sre_parse: _PATTERNENDERS,SRE_FLAG_UNICODE + sre_parse: __getslice__,_PATTERNENDERS,SRE_FLAG_UNICODE array: types: __module__,IntType,TypeType --------------------------------------------------- diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/modules.rst --- a/Doc/library/modules.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/modules.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,6 +12,7 @@ .. toctree:: + imp.rst zipimport.rst pkgutil.rst modulefinder.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/msilib.rst --- a/Doc/library/msilib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/msilib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -88,7 +88,8 @@ record according to the schema of the table. For optional fields, ``None`` can be passed. - Field values can be ints, strings, or instances of the Binary class. + Field values can be int or long numbers, strings, or instances of the Binary + class. .. class:: Binary(filename) @@ -120,9 +121,9 @@ .. seealso:: - `FCICreateFile `_ - `UuidCreate `_ - `UuidToString `_ + `FCICreateFile `_ + `UuidCreate `_ + `UuidToString `_ .. _database-objects: @@ -151,9 +152,9 @@ .. seealso:: - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ + `MSIDatabaseOpenView `_ + `MSIDatabaseCommit `_ + `MSIGetSummaryInformation `_ .. _view-objects: @@ -199,11 +200,11 @@ .. seealso:: - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ + `MsiViewExecute `_ + `MSIViewGetColumnInfo `_ + `MsiViewFetch `_ + `MsiViewModify `_ + `MsiViewClose `_ .. _summary-objects: @@ -243,10 +244,10 @@ .. seealso:: - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ + `MsiSummaryInfoGetProperty `_ + `MsiSummaryInfoGetPropertyCount `_ + `MsiSummaryInfoSetProperty `_ + `MsiSummaryInfoPersist `_ .. _record-objects: @@ -297,11 +298,11 @@ .. seealso:: - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClear `_ + `MsiRecordGetFieldCount `_ + `MsiRecordSetString `_ + `MsiRecordSetStream `_ + `MsiRecordSetInteger `_ + `MsiRecordClear `_ .. _msi-errors: @@ -393,10 +394,10 @@ .. seealso:: - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ + `Directory Table `_ + `File Table `_ + `Component Table `_ + `FeatureComponents Table `_ .. _features: @@ -421,7 +422,7 @@ .. seealso:: - `Feature Table `_ + `Feature Table `_ .. _msi-gui: @@ -429,9 +430,8 @@ ----------- :mod:`msilib` provides several classes that wrap the GUI tables in an MSI -database. However, no standard user interface is provided; use -:mod:`~distutils.command.bdist_msi` to create MSI files with a user-interface -for installing Python packages. +database. However, no standard user interface is provided; use :mod:`bdist_msi` +to create MSI files with a user-interface for installing Python packages. .. class:: Control(dlg, name) @@ -516,13 +516,13 @@ .. seealso:: - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ + `Dialog Table `_ + `Control Table `_ + `Control Types `_ + `ControlCondition Table `_ + `ControlEvent Table `_ + `EventMapping Table `_ + `RadioButton Table `_ .. _msi-tables: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/msvcrt.rst --- a/Doc/library/msvcrt.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/msvcrt.rst Mon Jan 25 17:05:13 2016 +0100 @@ -18,7 +18,7 @@ The module implements both the normal and wide char variants of the console I/O api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where -ever possible. +ever possible .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/multiprocessing.rst --- a/Doc/library/multiprocessing.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/multiprocessing.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,27 +16,41 @@ leverage multiple processors on a given machine. It runs on both Unix and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the -:class:`~multiprocessing.pool.Pool` object which offers a convenient means of -parallelizing the execution of a function across multiple input values, -distributing the input data across processes (data parallelism). The following -example demonstrates the common practice of defining such functions in a module -so that child processes can successfully import that module. This basic example -of data parallelism using :class:`~multiprocessing.pool.Pool`, :: - - from multiprocessing import Pool - - def f(x): - return x*x - - if __name__ == '__main__': - with Pool(5) as p: - print(p.map(f, [1, 2, 3])) - -will print to standard output :: - - [1, 4, 9] +.. note:: + + Some of this package's functionality requires a functioning shared semaphore + implementation on the host operating system. Without one, the + :mod:`multiprocessing.synchronize` module will be disabled, and attempts to + import it will result in an :exc:`ImportError`. See + :issue:`3770` for additional information. + +.. note:: + + Functionality within this package requires that the ``__main__`` module be + importable by the children. This is covered in :ref:`multiprocessing-programming` + however it is worth pointing out here. This means that some examples, such + as the :class:`multiprocessing.Pool` examples will not work in the + interactive interpreter. For example:: + + >>> from multiprocessing import Pool + >>> p = Pool(5) + >>> def f(x): + ... return x*x + ... + >>> p.map(f, [1,2,3]) + Process PoolWorker-1: + Process PoolWorker-2: + Process PoolWorker-3: + Traceback (most recent call last): + Traceback (most recent call last): + Traceback (most recent call last): + AttributeError: 'module' object has no attribute 'f' + AttributeError: 'module' object has no attribute 'f' + AttributeError: 'module' object has no attribute 'f' + + (If you try this it will actually output three full tracebacks + interleaved in a semi-random fashion, and then you may have to + stop the master process somehow.) The :class:`Process` class @@ -78,112 +92,11 @@ p.start() p.join() -For an explanation of why the ``if __name__ == '__main__'`` part is +For an explanation of why (on Windows) the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. _multiprocessing-start-methods: - -Depending on the platform, :mod:`multiprocessing` supports three ways -to start a process. These *start methods* are - - *spawn* - The parent process starts a fresh python interpreter process. The - child process will only inherit those resources necessary to run - the process objects :meth:`~Process.run` method. In particular, - unnecessary file descriptors and handles from the parent process - will not be inherited. Starting a process using this method is - rather slow compared to using *fork* or *forkserver*. - - Available on Unix and Windows. The default on Windows. - - *fork* - The parent process uses :func:`os.fork` to fork the Python - interpreter. The child process, when it begins, is effectively - identical to the parent process. All resources of the parent are - inherited by the child process. Note that safely forking a - multithreaded process is problematic. - - Available on Unix only. The default on Unix. - - *forkserver* - When the program starts and selects the *forkserver* start method, - a server process is started. From then on, whenever a new process - is needed, the parent process connects to the server and requests - that it fork a new process. The fork server process is single - threaded so it is safe for it to use :func:`os.fork`. No - unnecessary resources are inherited. - - Available on Unix platforms which support passing file descriptors - over Unix pipes. - -.. versionchanged:: 3.4 - *spawn* added on all unix platforms, and *forkserver* added for - some unix platforms. - Child processes no longer inherit all of the parents inheritable - handles on Windows. - -On Unix using the *spawn* or *forkserver* start methods will also -start a *semaphore tracker* process which tracks the unlinked named -semaphores created by processes of the program. When all processes -have exited the semaphore tracker unlinks any remaining semaphores. -Usually there should be none, but if a process was killed by a signal -there may some "leaked" semaphores. (Unlinking the named semaphores -is a serious matter since the system allows only a limited number, and -they will not be automatically unlinked until the next reboot.) - -To select a start method you use the :func:`set_start_method` in -the ``if __name__ == '__main__'`` clause of the main module. For -example:: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - mp.set_start_method('spawn') - q = mp.Queue() - p = mp.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -:func:`set_start_method` should not be used more than once in the -program. - -Alternatively, you can use :func:`get_context` to obtain a context -object. Context objects have the same API as the multiprocessing -module, and allow one to use multiple start methods in the same -program. :: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - ctx = mp.get_context('spawn') - q = ctx.Queue() - p = ctx.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -Note that objects related to one context may not be compatible with -processes for a different context. In particular, locks created using -the *fork* context cannot be passed to a processes started using the -*spawn* or *forkserver* start methods. - -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. - - Exchanging objects between processes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -207,7 +120,9 @@ print(q.get()) # prints "[42, None, 'hello']" p.join() - Queues are thread and process safe. + Queues are thread and process safe, but note that they must never + be instantiated as a side effect of importing a module: this can lead + to a deadlock! (see :ref:`threaded-imports`) **Pipes** @@ -247,10 +162,8 @@ def f(l, i): l.acquire() - try: - print('hello world', i) - finally: - l.release() + print('hello world', i) + l.release() if __name__ == '__main__': lock = Lock() @@ -315,11 +228,11 @@ holds Python objects and allows other processes to manipulate them using proxies. - A manager returned by :func:`Manager` will support types - :class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`, - :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, - :class:`Condition`, :class:`Event`, :class:`Barrier`, - :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: + A manager returned by :func:`Manager` will support types :class:`list`, + :class:`dict`, :class:`Namespace`, :class:`Lock`, :class:`RLock`, + :class:`Semaphore`, :class:`BoundedSemaphore`, :class:`Condition`, + :class:`Event`, :class:`Queue`, :class:`Value` and :class:`Array`. For + example, :: from multiprocessing import Process, Manager @@ -330,16 +243,17 @@ l.reverse() if __name__ == '__main__': - with Manager() as manager: - d = manager.dict() - l = manager.list(range(10)) - - p = Process(target=f, args=(d, l)) - p.start() - p.join() - - print(d) - print(l) + manager = Manager() + + d = manager.dict() + l = manager.list(range(10)) + + p = Process(target=f, args=(d, l)) + p.start() + p.join() + + print(d) + print(l) will print :: @@ -361,78 +275,16 @@ For example:: - from multiprocessing import Pool, TimeoutError - import time - import os + from multiprocessing import Pool def f(x): return x*x if __name__ == '__main__': - # start 4 worker processes - with Pool(processes=4) as pool: - - # print "[0, 1, 4,..., 81]" - print(pool.map(f, range(10))) - - # print same numbers in arbitrary order - for i in pool.imap_unordered(f, range(10)): - print(i) - - # evaluate "f(20)" asynchronously - res = pool.apply_async(f, (20,)) # runs in *only* one process - print(res.get(timeout=1)) # prints "400" - - # evaluate "os.getpid()" asynchronously - res = pool.apply_async(os.getpid, ()) # runs in *only* one process - print(res.get(timeout=1)) # prints the PID of that process - - # launching multiple evaluations asynchronously *may* use more processes - multiple_results = [pool.apply_async(os.getpid, ()) for i in range(4)] - print([res.get(timeout=1) for res in multiple_results]) - - # make a single worker sleep for 10 secs - res = pool.apply_async(time.sleep, (10,)) - try: - print(res.get(timeout=1)) - except TimeoutError: - print("We lacked patience and got a multiprocessing.TimeoutError") - - print("For the moment, the pool remains available for more work") - - # exiting the 'with'-block has stopped the pool - print("Now the pool is closed and no longer available") - -Note that the methods of a pool should only ever be used by the -process which created it. - -.. note:: - - Functionality within this package requires that the ``__main__`` module be - importable by the children. This is covered in :ref:`multiprocessing-programming` - however it is worth pointing out here. This means that some examples, such - as the :class:`multiprocessing.pool.Pool` examples will not work in the - interactive interpreter. For example:: - - >>> from multiprocessing import Pool - >>> p = Pool(5) - >>> def f(x): - ... return x*x - ... - >>> p.map(f, [1,2,3]) - Process PoolWorker-1: - Process PoolWorker-2: - Process PoolWorker-3: - Traceback (most recent call last): - Traceback (most recent call last): - Traceback (most recent call last): - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' - - (If you try this it will actually output three full tracebacks - interleaved in a semi-random fashion, and then you may have to - stop the master process somehow.) + pool = Pool(processes=4) # start 4 worker processes + result = pool.apply_async(f, [10]) # evaluate "f(10)" asynchronously + print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow + print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" Reference @@ -445,8 +297,7 @@ :class:`Process` and exceptions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ - *, daemon=None) +.. class:: Process([group[, target[, name[, args[, kwargs]]]]], *, daemon=None) Process objects represent activity that is run in a separate process. The :class:`Process` class has equivalents of all the methods of @@ -456,12 +307,14 @@ should always be ``None``; it exists solely for compatibility with :class:`threading.Thread`. *target* is the callable object to be invoked by the :meth:`run()` method. It defaults to ``None``, meaning nothing is - called. *name* is the process name (see :attr:`name` for more details). - *args* is the argument tuple for the target invocation. *kwargs* is a - dictionary of keyword arguments for the target invocation. If provided, - the keyword-only *daemon* argument sets the process :attr:`daemon` flag - to ``True`` or ``False``. If ``None`` (the default), this flag will be - inherited from the creating process. + called. *name* is the process name. By default, a unique name is constructed + of the form 'Process-N\ :sub:`1`:N\ :sub:`2`:...:N\ :sub:`k`' where N\ + :sub:`1`,N\ :sub:`2`,...,N\ :sub:`k` is a sequence of integers whose length + is determined by the *generation* of the process. *args* is the argument + tuple for the target invocation. *kwargs* is a dictionary of keyword + arguments for the target invocation. If provided, the keyword-only *daemon* argument + sets the process :attr:`daemon` flag to ``True`` or ``False``. If ``None`` + (the default), this flag will be inherited from the creating process. By default, no arguments are passed to *target*. @@ -501,14 +354,11 @@ .. attribute:: name - The process's name. The name is a string used for identification purposes - only. It has no semantics. Multiple processes may be given the same - name. - - The initial name is set by the constructor. If no explicit name is - provided to the constructor, a name of the form - 'Process-N\ :sub:`1`:N\ :sub:`2`:...:N\ :sub:`k`' is constructed, where - each N\ :sub:`k` is the N-th child of its parent. + The process's name. + + The name is a string used for identification purposes only. It has no + semantics. Multiple processes may be given the same name. The initial + name is set by the constructor. .. method:: is_alive @@ -533,7 +383,7 @@ Unix daemons or services, they are normal processes that will be terminated (and not joined) if non-daemonic processes have exited. - In addition to the :class:`threading.Thread` API, :class:`Process` objects + In addition to the :class:`Threading.Thread` API, :class:`Process` objects also support the following attributes and methods: .. attribute:: pid @@ -552,7 +402,7 @@ The process's authentication key (a byte string). When :mod:`multiprocessing` is initialized the main process is assigned a - random string using :func:`os.urandom`. + random string using :func:`os.random`. When a :class:`Process` object is created, it will inherit the authentication key of its parent process, although this may be changed by @@ -593,7 +443,7 @@ cause other processes to deadlock. Note that the :meth:`start`, :meth:`join`, :meth:`is_alive`, - :meth:`terminate` and :attr:`exitcode` methods should only be called by + :meth:`terminate` and :attr:`exit_code` methods should only be called by the process that created the process object. Example usage of some of the methods of :class:`Process`: @@ -614,9 +464,6 @@ >>> p.exitcode == -signal.SIGTERM True -.. exception:: ProcessError - - The base class of all :mod:`multiprocessing` exceptions. .. exception:: BufferTooShort @@ -626,13 +473,6 @@ If ``e`` is an instance of :exc:`BufferTooShort` then ``e.args[0]`` will give the message as a byte string. -.. exception:: AuthenticationError - - Raised when there is an authentication error. - -.. exception:: TimeoutError - - Raised by methods with a timeout when the timeout expires. Pipes and Queues ~~~~~~~~~~~~~~~~ @@ -665,24 +505,6 @@ the :mod:`multiprocessing` namespace so you need to import them from :mod:`queue`. -.. note:: - - When an object is put on a queue, the object is pickled and a - background thread later flushes the pickled data to an underlying - pipe. This has some consequences which are a little surprising, - but should not cause any practical difficulties -- if they really - bother you then you can instead use a queue created with a - :ref:`manager `. - - (1) After putting an object on an empty queue there may be an - infinitesimal delay before the queue's :meth:`~Queue.empty` - method returns :const:`False` and :meth:`~Queue.get_nowait` can - return without raising :exc:`queue.Empty`. - - (2) If multiple processes are enqueuing objects, it is possible for - the objects to be received at the other end out-of-order. - However, objects enqueued by the same process will always be in - the expected order with respect to each other. .. warning:: @@ -694,8 +516,7 @@ .. warning:: As mentioned above, if a child process has put items on a queue (and it has - not used :meth:`JoinableQueue.cancel_join_thread - `), then that process will + not used :meth:`JoinableQueue.cancel_join_thread`), then that process will not terminate until all buffered items have been flushed to the pipe. This means that if you try joining that process you may get a deadlock unless @@ -728,7 +549,7 @@ thread is started which transfers objects from a buffer into the pipe. The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the - standard library's :mod:`queue` module are raised to signal timeouts. + standard library's :mod:`Queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. @@ -777,6 +598,7 @@ :exc:`queue.Empty` exception (*timeout* is ignored in that case). .. method:: get_nowait() + get_no_wait() Equivalent to ``get(False)``. @@ -807,21 +629,6 @@ the background thread from being joined automatically when the process exits -- see :meth:`join_thread`. - A better name for this method might be - ``allow_exit_without_flush()``. It is likely to cause enqueued - data to lost, and you almost certainly will not need to use it. - It is really only there if you need the current process to exit - immediately without waiting to flush enqueued data to the - underlying pipe, and you don't care about lost data. - - .. note:: - - This class's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - functionality in this class will be disabled, and attempts to - instantiate a :class:`Queue` will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. The same holds true for any - of the specialized queue types listed below. .. class:: SimpleQueue() @@ -847,12 +654,12 @@ .. method:: task_done() - Indicate that a formerly enqueued task is complete. Used by queue - consumers. For each :meth:`~Queue.get` used to fetch a task, a subsequent + Indicate that a formerly enqueued task is complete. Used by queue consumer + threads. For each :meth:`~Queue.get` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue that the processing on the task is complete. - If a :meth:`~queue.Queue.join` is currently blocking, it will resume when all + If a :meth:`~Queue.join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`~Queue.put` into the queue). @@ -865,10 +672,10 @@ Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls + queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, - :meth:`~queue.Queue.join` unblocks. + :meth:`~Queue.join` unblocks. Miscellaneous @@ -878,21 +685,13 @@ Return list of all live children of the current process. - Calling this has the side effect of "joining" any processes which have + Calling this has the side affect of "joining" any processes which have already finished. .. function:: cpu_count() - Return the number of CPUs in the system. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` - - May raise :exc:`NotImplementedError`. - - .. seealso:: - :func:`os.cpu_count` + Return the number of CPUs in the system. May raise + :exc:`NotImplementedError`. .. function:: current_process() @@ -921,47 +720,8 @@ If the ``freeze_support()`` line is omitted then trying to run the frozen executable will raise :exc:`RuntimeError`. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. - -.. function:: get_all_start_methods() - - Returns a list of the supported start methods, the first of which - is the default. The possible start methods are ``'fork'``, - ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is - available. On Unix ``'fork'`` and ``'spawn'`` are always - supported, with ``'fork'`` being the default. - - .. versionadded:: 3.4 - -.. function:: get_context(method=None) - - Return a context object which has the same attributes as the - :mod:`multiprocessing` module. - - If *method* is *None* then the default context is returned. - Otherwise *method* should be ``'fork'``, ``'spawn'``, - ``'forkserver'``. :exc:`ValueError` is raised if the specified - start method is not available. - - .. versionadded:: 3.4 - -.. function:: get_start_method(allow_none=False) - - Return the name of start method used for starting processes. - - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then *None* is returned. - - The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` - or *None*. ``'fork'`` is the default on Unix, while ``'spawn'`` is - the default on Windows. - - .. versionadded:: 3.4 + If the module is being run normally by the Python interpreter then + :func:`freeze_support` has no effect. .. function:: set_executable() @@ -971,21 +731,8 @@ set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) - before they can create child processes. - - .. versionchanged:: 3.4 - Now supported on Unix when the ``'spawn'`` start method is used. - -.. function:: set_start_method(method) - - Set the method which should be used to start child processes. - *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. - - Note that this should be called at most once, and it should be - protected inside the ``if __name__ == '__main__'`` clause of the - main module. - - .. versionadded:: 3.4 + before they can create child processes. (Windows only) + .. note:: @@ -1044,7 +791,8 @@ .. method:: send_bytes(buffer[, offset[, size]]) - Send byte data from a :term:`bytes-like object` as a complete message. + Send byte data from an object supporting the buffer interface as a + complete message. If *offset* is given then data is read from that position in *buffer*. If *size* is given then that many bytes will be read from buffer. Very large @@ -1063,7 +811,7 @@ readable. .. versionchanged:: 3.3 - This function used to raise :exc:`IOError`, which is now an + This function used to raise a :exc:`IOError`, which is now an alias of :exc:`OSError`. @@ -1075,7 +823,7 @@ :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buffer* must be an object satisfying the writable buffer interface. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the length of *buffer* (in bytes). @@ -1084,14 +832,6 @@ raised and the complete message is available as ``e.args[0]`` where ``e`` is the exception instance. - .. versionchanged:: 3.3 - Connection objects themselves can now be transferred between processes - using :meth:`Connection.send` and :meth:`Connection.recv`. - - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. For example: @@ -1143,157 +883,41 @@ Note that one can also create synchronization primitives by using a manager object -- see :ref:`multiprocessing-managers`. -.. class:: Barrier(parties[, action[, timeout]]) - - A barrier object: a clone of :class:`threading.Barrier`. - - .. versionadded:: 3.3 - .. class:: BoundedSemaphore([value]) - A bounded semaphore object: a close analog of - :class:`threading.BoundedSemaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. - - .. note:: - On Mac OS X, this is indistinguishable from :class:`Semaphore` because - ``sem_getvalue()`` is not implemented on that platform. + A bounded semaphore object: a clone of :class:`threading.BoundedSemaphore`. + + (On Mac OS X, this is indistinguishable from :class:`Semaphore` because + ``sem_getvalue()`` is not implemented on that platform). .. class:: Condition([lock]) - A condition variable: an alias for :class:`threading.Condition`. + A condition variable: a clone of :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` object from :mod:`multiprocessing`. - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - .. class:: Event() A clone of :class:`threading.Event`. - + This method returns the state of the internal semaphore on exit, so it + will always return ``True`` except if a timeout is given and the operation + times out. + + .. versionchanged:: 3.1 + Previously, the method always returned ``None``. .. class:: Lock() - A non-recursive lock object: a close analog of :class:`threading.Lock`. - Once a process or thread has acquired a lock, subsequent attempts to - acquire it from any process or thread will block until it is released; - any process or thread may release it. The concepts and behaviors of - :class:`threading.Lock` as it applies to threads are replicated here in - :class:`multiprocessing.Lock` as it applies to either processes or threads, - except as noted. - - Note that :class:`Lock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.Lock`` initialized with a - default context. - - :class:`Lock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - With the *block* argument set to ``True`` (the default), the method call - will block until the lock is in an unlocked state, then set it to locked - and return ``True``. Note that the name of this first argument differs - from that in :meth:`threading.Lock.acquire`. - - With the *block* argument set to ``False``, the method call does not - block. If the lock is currently in a locked state, return ``False``; - otherwise set the lock to a locked state and return ``True``. - - When invoked with a positive, floating-point value for *timeout*, block - for at most the number of seconds specified by *timeout* as long as - the lock can not be acquired. Invocations with a negative value for - *timeout* are equivalent to a *timeout* of zero. Invocations with a - *timeout* value of ``None`` (the default) set the timeout period to - infinite. Note that the treatment of negative or ``None`` values for - *timeout* differs from the implemented behavior in - :meth:`threading.Lock.acquire`. The *timeout* argument has no practical - implications if the *block* argument is set to ``False`` and is thus - ignored. Returns ``True`` if the lock has been acquired or ``False`` if - the timeout period has elapsed. - - - .. method:: release() - - Release a lock. This can be called from any process or thread, not only - the process or thread which originally acquired the lock. - - Behavior is the same as in :meth:`threading.Lock.release` except that - when invoked on an unlocked lock, a :exc:`ValueError` is raised. - + A non-recursive lock object: a clone of :class:`threading.Lock`. .. class:: RLock() - A recursive lock object: a close analog of :class:`threading.RLock`. A - recursive lock must be released by the process or thread that acquired it. - Once a process or thread has acquired a recursive lock, the same process - or thread may acquire it again without blocking; that process or thread - must release it once for each time it has been acquired. - - Note that :class:`RLock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.RLock`` initialized with a - default context. - - :class:`RLock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - When invoked with the *block* argument set to ``True``, block until the - lock is in an unlocked state (not owned by any process or thread) unless - the lock is already owned by the current process or thread. The current - process or thread then takes ownership of the lock (if it does not - already have ownership) and the recursion level inside the lock increments - by one, resulting in a return value of ``True``. Note that there are - several differences in this first argument's behavior compared to the - implementation of :meth:`threading.RLock.acquire`, starting with the name - of the argument itself. - - When invoked with the *block* argument set to ``False``, do not block. - If the lock has already been acquired (and thus is owned) by another - process or thread, the current process or thread does not take ownership - and the recursion level within the lock is not changed, resulting in - a return value of ``False``. If the lock is in an unlocked state, the - current process or thread takes ownership and the recursion level is - incremented, resulting in a return value of ``True``. - - Use and behaviors of the *timeout* argument are the same as in - :meth:`Lock.acquire`. Note that some of these behaviors of *timeout* - differ from the implemented behaviors in :meth:`threading.RLock.acquire`. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the - decrement the recursion level is zero, reset the lock to unlocked (not - owned by any process or thread) and if any other processes or threads - are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling process or - thread. - - Only call this method when the calling process or thread owns the lock. - An :exc:`AssertionError` is raised if this method is called by a process - or thread other than the owner or if the lock is in an unlocked (unowned) - state. Note that the type of exception raised in this situation - differs from the implemented behavior in :meth:`threading.RLock.release`. - + A recursive lock object: a clone of :class:`threading.RLock`. .. class:: Semaphore([value]) - A semaphore object: a close analog of :class:`threading.Semaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + A semaphore object: a clone of :class:`threading.Semaphore`. .. note:: @@ -1302,7 +926,7 @@ .. note:: - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is + If the SIGINT signal generated by Ctrl-C arrives while the main thread is blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` or :meth:`Condition.wait` then the call will be immediately interrupted and @@ -1311,14 +935,6 @@ This differs from the behaviour of :mod:`threading` where SIGINT will be ignored while the equivalent blocking calls are in progress. -.. note:: - - Some of this package's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - :mod:`multiprocessing.synchronize` module will be disabled, and attempts to - import it will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. - Shared :mod:`ctypes` Objects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1326,34 +942,21 @@ It is possible to create shared objects using shared memory which can be inherited by child processes. -.. function:: Value(typecode_or_type, *args, lock=True) +.. function:: Value(typecode_or_type, *args[, lock]) Return a :mod:`ctypes` object allocated from shared memory. By default the - return value is actually a synchronized wrapper for the object. The object - itself can be accessed via the *value* attribute of a :class:`Value`. + return value is actually a synchronized wrapper for the object. *typecode_or_type* determines the type of the returned object: it is either a ctypes type or a one character typecode of the kind used by the :mod:`array` module. *\*args* is passed on to the constructor for the type. - If *lock* is ``True`` (the default) then a new recursive lock - object is created to synchronize access to the value. If *lock* is - a :class:`Lock` or :class:`RLock` object then that will be used to - synchronize access to the value. If *lock* is ``False`` then - access to the returned object will not be automatically protected - by a lock, so it will not necessarily be "process-safe". - - Operations like ``+=`` which involve a read and write are not - atomic. So if, for instance, you want to atomically increment a - shared value it is insufficient to just do :: - - counter.value += 1 - - Assuming the associated lock is recursive (which it is by default) - you can instead do :: - - with counter.get_lock(): - counter.value += 1 + If *lock* is ``True`` (the default) then a new lock object is created to + synchronize access to the value. If *lock* is a :class:`Lock` or + :class:`RLock` object then that will be used to synchronize access to the + value. If *lock* is ``False`` then access to the returned object will not be + automatically protected by a lock, so it will not necessarily be + "process-safe". Note that *lock* is a keyword-only argument. @@ -1431,31 +1034,30 @@ attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) +.. function:: Array(typecode_or_type, size_or_initializer, *args[, lock]) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes array. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a - :class:`~multiprocessing.Lock` or :class:`~multiprocessing.RLock` object - then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a :class:`Lock` or + :class:`RLock` object then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". Note that *lock* is a keyword-only argument. -.. function:: Value(typecode_or_type, *args, lock=True) +.. function:: Value(typecode_or_type, *args[, lock]) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes object. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`~multiprocessing.Lock` or - :class:`~multiprocessing.RLock` object then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a :class:`Lock` or + :class:`RLock` object then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". @@ -1480,9 +1082,6 @@ Note that accessing the ctypes object through the wrapper can be a lot slower than accessing the raw ctypes object. - .. versionchanged:: 3.5 - Synchronized objects support the :term:`context manager` protocol. - The table below compares the syntax for creating shared ctypes objects from shared memory with the normal ctypes syntax. (In the table ``MyStruct`` is some @@ -1521,7 +1120,7 @@ n = Value('i', 7) x = Value(c_double, 1.0/3.0, lock=False) - s = Array('c', b'hello world', lock=lock) + s = Array('c', 'hello world', lock=lock) A = Array(Point, [(1.875,-6.25), (-5.75,2.0), (2.375,9.5)], lock=lock) p = Process(target=modify, args=(n, x, s, A)) @@ -1543,7 +1142,7 @@ HELLO WORLD [(3.515625, 39.0625), (33.0625, 4.0), (5.640625, 90.25)] -.. highlight:: python3 +.. highlight:: python .. _multiprocessing-managers: @@ -1552,10 +1151,8 @@ ~~~~~~~~ Managers provide a way to create data which can be shared between different -processes, including sharing over a network between processes running on -different machines. A manager object controls a server process which manages -*shared objects*. Other processes can access the shared objects by using -proxies. +processes. A manager object controls a server process which manages *shared +objects*. Other processes can access the shared objects by using proxies. .. function:: multiprocessing.Manager() @@ -1581,10 +1178,10 @@ *address* is the address on which the manager process listens for new connections. If *address* is ``None`` then an arbitrary one is chosen. - *authkey* is the authentication key which will be used to check the - validity of incoming connections to the server process. If - *authkey* is ``None`` then ``current_process().authkey`` is used. - Otherwise *authkey* is used and it must be a byte string. + *authkey* is the authentication key which will be used to check the validity + of incoming connections to the server process. If *authkey* is ``None`` then + ``current_process().authkey``. Otherwise *authkey* is used and it + must be a string. .. method:: start([initializer[, initargs]]) @@ -1598,7 +1195,7 @@ :meth:`serve_forever` method:: >>> from multiprocessing.managers import BaseManager - >>> manager = BaseManager(address=('', 50000), authkey=b'abc') + >>> manager = BaseManager(address=('', 50000), authkey='abc') >>> server = manager.get_server() >>> server.serve_forever() @@ -1609,7 +1206,7 @@ Connect a local manager object to a remote manager process:: >>> from multiprocessing.managers import BaseManager - >>> m = BaseManager(address=('127.0.0.1', 5000), authkey=b'abc') + >>> m = BaseManager(address=('127.0.0.1', 5000), authkey='abc') >>> m.connect() .. method:: shutdown() @@ -1628,10 +1225,9 @@ type of shared object. This must be a string. *callable* is a callable used for creating objects for this type - identifier. If a manager instance will be connected to the - server using the :meth:`connect` method, or if the - *create_method* argument is ``False`` then this can be left as - ``None``. + identifier. If a manager instance will be created using the + :meth:`from_address` classmethod or if the *create_method* argument is + ``False`` then this can be left as ``None``. *proxytype* is a subclass of :class:`BaseProxy` which is used to create proxies for shared objects with this *typeid*. If ``None`` then a proxy @@ -1639,12 +1235,12 @@ *exposed* is used to specify a sequence of method names which proxies for this typeid should be allowed to access using - :meth:`BaseProxy._callmethod`. (If *exposed* is ``None`` then + :meth:`BaseProxy._callMethod`. (If *exposed* is ``None`` then :attr:`proxytype._exposed_` is used instead if it exists.) In the case where no exposed list is specified, all "public methods" of the shared object will be accessible. (Here a "public method" means any attribute - which has a :meth:`~object.__call__` method and whose name does not begin - with ``'_'``.) + which has a :meth:`__call__` method and whose name does not begin with + ``'_'``.) *method_to_typeid* is a mapping used to specify the return type of those exposed methods which should return a proxy. It maps method names to @@ -1663,14 +1259,6 @@ The address used by the manager. - .. versionchanged:: 3.3 - Manager objects support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` starts the - server process (if it has not already started) and then returns the - manager object. :meth:`~contextmanager.__exit__` calls :meth:`shutdown`. - - In previous versions :meth:`~contextmanager.__enter__` did not start the - manager's server process if it was not already started. .. class:: SyncManager @@ -1680,13 +1268,6 @@ It also supports creation of shared lists and dictionaries. - .. method:: Barrier(parties[, action[, timeout]]) - - Create a shared :class:`threading.Barrier` object and return a - proxy for it. - - .. versionadded:: 3.3 - .. method:: BoundedSemaphore([value]) Create a shared :class:`threading.BoundedSemaphore` object and return a @@ -1700,9 +1281,6 @@ If *lock* is supplied then it should be a proxy for a :class:`threading.Lock` or :class:`threading.RLock` object. - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - .. method:: Event() Create a shared :class:`threading.Event` object and return a proxy for it. @@ -1767,26 +1345,24 @@ lproxy[0] = d -.. class:: Namespace - - A type that can register with :class:`SyncManager`. - - A namespace object has no public methods, but does have writable attributes. - Its representation shows the values of its attributes. - - However, when using a proxy for a namespace object, an attribute beginning - with ``'_'`` will be an attribute of the proxy and not an attribute of the - referent: - - .. doctest:: - - >>> manager = multiprocessing.Manager() - >>> Global = manager.Namespace() - >>> Global.x = 10 - >>> Global.y = 'hello' - >>> Global._z = 12.3 # this is an attribute of the proxy - >>> print(Global) - Namespace(x=10, y='hello') +Namespace objects +>>>>>>>>>>>>>>>>> + +A namespace object has no public methods, but does have writable attributes. +Its representation shows the values of its attributes. + +However, when using a proxy for a namespace object, an attribute beginning with +``'_'`` will be an attribute of the proxy and not an attribute of the referent: + +.. doctest:: + + >>> manager = multiprocessing.Manager() + >>> Global = manager.Namespace() + >>> Global.x = 10 + >>> Global.y = 'hello' + >>> Global._z = 12.3 # this is an attribute of the proxy + >>> print(Global) + Namespace(x=10, y='hello') Customized managers @@ -1810,10 +1386,11 @@ MyManager.register('Maths', MathsClass) if __name__ == '__main__': - with MyManager() as manager: - maths = manager.Maths() - print(maths.add(4, 3)) # prints 7 - print(maths.mul(7, 8)) # prints 56 + manager = MyManager() + manager.start() + maths = manager.Maths() + print(maths.add(4, 3)) # prints 7 + print(maths.mul(7, 8)) # prints 56 Using a remote manager @@ -1830,7 +1407,7 @@ >>> queue = queue.Queue() >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue', callable=lambda:queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') + >>> m = QueueManager(address=('', 50000), authkey='abracadabra') >>> s = m.get_server() >>> s.serve_forever() @@ -1839,7 +1416,7 @@ >>> from multiprocessing.managers import BaseManager >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') + >>> m = QueueManager(address=('foo.bar.org', 50000), authkey='abracadabra') >>> m.connect() >>> queue = m.get_queue() >>> queue.put('hello') @@ -1849,7 +1426,7 @@ >>> from multiprocessing.managers import BaseManager >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') + >>> m = QueueManager(address=('foo.bar.org', 50000), authkey='abracadabra') >>> m.connect() >>> queue = m.get_queue() >>> queue.get() @@ -1873,7 +1450,7 @@ >>> class QueueManager(BaseManager): pass ... >>> QueueManager.register('get_queue', callable=lambda: queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') + >>> m = QueueManager(address=('', 50000), authkey='abracadabra') >>> s = m.get_server() >>> s.serve_forever() @@ -1963,7 +1540,7 @@ raised by :meth:`_callmethod`. Note in particular that an exception will be raised if *methodname* has - not been *exposed*. + not been *exposed* An example of the usage of :meth:`_callmethod`: @@ -1972,9 +1549,9 @@ >>> l = manager.list(range(10)) >>> l._callmethod('__len__') 10 - >>> l._callmethod('__getitem__', (slice(2, 7),)) # equivalent to l[2:7] + >>> l._callmethod('__getslice__', (2, 7)) # equiv to `l[2:7]` [2, 3, 4, 5, 6] - >>> l._callmethod('__getitem__', (20,)) # equivalent to l[20] + >>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]` Traceback (most recent call last): ... IndexError: list index out of range @@ -2013,37 +1590,22 @@ One can create a pool of processes which will carry out tasks submitted to it with the :class:`Pool` class. -.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild [, context]]]]]) +.. class:: multiprocessing.Pool([processes[, initializer[, initargs[, maxtasksperchild]]]]) A process pool object which controls a pool of worker processes to which jobs can be submitted. It supports asynchronous results with timeouts and callbacks and has a parallel map implementation. *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. - - If *initializer* is not ``None`` then each worker process will call + ``None`` then the number returned by :func:`cpu_count` is used. If + *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. - *maxtasksperchild* is the number of tasks a worker process can complete - before it will exit and be replaced with a fresh worker process, to enable - unused resources to be freed. The default *maxtasksperchild* is None, which - means worker processes will live as long as the pool. - - *context* can be used to specify the context used for starting - the worker processes. Usually a pool is created using the - function :func:`multiprocessing.Pool` or the :meth:`Pool` method - of a context object. In both cases *context* is set - appropriately. - - Note that the methods of the pool object should only be called by - the process which created the pool. - .. versionadded:: 3.2 - *maxtasksperchild* - - .. versionadded:: 3.4 - *context* + *maxtasksperchild* is the number of tasks a worker process can complete + before it will exit and be replaced with a fresh worker process, to enable + unused resources to be freed. The default *maxtasksperchild* is None, which + means worker processes will live as long as the pool. .. note:: @@ -2069,7 +1631,7 @@ If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to it, that is unless the call failed, in which case the *error_callback* - is applied instead. + is applied instead If *error_callback* is specified then it should be a callable which accepts a single argument. If the target function fails, then @@ -2094,7 +1656,7 @@ If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to it, that is unless the call failed, in which case the *error_callback* - is applied instead. + is applied instead If *error_callback* is specified then it should be a callable which accepts a single argument. If the target function fails, then @@ -2125,18 +1687,18 @@ .. method:: starmap(func, iterable[, chunksize]) - Like :meth:`map` except that the elements of the *iterable* are expected + Like :meth:`map` except that the elements of the `iterable` are expected to be iterables that are unpacked as arguments. - Hence an *iterable* of ``[(1,2), (3, 4)]`` results in ``[func(1,2), - func(3,4)]``. + Hence an `iterable` of `[(1,2), (3, 4)]` results in `[func(1,2), + func(3,4)]`. .. versionadded:: 3.3 .. method:: starmap_async(func, iterable[, chunksize[, callback[, error_back]]]) A combination of :meth:`starmap` and :meth:`map_async` that iterates over - *iterable* of iterables and calls *func* with the iterables unpacked. + `iterable` of iterables and calls `func` with the iterables unpacked. Returns a result object. .. versionadded:: 3.3 @@ -2157,11 +1719,6 @@ Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 - Pool objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. - .. class:: AsyncResult @@ -2191,25 +1748,26 @@ The following example demonstrates the use of a pool:: from multiprocessing import Pool - import time def f(x): return x*x if __name__ == '__main__': - with Pool(processes=4) as pool: # start 4 worker processes - result = pool.apply_async(f, (10,)) # evaluate "f(10)" asynchronously in a single process - print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow - - print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" - - it = pool.imap(f, range(10)) - print(next(it)) # prints "0" - print(next(it)) # prints "1" - print(it.next(timeout=1)) # prints "4" unless your computer is *very* slow - - result = pool.apply_async(time.sleep, (10,)) - print(result.get(timeout=1)) # raises multiprocessing.TimeoutError + pool = Pool(processes=4) # start 4 worker processes + + result = pool.apply_async(f, (10,)) # evaluate "f(10)" asynchronously + print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow + + print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" + + it = pool.imap(f, range(10)) + print(next(it)) # prints "0" + print(next(it)) # prints "1" + print(it.next(timeout=1)) # prints "4" unless your computer is *very* slow + + import time + result = pool.apply_async(time.sleep, (10,)) + print(result.get(timeout=1)) # raises TimeoutError .. _multiprocessing-listeners-clients: @@ -2221,8 +1779,7 @@ :synopsis: API for dealing with sockets. Usually message passing between processes is done using queues or by using -:class:`~multiprocessing.Connection` objects returned by -:func:`~multiprocessing.Pipe`. +:class:`Connection` objects returned by :func:`Pipe`. However, the :mod:`multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing @@ -2238,15 +1795,15 @@ If the reply matches the digest of the message using *authkey* as the key then a welcome message is sent to the other end of the connection. Otherwise - :exc:`~multiprocessing.AuthenticationError` is raised. - -.. function:: answer_challenge(connection, authkey) + :exc:`AuthenticationError` is raised. + +.. function:: answerChallenge(connection, authkey) Receive a message, calculate the digest of the message using *authkey* as the key, and then send the digest back. - If a welcome message is not received, then - :exc:`~multiprocessing.AuthenticationError` is raised. + If a welcome message is not received, then :exc:`AuthenticationError` is + raised. .. function:: Client(address[, family[, authenticate[, authkey]]]) @@ -2257,11 +1814,10 @@ generally be omitted since it can usually be inferred from the format of *address*. (See :ref:`multiprocessing-address-formats`) - If *authenticate* is ``True`` or *authkey* is a byte string then digest + If *authenticate* is ``True`` or *authkey* is a string then digest authentication is used. The key used for authentication will be either - *authkey* or ``current_process().authkey`` if *authkey* is ``None``. - If authentication fails then - :exc:`~multiprocessing.AuthenticationError` is raised. See + *authkey* or ``current_process().authkey)`` if *authkey* is ``None``. + If authentication fails then :exc:`AuthenticationError` is raised. See :ref:`multiprocessing-auth-keys`. .. class:: Listener([address[, family[, backlog[, authenticate[, authkey]]]]]) @@ -2290,28 +1846,25 @@ private temporary directory created using :func:`tempfile.mkstemp`. If the listener object uses a socket then *backlog* (1 by default) is passed - to the :meth:`~socket.socket.listen` method of the socket once it has been - bound. + to the :meth:`listen` method of the socket once it has been bound. If *authenticate* is ``True`` (``False`` by default) or *authkey* is not ``None`` then digest authentication is used. - If *authkey* is a byte string then it will be used as the - authentication key; otherwise it must be *None*. + If *authkey* is a string then it will be used as the authentication key; + otherwise it must be *None*. If *authkey* is ``None`` and *authenticate* is ``True`` then ``current_process().authkey`` is used as the authentication key. If *authkey* is ``None`` and *authenticate* is ``False`` then no authentication is done. If authentication fails then - :exc:`~multiprocessing.AuthenticationError` is raised. - See :ref:`multiprocessing-auth-keys`. + :exc:`AuthenticationError` is raised. See :ref:`multiprocessing-auth-keys`. .. method:: accept() Accept a connection on the bound socket or named pipe of the listener - object and return a :class:`~multiprocessing.Connection` object. If - authentication is attempted and fails, then - :exc:`~multiprocessing.AuthenticationError` is raised. + object and return a :class:`Connection` object. If authentication is + attempted and fails, then :exc:`AuthenticationError` is raised. .. method:: close() @@ -2330,18 +1883,12 @@ The address from which the last accepted connection came. If this is unavailable then it is ``None``. - .. versionadded:: 3.3 - Listener objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. - .. function:: wait(object_list, timeout=None) Wait till an object in *object_list* is ready. Returns the list of those objects in *object_list* which are ready. If *timeout* is a float then the call blocks for at most that many seconds. If *timeout* is ``None`` then it will block for an unlimited period. - A negative timeout is equivalent to a zero timeout. For both Unix and Windows, an object can appear in *object_list* if it is @@ -2369,6 +1916,12 @@ .. versionadded:: 3.3 +The module defines two exceptions: + +.. exception:: AuthenticationError + + Exception raised when there is an authentication error. + **Examples** @@ -2380,16 +1933,19 @@ from array import array address = ('localhost', 6000) # family is deduced to be 'AF_INET' - - with Listener(address, authkey=b'secret password') as listener: - with listener.accept() as conn: - print('connection accepted from', listener.last_accepted) - - conn.send([2.25, None, 'junk', float]) - - conn.send_bytes(b'hello') - - conn.send_bytes(array('i', [42, 1729])) + listener = Listener(address, authkey=b'secret password') + + conn = listener.accept() + print('connection accepted from', listener.last_accepted) + + conn.send([2.25, None, 'junk', float]) + + conn.send_bytes(b'hello') + + conn.send_bytes(array('i', [42, 1729])) + + conn.close() + listener.close() The following code connects to the server and receives some data from the server:: @@ -2398,15 +1954,17 @@ from array import array address = ('localhost', 6000) - - with Client(address, authkey=b'secret password') as conn: - print(conn.recv()) # => [2.25, None, 'junk', float] - - print(conn.recv_bytes()) # => 'hello' - - arr = array('i', [0, 0, 0, 0, 0]) - print(conn.recv_bytes_into(arr)) # => 8 - print(arr) # => array('i', [42, 1729, 0, 0, 0]) + conn = Client(address, authkey=b'secret password') + + print(conn.recv()) # => [2.25, None, 'junk', float] + + print(conn.recv_bytes()) # => 'hello' + + arr = array('i', [0, 0, 0, 0, 0]) + print(conn.recv_bytes_into(arr)) # => 8 + print(arr) # => array('i', [42, 1729, 0, 0, 0]) + + conn.close() The following code uses :func:`~multiprocessing.connection.wait` to wait for messages from multiple processes at once:: @@ -2469,19 +2027,17 @@ Authentication keys ~~~~~~~~~~~~~~~~~~~ -When one uses :meth:`Connection.recv `, the -data received is automatically +When one uses :meth:`Connection.recv`, the data received is automatically unpickled. Unfortunately unpickling data from an untrusted source is a security risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module to provide digest authentication. -An authentication key is a byte string which can be thought of as a -password: once a connection is established both ends will demand proof -that the other knows the authentication key. (Demonstrating that both -ends are using the same key does **not** involve sending the key over -the connection.) - -If authentication is requested but no authentication key is specified then the +An authentication key is a string which can be thought of as a password: once a +connection is established both ends will demand proof that the other knows the +authentication key. (Demonstrating that both ends are using the same key does +**not** involve sending the key over the connection.) + +If authentication is requested but do authentication key is specified then the return value of ``current_process().authkey`` is used (see :class:`~multiprocessing.Process`). This value will automatically inherited by any :class:`~multiprocessing.Process` object that the current process creates. @@ -2536,8 +2092,43 @@ [INFO/MainProcess] sending shutdown message to manager [INFO/SyncManager-...] manager exiting with exitcode 0 +In addition to having these two logging functions, the multiprocessing also +exposes two additional logging level attributes. These are :const:`SUBWARNING` +and :const:`SUBDEBUG`. The table below illustrates where theses fit in the +normal level hierarchy. + ++----------------+----------------+ +| Level | Numeric value | ++================+================+ +| ``SUBWARNING`` | 25 | ++----------------+----------------+ +| ``SUBDEBUG`` | 5 | ++----------------+----------------+ + For a full table of logging levels, see the :mod:`logging` module. +These additional logging levels are used primarily for certain debug messages +within the multiprocessing module. Below is the same example as above, except +with :const:`SUBDEBUG` enabled:: + + >>> import multiprocessing, logging + >>> logger = multiprocessing.log_to_stderr() + >>> logger.setLevel(multiprocessing.SUBDEBUG) + >>> logger.warning('doomed') + [WARNING/MainProcess] doomed + >>> m = multiprocessing.Manager() + [INFO/SyncManager-...] child process calling self.run() + [INFO/SyncManager-...] created temp directory /.../pymp-... + [INFO/SyncManager-...] manager serving at '/.../pymp-djGBXN/listener-...' + >>> del m + [SUBDEBUG/MainProcess] finalizer calling ... + [INFO/MainProcess] sending shutdown message to manager + [DEBUG/SyncManager-...] manager received shutdown message + [SUBDEBUG/SyncManager-...] calling ... + [SUBDEBUG/SyncManager-...] calling + [SUBDEBUG/SyncManager-...] finalizer calling ... + [INFO/SyncManager-...] manager exiting with exitcode 0 The :mod:`multiprocessing.dummy` module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2558,10 +2149,8 @@ :mod:`multiprocessing`. -All start methods -~~~~~~~~~~~~~~~~~ - -The following applies to all start methods. +All platforms +~~~~~~~~~~~~~ Avoid shared state @@ -2570,7 +2159,7 @@ It is probably best to stick to using queues or pipes for communication between processes rather than using the lower level synchronization - primitives. + primitives from the :mod:`threading` module. Picklability @@ -2587,47 +2176,41 @@ On Unix when a process finishes but has not been joined it becomes a zombie. There should never be very many because each time a new process starts (or - :func:`~multiprocessing.active_children` is called) all completed processes - which have not yet been joined will be joined. Also calling a finished - process's :meth:`Process.is_alive ` will - join the process. Even so it is probably good + :func:`active_children` is called) all completed processes which have not + yet been joined will be joined. Also calling a finished process's + :meth:`Process.is_alive` will join the process. Even so it is probably good practice to explicitly join all the processes that you start. Better to inherit than pickle/unpickle - When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child - processes can use them. However, one should generally avoid - sending shared objects to other processes using pipes or queues. - Instead you should arrange the program so that a process which - needs access to a shared resource created elsewhere can inherit it - from an ancestor process. + On Windows many types from :mod:`multiprocessing` need to be picklable so + that child processes can use them. However, one should generally avoid + sending shared objects to other processes using pipes or queues. Instead + you should arrange the program so that a process which needs access to a + shared resource created elsewhere can inherit it from an ancestor process. Avoid terminating processes - Using the :meth:`Process.terminate ` - method to stop a process is liable to + Using the :meth:`Process.terminate` method to stop a process is liable to cause any shared resources (such as locks, semaphores, pipes and queues) currently being used by the process to become broken or unavailable to other processes. Therefore it is probably best to only consider using - :meth:`Process.terminate ` on processes - which never use any shared resources. + :meth:`Process.terminate` on processes which never use any shared resources. Joining processes that use queues Bear in mind that a process that has put items in a queue will wait before terminating until all the buffered items are fed by the "feeder" thread to the underlying pipe. (The child process can call the - :meth:`Queue.cancel_join_thread ` - method of the queue to avoid this behaviour.) + :meth:`Queue.cancel_join_thread` method of the queue to avoid this behaviour.) This means that whenever you use a queue you need to make sure that all items which have been put on the queue will eventually be removed before the process is joined. Otherwise you cannot be sure that processes which have put items on the queue will terminate. Remember also that non-daemonic - processes will be joined automatically. + processes will be automatically be joined. An example which will deadlock is the following:: @@ -2643,22 +2226,20 @@ p.join() # this deadlocks obj = queue.get() - A fix here would be to swap the last two lines (or simply remove the + A fix here would be to swap the last two lines round (or simply remove the ``p.join()`` line). Explicitly pass resources to child processes - On Unix using the *fork* start method, a child process can make - use of a shared resource created in a parent process using a - global resource. However, it is better to pass the object as an - argument to the constructor for the child process. - - Apart from making the code (potentially) compatible with Windows - and the other start methods this also ensures that as long as the - child process is still alive the object will not be garbage - collected in the parent process. This might be important if some - resource is freed when the object is garbage collected in the - parent process. + On Unix a child process can make use of a shared resource created in a + parent process using a global resource. However, it is better to pass the + object as an argument to the constructor for the child process. + + Apart from making the code (potentially) compatible with Windows this also + ensures that as long as the child process is still alive the object will not + be garbage collected in the parent process. This might be important if some + resource is freed when the object is garbage collected in the parent + process. So for instance :: @@ -2700,7 +2281,7 @@ resulting in a bad file descriptor error, but introduces a potential danger to applications which replace :func:`sys.stdin` with a "file-like object" with output buffering. This danger is that if multiple processes call - :meth:`~io.IOBase.close()` on this file-like object, it could result in the same + :func:`close()` on this file-like object, it could result in the same data being flushed to the object multiple times, resulting in corruption. If you write a file-like object and implement your own caching, you can @@ -2717,30 +2298,26 @@ For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` -The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are a few extra restriction which don't apply to the *fork* -start method. +Windows +~~~~~~~ + +Since Windows lacks :func:`os.fork` it has a few extra restrictions: More picklability - Ensure that all arguments to :meth:`Process.__init__` are - picklable. This means, in particular, that bound or unbound - methods cannot be used directly as the ``target`` (unless you use - the *fork* start method) --- just define a function and use that - instead. - - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. + Ensure that all arguments to :meth:`Process.__init__` are picklable. This + means, in particular, that bound or unbound methods cannot be used directly + as the ``target`` argument on Windows --- just define a function and use + that instead. + + Also, if you subclass :class:`Process` then make sure that instances will be + picklable when the :meth:`Process.start` method is called. Global variables Bear in mind that if code run in a child process tries to access a global variable, then the value it sees (if any) may not be the same as the value - in the parent process at the time that :meth:`Process.start - ` was called. + in the parent process at the time that :meth:`Process.start` was called. However, global variables which are just module level constants cause no problems. @@ -2751,8 +2328,7 @@ interpreter without causing unintended side effects (such a starting a new process). - For example, using the *spawn* or *forkserver* start method - running the following module would fail with a + For example, under Windows running the following module would fail with a :exc:`RuntimeError`:: from multiprocessing import Process @@ -2766,14 +2342,13 @@ Instead one should protect the "entry point" of the program by using ``if __name__ == '__main__':`` as follows:: - from multiprocessing import Process, freeze_support, set_start_method + from multiprocessing import Process, freeze_support def foo(): print('hello') if __name__ == '__main__': freeze_support() - set_start_method('spawn') p = Process(target=foo) p.start() @@ -2795,16 +2370,32 @@ Demonstration of how to create and use customized managers and proxies: .. literalinclude:: ../includes/mp_newtype.py - :language: python3 - - -Using :class:`~multiprocessing.pool.Pool`: + + +Using :class:`Pool`: .. literalinclude:: ../includes/mp_pool.py - :language: python3 + + +Synchronization types like locks, conditions and queues: + +.. literalinclude:: ../includes/mp_synchronize.py An example showing how to use queues to feed tasks to a collection of worker processes and collect the results: .. literalinclude:: ../includes/mp_workers.py + + +An example of how a pool of worker processes can each run a +:class:`~http.server.SimpleHTTPRequestHandler` instance while sharing a single +listening socket. + +.. literalinclude:: ../includes/mp_webserver.py + + +Some simple benchmarks comparing :mod:`multiprocessing` with :mod:`threading`: + +.. literalinclude:: ../includes/mp_benchmarks.py + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/netrc.rst --- a/Doc/library/netrc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/netrc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -22,14 +22,6 @@ no argument is given, the file :file:`.netrc` in the user's home directory will be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. - If no argument is specified on a POSIX system, the presence of passwords in - the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file - ownership or permissions are insecure (owned by a user other than the user - running the process, or accessible for read or write by any other user). - This implements security behavior equivalent to that of ftp and other - programs that use :file:`.netrc`. - - .. versionchanged:: 3.4 Added the POSIX permission check. .. exception:: NetrcParseError diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/nis.rst --- a/Doc/library/nis.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/nis.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ The :mod:`nis` module defines the following functions: -.. function:: match(key, mapname, domain=default_domain) +.. function:: match(key, mapname[, domain=default_domain]) Return the match for *key* in map *mapname*, or raise an error (:exc:`nis.error`) if there is none. Both should be strings, *key* is 8-bit @@ -30,7 +30,7 @@ unspecified, lookup is in the default NIS domain. -.. function:: cat(mapname, domain=default_domain) +.. function:: cat(mapname[, domain=default_domain]) Return a dictionary mapping *key* to *value* such that ``match(key, mapname)==value``. Note that both keys and values of the dictionary are @@ -42,7 +42,7 @@ unspecified, lookup is in the default NIS domain. -.. function:: maps(domain=default_domain) +.. function:: maps([domain=default_domain]) Return a list of all valid maps. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/nntplib.rst --- a/Doc/library/nntplib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/nntplib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,4 @@ + :mod:`nntplib` --- NNTP protocol client ======================================= @@ -46,7 +47,7 @@ headers, and that you have right to post on the particular newsgroup):: >>> s = nntplib.NNTP('news.gmane.org') - >>> f = open('article.txt', 'rb') + >>> f = open('/tmp/article.txt', 'rb') >>> s.post(f) '240 Article posted successfully.' >>> s.quit() @@ -69,9 +70,9 @@ connecting to an NNTP server on the local machine and intend to call reader-specific commands, such as ``group``. If you get unexpected :exc:`NNTPPermanentError`\ s, you might need to set *readermode*. - The :class:`NNTP` class supports the :keyword:`with` statement to - unconditionally consume :exc:`OSError` exceptions and to close the NNTP - connection when done, e.g.: + :class:`NNTP` class supports the :keyword:`with` statement to + unconditionally consume :exc:`socket.error` exceptions and to close the NNTP + connection when done. Here is a sample on how using it: >>> from nntplib import NNTP >>> with NNTP('news.gmane.org') as n: @@ -82,7 +83,7 @@ .. versionchanged:: 3.2 - *usenetrc* is now ``False`` by default. + *usenetrc* is now False by default. .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -94,7 +95,6 @@ port *port*. :class:`NNTP_SSL` objects have the same methods as :class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used. *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object. - Please read :ref:`ssl-security` for best practices. All other parameters behave the same as for :class:`NNTP`. Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of @@ -103,10 +103,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). .. exception:: NNTPError @@ -221,7 +217,7 @@ .. method:: NNTP.login(user=None, password=None, usenetrc=True) Send ``AUTHINFO`` commands with the user name and password. If *user* - and *password* are None and *usenetrc* is true, credentials from + and *password* are None and *usenetrc* is True, credentials from ``~/.netrc`` will be used if possible. Unless intentionally delayed, login is normally performed during the @@ -235,10 +231,9 @@ .. method:: NNTP.starttls(ssl_context=None) - Send a ``STARTTLS`` command. This will enable encryption on the NNTP - connection. The *ssl_context* argument is optional and should be a - :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best - practices. + Send a ``STARTTLS`` command. The *ssl_context* argument is optional + and should be a :class:`ssl.SSLContext` object. This will enable + encryption on the NNTP connection. Note that this may not be done after authentication information has been transmitted, and authentication occurs by default if possible during a @@ -247,10 +242,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). .. method:: NNTP.newgroups(date, *, file=None) @@ -341,7 +332,7 @@ .. method:: NNTP.over(message_spec, *, file=None) - Send an ``OVER`` command, or an ``XOVER`` command on legacy servers. + Send a ``OVER`` command, or a ``XOVER`` command on legacy servers. *message_spec* can be either a string representing a message id, or a ``(first, last)`` tuple of numbers indicating a range of articles in the current group, or a ``(first, None)`` tuple indicating a range of @@ -404,18 +395,18 @@ .. method:: NNTP.next() - Send a ``NEXT`` command. Return as for :meth:`.stat`. + Send a ``NEXT`` command. Return as for :meth:`stat`. .. method:: NNTP.last() - Send a ``LAST`` command. Return as for :meth:`.stat`. + Send a ``LAST`` command. Return as for :meth:`stat`. .. method:: NNTP.article(message_spec=None, *, file=None) Send an ``ARTICLE`` command, where *message_spec* has the same meaning as - for :meth:`.stat`. Return a tuple ``(response, info)`` where *info* + for :meth:`stat`. Return a tuple ``(response, info)`` where *info* is a :class:`~collections.namedtuple` with three attributes *number*, *message_id* and *lines* (in that order). *number* is the article number in the group (or 0 if the information is not available), *message_id* the @@ -496,10 +487,10 @@ them have been superseded by newer commands in :rfc:`3977`. -.. method:: NNTP.xhdr(hdr, str, *, file=None) +.. method:: NNTP.xhdr(header, string, *, file=None) - Send an ``XHDR`` command. The *hdr* argument is a header keyword, e.g. - ``'subject'``. The *str* argument should have the form ``'first-last'`` + Send an ``XHDR`` command. The *header* argument is a header keyword, e.g. + ``'subject'``. The *string* argument should have the form ``'first-last'`` where *first* and *last* are the first and last article numbers to search. Return a pair ``(response, list)``, where *list* is a list of pairs ``(id, text)``, where *id* is an article number (as a string) and *text* is the text of diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/numbers.rst --- a/Doc/library/numbers.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/numbers.rst Mon Jan 25 17:05:13 2016 +0100 @@ -35,7 +35,7 @@ Abstract. Retrieves the imaginary component of this number. - .. abstractmethod:: conjugate() + .. method:: conjugate() Abstract. Returns the complex conjugate. For example, ``(1+3j).conjugate() == (1-3j)``. @@ -71,10 +71,10 @@ .. class:: Integral - Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides - defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`. Adds abstract methods for ``**`` and - bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``. + Subtypes :class:`Rational` and adds a conversion to :class:`int`. + Provides defaults for :func:`float`, :attr:`~Rational.numerator`, and + :attr:`~Rational.denominator`, and bit-string operations: ``<<``, + ``>>``, ``&``, ``^``, ``|``, ``~``. Notes for type implementors @@ -110,8 +110,6 @@ MyFoo.register(Real) -.. _implementing-the-arithmetic-operations: - Implementing the arithmetic operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/numeric.rst --- a/Doc/library/numeric.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/numeric.rst Mon Jan 25 17:05:13 2016 +0100 @@ -23,4 +23,3 @@ decimal.rst fractions.rst random.rst - statistics.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/operator.rst --- a/Doc/library/operator.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/operator.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,9 +11,6 @@ import operator from operator import itemgetter, iadd -**Source code:** :source:`Lib/operator.py` - --------------- The :mod:`operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is @@ -138,14 +135,6 @@ Return ``a * b``, for *a* and *b* numbers. -.. function:: matmul(a, b) - __matmul__(a, b) - - Return ``a @ b``. - - .. versionadded:: 3.5 - - .. function:: neg(obj) __neg__(obj) @@ -236,14 +225,15 @@ Set the value of *a* at index *b* to *c*. +Example: Build a dictionary that maps the ordinals from ``0`` to ``255`` to +their character equivalents. -.. function:: length_hint(obj, default=0) + >>> d = {} + >>> keys = range(256) + >>> vals = map(chr, keys) + >>> map(operator.setitem, [d]*len(keys), keys, vals) # doctest: +SKIP - Return an estimated length for the object *o*. First try to return its - actual length, then an estimate using :meth:`object.__length_hint__`, and - finally return the default value. - - .. versionadded:: 3.4 +.. XXX: find a better, readable, example The :mod:`operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for @@ -251,22 +241,13 @@ expect a function argument. -.. function:: attrgetter(attr) - attrgetter(*attrs) +.. function:: attrgetter(attr[, args...]) - Return a callable object that fetches *attr* from its operand. - If more than one attribute is requested, returns a tuple of attributes. - The attribute names can also contain dots. For example: - - * After ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. - - * After ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns - ``(b.name, b.date)``. - - * After ``f = attrgetter('name.first', 'name.last')``, the call ``f(b)`` - returns ``(b.name.first, b.name.last)``. - - Equivalent to:: + Return a callable object that fetches *attr* from its operand. If more than one + attribute is requested, returns a tuple of attributes. After, + ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. After, + ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns ``(b.name, + b.date)``. Equivalent to:: def attrgetter(*items): if any(not isinstance(item, str) for item in items): @@ -277,7 +258,7 @@ return resolve_attr(obj, attr) else: def g(obj): - return tuple(resolve_attr(obj, attr) for attr in items) + return tuple(resolve_att(obj, attr) for attr in items) return g def resolve_attr(obj, attr): @@ -286,19 +267,14 @@ return obj -.. function:: itemgetter(item) - itemgetter(*items) + The attribute names can also contain dots; after ``f = attrgetter('date.month')``, + the call ``f(b)`` returns ``b.date.month``. + +.. function:: itemgetter(item[, args...]) Return a callable object that fetches *item* from its operand using the operand's :meth:`__getitem__` method. If multiple items are specified, - returns a tuple of lookup values. For example: - - * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. - - * After ``g = itemgetter(2, 5, 3)``, the call ``g(r)`` returns - ``(r[2], r[5], r[3])``. - - Equivalent to:: + returns a tuple of lookup values. Equivalent to:: def itemgetter(*items): if len(items) == 1: @@ -337,14 +313,9 @@ Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given - to the method as well. For example: - - * After ``f = methodcaller('name')``, the call ``f(b)`` returns ``b.name()``. - - * After ``f = methodcaller('name', 'foo', bar=1)``, the call ``f(b)`` - returns ``b.name('foo', bar=1)``. - - Equivalent to:: + to the method as well. After ``f = methodcaller('name')``, the call ``f(b)`` + returns ``b.name()``. After ``f = methodcaller('name', 'foo', bar=1)``, the + call ``f(b)`` returns ``b.name('foo', bar=1)``. Equivalent to:: def methodcaller(name, *args, **kwargs): def caller(obj): @@ -399,8 +370,6 @@ +-----------------------+-------------------------+---------------------------------------+ | Multiplication | ``a * b`` | ``mul(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Matrix Multiplication | ``a @ b`` | ``matmul(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ | Negation (Arithmetic) | ``- a`` | ``neg(a)`` | +-----------------------+-------------------------+---------------------------------------+ | Negation (Logical) | ``not a`` | ``not_(a)`` | @@ -435,7 +404,7 @@ +-----------------------+-------------------------+---------------------------------------+ Inplace Operators ------------------ +================= Many operations have an "in-place" version. Listed below are functions providing a more primitive access to in-place operators than the usual syntax @@ -509,14 +478,6 @@ ``a = imul(a, b)`` is equivalent to ``a *= b``. -.. function:: imatmul(a, b) - __imatmul__(a, b) - - ``a = imatmul(a, b)`` is equivalent to ``a @= b``. - - .. versionadded:: 3.5 - - .. function:: ior(a, b) __ior__(a, b) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/optparse.rst --- a/Doc/library/optparse.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/optparse.rst Mon Jan 25 17:05:13 2016 +0100 @@ -8,8 +8,8 @@ .. sectionauthor:: Greg Ward .. deprecated:: 3.2 - The :mod:`optparse` module is deprecated and will not be developed further; - development will continue with the :mod:`argparse` module. + The :mod:`optparse` module is deprecated and will not be developed further; + development will continue with the :mod:`argparse` module. **Source code:** :source:`Lib/optparse.py` @@ -171,10 +171,10 @@ For example, consider this hypothetical command-line:: - prog -v --report report.txt foo bar + prog -v --report /tmp/report.txt foo bar ``-v`` and ``--report`` are both options. Assuming that ``--report`` -takes one argument, ``report.txt`` is an option argument. ``foo`` and +takes one argument, ``/tmp/report.txt`` is an option argument. ``foo`` and ``bar`` are positional arguments. @@ -273,8 +273,7 @@ strings as you like (including zero), as long as there is at least one option string overall. -The option strings passed to :meth:`OptionParser.add_option` are effectively -labels for the +The option strings passed to :meth:`add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to *encountering an option* on the command line; in reality, :mod:`optparse` encounters *option strings* and looks up options from them. @@ -893,8 +892,7 @@ The canonical way to create an :class:`Option` instance is with the :meth:`add_option` method of :class:`OptionParser`. -.. method:: OptionParser.add_option(option) - OptionParser.add_option(*opt_str, attr=value, ...) +.. method:: OptionParser.add_option(opt_str[, ...], attr=value, ...) To define an option with only a short option string:: @@ -1167,17 +1165,6 @@ options.tracks.append(int("4")) - The ``append`` action calls the ``append`` method on the current value of the - option. This means that any default value specified must have an ``append`` - method. It also means that if the default value is non-empty, the default - elements will be present in the parsed value for the option, with any values - from the command line appended after those default values:: - - >>> parser.add_option("--files", action="append", default=['~/.mypkg/defaults']) - >>> opts, args = parser.parse_args(['--files', 'overrides.mypkg']) - >>> opts.files - ['~/.mypkg/defaults', 'overrides.mypkg'] - * ``"append_const"`` [required: :attr:`~Option.const`; relevant: :attr:`~Option.dest`] @@ -1324,7 +1311,7 @@ the list of arguments to process (default: ``sys.argv[1:]``) ``values`` - an :class:`optparse.Values` object to store option arguments in (default: a + a :class:`optparse.Values` object to store option arguments in (default: a new instance of :class:`Values`) -- if you give an existing object, the option defaults will not be initialized on it diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/os.path.rst --- a/Doc/library/os.path.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/os.path.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,16 +17,6 @@ names on Windows (in the standard ``mbcs`` encoding), hence Windows applications should use string objects to access all files. -Unlike a unix shell, Python does not do any *automatic* path expansions. -Functions such as :func:`expanduser` and :func:`expandvars` can be invoked -explicitly when an application desires shell-like path expansion. (See also -the :mod:`glob` module.) - - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - .. note:: All of these functions accept either only bytes or only string objects as @@ -47,76 +37,44 @@ * :mod:`posixpath` for UNIX-style paths * :mod:`ntpath` for Windows paths * :mod:`macpath` for old-style MacOS paths + * :mod:`os2emxpath` for OS/2 EMX paths .. function:: abspath(path) Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. + platforms, this is equivalent to ``normpath(join(os.getcwd(), path))``. .. function:: basename(path) - Return the base name of pathname *path*. This is the second element of the - pair returned by passing *path* to the function :func:`split`. Note that - the result of this function is different + Return the base name of pathname *path*. This is the second half of the pair + returned by ``split(path)``. Note that the result of this function is different from the Unix :program:`basename` program; where :program:`basename` for ``'/foo/bar/'`` returns ``'bar'``, the :func:`basename` function returns an empty string (``''``). -.. function:: commonpath(paths) - - Return the longest common sub-path of each pathname in the sequence - *paths*. Raise ValueError if *paths* contains both absolute and relative - pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this - returns a valid path. - - Availability: Unix, Windows - - .. versionadded:: 3.5 - - .. function:: commonprefix(list) - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string - (``''``). - - .. note:: - - This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. - - :: - - >>> os.path.commonprefix(['/usr/lib', '/usr/local/lib']) - '/usr/l' - - >>> os.path.commonpath(['/usr/lib', '/usr/local/lib']) - '/usr' + Return the longest path prefix (taken character-by-character) that is a prefix + of all paths in *list*. If *list* is empty, return the empty string (``''``). + Note that this may return invalid paths because it works a character at a time. .. function:: dirname(path) - Return the directory name of pathname *path*. This is the first element of - the pair returned by passing *path* to the function :func:`split`. + Return the directory name of pathname *path*. This is the first half of the + pair returned by ``split(path)``. .. function:: exists(path) - Return ``True`` if *path* refers to an existing path or an open - file descriptor. Returns ``False`` for broken symbolic links. On - some platforms, this function may return ``False`` if permission is - not granted to execute :func:`os.stat` on the requested file, even + Return ``True`` if *path* refers to an existing path. Returns ``False`` for + broken symbolic links. On some platforms, this function may return ``False`` if + permission is not granted to execute :func:`os.stat` on the requested file, even if the *path* physically exists. - .. versionchanged:: 3.3 - *path* can now be an integer: ``True`` is returned if it is an - open file descriptor, ``False`` otherwise. - .. function:: lexists(path) @@ -161,9 +119,9 @@ Return the time of last access of *path*. The return value is a number giving the number of seconds since the epoch (see the :mod:`time` module). Raise - :exc:`OSError` if the file does not exist or is inaccessible. + :exc:`os.error` if the file does not exist or is inaccessible. - If :func:`os.stat_float_times` returns ``True``, the result is a floating point + If :func:`os.stat_float_times` returns True, the result is a floating point number. @@ -171,24 +129,24 @@ Return the time of last modification of *path*. The return value is a number giving the number of seconds since the epoch (see the :mod:`time` module). - Raise :exc:`OSError` if the file does not exist or is inaccessible. + Raise :exc:`os.error` if the file does not exist or is inaccessible. - If :func:`os.stat_float_times` returns ``True``, the result is a floating point + If :func:`os.stat_float_times` returns True, the result is a floating point number. .. function:: getctime(path) Return the system's ctime which, on some systems (like Unix) is the time of the - last metadata change, and, on others (like Windows), is the creation time for *path*. + last change, and, on others (like Windows), is the creation time for *path*. The return value is a number giving the number of seconds since the epoch (see - the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or + the :mod:`time` module). Raise :exc:`os.error` if the file does not exist or is inaccessible. .. function:: getsize(path) - Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does + Return the size, in bytes, of *path*. Raise :exc:`os.error` if the file does not exist or is inaccessible. @@ -214,40 +172,29 @@ .. function:: islink(path) Return ``True`` if *path* refers to a directory entry that is a symbolic link. - Always ``False`` if symbolic links are not supported by the python runtime. + Always ``False`` if symbolic links are not supported. .. function:: ismount(path) - Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. + Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a file + system where a different file system has been mounted. The function checks + whether *path*'s parent, :file:`path/..`, is on a different device than *path*, + or whether :file:`path/..` and *path* point to the same i-node on the same + device --- this should detect mount points for all Unix and POSIX variants. - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. +.. function:: join(path1[, path2[, ...]]) -.. function:: join(path, *paths) - - Join one or more path components intelligently. The return value is the - concatenation of *path* and any members of *\*paths* with exactly one - directory separator (``os.sep``) following each non-empty part except the - last, meaning that the result will only end in a separator if the last - part is empty. If a component is an absolute path, all previous - components are thrown away and joining continues from the absolute path - component. - - On Windows, the drive letter is not reset when an absolute path component - (e.g., ``r'\foo'``) is encountered. If a component contains a drive - letter, all previous components are thrown away and the drive letter is - reset. Note that since there is a current directory for each drive, - ``os.path.join("c:", "foo")`` represents a path relative to the current - directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. + Join one or more path components intelligently. If any component is an absolute + path, all previous components (on Windows, including the previous drive letter, + if there was one) are thrown away, and joining continues. The return value is + the concatenation of *path1*, and optionally *path2*, etc., with exactly one + directory separator (``os.sep``) following each non-empty part except the last. + (This means that an empty last part will result in a path that ends with a + separator.) Note that on Windows, since there is a current directory for + each drive, ``os.path.join("c:", "foo")`` represents a path relative to the + current directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. .. function:: normcase(path) @@ -260,11 +207,13 @@ .. function:: normpath(path) - Normalize a pathname by collapsing redundant separators and up-level - references so that ``A//B``, ``A/B/``, ``A/./B`` and ``A/foo/../B`` all - become ``A/B``. This string manipulation may change the meaning of a path - that contains symbolic links. On Windows, it converts forward slashes to - backward slashes. To normalize case, use :func:`normcase`. + Normalize a pathname. This collapses redundant separators and up-level + references so that ``A//B``, ``A/B/``, ``A/./B`` and ``A/foo/../B`` all become + ``A/B``. + + It does not normalize the case (use :func:`normcase` for that). On Windows, it + converts forward slashes to backward slashes. It should be understood that this + may change the meaning of the path if it contains symbolic links! .. function:: realpath(path) @@ -273,12 +222,10 @@ links encountered in the path (if they are supported by the operating system). -.. function:: relpath(path, start=os.curdir) +.. function:: relpath(path, start=None) - Return a relative filepath to *path* either from the current directory or - from an optional *start* directory. This is a path computation: the - filesystem is not accessed to confirm the existence or nature of *path* or - *start*. + Return a relative filepath to *path* either from the current directory or from + an optional *start* point. *start* defaults to :attr:`os.curdir`. @@ -288,17 +235,18 @@ .. function:: samefile(path1, path2) Return ``True`` if both pathname arguments refer to the same file or directory. - This is determined by the device number and i-node number and raises an - exception if an :func:`os.stat` call on either pathname fails. + On Unix, this is determined by the device number and i-node number and raises an + exception if a :func:`os.stat` call on either pathname fails. + + On Windows, two files are the same if they resolve to the same final path + name using the Windows API call GetFinalPathNameByHandle. This function + raises an exception if handles cannot be obtained to either file. Availability: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. - .. versionchanged:: 3.4 - Windows now uses the same implementation as all other platforms. - .. function:: sameopenfile(fp1, fp2) @@ -306,21 +254,17 @@ Availability: Unix, Windows. - .. versionchanged:: 3.2 - Added Windows support. + .. versionchanged:: 3.2 Added Windows support. .. function:: samestat(stat1, stat2) Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. - These structures may have been returned by :func:`os.fstat`, - :func:`os.lstat`, or :func:`os.stat`. This function implements the - underlying comparison used by :func:`samefile` and :func:`sameopenfile`. + These structures may have been returned by :func:`fstat`, :func:`lstat`, or + :func:`stat`. This function implements the underlying comparison used by + :func:`samefile` and :func:`sameopenfile`. - Availability: Unix, Windows. - - .. versionchanged:: 3.4 - Added Windows support. + Availability: Unix. .. function:: split(path) @@ -332,8 +276,7 @@ *path* is empty, both *head* and *tail* are empty. Trailing slashes are stripped from *head* unless it is the root (one or more slashes only). In all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. + (but the strings may differ). .. function:: splitdrive(path) @@ -377,5 +320,5 @@ .. data:: supports_unicode_filenames - ``True`` if arbitrary Unicode strings can be used as file names (within limitations + True if arbitrary Unicode strings can be used as file names (within limitations imposed by the file system). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/os.rst --- a/Doc/library/os.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/os.rst Mon Jan 25 17:05:13 2016 +0100 @@ -53,8 +53,8 @@ .. data:: name The name of the operating system dependent module imported. The following - names have currently been registered: ``'posix'``, ``'nt'``, - ``'ce'``, ``'java'``. + names have currently been registered: ``'posix'``, ``'nt'``, ``'mac'``, + ``'os2'``, ``'ce'``, ``'java'``. .. seealso:: :attr:`sys.platform` has a finer granularity. :func:`os.uname` gives @@ -65,7 +65,6 @@ .. _os-filenames: -.. _filesystem-encoding: File Names, Command Line Arguments, and Environment Variables ------------------------------------------------------------- @@ -78,10 +77,9 @@ .. versionchanged:: 3.1 On some systems, conversion using the file system encoding may fail. In this - case, Python uses the :ref:`surrogateescape encoding error handler - `, which means that undecodable bytes are replaced by a - Unicode character U+DCxx on decoding, and these are again translated to the - original byte on encoding. + case, Python uses the ``surrogateescape`` encoding error handler, which means + that undecodable bytes are replaced by a Unicode character U+DCxx on + decoding, and these are again translated to the original byte on encoding. The file system encoding must guarantee to successfully decode all bytes @@ -98,16 +96,9 @@ process and user. -.. function:: ctermid() - - Return the filename corresponding to the controlling terminal of the process. - - Availability: Unix. - - .. data:: environ - A :term:`mapping` object representing the string environment. For example, + A mapping object representing the string environment. For example, ``environ['HOME']`` is the pathname of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. @@ -147,7 +138,7 @@ .. data:: environb - Bytes version of :data:`environ`: a :term:`mapping` object representing the + Bytes version of :data:`environ`: a mapping object representing the environment as byte strings. :data:`environ` and :data:`environb` are synchronized (modify :data:`environb` updates :data:`environ`, and vice versa). @@ -186,28 +177,6 @@ .. versionadded:: 3.2 -.. function:: getenv(key, default=None) - - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are str. - - On Unix, keys and values are decoded with :func:`sys.getfilesystemencoding` - and ``'surrogateescape'`` error handler. Use :func:`os.getenvb` if you - would like to use a different encoding. - - Availability: most flavors of Unix, Windows. - - -.. function:: getenvb(key, default=None) - - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are bytes. - - Availability: most flavors of Unix. - - .. versionadded:: 3.2 - - .. function:: get_exec_path(env=None) Returns the list of directories that will be searched for a named @@ -219,6 +188,13 @@ .. versionadded:: 3.2 +.. function:: ctermid() + + Return the filename corresponding to the controlling terminal of the process. + + Availability: Unix. + + .. function:: getegid() Return the effective group id of the current process. This corresponds to the @@ -262,30 +238,25 @@ Availability: Unix. - .. note:: - - On Mac OS X, :func:`getgroups` behavior differs somewhat from - other Unix platforms. If the Python interpreter was built with a - deployment target of :const:`10.5` or earlier, :func:`getgroups` returns - the list of effective group ids associated with the current user process; - this list is limited to a system-defined number of entries, typically 16, - and may be modified by calls to :func:`setgroups` if suitably privileged. - If built with a deployment target greater than :const:`10.5`, - :func:`getgroups` returns the current group access list for the user - associated with the effective user id of the process; the group access - list may change over the lifetime of the process, it is not affected by - calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. + +.. function:: initgroups(username, gid) + + Call the system initgroups() to initialize the group access list with all of + the groups of which the specified username is a member, plus the specified + group id. + + Availability: Unix. + + .. versionadded:: 3.2 .. function:: getlogin() Return the name of the user logged in on the controlling terminal of the - process. For most purposes, it is more useful to use the environment - variables :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user - is, or ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the current - real user id. + process. For most purposes, it is more useful to use the environment variables + :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user is, or + ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the currently + effective user id. Availability: Unix, Windows. @@ -312,6 +283,8 @@ Return the current process id. + Availability: Unix, Windows. + .. function:: getppid() @@ -321,80 +294,87 @@ the id returned is the one of the init process (1), on Windows it is still the same id, which may be already reused by another process. - Availability: Unix, Windows. + Availability: Unix, Windows .. versionchanged:: 3.2 Added support for Windows. - .. function:: getpriority(which, who) .. index:: single: process; scheduling priority - Get program scheduling priority. The value *which* is one of + Get program scheduling priority. The value *which* is one of :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who* is interpreted relative to *which* (a process identifier for :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a - user ID for :const:`PRIO_USER`). A zero value for *who* denotes + user ID for :const:`PRIO_USER`). A zero value for *who* denotes (respectively) the calling process, the process group of the calling process, or the real user ID of the calling process. + Availability: Unix + + .. versionadded:: 3.3 + +.. function:: getresuid() + + Return a tuple (ruid, euid, suid) denoting the current process's + real, effective, and saved user ids. + Availability: Unix. - .. versionadded:: 3.3 - + .. versionadded:: 3.2 + + +.. function:: getresgid() + + Return a tuple (rgid, egid, sgid) denoting the current process's + real, effective, and saved group ids. + + Availability: Unix. + + .. versionadded:: 3.2 + + +.. function:: getuid() + + .. index:: single: user; id + + Return the current process's user id. + + Availability: Unix. + + +.. function:: getenv(key, default=None) + + Return the value of the environment variable *key* if it exists, or + *default* if it doesn't. *key*, *default* and the result are str. + + On Unix, keys and values are decoded with :func:`sys.getfilesystemencoding` + and ``'surrogateescape'`` error handler. Use :func:`os.getenvb` if you + would like to use a different encoding. + + Availability: most flavors of Unix, Windows. + + +.. function:: getenvb(key, default=None) + + Return the value of the environment variable *key* if it exists, or + *default* if it doesn't. *key*, *default* and the result are bytes. + + Availability: most flavors of Unix. + + .. versionadded:: 3.2 .. data:: PRIO_PROCESS PRIO_PGRP PRIO_USER - Parameters for the :func:`getpriority` and :func:`setpriority` functions. + Parameters for :func:`getpriority` and :func:`setpriority` functions. Availability: Unix. .. versionadded:: 3.3 - -.. function:: getresuid() - - Return a tuple (ruid, euid, suid) denoting the current process's - real, effective, and saved user ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: getresgid() - - Return a tuple (rgid, egid, sgid) denoting the current process's - real, effective, and saved group ids. - - Availability: Unix. - - .. versionadded:: 3.2 - - -.. function:: getuid() - - .. index:: single: user; id - - Return the current process's real user id. - - Availability: Unix. - - -.. function:: initgroups(username, gid) - - Call the system initgroups() to initialize the group access list with all of - the groups of which the specified username is a member, plus the specified - group id. - - Availability: Unix. - - .. versionadded:: 3.2 - - .. function:: putenv(key, value) .. index:: single: environment variables; setting @@ -445,14 +425,10 @@ Availability: Unix. - .. note:: On Mac OS X, the length of *groups* may not exceed the - system-defined maximum number of effective group ids, typically 16. - See the documentation for :func:`getgroups` for cases where it may not - return the same group list set by calling setgroups(). .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`setpgrp` or :c:func:`setpgrp(0, 0)` depending on which version is implemented (if any). See the Unix manual for the semantics. Availability: Unix. @@ -548,10 +524,12 @@ On platforms where :c:func:`strerror` returns ``NULL`` when given an unknown error number, :exc:`ValueError` is raised. + Availability: Unix, Windows. + .. data:: supports_bytes_environ - ``True`` if the native OS type of the environment is bytes (eg. ``False`` on + True if the native OS type of the environment is bytes (eg. False on Windows). .. versionadded:: 3.2 @@ -561,6 +539,8 @@ Set the current numeric umask and return the previous umask. + Availability: Unix, Windows. + .. function:: uname() @@ -568,31 +548,15 @@ single: gethostname() (in module socket) single: gethostbyaddr() (in module socket) - Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the + Return a 5-tuple containing information identifying the current operating + system. The tuple contains 5 strings: ``(sysname, nodename, release, version, + machine)``. Some systems truncate the nodename to 8 characters or to the leading component; a better way to get the hostname is :func:`socket.gethostname` or even ``socket.gethostbyaddr(socket.gethostname())``. Availability: recent flavors of Unix. - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. - .. function:: unsetenv(key) @@ -615,17 +579,29 @@ File Object Creation -------------------- -This function creates new :term:`file objects `. (See also -:func:`~os.open` for opening file descriptors.) - - -.. function:: fdopen(fd, *args, **kwargs) - - Return an open file object connected to the file descriptor *fd*. This is an - alias of the :func:`open` built-in function and accepts the same arguments. - The only difference is that the first argument of :func:`fdopen` must always - be an integer. - +These functions create new :term:`file objects `. (See also :func:`open`.) + + +.. function:: fdopen(fd[, mode[, bufsize]]) + + .. index:: single: I/O control; buffering + + Return an open file object connected to the file descriptor *fd*. The *mode* + and *bufsize* arguments have the same meaning as the corresponding arguments to + the built-in :func:`open` function. + + When specified, the *mode* argument must start with one of the letters + ``'r'``, ``'w'``, ``'x'`` or ``'a'``, otherwise a :exc:`ValueError` is + raised. + + On Unix, when the *mode* argument starts with ``'a'``, the *O_APPEND* flag is + set on the file descriptor (which the :c:func:`fdopen` implementation already + does on most platforms). + + Availability: Unix, Windows. + + .. versionchanged:: 3.3 + The ``'x'`` mode was added. .. _os-fd-ops: @@ -641,28 +617,44 @@ is slightly deceptive; on Unix platforms, sockets and pipes are also referenced by file descriptors. -The :meth:`~io.IOBase.fileno` method can be used to obtain the file descriptor +The :meth:`~file.fileno` method can be used to obtain the file descriptor associated with a :term:`file object` when required. Note that using the file descriptor directly will bypass the file object methods, ignoring aspects such as internal buffering of data. +.. data:: AT_SYMLINK_NOFOLLOW + AT_EACCESS + AT_FDCWD + AT_REMOVEDIR + AT_SYMLINK_FOLLOW + UTIME_NOW + UTIME_OMIT + + These parameters are used as flags to the \*at family of functions. + + Availability: Unix. + + .. versionadded:: 3.3 + .. function:: close(fd) Close file descriptor *fd*. + Availability: Unix, Windows. + .. note:: This function is intended for low-level I/O and must be applied to a file descriptor as returned by :func:`os.open` or :func:`pipe`. To close a "file object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, use its :meth:`~io.IOBase.close` method. + :func:`fdopen`, use its :meth:`~file.close` method. .. function:: closerange(fd_low, fd_high) Close all file descriptors from *fd_low* (inclusive) to *fd_high* (exclusive), - ignoring errors. Equivalent to (but much faster than):: + ignoring errors. Equivalent to:: for fd in range(fd_low, fd_high): try: @@ -670,6 +662,8 @@ except OSError: pass + Availability: Unix, Windows. + .. function:: device_encoding(fd) @@ -679,46 +673,71 @@ .. function:: dup(fd) - Return a duplicate of file descriptor *fd*. The new file descriptor is - :ref:`non-inheritable `. - - On Windows, when duplicating a standard stream (0: stdin, 1: stdout, - 2: stderr), the new file descriptor is :ref:`inheritable - `. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - -.. function:: dup2(fd, fd2, inheritable=True) + Return a duplicate of file descriptor *fd*. + + Availability: Unix, Windows. + + +.. function:: dup2(fd, fd2) Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary. - The file descriptor *fd2* is :ref:`inheritable ` by default, - or non-inheritable if *inheritable* is ``False``. - - .. versionchanged:: 3.4 - Add the optional *inheritable* parameter. + + Availability: Unix, Windows. + + +.. function:: faccessat(dirfd, path, mode, flags=0) + + Like :func:`access` but if *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and can be constructed by ORing together zero or more + of these values: :data:`AT_SYMLINK_NOFOLLOW`, :data:`AT_EACCESS`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 .. function:: fchmod(fd, mode) - Change the mode of the file given by *fd* to the numeric *mode*. See the - docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this - is equivalent to ``os.chmod(fd, mode)``. + Change the mode of the file given by *fd* to the numeric *mode*. See the docs + for :func:`chmod` for possible values of *mode*. Availability: Unix. +.. function:: fchmodat(dirfd, path, mode, flags=0) + + Like :func:`chmod` but if *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + .. function:: fchown(fd, uid, gid) Change the owner and group id of the file given by *fd* to the numeric *uid* - and *gid*. To leave one of the ids unchanged, set it to -1. See - :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, - gid)``. + and *gid*. To leave one of the ids unchanged, set it to -1. Availability: Unix. +.. function:: fchownat(dirfd, path, uid, gid, flags=0) + + Like :func:`chown` but if *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + .. function:: fdatasync(fd) Force write of file with filedescriptor *fd* to disk. Does not force update of @@ -730,6 +749,47 @@ This function is not available on MacOS. +.. function:: fgetxattr(fd, attr) + + This works exactly like :func:`getxattr` but operates on a file descriptor, + *fd*, instead of a path. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: flistxattr(fd) + + This is exactly like :func:`listxattr` but operates on a file descriptor, + *fd*, instead of a path. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: flistdir(fd) + + Like :func:`listdir`, but uses a file descriptor instead and always returns + strings. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: fexecve(fd, args, env) + + Execute the program specified by a file descriptor *fd* with arguments given + by *args* and environment given by *env*, replacing the current process. + *args* and *env* are given as in :func:`execve`. + + Availability: Unix. + + .. versionadded:: 3.3 + + .. function:: fpathconf(fd, name) Return system configuration information relevant to an open file. *name* @@ -745,28 +805,31 @@ included in ``pathconf_names``, an :exc:`OSError` is raised with :const:`errno.EINVAL` for the error number. - As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - Availability: Unix. .. function:: fstat(fd) - Get the status of the file descriptor *fd*. Return a :class:`stat_result` - object. - - As of Python 3.3, this is equivalent to ``os.stat(fd)``. - - .. seealso:: - - The :func:`.stat` function. + Return status for file descriptor *fd*, like :func:`~os.stat`. + + Availability: Unix, Windows. + +.. function:: fstatat(dirfd, path, flags=0) + + Like :func:`stat` but if *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and may be 0 or :data:`AT_SYMLINK_NOFOLLOW`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 .. function:: fstatvfs(fd) - Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + Return information about the filesystem containing the file associated with file + descriptor *fd*, like :func:`statvfs`. Availability: Unix. @@ -780,36 +843,112 @@ ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal buffers associated with *f* are written to disk. - Availability: Unix, Windows. + Availability: Unix, and Windows. .. function:: ftruncate(fd, length) - Truncate the file corresponding to file descriptor *fd*, so that it is at - most *length* bytes in size. As of Python 3.3, this is equivalent to - ``os.truncate(fd, length)``. - - Availability: Unix, Windows. - - .. versionchanged:: 3.5 - Added support for Windows - -.. function:: get_blocking(fd) - - Get the blocking mode of the file descriptor: ``False`` if the - :data:`O_NONBLOCK` flag is set, ``True`` if the flag is cleared. - - See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. + Truncate the file corresponding to file descriptor *fd*, so that it is at most + *length* bytes in size. Availability: Unix. - .. versionadded:: 3.5 + +.. function:: fremovexattr(fd, attr) + + This works exactly like :func:`removexattr` but operates on a file + descriptor, *fd*, instead of a path. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: fsetxattr(fd, attr, value, flags=0) + + This works exactly like :func:`setxattr` but on a file descriptor, *fd*, + instead of a path. + + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: futimesat(dirfd, path[, times]) + + Like :func:`utime` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. *times* must be a + 2-tuple of numbers, of the form ``(atime, mtime)``, or None. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: futimens(fd[, atimes, mtimes]) + + Updates the timestamps of a file specified by the file descriptor *fd*, with + nanosecond precision. + If no second argument is given, set *atime* and *mtime* to the current time. + *atimes* and *mtimes* must be 2-tuples of numbers, of the form + ``(atime_sec, atime_nsec)`` and ``(mtime_sec, mtime_nsec)`` respectively, + or ``None``. + If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_NOW`, the corresponding + timestamp is updated to the current time. + If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_OMIT`, the corresponding + timestamp is not updated. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: UTIME_NOW + UTIME_OMIT + + Flags used with :func:`futimens` to specify that the timestamp must be + updated either to the current time or not updated at all. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: futimes(fd[, times]) + + Set the access and modified time of the file specified by the file + descriptor *fd* to the given values. *atimes* must be a 2-tuple of numbers, + of the form ``(atime, mtime)``, or None. If no second argument is used, + set the access and modified times to the current time. + + Availability: Unix. + + .. versionadded:: 3.3 + .. function:: isatty(fd) Return ``True`` if the file descriptor *fd* is open and connected to a tty(-like) device, else ``False``. + Availability: Unix. + + +.. function:: linkat(srcfd, srcpath, dstfd, dstpath, flags=0) + + Like :func:`link` but if *srcpath* is relative, it is taken as relative to *srcfd* + and if *dstpath* is relative, it is taken as relative to *dstfd*. + *flags* is optional and may be 0 or :data:`AT_SYMLINK_FOLLOW`. + If *srcpath* is relative and *srcfd* is the special value :data:`AT_FDCWD`, then + *srcpath* is interpreted relative to the current working directory. This + also applies for *dstpath*. + + Availability: Unix. + + .. versionadded:: 3.3 + .. function:: lockf(fd, cmd, len) @@ -835,45 +974,71 @@ .. versionadded:: 3.3 - .. function:: lseek(fd, pos, how) Set the current position of file descriptor *fd* to position *pos*, modified by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of + current position; :const:`os.SEEK_END` or ``2`` to set it relative to the end of the file. Return the new cursor position in bytes, starting from the beginning. + Availability: Unix, Windows. + .. data:: SEEK_SET SEEK_CUR SEEK_END Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + respectively. Availability: Windows, Unix. + + +.. function:: mkdirat(dirfd, path, mode=0o777) + + Like :func:`mkdir` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. .. versionadded:: 3.3 - Some operating systems could support additional values, like - :data:`os.SEEK_HOLE` or :data:`os.SEEK_DATA`. - - -.. function:: open(path, flags, mode=0o777, *, dir_fd=None) - - Open the file *path* and set various flags according to *flags* and possibly - its mode according to *mode*. When computing *mode*, the current umask value - is first masked out. Return the file descriptor for the newly opened file. - The new file descriptor is :ref:`non-inheritable `. + + +.. function:: mkfifoat(dirfd, path, mode=0o666) + + Like :func:`mkfifo` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: mknodat(dirfd, path, mode=0o600, device=0) + + Like :func:`mknod` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: open(file, flags[, mode]) + + Open the file *file* and set various flags according to *flags* and possibly + its mode according to *mode*. The default *mode* is ``0o777`` (octal), and + the current umask value is first masked out. Return the file descriptor for + the newly opened file. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + this module too (see :ref:`open-constants`). In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. - This function can support :ref:`paths relative to directory descriptors - ` with the *dir_fd* parameter. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. + Availability: Unix, Windows. .. note:: @@ -882,13 +1047,302 @@ :meth:`~file.read` and :meth:`~file.write` methods (and many more). To wrap a file descriptor in a file object, use :func:`fdopen`. + +.. function:: openat(dirfd, path, flags, mode=0o777) + + Like :func:`open` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + +.. function:: openpty() + + .. index:: module: pty + + Open a new pseudo-terminal pair. Return a pair of file descriptors ``(master, + slave)`` for the pty and the tty, respectively. For a (slightly) more portable + approach, use the :mod:`pty` module. + + Availability: some flavors of Unix. + + +.. function:: pipe() + + Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for reading + and writing, respectively. + + Availability: Unix, Windows. + + +.. function:: pipe2(flags) + + Create a pipe with *flags* set atomically. + *flags* can be constructed by ORing together one or more of these values: + :data:`O_NONBLOCK`, :data:`O_CLOEXEC`. + Return a pair of file descriptors ``(r, w)`` usable for reading and writing, + respectively. + + Availability: some flavors of Unix. + + .. versionadded:: 3.3 + + +.. function:: posix_fallocate(fd, offset, len) + + Ensures that enough disk space is allocated for the file specified by *fd* + starting from *offset* and continuing for *len* bytes. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: posix_fadvise(fd, offset, len, advice) + + Announces an intention to access data in a specific pattern thus allowing + the kernel to make optimizations. + The advice applies to the region of the file specified by *fd* starting at + *offset* and continuing for *len* bytes. + *advice* is one of :data:`POSIX_FADV_NORMAL`, :data:`POSIX_FADV_SEQUENTIAL`, + :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, + :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: POSIX_FADV_NORMAL + POSIX_FADV_SEQUENTIAL + POSIX_FADV_RANDOM + POSIX_FADV_NOREUSE + POSIX_FADV_WILLNEED + POSIX_FADV_DONTNEED + + Flags that can be used in *advice* in :func:`posix_fadvise` that specify + the access pattern that is likely to be used. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: pread(fd, buffersize, offset) + + Read from a file descriptor, *fd*, at a position of *offset*. It will read up + to *buffersize* number of bytes. The file offset remains unchanged. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: pwrite(fd, string, offset) + + Write *string* to a file descriptor, *fd*, from *offset*, leaving the file + offset unchanged. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: read(fd, n) + + Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the + bytes read. If the end of the file referred to by *fd* has been reached, an + empty bytes object is returned. + + Availability: Unix, Windows. + + .. note:: + + This function is intended for low-level I/O and must be applied to a file + descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" + returned by the built-in function :func:`open` or by :func:`popen` or + :func:`fdopen`, or :data:`sys.stdin`, use its :meth:`~file.read` or + :meth:`~file.readline` methods. + + +.. function:: sendfile(out, in, offset, nbytes) + sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0) + + Copy *nbytes* bytes from file descriptor *in* to file descriptor *out* + starting at *offset*. + Return the number of bytes sent. When EOF is reached return 0. + + The first function notation is supported by all platforms that define + :func:`sendfile`. + + On Linux, if *offset* is given as ``None``, the bytes are read from the + current position of *in* and the position of *in* is updated. + + The second case may be used on Mac OS X and FreeBSD where *headers* and + *trailers* are arbitrary sequences of buffers that are written before and + after the data from *in* is written. It returns the same as the first case. + + On Mac OS X and FreeBSD, a value of 0 for *nbytes* specifies to send until + the end of *in* is reached. + + On Solaris, *out* may be the file descriptor of a regular file or the file + descriptor of a socket. On all other platforms, *out* must be the file + descriptor of an open socket. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: SF_NODISKIO + SF_MNOWAIT + SF_SYNC + + Parameters to the :func:`sendfile` function, if the implementation supports + them. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: readlinkat(dirfd, path) + + Like :func:`readlink` but if *path* is relative, it is taken as relative to *dirfd*. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: renameat(olddirfd, oldpath, newdirfd, newpath) + + Like :func:`rename` but if *oldpath* is relative, it is taken as relative to + *olddirfd* and if *newpath* is relative, it is taken as relative to *newdirfd*. + If *oldpath* is relative and *olddirfd* is the special value :data:`AT_FDCWD`, then + *oldpath* is interpreted relative to the current working directory. This + also applies for *newpath*. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: symlinkat(src, dstfd, dst) + + Like :func:`symlink` but if *dst* is relative, it is taken as relative to *dstfd*. + If *dst* is relative and *dstfd* is the special value :data:`AT_FDCWD`, then *dst* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: readv(fd, buffers) + + Read from a file descriptor into a number of writable buffers. *buffers* is + an arbitrary sequence of writable buffers. Returns the total number of bytes + read. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: tcgetpgrp(fd) + + Return the process group associated with the terminal given by *fd* (an open + file descriptor as returned by :func:`os.open`). + + Availability: Unix. + + +.. function:: tcsetpgrp(fd, pg) + + Set the process group associated with the terminal given by *fd* (an open file + descriptor as returned by :func:`os.open`) to *pg*. + + Availability: Unix. + + +.. function:: ttyname(fd) + + Return a string which specifies the terminal device associated with + file descriptor *fd*. If *fd* is not associated with a terminal device, an + exception is raised. + + Availability: Unix. + + +.. function:: unlinkat(dirfd, path, flags=0) + + Like :func:`unlink` but if *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and may be 0 or :data:`AT_REMOVEDIR`. If :data:`AT_REMOVEDIR` is + specified, :func:`unlinkat` behaves like :func:`rmdir`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: utimensat(dirfd, path[, atime=(atime_sec, atime_nsec), mtime=(mtime_sec, mtime_nsec), flags=0]) + + Updates the timestamps of a file with nanosecond precision. + The *atime* and *mtime* tuples default to ``None``, which sets those + values to the current time. + If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_NOW`, the corresponding + timestamp is updated to the current time. + If *atime_nsec* or *mtime_nsec* is specified as :data:`UTIME_OMIT`, the corresponding + timestamp is not updated. + If *path* is relative, it is taken as relative to *dirfd*. + *flags* is optional and may be 0 (the default) or :data:`AT_SYMLINK_NOFOLLOW`. + If *path* is relative and *dirfd* is the special value :data:`AT_FDCWD`, then *path* + is interpreted relative to the current working directory. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: write(fd, str) + + Write the bytestring in *str* to file descriptor *fd*. Return the number of + bytes actually written. + + Availability: Unix, Windows. + + .. note:: + + This function is intended for low-level I/O and must be applied to a file + descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file + object" returned by the built-in function :func:`open` or by :func:`popen` or + :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its + :meth:`~file.write` method. + + +.. function:: writev(fd, buffers) + + Write the contents of *buffers* to file descriptor *fd*, where *buffers* + is an arbitrary sequence of buffers. + Returns the total number of bytes written. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. _open-constants: + +``open()`` flag constants +~~~~~~~~~~~~~~~~~~~~~~~~~ The following constants are options for the *flags* parameter to the :func:`~os.open` function. They can be combined using the bitwise OR operator @@ -939,264 +1393,20 @@ O_DIRECTORY O_NOFOLLOW O_NOATIME - O_PATH - O_TMPFILE These constants are GNU extensions and not present if they are not defined by the C library. - .. versionchanged:: 3.4 - Add :data:`O_PATH` on systems that support it. - Add :data:`O_TMPFILE`, only available on Linux Kernel 3.11 - or newer. - - -.. function:: openpty() - - .. index:: module: pty - - Open a new pseudo-terminal pair. Return a pair of file descriptors - ``(master, slave)`` for the pty and the tty, respectively. The new file - descriptors are :ref:`non-inheritable `. For a (slightly) more - portable approach, use the :mod:`pty` module. - - Availability: some flavors of Unix. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe() - - Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for - reading and writing, respectively. The new file descriptor is - :ref:`non-inheritable `. - - Availability: Unix, Windows. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe2(flags) - - Create a pipe with *flags* set atomically. - *flags* can be constructed by ORing together one or more of these values: - :data:`O_NONBLOCK`, :data:`O_CLOEXEC`. - Return a pair of file descriptors ``(r, w)`` usable for reading and writing, - respectively. - - Availability: some flavors of Unix. - - .. versionadded:: 3.3 - - -.. function:: posix_fallocate(fd, offset, len) - - Ensures that enough disk space is allocated for the file specified by *fd* - starting from *offset* and continuing for *len* bytes. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: posix_fadvise(fd, offset, len, advice) - - Announces an intention to access data in a specific pattern thus allowing - the kernel to make optimizations. - The advice applies to the region of the file specified by *fd* starting at - *offset* and continuing for *len* bytes. - *advice* is one of :data:`POSIX_FADV_NORMAL`, :data:`POSIX_FADV_SEQUENTIAL`, - :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, - :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. data:: POSIX_FADV_NORMAL - POSIX_FADV_SEQUENTIAL - POSIX_FADV_RANDOM - POSIX_FADV_NOREUSE - POSIX_FADV_WILLNEED - POSIX_FADV_DONTNEED - - Flags that can be used in *advice* in :func:`posix_fadvise` that specify - the access pattern that is likely to be used. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: pread(fd, buffersize, offset) - - Read from a file descriptor, *fd*, at a position of *offset*. It will read up - to *buffersize* number of bytes. The file offset remains unchanged. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: pwrite(fd, str, offset) - - Write *bytestring* to a file descriptor, *fd*, from *offset*, - leaving the file offset unchanged. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: read(fd, n) - - Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the - bytes read. If the end of the file referred to by *fd* has been reached, an - empty bytes object is returned. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To read a - "file object" returned by the built-in function :func:`open` or by - :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: sendfile(out, in, offset, count) - sendfile(out, in, offset, count, [headers], [trailers], flags=0) - - Copy *count* bytes from file descriptor *in* to file descriptor *out* - starting at *offset*. - Return the number of bytes sent. When EOF is reached return 0. - - The first function notation is supported by all platforms that define - :func:`sendfile`. - - On Linux, if *offset* is given as ``None``, the bytes are read from the - current position of *in* and the position of *in* is updated. - - The second case may be used on Mac OS X and FreeBSD where *headers* and - *trailers* are arbitrary sequences of buffers that are written before and - after the data from *in* is written. It returns the same as the first case. - - On Mac OS X and FreeBSD, a value of 0 for *count* specifies to send until - the end of *in* is reached. - - All platforms support sockets as *out* file descriptor, and some platforms - allow other types (e.g. regular file, pipe) as well. - - Cross-platform applications should not use *headers*, *trailers* and *flags* - arguments. - - Availability: Unix. - - .. note:: - - For a higher-level wrapper of :func:`sendfile`, see - :meth:`socket.socket.sendfile`. - - .. versionadded:: 3.3 - - -.. function:: set_blocking(fd, blocking) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - Availability: Unix. - - .. versionadded:: 3.5 - - -.. data:: SF_NODISKIO - SF_MNOWAIT - SF_SYNC - - Parameters to the :func:`sendfile` function, if the implementation supports - them. - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: readv(fd, buffers) - - Read from a file descriptor *fd* into a number of mutable :term:`bytes-like - objects ` *buffers*. :func:`~os.readv` will transfer data - into each buffer until it is full and then move on to the next buffer in the - sequence to hold the rest of the data. :func:`~os.readv` returns the total - number of bytes read (which may be less than the total capacity of all the - objects). - - Availability: Unix. - - .. versionadded:: 3.3 - - -.. function:: tcgetpgrp(fd) - - Return the process group associated with the terminal given by *fd* (an open - file descriptor as returned by :func:`os.open`). - - Availability: Unix. - - -.. function:: tcsetpgrp(fd, pg) - - Set the process group associated with the terminal given by *fd* (an open file - descriptor as returned by :func:`os.open`) to *pg*. - - Availability: Unix. - - -.. function:: ttyname(fd) - - Return a string which specifies the terminal device associated with - file descriptor *fd*. If *fd* is not associated with a terminal device, an - exception is raised. - - Availability: Unix. - - -.. function:: write(fd, str) - - Write the bytestring in *str* to file descriptor *fd*. Return the number of - bytes actually written. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file - object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: writev(fd, buffers) - - Write the contents of *buffers* to file descriptor *fd*. *buffers* must be a - sequence of :term:`bytes-like objects `. - :func:`~os.writev` writes the contents of each object to the file descriptor - and returns the total number of bytes written. - - Availability: Unix. + +.. data:: RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + RTLD_NODELETE + RTLD_NOLOAD + RTLD_DEEPBIND + + See the Unix manual page :manpage:`dlopen(3)`. .. versionadded:: 3.3 @@ -1217,7 +1427,7 @@ output) specifies which file descriptor should be queried. If the file descriptor is not connected to a terminal, an :exc:`OSError` - is raised. + is thrown. :func:`shutil.get_terminal_size` is the high-level function which should normally be used, ``os.get_terminal_size`` is the low-level @@ -1225,9 +1435,9 @@ Availability: Unix, Windows. -.. class:: terminal_size - - A subclass of tuple, holding ``(columns, lines)`` of the terminal window size. +.. class:: terminal_size(tuple) + + A tuple of ``(columns, lines)`` for holding terminal window size. .. attribute:: columns @@ -1238,99 +1448,12 @@ Height of the terminal window in characters. -.. _fd_inheritance: - -Inheritance of File Descriptors -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.4 - -A file descriptor has an "inheritable" flag which indicates if the file descriptor -can be inherited by child processes. Since Python 3.4, file descriptors -created by Python are non-inheritable by default. - -On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. - -On Windows, non-inheritable handles and file descriptors are closed in child -processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout -and stderr), which are always inherited. Using :func:`spawn\* ` functions, -all inheritable handles and all inheritable file descriptors are inherited. -Using the :mod:`subprocess` module, all file descriptors except standard -streams are closed, and inheritable handles are only inherited if the -*close_fds* parameter is ``False``. - -.. function:: get_inheritable(fd) - - Get the "inheritable" flag of the specified file descriptor (a boolean). - -.. function:: set_inheritable(fd, inheritable) - - Set the "inheritable" flag of the specified file descriptor. - -.. function:: get_handle_inheritable(handle) - - Get the "inheritable" flag of the specified handle (a boolean). - - Availability: Windows. - -.. function:: set_handle_inheritable(handle, inheritable) - - Set the "inheritable" flag of the specified handle. - - Availability: Windows. - - .. _os-file-dir: Files and Directories --------------------- -On some Unix platforms, many of these functions support one or more of these -features: - -.. _path_fd: - -* **specifying a file descriptor:** - For some functions, the *path* argument can be not only a string giving a path - name, but also a file descriptor. The function will then operate on the file - referred to by the descriptor. (For POSIX systems, Python will call the - ``f...`` version of the function.) - - You can check whether or not *path* can be specified as a file descriptor on - your platform using :data:`os.supports_fd`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. - - If the function also supports *dir_fd* or *follow_symlinks* arguments, it is - an error to specify one of those when supplying *path* as a file descriptor. - -.. _dir_fd: - -* **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it - should be a file descriptor referring to a directory, and the path to operate - on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call - the ``...at`` or ``f...at`` version of the function.) - - You can check whether or not *dir_fd* is supported on your platform using - :data:`os.supports_dir_fd`. If it is unavailable, using it will raise a - :exc:`NotImplementedError`. - -.. _follow_symlinks: - -* **not following symlinks:** If *follow_symlinks* is - ``False``, and the last element of the path to operate on is a symbolic link, - the function will operate on the symbolic link itself instead of the file the - link points to. (For POSIX systems, Python will call the ``l...`` version of - the function.) - - You can check whether or not *follow_symlinks* is supported on your platform - using :data:`os.supports_follow_symlinks`. If it is unavailable, using it - will raise a :exc:`NotImplementedError`. - - - -.. function:: access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True) +.. function:: access(path, mode) Use the real uid/gid to test for access to *path*. Note that most operations will use the effective uid/gid, therefore this routine can be used in a @@ -1341,14 +1464,7 @@ :const:`False` if not. See the Unix man page :manpage:`access(2)` for more information. - This function can support specifying :ref:`paths relative to directory - descriptors ` and :ref:`not following symlinks `. - - If *effective_ids* is ``True``, :func:`access` will perform its access - checks using the effective uid/gid instead of the real uid/gid. - *effective_ids* may not be supported on your platform; you can check whether - or not it is available using :data:`os.supports_effective_ids`. If it is - unavailable, using it will raise a :exc:`NotImplementedError`. + Availability: Unix, Windows. .. note:: @@ -1379,18 +1495,29 @@ succeed, particularly for operations on network filesystems which may have permissions semantics beyond the usual POSIX permission-bit model. - .. versionchanged:: 3.3 - Added the *dir_fd*, *effective_ids*, and *follow_symlinks* parameters. - .. data:: F_OK - R_OK - W_OK - X_OK - - Values to pass as the *mode* parameter of :func:`access` to test the - existence, readability, writability and executability of *path*, - respectively. + + Value to pass as the *mode* parameter of :func:`access` to test the existence of + *path*. + + +.. data:: R_OK + + Value to include in the *mode* parameter of :func:`access` to test the + readability of *path*. + + +.. data:: W_OK + + Value to include in the *mode* parameter of :func:`access` to test the + writability of *path*. + + +.. data:: X_OK + + Value to include in the *mode* parameter of :func:`access` to determine if + *path* can be executed. .. function:: chdir(path) @@ -1399,15 +1526,33 @@ Change the current working directory to *path*. - This function can support :ref:`specifying a file descriptor `. The - descriptor must refer to an opened directory, not an open file. - - .. versionadded:: 3.3 - Added support for specifying *path* as a file descriptor - on some platforms. - - -.. function:: chflags(path, flags, *, follow_symlinks=True) + Availability: Unix, Windows. + + +.. function:: fchdir(fd) + + Change the current working directory to the directory represented by the file + descriptor *fd*. The descriptor must refer to an opened directory, not an open + file. + + Availability: Unix. + + +.. function:: getcwd() + + Return a string representing the current working directory. + + Availability: Unix, Windows. + + +.. function:: getcwdb() + + Return a bytestring representing the current working directory. + + Availability: Unix, Windows. + + +.. function:: chflags(path, flags) Set the flags of *path* to the numeric *flags*. *flags* may take a combination (bitwise OR) of the following values (as defined in the :mod:`stat` module): @@ -1425,15 +1570,16 @@ * :data:`stat.SF_NOUNLINK` * :data:`stat.SF_SNAPSHOT` - This function can support :ref:`not following symlinks `. - Availability: Unix. - .. versionadded:: 3.3 - The *follow_symlinks* argument. - - -.. function:: chmod(path, mode, *, dir_fd=None, follow_symlinks=True) + +.. function:: chroot(path) + + Change the root directory of the current process to *path*. Availability: + Unix. + + +.. function:: chmod(path, mode) Change the mode of *path* to the numeric *mode*. *mode* may take one of the following values (as defined in the :mod:`stat` module) or bitwise ORed @@ -1459,224 +1605,162 @@ * :data:`stat.S_IWOTH` * :data:`stat.S_IXOTH` - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. + Availability: Unix, Windows. .. note:: - Although Windows supports :func:`chmod`, you can only set the file's - read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` - constants or a corresponding integer value). All other bits are ignored. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor, - and the *dir_fd* and *follow_symlinks* arguments. - - -.. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. To - leave one of the ids unchanged, set it to -1. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. + Although Windows supports :func:`chmod`, you can only set the file's read-only + flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` + constants or a corresponding integer value). All other bits are + ignored. + + +.. function:: chown(path, uid, gid) + + Change the owner and group id of *path* to the numeric *uid* and *gid*. To leave + one of the ids unchanged, set it to -1. See :func:`shutil.chown` for a higher-level function that accepts names in addition to numeric ids. Availability: Unix. + +.. function:: getxattr(path, attr) + + Return the value of the extended filesystem attribute *attr* for + *path*. *attr* can be bytes or str. If it is str, it is encoded with the + filesystem encoding. + + Availability: Linux + .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, - and the *dir_fd* and *follow_symlinks* arguments. - - -.. function:: chroot(path) - - Change the root directory of the current process to *path*. + + +.. function:: lchflags(path, flags) + + Set the flags of *path* to the numeric *flags*, like :func:`chflags`, but do not + follow symbolic links. Availability: Unix. -.. function:: fchdir(fd) - - Change the current working directory to the directory represented by the file - descriptor *fd*. The descriptor must refer to an opened directory, not an - open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. +.. function:: lchmod(path, mode) + + Change the mode of *path* to the numeric *mode*. If path is a symlink, this + affects the symlink rather than the target. See the docs for :func:`chmod` + for possible values of *mode*. Availability: Unix. -.. function:: getcwd() - - Return a string representing the current working directory. - - -.. function:: getcwdb() - - Return a bytestring representing the current working directory. - - -.. function:: lchflags(path, flags) - - Set the flags of *path* to the numeric *flags*, like :func:`chflags`, but do - not follow symbolic links. As of Python 3.3, this is equivalent to - ``os.chflags(path, flags, follow_symlinks=False)``. +.. function:: lchown(path, uid, gid) + + Change the owner and group id of *path* to the numeric *uid* and *gid*. This + function will not follow symbolic links. Availability: Unix. -.. function:: lchmod(path, mode) - - Change the mode of *path* to the numeric *mode*. If path is a symlink, this - affects the symlink rather than the target. See the docs for :func:`chmod` - for possible values of *mode*. As of Python 3.3, this is equivalent to - ``os.chmod(path, mode, follow_symlinks=False)``. - - Availability: Unix. - - -.. function:: lchown(path, uid, gid) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. This - function will not follow symbolic links. As of Python 3.3, this is equivalent - to ``os.chown(path, uid, gid, follow_symlinks=False)``. - - Availability: Unix. - - -.. function:: link(src, dst, *, src_dir_fd=None, dst_dir_fd=None, follow_symlinks=True) - - Create a hard link pointing to *src* named *dst*. - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `, and :ref:`not - following symlinks `. +.. function:: lgetxattr(path, attr) + + This works exactly like :func:`getxattr` but doesn't follow symlinks. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: link(source, link_name) + + Create a hard link pointing to *source* named *link_name*. Availability: Unix, Windows. .. versionchanged:: 3.2 Added Windows support. - .. versionadded:: 3.3 - Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* arguments. - .. function:: listdir(path='.') Return a list containing the names of the entries in the directory given by - *path*. The list is in arbitrary order, and does not include the special + *path* (default: ``'.'``). The list is in arbitrary order. It does not include the special entries ``'.'`` and ``'..'`` even if they are present in the directory. - *path* may be either of type ``str`` or of type ``bytes``. If *path* - is of type ``bytes``, the filenames returned will also be of type ``bytes``; - in all other circumstances, they will be of type ``str``. - - This function can also support :ref:`specifying a file descriptor - `; the file descriptor must refer to a directory. - - .. note:: - To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. - - .. seealso:: - - The :func:`scandir` function returns directory entries along with - file attribute information, giving better performance for many - common use cases. + This function can be called with a bytes or string argument, and returns + filenames of the same datatype. + + Availability: Unix, Windows. .. versionchanged:: 3.2 The *path* parameter became optional. + +.. function:: listxattr(path) + + Return a list of the extended filesystem attributes on *path*. Attributes are + returned as string decoded with the filesystem encoding. + + Availability: Linux + .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. - - -.. function:: lstat(path, \*, dir_fd=None) + + +.. function:: llistxattr(path) + + This works exactly like :func:`listxattr` but doesn't follow symlinks. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: lremovexattr(path, attr) + + This works exactly like :func:`removexattr` but doesn't follow symlinks. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: lsetxattr(path, attr, value, flags=0) + + This works exactly like :func:`setxattr` but doesn't follow symlinks. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: lstat(path) Perform the equivalent of an :c:func:`lstat` system call on the given path. - Similar to :func:`~os.stat`, but does not follow symbolic links. Return a - :class:`stat_result` object. - - On platforms that do not support symbolic links, this is an alias for + Similar to :func:`~os.stat`, but does not follow symbolic links. On + platforms that do not support symbolic links, this is an alias for :func:`~os.stat`. - As of Python 3.3, this is equivalent to ``os.stat(path, dir_fd=dir_fd, - follow_symlinks=False)``. - - This function can also support :ref:`paths relative to directory descriptors - `. - - .. seealso:: - - The :func:`.stat` function. - .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionchanged:: 3.3 - Added the *dir_fd* parameter. - - -.. function:: mkdir(path, mode=0o777, *, dir_fd=None) - - Create a directory named *path* with numeric mode *mode*. - - On some systems, *mode* is ignored. Where it is used, the current umask - value is first masked out. If the directory already exists, :exc:`OSError` - is raised. - - This function can also support :ref:`paths relative to directory descriptors - `. - - It is also possible to create temporary directories; see the - :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. + +.. function:: lutimes(path[, times]) + + Like :func:`utime`, but if *path* is a symbolic link, it is not + dereferenced. *times* must be a 2-tuple of numbers, of the form + ``(atime, mtime)``, or None. + + + Availability: Unix. .. versionadded:: 3.3 - The *dir_fd* argument. - - -.. function:: makedirs(name, mode=0o777, exist_ok=False) - - .. index:: - single: directory; creating - single: UNC paths; and os.makedirs() - - Recursive directory creation function. Like :func:`mkdir`, but makes all - intermediate-level directories needed to contain the leaf directory. - - The default *mode* is ``0o777`` (octal). On some systems, *mode* is - ignored. Where it is used, the current umask value is first masked out. - - If *exist_ok* is ``False`` (the default), an :exc:`OSError` is raised if the - target directory already exists. - - .. note:: - - :func:`makedirs` will become confused if the path elements to create - include :data:`pardir` (eg. ".." on UNIX systems). - - This function handles UNC paths correctly. - - .. versionadded:: 3.2 - The *exist_ok* parameter. - - .. versionchanged:: 3.4.1 - - Before Python 3.4.1, if *exist_ok* was ``True`` and the directory existed, - :func:`makedirs` would still raise an error if *mode* did not match the - mode of the existing directory. Since this behavior was impossible to - implement safely, it was removed in Python 3.4.1. See :issue:`21082`. - - -.. function:: mkfifo(path, mode=0o666, *, dir_fd=None) - - Create a FIFO (a named pipe) named *path* with numeric mode *mode*. - The current umask value is first masked out from the mode. - - This function can also support :ref:`paths relative to directory descriptors - `. + + +.. function:: mkfifo(path[, mode]) + + Create a FIFO (a named pipe) named *path* with numeric mode *mode*. The + default *mode* is ``0o666`` (octal). The current umask value is first masked + out from the mode. FIFOs are pipes that can be accessed like regular files. FIFOs exist until they are deleted (for example with :func:`os.unlink`). Generally, FIFOs are used as @@ -1686,26 +1770,17 @@ Availability: Unix. - .. versionadded:: 3.3 - The *dir_fd* argument. - - -.. function:: mknod(path, mode=0o600, device=0, *, dir_fd=None) + +.. function:: mknod(filename[, mode=0o600[, device]]) Create a filesystem node (file, device special file or named pipe) named - *path*. *mode* specifies both the permissions to use and the type of node + *filename*. *mode* specifies both the permissions to use and the type of node to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, *device* defines the newly created device special file (probably using :func:`os.makedev`), otherwise it is ignored. - This function can also support :ref:`paths relative to directory descriptors - `. - - .. versionadded:: 3.3 - The *dir_fd* argument. - .. function:: major(device) @@ -1724,6 +1799,45 @@ Compose a raw device number from the major and minor device numbers. +.. function:: mkdir(path[, mode]) + + Create a directory named *path* with numeric mode *mode*. The default *mode* + is ``0o777`` (octal). On some systems, *mode* is ignored. Where it is used, + the current umask value is first masked out. If the directory already + exists, :exc:`OSError` is raised. + + It is also possible to create temporary directories; see the + :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. + + Availability: Unix, Windows. + + +.. function:: makedirs(path, mode=0o777, exist_ok=False) + + .. index:: + single: directory; creating + single: UNC paths; and os.makedirs() + + Recursive directory creation function. Like :func:`mkdir`, but makes all + intermediate-level directories needed to contain the leaf directory. If + the target directory with the same mode as specified already exists, + raises an :exc:`OSError` exception if *exist_ok* is False, otherwise no + exception is raised. If the directory cannot be created in other cases, + raises an :exc:`OSError` exception. The default *mode* is ``0o777`` (octal). + On some systems, *mode* is ignored. Where it is used, the current umask + value is first masked out. + + .. note:: + + :func:`makedirs` will become confused if the path elements to create + include :data:`pardir`. + + This function handles UNC paths correctly. + + .. versionadded:: 3.2 + The *exist_ok* parameter. + + .. function:: pathconf(path, name) Return system configuration information relevant to a named file. *name* @@ -1739,9 +1853,6 @@ included in ``pathconf_names``, an :exc:`OSError` is raised with :const:`errno.EINVAL` for the error number. - This function can support :ref:`specifying a file descriptor - `. - Availability: Unix. @@ -1749,53 +1860,40 @@ Dictionary mapping names accepted by :func:`pathconf` and :func:`fpathconf` to the integer values defined for those names by the host operating system. This - can be used to determine the set of names known to the system. - - Availability: Unix. - - -.. function:: readlink(path, *, dir_fd=None) + can be used to determine the set of names known to the system. Availability: + Unix. + + +.. function:: readlink(path) Return a string representing the path to which the symbolic link points. The - result may be either an absolute or relative pathname; if it is relative, it - may be converted to an absolute pathname using - ``os.path.join(os.path.dirname(path), result)``. + result may be either an absolute or relative pathname; if it is relative, it may + be converted to an absolute pathname using ``os.path.join(os.path.dirname(path), + result)``. If the *path* is a string object, the result will also be a string object, and the call may raise an UnicodeDecodeError. If the *path* is a bytes object, the result will be a bytes object. - This function can also support :ref:`paths relative to directory descriptors - `. - Availability: Unix, Windows .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - The *dir_fd* argument. - - -.. function:: remove(path, *, dir_fd=None) + +.. function:: remove(path) Remove (delete) the file *path*. If *path* is a directory, :exc:`OSError` is - raised. Use :func:`rmdir` to remove directories. - - This function can support :ref:`paths relative to directory descriptors - `. - - On Windows, attempting to remove a file that is in use causes an exception to - be raised; on Unix, the directory entry is removed but the storage allocated - to the file is not made available until the original file is no longer in use. - - This function is semantically identical to :func:`unlink`. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - -.. function:: removedirs(name) + raised; see :func:`rmdir` below to remove a directory. This is identical to + the :func:`unlink` function documented below. On Windows, attempting to + remove a file that is in use causes an exception to be raised; on Unix, the + directory entry is removed but the storage allocated to the file is not made + available until the original file is no longer in use. + + Availability: Unix, Windows. + + +.. function:: removedirs(path) .. index:: single: directory; deleting @@ -1809,7 +1907,18 @@ successfully removed. -.. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) +.. function:: removexattr(path, attr) + + Removes the extended filesystem attribute *attr* from *path*. *attr* should + be bytes or str. If it is a string, it is encoded with the filesystem + encoding. + + Availability: Linux + + .. versionadded:: 3.3 + + +.. function:: rename(src, dst) Rename the file or directory *src* to *dst*. If *dst* is a directory, :exc:`OSError` will be raised. On Unix, if *dst* exists and is a file, it will @@ -1819,13 +1928,9 @@ Windows, if *dst* already exists, :exc:`OSError` will be raised even if it is a file. - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. - If you want cross-platform overwriting of the destination, use :func:`replace`. - .. versionadded:: 3.3 - The *src_dir_fd* and *dst_dir_fd* arguments. + Availability: Unix, Windows. .. function:: renames(old, new) @@ -1841,7 +1946,7 @@ permissions needed to remove the leaf directory or file. -.. function:: replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None) +.. function:: replace(src, dst) Rename the file or directory *src* to *dst*. If *dst* is a directory, :exc:`OSError` will be raised. If *dst* exists and is a file, it will @@ -1849,309 +1954,97 @@ if *src* and *dst* are on different filesystems. If successful, the renaming will be an atomic operation (this is a POSIX requirement). - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. + Availability: Unix, Windows .. versionadded:: 3.3 -.. function:: rmdir(path, *, dir_fd=None) +.. function:: rmdir(path) Remove (delete) the directory *path*. Only works when the directory is empty, otherwise, :exc:`OSError` is raised. In order to remove whole directory trees, :func:`shutil.rmtree` can be used. - This function can support :ref:`paths relative to directory descriptors - `. + Availability: Unix, Windows. + + +.. data:: XATTR_SIZE_MAX + + The maximum size the value of an extended attribute can be. Currently, this + is 64 kilobytes on Linux. + + +.. data:: XATTR_CREATE + + This is a possible value for the flags argument in :func:`setxattr`. It + indicates the operation must create an attribute. + + +.. data:: XATTR_REPLACE + + This is a possible value for the flags argument in :func:`setxattr`. It + indicates the operation must replace an existing attribute. + + +.. function:: setxattr(path, attr, value, flags=0) + + Set the extended filesystem attribute *attr* on *path* to *value*. *attr* + must be a bytes or str with no embedded NULs. If it is str, it is encoded + with the filesystem encoding. *flags* may be :data:`XATTR_REPLACE` or + :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is given and the attribute + does not exist, ``EEXISTS`` will be raised. If :data:`XATTR_CREATE` is given + and the attribute already exists, the attribute will not be created and + ``ENODATA`` will be raised. + + Availability: Linux + + .. note:: + + A bug in Linux kernel versions less than 2.6.39 caused the flags argument + to be ignored on some filesystems. .. versionadded:: 3.3 - The *dir_fd* parameter. - - -.. function:: scandir(path='.') - - Return an iterator of :class:`DirEntry` objects corresponding to the entries - in the directory given by *path*. The entries are yielded in arbitrary - order, and the special entries ``'.'`` and ``'..'`` are not included. - - Using :func:`scandir` instead of :func:`listdir` can significantly - increase the performance of code that also needs file type or file - attribute information, because :class:`DirEntry` objects expose this - information if the operating system provides it when scanning a directory. - All :class:`DirEntry` methods may perform a system call, but - :func:`~DirEntry.is_dir` and :func:`~DirEntry.is_file` usually only - require a system call for symbolic links; :func:`DirEntry.stat` - always requires a system call on Unix but only requires one for - symbolic links on Windows. - - On Unix, *path* can be of type :class:`str` or :class:`bytes` (use - :func:`~os.fsencode` and :func:`~os.fsdecode` to encode and decode - :class:`bytes` paths). On Windows, *path* must be of type :class:`str`. - On both sytems, the type of the :attr:`~DirEntry.name` and - :attr:`~DirEntry.path` attributes of each :class:`DirEntry` will be of - the same type as *path*. - - The following example shows a simple use of :func:`scandir` to display all - the files (excluding directories) in the given *path* that don't start with - ``'.'``. The ``entry.is_file()`` call will generally not make an additional - system call:: - - for entry in os.scandir(path): - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) - - .. note:: - - On Unix-based systems, :func:`scandir` uses the system's - `opendir() `_ - and - `readdir() `_ - functions. On Windows, it uses the Win32 - `FindFirstFileW `_ - and - `FindNextFileW `_ - functions. - - .. versionadded:: 3.5 - - -.. class:: DirEntry - - Object yielded by :func:`scandir` to expose the file path and other file - attributes of a directory entry. - - :func:`scandir` will provide as much of this information as possible without - making additional system calls. When a ``stat()`` or ``lstat()`` system call - is made, the ``DirEntry`` object will cache the result. - - ``DirEntry`` instances are not intended to be stored in long-lived data - structures; if you know the file metadata has changed or if a long time has - elapsed since calling :func:`scandir`, call ``os.stat(entry.path)`` to fetch - up-to-date information. - - Because the ``DirEntry`` methods can make operating system calls, they may - also raise :exc:`OSError`. If you need very fine-grained - control over errors, you can catch :exc:`OSError` when calling one of the - ``DirEntry`` methods and handle as appropriate. - - Attributes and methods on a ``DirEntry`` instance are as follows: - - .. attribute:: name - - The entry's base filename, relative to the :func:`scandir` *path* - argument. - - The :attr:`name` attribute will be of the same type (``str`` or - ``bytes``) as the :func:`scandir` *path* argument. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. attribute:: path - - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. - - The :attr:`path` attribute will be of the same type (``str`` or - ``bytes``) as the :func:`scandir` *path* argument. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. method:: inode() - - Return the inode number of the entry. - - The result is cached on the ``DirEntry`` object, use ``os.stat(entry.path, - follow_symlinks=False).st_ino`` to fetch up-to-date information. - - On Unix, no system call is required. - - .. method:: is_dir(\*, follow_symlinks=True) - - If *follow_symlinks* is ``True`` (the default), return ``True`` if the - entry is a directory or a symbolic link pointing to a directory; - return ``False`` if it is or points to any other kind of file, or if it - doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a directory; return ``False`` if it is any other kind of file - or if it doesn't exist anymore. - - The result is cached on the ``DirEntry`` object. Call :func:`os.stat` - along with :func:`stat.S_ISDIR` to fetch up-to-date information. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - In most cases, no system call is required. - - .. method:: is_file(\*, follow_symlinks=True) - - If *follow_symlinks* is ``True`` (the default), return ``True`` if the - entry is a file or a symbolic link pointing to a file; return ``False`` - if it is or points to a directory or other non-file entry, or if it - doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a file; return ``False`` if it is a directory or other non-file entry, - or if it doesn't exist anymore. - - The result is cached on the ``DirEntry`` object. Call :func:`os.stat` - along with :func:`stat.S_ISREG` to fetch up-to-date information. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - In most cases, no system call is required. - - .. method:: is_symlink() - - Return ``True`` if this entry is a symbolic link (even if broken); - return ``False`` if it points to a directory or any kind of file, - or if it doesn't exist anymore. - - The result is cached on the ``DirEntry`` object. Call - :func:`os.path.islink` to fetch up-to-date information. - - The method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - In most cases, no system call is required. - - .. method:: stat(\*, follow_symlinks=True) - - Return a :class:`stat_result` object for this entry. This method - follows symbolic links by default; to stat a symbolic link add the - ``follow_symlinks=False`` argument. - - On Unix, this method always requires a system call. On Windows, - ``DirEntry.stat()`` requires a system call only if the - entry is a symbolic link, and ``DirEntry.stat(follow_symlinks=False)`` - never requires a system call. - - On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the - :class:`stat_result` are always set to zero. Call :func:`os.stat` to - get these attributes. - - The result is cached on the ``DirEntry`` object. Call :func:`os.stat` - to fetch up-to-date information. - - Note that there is a nice correspondence between several attributes - and methods of ``DirEntry`` and of :class:`pathlib.Path`. In - particular, the ``name`` and ``path`` attributes have the same - meaning, as do the ``is_dir()``, ``is_file()``, ``is_symlink()`` - and ``stat()`` methods. - - .. versionadded:: 3.5 - - -.. function:: stat(path, \*, dir_fd=None, follow_symlinks=True) - - Get the status of a file or a file descriptor. Perform the equivalent of a - :c:func:`stat` system call on the given path. *path* may be specified as - either a string or as an open file descriptor. Return a :class:`stat_result` - object. - - This function normally follows symlinks; to stat a symlink add the argument - ``follow_symlinks=False``, or use :func:`lstat`. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. index:: module: stat - - Example:: - - >>> import os - >>> statinfo = os.stat('somefile.txt') - >>> statinfo - os.stat_result(st_mode=33188, st_ino=7876932, st_dev=234881026, - st_nlink=1, st_uid=501, st_gid=501, st_size=264, st_atime=1297230295, - st_mtime=1297230027, st_ctime=1297230027) - >>> statinfo.st_size - 264 - - .. seealso:: - - :func:`fstat` and :func:`lstat` functions. - - .. versionadded:: 3.3 - Added the *dir_fd* and *follow_symlinks* arguments, specifying a file - descriptor instead of a path. - - -.. class:: stat_result - - Object whose attributes correspond roughly to the members of the - :c:type:`stat` structure. It is used for the result of :func:`os.stat`, - :func:`os.fstat` and :func:`os.lstat`. - - Attributes: - - .. attribute:: st_mode - - File mode: file type and file mode bits (permissions). - - .. attribute:: st_ino - - Inode number. - - .. attribute:: st_dev - - Identifier of the device on which this file resides. - - .. attribute:: st_nlink - - Number of hard links. - - .. attribute:: st_uid - - User identifier of the file owner. - - .. attribute:: st_gid - - Group identifier of the file owner. - - .. attribute:: st_size - - Size of the file in bytes, if it is a regular file or a symbolic link. - The size of a symbolic link is the length of the pathname it contains, - without a terminating null byte. - - Timestamps: - - .. attribute:: st_atime - - Time of most recent access expressed in seconds. - - .. attribute:: st_mtime - - Time of most recent content modification expressed in seconds. - - .. attribute:: st_ctime - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in seconds. - - .. attribute:: st_atime_ns - - Time of most recent access expressed in nanoseconds as an integer. - - .. attribute:: st_mtime_ns - - Time of most recent content modification expressed in nanoseconds as an - integer. - - .. attribute:: st_ctime_ns - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in nanoseconds as an - integer. - - See also the :func:`stat_float_times` function. + + +.. function:: stat(path) + + Perform the equivalent of a :c:func:`stat` system call on the given path. + (This function follows symlinks; to stat a symlink use :func:`lstat`.) + + The return value is an object whose attributes correspond to the members + of the :c:type:`stat` structure, namely: + + * :attr:`st_mode` - protection bits, + * :attr:`st_ino` - inode number, + * :attr:`st_dev` - device, + * :attr:`st_nlink` - number of hard links, + * :attr:`st_uid` - user id of owner, + * :attr:`st_gid` - group id of owner, + * :attr:`st_size` - size of file, in bytes, + * :attr:`st_atime` - time of most recent access, + * :attr:`st_mtime` - time of most recent content modification, + * :attr:`st_ctime` - platform dependent; time of most recent metadata change on + Unix, or the time of creation on Windows) + + On some Unix systems (such as Linux), the following attributes may also be + available: + + * :attr:`st_blocks` - number of blocks allocated for file + * :attr:`st_blksize` - filesystem blocksize + * :attr:`st_rdev` - type of device if an inode device + * :attr:`st_flags` - user defined flags for file + + On other Unix systems (such as FreeBSD), the following attributes may be + available (but may be only filled out if root tries to use them): + + * :attr:`st_gen` - file generation number + * :attr:`st_birthtime` - time of file creation + + On Mac OS systems, the following attributes may also be available: + + * :attr:`st_rsize` + * :attr:`st_creator` + * :attr:`st_type` .. note:: @@ -2162,89 +2055,31 @@ :attr:`st_atime` has only 1-day resolution. See your operating system documentation for details. - Similarly, although :attr:`st_atime_ns`, :attr:`st_mtime_ns`, - and :attr:`st_ctime_ns` are always expressed in nanoseconds, many - systems do not provide nanosecond precision. On systems that do - provide nanosecond precision, the floating-point object used to - store :attr:`st_atime`, :attr:`st_mtime`, and :attr:`st_ctime` - cannot preserve all of it, and as such will be slightly inexact. - If you need the exact timestamps you should always use - :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and :attr:`st_ctime_ns`. - - On some Unix systems (such as Linux), the following attributes may also be - available: - - .. attribute:: st_blocks - - Number of 512-byte blocks allocated for file. - This may be smaller than :attr:`st_size`/512 when the file has holes. - - .. attribute:: st_blksize - - "Preferred" blocksize for efficient file system I/O. Writing to a file in - smaller chunks may cause an inefficient read-modify-rewrite. - - .. attribute:: st_rdev - - Type of device if an inode device. - - .. attribute:: st_flags - - User defined flags for file. - - On other Unix systems (such as FreeBSD), the following attributes may be - available (but may be only filled out if root tries to use them): - - .. attribute:: st_gen - - File generation number. - - .. attribute:: st_birthtime - - Time of file creation. - - On Mac OS systems, the following attributes may also be available: - - .. attribute:: st_rsize - - Real size of the file. - - .. attribute:: st_creator - - Creator of the file. - - .. attribute:: st_type - - File type. - - On Windows systems, the following attribute is also available: - - .. attribute:: st_file_attributes - - Windows file attributes: ``dwFileAttributes`` member of the - ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` - constants in the :mod:`stat` module. - - The standard module :mod:`stat` defines functions and constants that are - useful for extracting information from a :c:type:`stat` structure. (On - Windows, some items are filled with dummy values.) - - For backward compatibility, a :class:`stat_result` instance is also - accessible as a tuple of at least 10 integers giving the most important (and - portable) members of the :c:type:`stat` structure, in the order - :attr:`st_mode`, :attr:`st_ino`, :attr:`st_dev`, :attr:`st_nlink`, - :attr:`st_uid`, :attr:`st_gid`, :attr:`st_size`, :attr:`st_atime`, - :attr:`st_mtime`, :attr:`st_ctime`. More items may be added at the end by - some implementations. For compatibility with older Python versions, - accessing :class:`stat_result` as a tuple always returns integers. - - .. versionadded:: 3.3 - Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and - :attr:`st_ctime_ns` members. - - .. versionadded:: 3.5 - Added the :attr:`st_file_attributes` member on Windows. + For backward compatibility, the return value of :func:`~os.stat` is also accessible + as a tuple of at least 10 integers giving the most important (and portable) + members of the :c:type:`stat` structure, in the order :attr:`st_mode`, + :attr:`st_ino`, :attr:`st_dev`, :attr:`st_nlink`, :attr:`st_uid`, + :attr:`st_gid`, :attr:`st_size`, :attr:`st_atime`, :attr:`st_mtime`, + :attr:`st_ctime`. More items may be added at the end by some implementations. + + .. index:: module: stat + + The standard module :mod:`stat` defines functions and constants that are useful + for extracting information from a :c:type:`stat` structure. (On Windows, some + items are filled with dummy values.) + + Example:: + + >>> import os + >>> statinfo = os.stat('somefile.txt') + >>> statinfo + posix.stat_result(st_mode=33188, st_ino=7876932, st_dev=234881026, + st_nlink=1, st_uid=501, st_gid=501, st_size=264, st_atime=1297230295, + st_mtime=1297230027, st_ctime=1297230027) + >>> statinfo.st_size + 264 + + Availability: Unix, Windows. .. function:: stat_float_times([newvalue]) @@ -2271,8 +2106,6 @@ are processed, this application should turn the feature off until the library has been corrected. - .. deprecated:: 3.3 - .. function:: statvfs(path) @@ -2288,138 +2121,36 @@ read-only, and if :const:`ST_NOSUID` is set, the semantics of setuid/setgid bits are disabled or not supported. - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). - - This function can support :ref:`specifying a file descriptor `. - .. versionchanged:: 3.2 The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - Availability: Unix. - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*. - - -.. data:: supports_dir_fd - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *dir_fd* parameter. Different platforms - provide different functionality, and an option that might work on one might - be unsupported on another. For consistency's sakes, functions that support - *dir_fd* always allow specifying the parameter, but will raise an exception - if the functionality is not actually available. - - To check whether a particular function permits use of its *dir_fd* - parameter, use the ``in`` operator on ``supports_dir_fd``. As an example, - this expression determines whether the *dir_fd* parameter of :func:`os.stat` - is locally available:: - - os.stat in os.supports_dir_fd - - Currently *dir_fd* parameters only work on Unix platforms; none of them work - on Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_effective_ids - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of the *effective_ids* parameter for - :func:`os.access`. If the local platform supports it, the collection will - contain :func:`os.access`, otherwise it will be empty. - - To check whether you can use the *effective_ids* parameter for - :func:`os.access`, use the ``in`` operator on ``supports_effective_ids``, - like so:: - - os.access in os.supports_effective_ids - - Currently *effective_ids* only works on Unix platforms; it does not work on - Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_fd - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file - descriptor. Different platforms provide different functionality, and an - option that might work on one might be unsupported on another. For - consistency's sakes, functions that support *fd* always allow specifying - the parameter, but will raise an exception if the functionality is not - actually available. - - To check whether a particular function permits specifying an open file - descriptor for its *path* parameter, use the ``in`` operator on - ``supports_fd``. As an example, this expression determines whether - :func:`os.chdir` accepts open file descriptors when called on your local - platform:: - - os.chdir in os.supports_fd - - .. versionadded:: 3.3 - - -.. data:: supports_follow_symlinks - - A :class:`~collections.abc.Set` object indicating which functions in the - :mod:`os` module permit use of their *follow_symlinks* parameter. Different - platforms provide different functionality, and an option that might work on - one might be unsupported on another. For consistency's sakes, functions that - support *follow_symlinks* always allow specifying the parameter, but will - raise an exception if the functionality is not actually available. - - To check whether a particular function permits use of its *follow_symlinks* - parameter, use the ``in`` operator on ``supports_follow_symlinks``. As an - example, this expression determines whether the *follow_symlinks* parameter - of :func:`os.stat` is locally available:: - - os.stat in os.supports_follow_symlinks - - .. versionadded:: 3.3 - - -.. function:: symlink(src, dst, target_is_directory=False, *, dir_fd=None) - - Create a symbolic link pointing to *src* named *dst*. - - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Window platforms, *target_is_directory* is ignored. + +.. function:: symlink(source, link_name) + symlink(source, link_name, target_is_directory=False) + + Create a symbolic link pointing to *source* named *link_name*. + + On Windows, symlink version takes an additional optional parameter, + *target_is_directory*, which defaults to ``False``. + + On Windows, a symlink represents a file or a directory, and does not morph to + the target dynamically. If *target_is_directory* is set to ``True``, the + symlink will be created as a directory symlink, otherwise as a file symlink + (the default). Symbolic link support was introduced in Windows 6.0 (Vista). :func:`symlink` will raise a :exc:`NotImplementedError` on Windows versions earlier than 6.0. - This function can support :ref:`paths relative to directory descriptors - `. - .. note:: - On Windows, the *SeCreateSymbolicLinkPrivilege* is required in order to - successfully create symlinks. This privilege is not typically granted to - regular users but is available to accounts which can escalate privileges - to the administrator level. Either obtaining the privilege or running your + The *SeCreateSymbolicLinkPrivilege* is required in order to successfully + create symlinks. This privilege is not typically granted to regular + users but is available to accounts which can escalate privileges to the + administrator level. Either obtaining the privilege or running your application as an administrator are ways to successfully create symlinks. - :exc:`OSError` is raised when the function is called by an unprivileged user. @@ -2428,10 +2159,6 @@ .. versionchanged:: 3.2 Added support for Windows 6.0 (Vista) symbolic links. - .. versionadded:: 3.3 - Added the *dir_fd* argument, and now allow *target_is_directory* - on non-Windows platforms. - .. function:: sync() @@ -2447,61 +2174,35 @@ Truncate the file corresponding to *path*, so that it is at most *length* bytes in size. - This function can support :ref:`specifying a file descriptor `. + Availability: Unix. + + .. versionadded:: 3.3 + + +.. function:: unlink(path) + + Remove (delete) the file *path*. This is the same function as + :func:`remove`; the :func:`unlink` name is its traditional Unix + name. Availability: Unix, Windows. - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - Added support for Windows - -.. function:: unlink(path, *, dir_fd=None) - - Remove (delete) the file *path*. This function is semantically - identical to :func:`remove`; the ``unlink`` name is its - traditional Unix name. Please see the documentation for - :func:`remove` for further information. - - .. versionadded:: 3.3 - The *dir_fd* parameter. - - -.. function:: utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True) - - Set the access and modified times of the file specified by *path*. - - :func:`utime` takes two optional parameters, *times* and *ns*. - These specify the times set on *path* and are used as follows: - - - If *ns* is specified, - it must be a 2-tuple of the form ``(atime_ns, mtime_ns)`` - where each member is an int expressing nanoseconds. - - If *times* is not ``None``, - it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. - - If *times* is ``None`` and *ns* is unspecified, - this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` - where both times are the current time. - - It is an error to specify tuples for both *times* and *ns*. - - Whether a directory can be given for *path* + +.. function:: utime(path[, times]) + + Set the access and modified times of the file specified by *path*. If *times* + is ``None`` or not specified, then the file's access and modified times are + set to the current time. (The effect is similar to running the Unix program + :program:`touch` on the path.) Otherwise, *times* must be a 2-tuple of + numbers, of the form ``(atime, mtime)`` which is used to set the access and + modified times, respectively. Whether a directory can be given for *path* depends on whether the operating system implements directories as files (for example, Windows does not). Note that the exact times you set here may not be returned by a subsequent :func:`~os.stat` call, depending on the resolution with which your operating system records access and modification - times; see :func:`~os.stat`. The best way to preserve exact times is to - use the *st_atime_ns* and *st_mtime_ns* fields from the :func:`os.stat` - result object with the *ns* parameter to `utime`. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path*, - and the *dir_fd*, *follow_symlinks*, and *ns* parameters. + times; see :func:`~os.stat`. + + Availability: Unix, Windows. .. function:: walk(top, topdown=True, onerror=None, followlinks=False) @@ -2524,20 +2225,18 @@ If optional argument *topdown* is ``True`` or not specified, the triple for a directory is generated before the triples for any of its subdirectories - (directories are generated top-down). If *topdown* is ``False``, the triple - for a directory is generated after the triples for all of its subdirectories - (directories are generated bottom-up). No matter the value of *topdown*, the - list of subdirectories is retrieved before the tuples for the directory and - its subdirectories are generated. + (directories are generated top-down). If *topdown* is ``False``, the triple for a + directory is generated after the triples for all of its subdirectories + (directories are generated bottom-up). When *topdown* is ``True``, the caller can modify the *dirnames* list in-place (perhaps using :keyword:`del` or slice assignment), and :func:`walk` will only recurse into the subdirectories whose names remain in *dirnames*; this can be used to prune the search, impose a specific order of visiting, or even to inform :func:`walk` about directories the caller creates or renames before it resumes - :func:`walk` again. Modifying *dirnames* when *topdown* is ``False`` has - no effect on the behavior of the walk, because in bottom-up mode the directories - in *dirnames* are generated before *dirpath* itself is generated. + :func:`walk` again. Modifying *dirnames* when *topdown* is ``False`` is + ineffective, because in bottom-up mode the directories in *dirnames* are + generated before *dirpath* itself is generated. By default, errors from the :func:`listdir` call are ignored. If optional argument *onerror* is specified, it should be a function; it will be called with @@ -2551,9 +2250,9 @@ .. note:: - Be aware that setting *followlinks* to ``True`` can lead to infinite - recursion if a link points to a parent directory of itself. :func:`walk` - does not keep track of the directories it visited already. + Be aware that setting *followlinks* to ``True`` can lead to infinite recursion if a + link points to a parent directory of itself. :func:`walk` does not keep track of + the directories it visited already. .. note:: @@ -2574,9 +2273,8 @@ if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories - In the next example (simple implementation of :func:`shutil.rmtree`), - walking the tree bottom-up is essential, :func:`rmdir` doesn't allow - deleting a directory before the directory is empty:: + In the next example, walking the tree bottom-up is essential: :func:`rmdir` + doesn't allow deleting a directory before the directory is empty:: # Delete everything reachable from the directory named in "top", # assuming there are no symbolic links. @@ -2589,28 +2287,19 @@ for name in dirs: os.rmdir(os.path.join(root, name)) - .. versionchanged:: 3.5 - This function now calls :func:`os.scandir` instead of :func:`os.listdir`, - making it faster by reducing the number of calls to :func:`os.stat`. - - -.. function:: fwalk(top='.', topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None) + +.. function:: fwalk(top, topdown=True, onerror=None, followlinks=False) .. index:: single: directory; walking single: directory; traversal This behaves exactly like :func:`walk`, except that it yields a 4-tuple - ``(dirpath, dirnames, filenames, dirfd)``, and it supports ``dir_fd``. + ``(dirpath, dirnames, filenames, dirfd)``. *dirpath*, *dirnames* and *filenames* are identical to :func:`walk` output, and *dirfd* is a file descriptor referring to the directory *dirpath*. - This function always supports :ref:`paths relative to directory descriptors - ` and :ref:`not following symlinks `. Note however - that, unlike other functions, the :func:`fwalk` default value for - *follow_symlinks* is ``False``. - .. note:: Since :func:`fwalk` yields file descriptors, those are only valid until @@ -2624,14 +2313,14 @@ import os for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): print(root, "consumes", end="") - print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), + print(sum([os.fstatat(rootfd, name).st_size for name in files]), end="") print("bytes in", len(files), "non-directory files") if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories In the next example, walking the tree bottom-up is essential: - :func:`rmdir` doesn't allow deleting a directory before the directory is + :func:`unlinkat` doesn't allow deleting a directory before the directory is empty:: # Delete everything reachable from the directory named in "top", @@ -2641,90 +2330,15 @@ import os for root, dirs, files, rootfd in os.fwalk(top, topdown=False): for name in files: - os.unlink(name, dir_fd=rootfd) + os.unlinkat(rootfd, name) for name in dirs: - os.rmdir(name, dir_fd=rootfd) + os.unlinkat(rootfd, name, os.AT_REMOVEDIR) Availability: Unix. .. versionadded:: 3.3 -Linux extended attributes -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.3 - -These functions are all available on Linux only. - -.. function:: getxattr(path, attribute, *, follow_symlinks=True) - - Return the value of the extended filesystem attribute *attribute* for - *path*. *attribute* can be bytes or str. If it is str, it is encoded - with the filesystem encoding. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - -.. function:: listxattr(path=None, *, follow_symlinks=True) - - Return a list of the extended filesystem attributes on *path*. The - attributes in the list are represented as strings decoded with the filesystem - encoding. If *path* is ``None``, :func:`listxattr` will examine the current - directory. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - -.. function:: removexattr(path, attribute, *, follow_symlinks=True) - - Removes the extended filesystem attribute *attribute* from *path*. - *attribute* should be bytes or str. If it is a string, it is encoded - with the filesystem encoding. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - -.. function:: setxattr(path, attribute, value, flags=0, *, follow_symlinks=True) - - Set the extended filesystem attribute *attribute* on *path* to *value*. - *attribute* must be a bytes or str with no embedded NULs. If it is a str, - it is encoded with the filesystem encoding. *flags* may be - :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is - given and the attribute does not exist, ``EEXISTS`` will be raised. - If :data:`XATTR_CREATE` is given and the attribute already exists, the - attribute will not be created and ``ENODATA`` will be raised. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. note:: - - A bug in Linux kernel versions less than 2.6.39 caused the flags argument - to be ignored on some filesystems. - - -.. data:: XATTR_SIZE_MAX - - The maximum size the value of an extended attribute can be. Currently, this - is 64 KiB on Linux. - - -.. data:: XATTR_CREATE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must create an attribute. - - -.. data:: XATTR_REPLACE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must replace an existing attribute. - - .. _os-process: Process Management @@ -2732,7 +2346,7 @@ These functions may be used to create and manage processes. -The various :func:`exec\* ` functions take a list of arguments for the new +The various :func:`exec\*` functions take a list of arguments for the new program loaded into the process. In each case, the first of these arguments is passed to the new program as its own name rather than as an argument a user may have typed on a command line. For the C programmer, this is the ``argv[0]`` @@ -2749,6 +2363,8 @@ Python signal handler registered for :const:`SIGABRT` with :func:`signal.signal`. + Availability: Unix, Windows. + .. function:: execl(path, arg0, arg1, ...) execle(path, arg0, arg1, ..., env) @@ -2768,9 +2384,9 @@ descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using :func:`sys.stdout.flush` or :func:`os.fsync` before calling an - :func:`exec\* ` function. - - The "l" and "v" variants of the :func:`exec\* ` functions differ in how + :func:`exec\*` function. + + The "l" and "v" variants of the :func:`exec\*` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the individual parameters simply become additional parameters to the :func:`execl\*` @@ -2782,7 +2398,7 @@ The variants which include a "p" near the end (:func:`execlp`, :func:`execlpe`, :func:`execvp`, and :func:`execvpe`) will use the :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`exec\*e ` variants, + environment is being replaced (using one of the :func:`exec\*e` variants, discussed in the next paragraph), the new environment is used as the source of the :envvar:`PATH` variable. The other variants, :func:`execl`, :func:`execle`, :func:`execv`, and :func:`execve`, will not use the :envvar:`PATH` variable to @@ -2796,22 +2412,16 @@ :func:`execlp`, :func:`execv`, and :func:`execvp` all cause the new process to inherit the environment of the current process. - For :func:`execve` on some platforms, *path* may also be specified as an open - file descriptor. This functionality may not be supported on your platform; - you can check whether or not it is available using :data:`os.supports_fd`. - If it is unavailable, using it will raise a :exc:`NotImplementedError`. - Availability: Unix, Windows. - .. versionadded:: 3.3 - Added support for specifying an open file descriptor for *path* - for :func:`execve`. .. function:: _exit(n) Exit the process with status *n*, without calling cleanup handlers, flushing stdio buffers, etc. + Availability: Unix, Windows. + .. note:: The standard way to exit is ``sys.exit(n)``. :func:`_exit` should @@ -2959,13 +2569,9 @@ Fork a child process. Return ``0`` in the child and the child's process id in the parent. If an error occurs :exc:`OSError` is raised. - Note that some platforms including FreeBSD <= 6.3 and Cygwin have + Note that some platforms including FreeBSD <= 6.3, Cygwin and OS/2 EMX have known issues when using fork() from a thread. - .. warning:: - - See :mod:`ssl` for applications that use the SSL module with fork(). - Availability: Unix. @@ -3029,28 +2635,11 @@ Availability: Unix. -.. function:: popen(cmd, mode='r', buffering=-1) - - Open a pipe to or from command *cmd*. - The return value is an open file object - connected to the pipe, which can be read or written depending on whether *mode* - is ``'r'`` (default) or ``'w'``. The *buffering* argument has the same meaning as - the corresponding argument to the built-in :func:`open` function. The - returned file object reads or writes text strings rather than bytes. - - The ``close`` method returns :const:`None` if the subprocess exited - successfully, or the subprocess's return code if there was an - error. On POSIX systems, if the return code is positive it - represents the return value of the process left-shifted by one - byte. If the return code is negative, the process was terminated - by the signal given by the negated value of the return code. (For - example, the return value might be ``- signal.SIGKILL`` if the - subprocess was killed.) On Windows systems, the return value - contains the signed integer return code from the child process. - - This is implemented using :class:`subprocess.Popen`; see that class's - documentation for more powerful ways to manage and communicate with - subprocesses. +.. function:: popen(...) + :noindex: + + Run child processes, returning opened pipes for communications. These functions + are described in section :ref:`os-newstreams`. .. function:: spawnl(mode, path, ...) @@ -3075,7 +2664,7 @@ process. On Windows, the process id will actually be the process handle, so can be used with the :func:`waitpid` function. - The "l" and "v" variants of the :func:`spawn\* ` functions differ in how + The "l" and "v" variants of the :func:`spawn\*` functions differ in how command-line arguments are passed. The "l" variants are perhaps the easiest to work with if the number of parameters is fixed when the code is written; the individual parameters simply become additional parameters to the @@ -3087,7 +2676,7 @@ The variants which include a second "p" near the end (:func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp`, and :func:`spawnvpe`) will use the :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`spawn\*e ` variants, + environment is being replaced (using one of the :func:`spawn\*e` variants, discussed in the next paragraph), the new environment is used as the source of the :envvar:`PATH` variable. The other variants, :func:`spawnl`, :func:`spawnle`, :func:`spawnv`, and :func:`spawnve`, will not use the @@ -3121,7 +2710,7 @@ .. data:: P_NOWAIT P_NOWAITO - Possible values for the *mode* parameter to the :func:`spawn\* ` family of + Possible values for the *mode* parameter to the :func:`spawn\*` family of functions. If either of these values is given, the :func:`spawn\*` functions will return as soon as the new process has been created, with the process id as the return value. @@ -3131,7 +2720,7 @@ .. data:: P_WAIT - Possible value for the *mode* parameter to the :func:`spawn\* ` family of + Possible value for the *mode* parameter to the :func:`spawn\*` family of functions. If this is given as *mode*, the :func:`spawn\*` functions will not return until the new process has run to completion and will return the exit code of the process the run is successful, or ``-signal`` if a signal kills the @@ -3143,11 +2732,11 @@ .. data:: P_DETACH P_OVERLAY - Possible values for the *mode* parameter to the :func:`spawn\* ` family of + Possible values for the *mode* parameter to the :func:`spawn\*` family of functions. These are less portable than those listed above. :const:`P_DETACH` is similar to :const:`P_NOWAIT`, but the new process is detached from the console of the calling process. If :const:`P_OVERLAY` is used, the current - process will be replaced; the :func:`spawn\* ` function will not return. + process will be replaced; the :func:`spawn\*` function will not return. Availability: Windows. @@ -3174,10 +2763,6 @@ doesn't work if it is. Use the :func:`os.path.normpath` function to ensure that the path is properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` - function is not resolved until this function is first called. If the function - cannot be resolved, :exc:`NotImplementedError` will be raised. - Availability: Windows. @@ -3210,29 +2795,14 @@ .. function:: times() - Returns the current global process times. - The return value is an object with five attributes: - - * :attr:`user` - user time - * :attr:`system` - system time - * :attr:`children_user` - user time of all child processes - * :attr:`children_system` - system time of all child processes - * :attr:`elapsed` - elapsed real time since a fixed point in the past - - For backwards compatibility, this object also behaves like a five-tuple - containing :attr:`user`, :attr:`system`, :attr:`children_user`, - :attr:`children_system`, and :attr:`elapsed` in that order. - - See the Unix manual page + Return a 5-tuple of floating point numbers indicating accumulated (processor + or other) times, in seconds. The items are: user time, system time, + children's user time, children's system time, and elapsed real time since a + fixed point in the past, in that order. See the Unix manual page :manpage:`times(2)` or the corresponding Windows Platform API documentation. - On Windows, only :attr:`user` and :attr:`system` are known; the other - attributes are zero. - - Availability: Unix, Windows. - - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. + On Windows, only the first two items are filled, the others are zero. + + Availability: Unix, Windows .. function:: wait() @@ -3322,23 +2892,17 @@ (shifting makes cross-platform use of the function easier). A *pid* less than or equal to ``0`` has no special meaning on Windows, and raises an exception. The value of integer *options* has no effect. *pid* can refer to any process whose - id is known, not necessarily a child process. The :func:`spawn\* ` - functions called with :const:`P_NOWAIT` return suitable process handles. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: wait3(options) + id is known, not necessarily a child process. The :func:`spawn` functions called + with :const:`P_NOWAIT` return suitable process handles. + + +.. function:: wait3([options]) Similar to :func:`waitpid`, except no process id argument is given and a 3-element tuple containing the child's process id, exit status indication, and resource usage information is returned. Refer to :mod:`resource`.\ - :func:`~resource.getrusage` for details on resource usage information. The - option argument is the same as that provided to :func:`waitpid` and - :func:`wait4`. + :func:`getrusage` for details on resource usage information. The option + argument is the same as that provided to :func:`waitpid` and :func:`wait4`. Availability: Unix. @@ -3347,9 +2911,9 @@ Similar to :func:`waitpid`, except a 3-element tuple, containing the child's process id, exit status indication, and resource usage information is returned. - Refer to :mod:`resource`.\ :func:`~resource.getrusage` for details on - resource usage information. The arguments to :func:`wait4` are the same - as those provided to :func:`waitpid`. + Refer to :mod:`resource`.\ :func:`getrusage` for details on resource usage + information. The arguments to :func:`wait4` are the same as those provided to + :func:`waitpid`. Availability: Unix. @@ -3367,7 +2931,7 @@ This option causes child processes to be reported if they have been continued from a job control stop since their status was last reported. - Availability: some Unix systems. + Availability: Some Unix systems. .. data:: WUNTRACED @@ -3453,7 +3017,7 @@ .. versionadded:: 3.3 -The following scheduling policies are exposed if they are supported by the +The following scheduling policies are exposed if they are a supported by the operating system. .. data:: SCHED_OTHER @@ -3550,17 +3114,47 @@ Voluntarily relinquish the CPU. +.. class:: cpu_set(ncpus) + + :class:`cpu_set` represents a set of CPUs on which a process is eligible to + run. *ncpus* is the number of CPUs the set should describe. Methods on + :class:`cpu_set` allow CPUs to be add or removed. + + :class:`cpu_set` supports the AND, OR, and XOR bitwise operations. For + example, given two cpu_sets, ``one`` and ``two``, ``one | two`` returns a + :class:`cpu_set` containing the cpus enabled both in ``one`` and ``two``. + + .. method:: set(i) + + Enable CPU *i*. + + .. method:: clear(i) + + Remove CPU *i*. + + .. method:: isset(i) + + Return ``True`` if CPU *i* is enabled in the set. + + .. method:: count() + + Return the number of enabled CPUs in the set. + + .. method:: zero() + + Clear the set completely. + + .. function:: sched_setaffinity(pid, mask) - Restrict the process with PID *pid* (or the current process if zero) to a - set of CPUs. *mask* is an iterable of integers representing the set of - CPUs to which the process should be restricted. - - -.. function:: sched_getaffinity(pid) - - Return the set of CPUs the process with PID *pid* (or the current process - if zero) is restricted to. + Restrict the process with PID *pid* to a set of CPUs. *mask* is a + :class:`cpu_set` instance. + + +.. function:: sched_getaffinity(pid, size) + + Return the :class:`cpu_set` the process with PID *pid* is restricted to. The + result will contain *size* CPUs. .. _os-path: @@ -3587,7 +3181,7 @@ included in ``confstr_names``, an :exc:`OSError` is raised with :const:`errno.EINVAL` for the error number. - Availability: Unix. + Availability: Unix .. data:: confstr_names @@ -3599,18 +3193,6 @@ Availability: Unix. -.. function:: cpu_count() - - Return the number of CPUs in the system. Returns None if undetermined. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` - - - .. versionadded:: 3.4 - - .. function:: getloadavg() Return the number of processes in the system run queue averaged over the last @@ -3690,9 +3272,8 @@ .. data:: defpath - The default search path used by :func:`exec\*p\* ` and - :func:`spawn\*p\* ` if the environment doesn't have a ``'PATH'`` - key. Also available via :mod:`os.path`. + The default search path used by :func:`exec\*p\*` and :func:`spawn\*p\*` if the + environment doesn't have a ``'PATH'`` key. Also available via :mod:`os.path`. .. data:: linesep @@ -3709,19 +3290,6 @@ The file path of the null device. For example: ``'/dev/null'`` for POSIX, ``'nul'`` for Windows. Also available via :mod:`os.path`. -.. data:: RTLD_LAZY - RTLD_NOW - RTLD_GLOBAL - RTLD_LOCAL - RTLD_NODELETE - RTLD_NOLOAD - RTLD_DEEPBIND - - Flags for use with the :func:`~sys.setdlopenflags` and - :func:`~sys.getdlopenflags` functions. See the Unix manual page - :manpage:`dlopen(3)` for what the different flags mean. - - .. versionadded:: 3.3 .. _os-miscfunc: @@ -3735,16 +3303,6 @@ This function returns random bytes from an OS-specific randomness source. The returned data should be unpredictable enough for cryptographic applications, - though its exact quality depends on the OS implementation. On a Unix-like - system this will query ``/dev/urandom``, and on Windows it will use - ``CryptGenRandom()``. If a randomness source is not found, - :exc:`NotImplementedError` will be raised. - - For an easy-to-use interface to the random number generator - provided by your platform, please see :class:`random.SystemRandom`. - - .. versionchanged:: 3.5 - On Linux 3.17 and newer, the ``getrandom()`` syscall is now used - when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` - function is now used. These functions avoid the usage of an internal file - descriptor. + though its exact quality depends on the OS implementation. On a UNIX-like + system this will query /dev/urandom, and on Windows it will use CryptGenRandom. + If a randomness source is not found, :exc:`NotImplementedError` will be raised. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ossaudiodev.rst --- a/Doc/library/ossaudiodev.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/ossaudiodev.rst Mon Jan 25 17:05:13 2016 +0100 @@ -49,7 +49,7 @@ the official documentation for the OSS C API The module defines a large number of constants supplied by the OSS device - driver; see ```` on either Linux or FreeBSD for a listing. + driver; see ```` on either Linux or FreeBSD for a listing . :mod:`ossaudiodev` defines the following variables and functions: @@ -67,8 +67,7 @@ ``ossaudiodev.error``.) -.. function:: open(mode) - open(device, mode) +.. function:: open([device, ]mode) Open an audio device and return an OSS audio device object. This object supports many file-like methods, such as :meth:`read`, :meth:`write`, and @@ -148,41 +147,32 @@ .. method:: oss_audio_device.write(data) - Write a :term:`bytes-like object` *data* to the audio device and return the - number of bytes written. If the audio device is in blocking mode (the - default), the entire data is always written (again, this is different from - usual Unix device semantics). If the device is in non-blocking mode, some - data may not be written + Write the Python string *data* to the audio device and return the number of + bytes written. If the audio device is in blocking mode (the default), the + entire string is always written (again, this is different from usual Unix device + semantics). If the device is in non-blocking mode, some data may not be written ---see :meth:`writeall`. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. method:: oss_audio_device.writeall(data) - Write a :term:`bytes-like object` *data* to the audio device: waits until - the audio device is able to accept data, writes as much data as it will - accept, and repeats until *data* has been completely written. If the device - is in blocking mode (the default), this has the same effect as - :meth:`write`; :meth:`writeall` is only useful in non-blocking mode. Has - no return value, since the amount of data written is always equal to the - amount of data supplied. - - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - + Write the entire Python string *data* to the audio device: waits until the audio + device is able to accept data, writes as much data as it will accept, and + repeats until *data* has been completely written. If the device is in blocking + mode (the default), this has the same effect as :meth:`write`; :meth:`writeall` + is only useful in non-blocking mode. Has no return value, since the amount of + data written is always equal to the amount of data supplied. .. versionchanged:: 3.2 - Audio device objects also support the context management protocol, i.e. they can + Audio device objects also support the context manager protocol, i.e. they can be used in a :keyword:`with` statement. -The following methods each map to exactly one :c:func:`ioctl` system call. The +The following methods each map to exactly one :func:`ioctl` system call. The correspondence is obvious: for example, :meth:`setfmt` corresponds to the ``SNDCTL_DSP_SETFMT`` ioctl, and :meth:`sync` to ``SNDCTL_DSP_SYNC`` (this can be useful when consulting the OSS documentation). If the underlying -:c:func:`ioctl` fails, they all raise :exc:`OSError`. +:func:`ioctl` fails, they all raise :exc:`OSError`. .. method:: oss_audio_device.nonblock() @@ -291,7 +281,7 @@ simple calculations. -.. method:: oss_audio_device.setparameters(format, nchannels, samplerate[, strict=False]) +.. method:: oss_audio_device.setparameters(format, nchannels, samplerate [, strict=False]) Set the key audio sampling parameters---sample format, number of channels, and sampling rate---in one method call. *format*, *nchannels*, and *samplerate* @@ -311,7 +301,7 @@ fmt = dsp.setfmt(fmt) channels = dsp.channels(channels) - rate = dsp.rate(rate) + rate = dsp.rate(channels) .. method:: oss_audio_device.bufsize() @@ -366,7 +356,7 @@ Returns the file handle number of the open mixer device file. .. versionchanged:: 3.2 - Mixer objects also support the context management protocol. + Mixer objects also support the context manager protocol. The remaining methods are specific to audio mixing: @@ -416,7 +406,7 @@ (silent) to 100 (full volume). If the control is monophonic, a 2-tuple is still returned, but both volumes are the same. - Raises :exc:`OSSAudioError` if an invalid control is specified, or + Raises :exc:`OSSAudioError` if an invalid control was is specified, or :exc:`OSError` if an unsupported control is specified. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/othergui.rst --- a/Doc/library/othergui.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/othergui.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,7 @@ `PyGObject `_ provides introspection bindings for C libraries using - `GObject `_. One of + `GObject `_. One of these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online `Python GTK+ 3 Tutorial `_ @@ -19,10 +19,11 @@ `PyGTK `_ provides bindings for an older version of the library, GTK+ 2. It provides an object oriented interface that is slightly higher level than the C one. There are also bindings to - `GNOME `_. An online `tutorial + `GNOME `_. One well known PyGTK application is + `PythonCAD `_. An online `tutorial `_ is available. - `PyQt `_ + `PyQt `_ PyQt is a :program:`sip`\ -wrapped binding to the Qt toolkit. Qt is an extensive C++ GUI application development framework that is available for Unix, Windows and Mac OS X. :program:`sip` is a tool @@ -34,7 +35,7 @@ with Python and Qt `_, by Mark Summerfield. - `PySide `_ + `PySide `_ is a newer binding to the Qt toolkit, provided by Nokia. Compared to PyQt, its licensing scheme is friendlier to non-open source applications. @@ -50,13 +51,13 @@ low-level device context drawing, drag and drop, system clipboard access, an XML-based resource format and more, including an ever growing library of user-contributed modules. wxPython has a book, `wxPython in Action - `_, by Noel Rappin and + `_, by Noel Rappin and Robin Dunn. PyGTK, PyQt, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for Python, both cross-platform, and platform-specific. See the `GUI Programming -`_ page in the Python Wiki for a +`_ page in the Python Wiki for a much more complete list, and also for links to documents where the different GUI toolkits are compared. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging-misc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging-misc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,27 @@ +.. temporary file for modules that don't need a dedicated file yet + +:mod:`packaging.errors` --- Packaging exceptions +================================================ + +.. module:: packaging.errors + :synopsis: Packaging exceptions. + + +Provides exceptions used by the Packaging modules. Note that Packaging modules +may raise standard exceptions; in particular, SystemExit is usually raised for +errors that are obviously the end-user's fault (e.g. bad command-line arguments). + +This module is safe to use in ``from ... import *`` mode; it only exports +symbols whose names start with ``Packaging`` and end with ``Error``. + + +:mod:`packaging.manifest` --- The Manifest class +================================================ + +.. module:: packaging.manifest + :synopsis: The Manifest class, used for poking about the file system and + building lists of files. + + +This module provides the :class:`Manifest` class, used for poking about the +filesystem and building lists of files. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.command.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.command.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,111 @@ +:mod:`packaging.command` --- Standard Packaging commands +======================================================== + +.. module:: packaging.command + :synopsis: Standard packaging commands. + + +This subpackage contains one module for each standard Packaging command, such as +:command:`build` or :command:`upload`. Each command is implemented as a +separate module, with the command name as the name of the module and of the +class defined therein. + + + +:mod:`packaging.command.cmd` --- Abstract base class for Packaging commands +=========================================================================== + +.. module:: packaging.command.cmd + :synopsis: Abstract base class for commands. + + +This module supplies the abstract base class :class:`Command`. This class is +subclassed by the modules in the packaging.command subpackage. + + +.. class:: Command(dist) + + Abstract base class for defining command classes, the "worker bees" of the + Packaging. A useful analogy for command classes is to think of them as + subroutines with local variables called *options*. The options are declared + in :meth:`initialize_options` and defined (given their final values) in + :meth:`finalize_options`, both of which must be defined by every command + class. The distinction between the two is necessary because option values + might come from the outside world (command line, config file, ...), and any + options dependent on other options must be computed after these outside + influences have been processed --- hence :meth:`finalize_options`. The body + of the subroutine, where it does all its work based on the values of its + options, is the :meth:`run` method, which must also be implemented by every + command class. + + The class constructor takes a single argument *dist*, a + :class:`~packaging.dist.Distribution` instance. + + +Creating a new Packaging command +-------------------------------- + +This section outlines the steps to create a new Packaging command. + +.. XXX the following paragraph is focused on the stdlib; expand it to document + how to write and register a command in third-party projects + +A new command lives in a module in the :mod:`packaging.command` package. There +is a sample template in that directory called :file:`command_template`. Copy +this file to a new module with the same name as the new command you're +implementing. This module should implement a class with the same name as the +module (and the command). So, for instance, to create the command +``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy +:file:`command_template` to :file:`packaging/command/peel_banana.py`, then edit +it so that it's implementing the class :class:`peel_banana`, a subclass of +:class:`Command`. It must define the following methods: + +.. method:: Command.initialize_options() + + Set default values for all the options that this command supports. Note that + these defaults may be overridden by other commands, by the setup script, by + config files, or by the command line. Thus, this is not the place to code + dependencies between options; generally, :meth:`initialize_options` + implementations are just a bunch of ``self.foo = None`` assignments. + + +.. method:: Command.finalize_options() + + Set final values for all the options that this command supports. This is + always called as late as possible, i.e. after any option assignments from the + command line or from other commands have been done. Thus, this is the place + to code option dependencies: if *foo* depends on *bar*, then it is safe to + set *foo* from *bar* as long as *foo* still has the same value it was + assigned in :meth:`initialize_options`. + + +.. method:: Command.run() + + A command's raison d'etre: carry out the action it exists to perform, + controlled by the options initialized in :meth:`initialize_options`, + customized by other commands, the setup script, the command line, and config + files, and finalized in :meth:`finalize_options`. All terminal output and + filesystem interaction should be done by :meth:`run`. + + +Command classes may define this attribute: + + +.. attribute:: Command.sub_commands + + *sub_commands* formalizes the notion of a "family" of commands, + e.g. ``install_dist`` as the parent with sub-commands ``install_lib``, + ``install_headers``, etc. The parent of a family of commands defines + *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name, + predicate)``, with *command_name* a string and *predicate* a function, a + string or ``None``. *predicate* is a method of the parent command that + determines whether the corresponding command is applicable in the current + situation. (E.g. ``install_headers`` is only applicable if we have any C + header files to install.) If *predicate* is ``None``, that command is always + applicable. + + *sub_commands* is usually defined at the *end* of a class, because + predicates can be methods of the class, so they must already have been + defined. The canonical example is the :command:`install_dist` command. + +.. XXX document how to add a custom command to another one's subcommands diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.compiler.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.compiler.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,681 @@ +:mod:`packaging.compiler` --- Compiler classes +============================================== + +.. module:: packaging.compiler + :synopsis: Compiler classes to build C/C++ extensions or libraries. + + +This subpackage contains an abstract base class representing a compiler and +concrete implementations for common compilers. The compiler classes should not +be instantiated directly, but created using the :func:`new_compiler` factory +function. Compiler types provided by Packaging are listed in +:ref:`packaging-standard-compilers`. + + +Public functions +---------------- + +.. function:: new_compiler(plat=None, compiler=None, dry_run=False, force=False) + + Factory function to generate an instance of some + :class:`~.ccompiler.CCompiler` subclass for the requested platform or + compiler type. + + If no argument is given for *plat* and *compiler*, the default compiler type + for the platform (:attr:`os.name`) will be used: ``'unix'`` for Unix and + Mac OS X, ``'msvc'`` for Windows. + + If *plat* is given, it must be one of ``'posix'``, ``'darwin'`` or ``'nt'``. + An invalid value will not raise an exception but use the default compiler + type for the current platform. + + .. XXX errors should never pass silently; this behavior is particularly + harmful when a compiler type is given as first argument + + If *compiler* is given, *plat* will be ignored, allowing you to get for + example a ``'unix'`` compiler object under Windows or an ``'msvc'`` compiler + under Unix. However, not all compiler types can be instantiated on every + platform. + + +.. function:: customize_compiler(compiler) + + Do any platform-specific customization of a CCompiler instance. Mainly + needed on Unix to plug in the information that varies across Unices and is + stored in CPython's Makefile. + + +.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) + + Generate linker options for searching library directories and linking with + specific libraries. *libraries* and *library_dirs* are, respectively, lists + of library names (not filenames!) and search directories. Returns a list of + command-line options suitable for use with some compiler (depending on the + two format strings passed in). + + +.. function:: gen_preprocess_options(macros, include_dirs) + + Generate C preprocessor options (:option:`-D`, :option:`-U`, :option:`-I`) as + used by at least two types of compilers: the typical Unix compiler and Visual + C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)`` + means undefine (:option:`-U`) macro *name*, and ``(name, value)`` means + define (:option:`-D`) macro *name* to *value*. *include_dirs* is just a list + of directory names to be added to the header file search path (:option:`-I`). + Returns a list of command-line options suitable for either Unix compilers or + Visual C++. + + +.. function:: get_default_compiler(osname, platform) + + Determine the default compiler to use for the given platform. + + *osname* should be one of the standard Python OS names (i.e. the ones + returned by ``os.name``) and *platform* the common value returned by + ``sys.platform`` for the platform in question. + + The default values are ``os.name`` and ``sys.platform``. + + +.. function:: set_compiler(location) + + Add or change a compiler + + +.. function:: show_compilers() + + Print list of available compilers (used by the :option:`--help-compiler` + options to :command:`build`, :command:`build_ext`, :command:`build_clib`). + + +.. _packaging-standard-compilers: + +Standard compilers +------------------ + +Concrete subclasses of :class:`~.ccompiler.CCompiler` are provided in submodules +of the :mod:`packaging.compiler` package. You do not need to import them, using +:func:`new_compiler` is the public API to use. This table documents the +standard compilers; be aware that they can be replaced by other classes on your +platform. + +=============== ======================================================== ======= +name description notes +=============== ======================================================== ======= +``'unix'`` typical Unix-style command-line C compiler [#]_ +``'msvc'`` Microsoft compiler [#]_ +``'bcpp'`` Borland C++ compiler +``'cygwin'`` Cygwin compiler (Windows port of GCC) +``'mingw32'`` Mingw32 port of GCC (same as Cygwin in no-Cygwin mode) +=============== ======================================================== ======= + + +.. [#] The Unix compiler class assumes this behavior: + + * macros defined with :option:`-Dname[=value]` + + * macros undefined with :option:`-Uname` + + * include search directories specified with :option:`-Idir` + + * libraries specified with :option:`-llib` + + * library search directories specified with :option:`-Ldir` + + * compile handled by :program:`cc` (or similar) executable with + :option:`-c` option: compiles :file:`.c` to :file:`.o` + + * link static library handled by :program:`ar` command (possibly with + :program:`ranlib`) + + * link shared library handled by :program:`cc` :option:`-shared` + + +.. [#] On Windows, extension modules typically need to be compiled with the same + compiler that was used to compile CPython (for example Microsoft Visual + Studio .NET 2003 for CPython 2.4 and 2.5). The AMD64 and Itanium + binaries are created using the Platform SDK. + + Under the hood, there are actually two different subclasses of + :class:`~.ccompiler.CCompiler` defined: one is compatible with MSVC 2005 + and 2008, the other works with older versions. This should not be a + concern for regular use of the functions in this module. + + Packaging will normally choose the right compiler, linker etc. on its + own. To override this choice, the environment variables + *DISTUTILS_USE_SDK* and *MSSdk* must be both set. *MSSdk* indicates that + the current environment has been setup by the SDK's ``SetEnv.Cmd`` + script, or that the environment variables had been registered when the + SDK was installed; *DISTUTILS_USE_SDK* indicates that the user has made + an explicit choice to override the compiler selection done by Packaging. + + .. TODO document the envvars in Doc/using and the man page + + +:mod:`packaging.compiler.ccompiler` --- CCompiler base class +============================================================ + +.. module:: packaging.compiler.ccompiler + :synopsis: Abstract CCompiler class. + + +This module provides the abstract base class for the :class:`CCompiler` +classes. A :class:`CCompiler` instance can be used for all the compile and +link steps needed to build a single project. Methods are provided to set +options for the compiler --- macro definitions, include directories, link path, +libraries and the like. + +.. class:: CCompiler(dry_run=False, force=False) + + The abstract base class :class:`CCompiler` defines the interface that must be + implemented by real compiler classes. The class also has some utility + methods used by several compiler classes. + + The basic idea behind a compiler abstraction class is that each instance can + be used for all the compile/link steps in building a single project. Thus, + attributes common to all of those compile and link steps --- include + directories, macros to define, libraries to link against, etc. --- are + attributes of the compiler instance. To allow for variability in how + individual files are treated, most of those attributes may be varied on a + per-compilation or per-link basis. + + The constructor for each subclass creates an instance of the Compiler object. + Flags are *dry_run* (don't actually execute + the steps) and *force* (rebuild everything, regardless of dependencies). All + of these flags default to ``False`` (off). Note that you probably don't want to + instantiate :class:`CCompiler` or one of its subclasses directly - use the + :func:`new_compiler` factory function instead. + + The following methods allow you to manually alter compiler options for the + instance of the Compiler class. + + + .. method:: CCompiler.add_include_dir(dir) + + Add *dir* to the list of directories that will be searched for header + files. The compiler is instructed to search directories in the order in + which they are supplied by successive calls to :meth:`add_include_dir`. + + + .. method:: CCompiler.set_include_dirs(dirs) + + Set the list of directories that will be searched to *dirs* (a list of + strings). Overrides any preceding calls to :meth:`add_include_dir`; + subsequent calls to :meth:`add_include_dir` add to the list passed to + :meth:`set_include_dirs`. This does not affect any list of standard + include directories that the compiler may search by default. + + + .. method:: CCompiler.add_library(libname) + + Add *libname* to the list of libraries that will be included in all links + driven by this compiler object. Note that *libname* should *not* be the + name of a file containing a library, but the name of the library itself: + the actual filename will be inferred by the linker, the compiler, or the + compiler class (depending on the platform). + + The linker will be instructed to link against libraries in the order they + were supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is + perfectly valid to duplicate library names; the linker will be instructed + to link against libraries as many times as they are mentioned. + + + .. method:: CCompiler.set_libraries(libnames) + + Set the list of libraries to be included in all links driven by this + compiler object to *libnames* (a list of strings). This does not affect + any standard system libraries that the linker may include by default. + + + .. method:: CCompiler.add_library_dir(dir) + + Add *dir* to the list of directories that will be searched for libraries + specified to :meth:`add_library` and :meth:`set_libraries`. The linker + will be instructed to search for libraries in the order they are supplied + to :meth:`add_library_dir` and/or :meth:`set_library_dirs`. + + + .. method:: CCompiler.set_library_dirs(dirs) + + Set the list of library search directories to *dirs* (a list of strings). + This does not affect any standard library search path that the linker may + search by default. + + + .. method:: CCompiler.add_runtime_library_dir(dir) + + Add *dir* to the list of directories that will be searched for shared + libraries at runtime. + + + .. method:: CCompiler.set_runtime_library_dirs(dirs) + + Set the list of directories to search for shared libraries at runtime to + *dirs* (a list of strings). This does not affect any standard search path + that the runtime linker may search by default. + + + .. method:: CCompiler.define_macro(name[, value=None]) + + Define a preprocessor macro for all compilations driven by this compiler + object. The optional parameter *value* should be a string; if it is not + supplied, then the macro will be defined without an explicit value and the + exact outcome depends on the compiler used (XXX true? does ANSI say + anything about this?) + + + .. method:: CCompiler.undefine_macro(name) + + Undefine a preprocessor macro for all compilations driven by this compiler + object. If the same macro is defined by :meth:`define_macro` and + undefined by :meth:`undefine_macro` the last call takes precedence + (including multiple redefinitions or undefinitions). If the macro is + redefined/undefined on a per-compilation basis (i.e. in the call to + :meth:`compile`), then that takes precedence. + + + .. method:: CCompiler.add_link_object(object) + + Add *object* to the list of object files (or analogues, such as explicitly + named library files or the output of "resource compilers") to be included + in every link driven by this compiler object. + + + .. method:: CCompiler.set_link_objects(objects) + + Set the list of object files (or analogues) to be included in every link + to *objects*. This does not affect any standard object files that the + linker may include by default (such as system libraries). + + The following methods implement methods for autodetection of compiler + options, providing some functionality similar to GNU :program:`autoconf`. + + + .. method:: CCompiler.detect_language(sources) + + Detect the language of a given file, or list of files. Uses the instance + attributes :attr:`language_map` (a dictionary), and :attr:`language_order` + (a list) to do the job. + + + .. method:: CCompiler.find_library_file(dirs, lib[, debug=0]) + + Search the specified list of directories for a static or shared library file + *lib* and return the full path to that file. If *debug* is true, look for a + debugging version (if that makes sense on the current platform). Return + ``None`` if *lib* wasn't found in any of the specified directories. + + + .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None]) + + Return a boolean indicating whether *funcname* is supported on the current + platform. The optional arguments can be used to augment the compilation + environment by providing additional include files and paths and libraries and + paths. + + + .. method:: CCompiler.library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + libraries. + + + .. method:: CCompiler.library_option(lib) + + Return the compiler option to add *dir* to the list of libraries linked into the + shared library or executable. + + + .. method:: CCompiler.runtime_library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + runtime libraries. + + + .. method:: CCompiler.set_executables(**args) + + Define the executables (and options for them) that will be run to perform the + various stages of compilation. The exact set of executables that may be + specified here depends on the compiler class (via the 'executables' class + attribute), but most will have: + + +--------------+------------------------------------------+ + | attribute | description | + +==============+==========================================+ + | *compiler* | the C/C++ compiler | + +--------------+------------------------------------------+ + | *linker_so* | linker used to create shared objects and | + | | libraries | + +--------------+------------------------------------------+ + | *linker_exe* | linker used to create binary executables | + +--------------+------------------------------------------+ + | *archiver* | static library creator | + +--------------+------------------------------------------+ + + On platforms with a command line (Unix, DOS/Windows), each of these is a string + that will be split into executable name and (optional) list of arguments. + (Splitting the string is done similarly to how Unix shells operate: words are + delimited by spaces, but quotes and backslashes can override this. See + :func:`packaging.util.split_quoted`.) + + The following methods invoke stages in the build process. + + + .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None]) + + Compile one or more source files. Generates object files (e.g. transforms a + :file:`.c` file to a :file:`.o` file.) + + *sources* must be a list of filenames, most likely C/C++ files, but in reality + anything that can be handled by a particular compiler and compiler class (e.g. + an ``'msvc'`` compiler can handle resource files in *sources*). Return a list of + object filenames, one per source filename in *sources*. Depending on the + implementation, not all source files will necessarily be compiled, but all + corresponding object filenames will be returned. + + If *output_dir* is given, object files will be put under it, while retaining + their original path component. That is, :file:`foo/bar.c` normally compiles to + :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then + it would compile to :file:`build/foo/bar.o`. + + *macros*, if given, must be a list of macro definitions. A macro definition is + either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines + a macro; if the value is ``None``, the macro is defined without an explicit + value. The 1-tuple case undefines a macro. Later + definitions/redefinitions/undefinitions take precedence. + + *include_dirs*, if given, must be a list of strings, the directories to add to + the default include file search path for this compilation only. + + *debug* is a boolean; if true, the compiler will be instructed to output debug + symbols in (or alongside) the object file(s). + + *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms + that have the notion of a command line (e.g. Unix, DOS/Windows), they are most + likely lists of strings: extra command-line arguments to prepend/append to the + compiler command line. On other platforms, consult the implementation class + documentation. In any event, they are intended as an escape hatch for those + occasions when the abstract compiler framework doesn't cut the mustard. + + *depends*, if given, is a list of filenames that all targets depend on. If a + source file is older than any file in depends, then the source file will be + recompiled. This supports dependency tracking, but only at a coarse + granularity. + + Raises :exc:`CompileError` on failure. + + + .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None]) + + Link a bunch of stuff together to create a static library file. The "bunch of + stuff" consists of the list of object files supplied as *objects*, the extra + object files supplied to :meth:`add_link_object` and/or + :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or + :meth:`set_libraries`, and the libraries supplied as *libraries* (if any). + + *output_libname* should be a library name, not a filename; the filename will be + inferred from the library name. *output_dir* is the directory where the library + file will be put. XXX defaults to what? + + *debug* is a boolean; if true, debugging information will be included in the + library (note that on most platforms, it is the compile step where this matters: + the *debug* flag is included here just for consistency). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LibError` on failure. + + + .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a bunch of stuff together to create an executable or shared library file. + + The "bunch of stuff" consists of the list of object files supplied as *objects*. + *output_filename* should be a filename. If *output_dir* is supplied, + *output_filename* is relative to it (i.e. *output_filename* can provide + directory components if needed). + + *libraries* is a list of libraries to link against. These are library names, + not filenames, since they're translated into filenames in a platform-specific + way (e.g. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on + DOS/Windows). However, they can include a directory component, which means the + linker will look in that specific directory rather than searching all the normal + locations. + + *library_dirs*, if supplied, should be a list of directories to search for + libraries that were specified as bare library names (i.e. no directory + component). These are on top of the system default and those supplied to + :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs* + is a list of directories that will be embedded into the shared library and used + to search for other shared libraries that \*it\* depends on at run-time. (This + may only be relevant on Unix.) + + *export_symbols* is a list of symbols that the shared library will export. + (This appears to be relevant only on Windows.) + + *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the + slight distinction that it actually matters on most platforms (as opposed to + :meth:`create_static_lib`, which includes a *debug* flag mostly for form's + sake). + + *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of + course that they supply command-line arguments for the particular linker being + used). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LinkError` on failure. + + + .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None]) + + Link an executable. *output_progname* is the name of the file executable, while + *objects* are a list of object filenames to link in. Other arguments are as for + the :meth:`link` method. + + + .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared library. *output_libname* is the name of the output library, + while *objects* is a list of object filenames to link in. Other arguments are + as for the :meth:`link` method. + + + .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared object. *output_filename* is the name of the shared object that + will be created, while *objects* is a list of object filenames to link in. + Other arguments are as for the :meth:`link` method. + + + .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None]) + + Preprocess a single C/C++ source file, named in *source*. Output will be written + to file named *output_file*, or *stdout* if *output_file* not supplied. + *macros* is a list of macro definitions as for :meth:`compile`, which will + augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`. + *include_dirs* is a list of directory names that will be added to the default + list, in the same way as :meth:`add_include_dir`. + + Raises :exc:`PreprocessError` on failure. + + The following utility methods are defined by the :class:`CCompiler` class, for + use by the various concrete subclasses. + + + .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir='']) + + Returns the filename of the executable for the given *basename*. Typically for + non-Windows platforms this is the same as the basename, while Windows will get + a :file:`.exe` added. + + + .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir='']) + + Returns the filename for the given library name on the current platform. On Unix + a library with *lib_type* of ``'static'`` will typically be of the form + :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form + :file:`liblibname.so`. + + + .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir='']) + + Returns the name of the object files for the given source files. + *source_filenames* should be a list of filenames. + + + .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir='']) + + Returns the name of a shared object file for the given file name *basename*. + + + .. method:: CCompiler.execute(func, args[, msg=None, level=1]) + + Invokes :func:`packaging.util.execute` This method invokes a Python function + *func* with the given arguments *args*, after logging and taking into account + the *dry_run* flag. XXX see also. + + + .. method:: CCompiler.spawn(cmd) + + Invokes :func:`packaging.util.spawn`. This invokes an external process to run + the given command. XXX see also. + + + .. method:: CCompiler.mkpath(name[, mode=511]) + + Invokes :func:`packaging.dir_util.mkpath`. This creates a directory and any + missing ancestor directories. XXX see also. + + + .. method:: CCompiler.move_file(src, dst) + + Invokes :meth:`packaging.file_util.move_file`. Renames *src* to *dst*. XXX see + also. + + +:mod:`packaging.compiler.extension` --- The Extension class +=========================================================== + +.. module:: packaging.compiler.extension + :synopsis: Class used to represent C/C++ extension modules. + + +This module provides the :class:`Extension` class, used to represent C/C++ +extension modules. + +.. class:: Extension + + The Extension class describes a single C or C++ extension module. It accepts + the following keyword arguments in its constructor: + + +------------------------+--------------------------------+---------------------------+ + | argument name | value | type | + +========================+================================+===========================+ + | *name* | the full name of the | string | + | | extension, including any | | + | | packages --- i.e. *not* a | | + | | filename or pathname, but | | + | | Python dotted name | | + +------------------------+--------------------------------+---------------------------+ + | *sources* | list of source filenames, | list of strings | + | | relative to the distribution | | + | | root (where the setup script | | + | | lives), in Unix form (slash- | | + | | separated) for portability. | | + | | Source files may be C, C++, | | + | | SWIG (.i), platform-specific | | + | | resource files, or whatever | | + | | else is recognized by the | | + | | :command:`build_ext` command | | + | | as source for a Python | | + | | extension. | | + +------------------------+--------------------------------+---------------------------+ + | *include_dirs* | list of directories to search | list of strings | + | | for C/C++ header files (in | | + | | Unix form for portability) | | + +------------------------+--------------------------------+---------------------------+ + | *define_macros* | list of macros to define; each | list of tuples | + | | macro is defined using a | | + | | 2-tuple ``(name, value)``, | | + | | where *value* is | | + | | either the string to define it | | + | | to or ``None`` to define it | | + | | without a particular value | | + | | (equivalent of ``#define FOO`` | | + | | in source or :option:`-DFOO` | | + | | on Unix C compiler command | | + | | line) | | + +------------------------+--------------------------------+---------------------------+ + | *undef_macros* | list of macros to undefine | list of strings | + | | explicitly | | + +------------------------+--------------------------------+---------------------------+ + | *library_dirs* | list of directories to search | list of strings | + | | for C/C++ libraries at link | | + | | time | | + +------------------------+--------------------------------+---------------------------+ + | *libraries* | list of library names (not | list of strings | + | | filenames or paths) to link | | + | | against | | + +------------------------+--------------------------------+---------------------------+ + | *runtime_library_dirs* | list of directories to search | list of strings | + | | for C/C++ libraries at run | | + | | time (for shared extensions, | | + | | this is when the extension is | | + | | loaded) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_objects* | list of extra files to link | list of strings | + | | with (e.g. object files not | | + | | implied by 'sources', static | | + | | library that must be | | + | | explicitly specified, binary | | + | | resource files, etc.) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_compile_args* | any extra platform- and | list of strings | + | | compiler-specific information | | + | | to use when compiling the | | + | | source files in 'sources'. For | | + | | platforms and compilers where | | + | | a command line makes sense, | | + | | this is typically a list of | | + | | command-line arguments, but | | + | | for other platforms it could | | + | | be anything. | | + +------------------------+--------------------------------+---------------------------+ + | *extra_link_args* | any extra platform- and | list of strings | + | | compiler-specific information | | + | | to use when linking object | | + | | files together to create the | | + | | extension (or to create a new | | + | | static Python interpreter). | | + | | Similar interpretation as for | | + | | 'extra_compile_args'. | | + +------------------------+--------------------------------+---------------------------+ + | *export_symbols* | list of symbols to be exported | list of strings | + | | from a shared extension. Not | | + | | used on all platforms, and not | | + | | generally necessary for Python | | + | | extensions, which typically | | + | | export exactly one symbol: | | + | | ``init`` + extension_name. | | + +------------------------+--------------------------------+---------------------------+ + | *depends* | list of files that the | list of strings | + | | extension depends on | | + +------------------------+--------------------------------+---------------------------+ + | *language* | extension language (i.e. | string | + | | ``'c'``, ``'c++'``, | | + | | ``'objc'``). Will be detected | | + | | from the source extensions if | | + | | not provided. | | + +------------------------+--------------------------------+---------------------------+ + | *optional* | specifies that a build failure | boolean | + | | in the extension should not | | + | | abort the build process, but | | + | | simply skip the extension. | | + +------------------------+--------------------------------+---------------------------+ + +To distribute extension modules that live in a package (e.g. ``package.ext``), +you need to create a :file:`{package}/__init__.py` file to let Python recognize +and import your module. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.database.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.database.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,345 @@ +:mod:`packaging.database` --- Database of installed distributions +================================================================= + +.. module:: packaging.database + :synopsis: Functions to query and manipulate installed distributions. + + +This module provides an implementation of :PEP:`376`. It was originally +intended to land in :mod:`pkgutil`, but with the inclusion of Packaging in the +standard library, it was thought best to include it in a submodule of +:mod:`packaging`, leaving :mod:`pkgutil` to deal with imports. + +Installed Python distributions are represented by instances of +:class:`Distribution`, or :class:`EggInfoDistribution` for legacy egg formats. +Most functions also provide an extra argument ``use_egg_info`` to take legacy +distributions into account. + +For the purpose of this module, "installed" means that the distribution's +:file:`.dist-info`, :file:`.egg-info` or :file:`egg` directory or file is found +on :data:`sys.path`. For example, if the parent directory of a +:file:`dist-info` directory is added to :envvar:`PYTHONPATH`, then it will be +available in the database. + +Classes representing installed distributions +-------------------------------------------- + +.. class:: Distribution(path) + + Class representing an installed distribution. It is different from + :class:`packaging.dist.Distribution` which holds the list of files, the + metadata and options during the run of a Packaging command. + + Instantiate with the *path* to a ``.dist-info`` directory. Instances can be + compared and sorted. Other available methods are: + + .. XXX describe how comparison works + + .. method:: get_distinfo_file(path, binary=False) + + Return a read-only file object for a file located at + :file:`{project}-{version}.dist-info/{path}`. *path* should be a + ``'/'``-separated path relative to the ``.dist-info`` directory or an + absolute path; if it is an absolute path and doesn't start with the path + to the :file:`.dist-info` directory, a :class:`PackagingError` is raised. + + If *binary* is ``True``, the file is opened in binary mode. + + .. method:: get_resource_path(relative_path) + + .. TODO + + .. method:: list_distinfo_files(local=False) + + Return an iterator over all files located in the :file:`.dist-info` + directory. If *local* is ``True``, each returned path is transformed into + a local absolute path, otherwise the raw value found in the :file:`RECORD` + file is returned. + + .. method:: list_installed_files(local=False) + + Iterate over the files installed with the distribution and registered in + the :file:`RECORD` file and yield a tuple ``(path, md5, size)`` for each + line. If *local* is ``True``, the returned path is transformed into a + local absolute path, otherwise the raw value is returned. + + A local absolute path is an absolute path in which occurrences of ``'/'`` + have been replaced by :data:`os.sep`. + + .. method:: uses(path) + + Check whether *path* was installed by this distribution (i.e. if the path + is present in the :file:`RECORD` file). *path* can be a local absolute + path or a relative ``'/'``-separated path. Returns a boolean. + + Available attributes: + + .. attribute:: metadata + + Instance of :class:`packaging.metadata.Metadata` filled with the contents + of the :file:`{project}-{version}.dist-info/METADATA` file. + + .. attribute:: name + + Shortcut for ``metadata['Name']``. + + .. attribute:: version + + Shortcut for ``metadata['Version']``. + + .. attribute:: requested + + Boolean indicating whether this distribution was requested by the user of + automatically installed as a dependency. + + +.. class:: EggInfoDistribution(path) + + Class representing a legacy distribution. It is compatible with distutils' + and setuptools' :file:`.egg-info` and :file:`.egg` files and directories. + + .. FIXME should be named EggDistribution + + Instantiate with the *path* to an egg file or directory. Instances can be + compared and sorted. Other available methods are: + + .. method:: list_installed_files(local=False) + + .. method:: uses(path) + + Available attributes: + + .. attribute:: metadata + + Instance of :class:`packaging.metadata.Metadata` filled with the contents + of the :file:`{project-version}.egg-info/PKG-INFO` or + :file:`{project-version}.egg` file. + + .. attribute:: name + + Shortcut for ``metadata['Name']``. + + .. attribute:: version + + Shortcut for ``metadata['Version']``. + + +Functions to work with the database +----------------------------------- + +.. function:: get_distribution(name, use_egg_info=False, paths=None) + + Return an instance of :class:`Distribution` or :class:`EggInfoDistribution` + for the first installed distribution matching *name*. Egg distributions are + considered only if *use_egg_info* is true; if both a dist-info and an egg + file are found, the dist-info prevails. The directories to be searched are + given in *paths*, which defaults to :data:`sys.path`. Returns ``None`` if no + matching distribution is found. + + .. FIXME param should be named use_egg + + +.. function:: get_distributions(use_egg_info=False, paths=None) + + Return an iterator of :class:`Distribution` instances for all installed + distributions found in *paths* (defaults to :data:`sys.path`). If + *use_egg_info* is true, also return instances of :class:`EggInfoDistribution` + for legacy distributions found. + + +.. function:: get_file_users(path) + + Return an iterator over all distributions using *path*, a local absolute path + or a relative ``'/'``-separated path. + + .. XXX does this work with prefixes or full file path only? + + +.. function:: obsoletes_distribution(name, version=None, use_egg_info=False) + + Return an iterator over all distributions that declare they obsolete *name*. + *version* is an optional argument to match only specific releases (see + :mod:`packaging.version`). If *use_egg_info* is true, legacy egg + distributions will be considered as well. + + +.. function:: provides_distribution(name, version=None, use_egg_info=False) + + Return an iterator over all distributions that declare they provide *name*. + *version* is an optional argument to match only specific releases (see + :mod:`packaging.version`). If *use_egg_info* is true, legacy egg + distributions will be considered as well. + + +Utility functions +----------------- + +.. function:: distinfo_dirname(name, version) + + Escape *name* and *version* into a filename-safe form and return the + directory name built from them, for example + :file:`{safename}-{safeversion}.dist-info.` In *name*, runs of + non-alphanumeric characters are replaced with one ``'_'``; in *version*, + spaces become dots, and runs of other non-alphanumeric characters (except + dots) a replaced by one ``'-'``. + + .. XXX wth spaces in version numbers? + +For performance purposes, the list of distributions is being internally +cached. Caching is enabled by default, but you can control it with these +functions: + +.. function:: clear_cache() + + Clear the cache. + +.. function:: disable_cache() + + Disable the cache, without clearing it. + +.. function:: enable_cache() + + Enable the internal cache, without clearing it. + + +Examples +-------- + +Printing all information about a distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the name of an installed distribution, we shall print out all +information that can be obtained using functions provided in this module:: + + import sys + import packaging.database + + try: + name = sys.argv[1] + except ValueError: + sys.exit('Not enough arguments') + + # first create the Distribution instance + dist = packaging.database.Distribution(path) + if dist is None: + sys.exit('No such distribution') + + print('Information about %r' % dist.name) + print() + + print('Files') + print('=====') + for path, md5, size in dist.list_installed_files(): + print('* Path: %s' % path) + print(' Hash %s, Size: %s bytes' % (md5, size)) + print() + + print('Metadata') + print('========') + for key, value in dist.metadata.items(): + print('%20s: %s' % (key, value)) + print() + + print('Extra') + print('=====') + if dist.requested: + print('* It was installed by user request') + else: + print('* It was installed as a dependency') + +If we save the script above as ``print_info.py``, we can use it to extract +information from a :file:`.dist-info` directory. By typing in the console: + +.. code-block:: sh + + python print_info.py choxie + +we get the following output: + +.. code-block:: none + + Information about 'choxie' + + Files + ===== + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/truffles.py + Hash 5e052db6a478d06bad9ae033e6bc08af, Size: 111 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py + Hash ac56bf496d8d1d26f866235b95f31030, Size: 214 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py + Hash 416aab08dfa846f473129e89a7625bbc, Size: 25 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER + Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA + Hash 696a209967fef3c8b8f5a7bb10386385, Size: 225 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED + Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD + Hash None, Size: None bytes + + Metadata + ======== + Metadata-Version: 1.2 + Name: choxie + Version: 2.0.0.9 + Platform: [] + Supported-Platform: UNKNOWN + Summary: Chocolate with a kick! + Description: UNKNOWN + Keywords: [] + Home-page: UNKNOWN + Author: UNKNOWN + Author-email: UNKNOWN + Maintainer: UNKNOWN + Maintainer-email: UNKNOWN + License: UNKNOWN + Classifier: [] + Download-URL: UNKNOWN + Obsoletes-Dist: ['truffles (<=0.8,>=0.5)', 'truffles (<=0.9,>=0.6)'] + Project-URL: [] + Provides-Dist: ['truffles (1.0)'] + Requires-Dist: ['towel-stuff (0.1)'] + Requires-Python: UNKNOWN + Requires-External: [] + + Extra + ===== + * It was installed as a dependency + + +Getting metadata about a distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes you're not interested about the packaging information contained in a +full :class:`Distribution` object but just want to do something with its +:attr:`~Distribution.metadata`:: + + >>> from packaging.database import get_distribution + >>> info = get_distribution('chocolate').metadata + >>> info['Keywords'] + ['cooking', 'happiness'] + + +Finding out obsoleted distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now, we tackle a different problem, we are interested in finding out +which distributions have been obsoleted. This can be easily done as follows:: + + import packaging.database + + # iterate over all distributions in the system + for dist in packaging.database.get_distributions(): + name, version = dist.name, dist.version + # find out which distributions obsolete this name/version combination + replacements = packaging.database.obsoletes_distribution(name, version) + if replacements: + print('%r %s is obsoleted by' % (name, version), + ', '.join(repr(r.name) for r in replacements)) + +This is how the output might look like: + +.. code-block:: none + + 'strawberry' 0.6 is obsoleted by 'choxie' + 'grammar' 1.0a4 is obsoleted by 'towel-stuff' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.depgraph.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.depgraph.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,199 @@ +:mod:`packaging.depgraph` --- Dependency graph builder +====================================================== + +.. module:: packaging.depgraph + :synopsis: Graph builder for dependencies between releases. + + +This module provides the means to analyse the dependencies between various +distributions and to create a graph representing these dependency relationships. +In this document, "distribution" refers to an instance of +:class:`packaging.database.Distribution` or +:class:`packaging.database.EggInfoDistribution`. + +.. XXX terminology problem with dist vs. release: dists are installed, but deps + use releases + +.. XXX explain how to use it with dists not installed: Distribution can only be + instantiated with a path, but this module is useful for remote dist too + +.. XXX functions should accept and return iterators, not lists + + +The :class:`DependencyGraph` class +---------------------------------- + +.. class:: DependencyGraph + + Represent a dependency graph between releases. The nodes are distribution + instances; the edge model dependencies. An edge from ``a`` to ``b`` means + that ``a`` depends on ``b``. + + .. method:: add_distribution(distribution) + + Add *distribution* to the graph. + + .. method:: add_edge(x, y, label=None) + + Add an edge from distribution *x* to distribution *y* with the given + *label* (string). + + .. method:: add_missing(distribution, requirement) + + Add a missing *requirement* (string) for the given *distribution*. + + .. method:: repr_node(dist, level=1) + + Print a subgraph starting from *dist*. *level* gives the depth of the + subgraph. + + Direct access to the graph nodes and edges is provided through these + attributes: + + .. attribute:: adjacency_list + + Dictionary mapping distributions to a list of ``(other, label)`` tuples + where ``other`` is a distribution and the edge is labeled with ``label`` + (i.e. the version specifier, if such was provided). + + .. attribute:: reverse_list + + Dictionary mapping distributions to a list of predecessors. This allows + efficient traversal. + + .. attribute:: missing + + Dictionary mapping distributions to a list of requirements that were not + provided by any distribution. + + +Auxiliary functions +------------------- + +.. function:: dependent_dists(dists, dist) + + Recursively generate a list of distributions from *dists* that are dependent + on *dist*. + + .. XXX what does member mean here: "dist is a member of *dists* for which we + are interested" + +.. function:: generate_graph(dists) + + Generate a :class:`DependencyGraph` from the given list of distributions. + + .. XXX make this alternate constructor a DepGraph classmethod or rename; + 'generate' can suggest it creates a file or an image, use 'make' + +.. function:: graph_to_dot(graph, f, skip_disconnected=True) + + Write a DOT output for the graph to the file-like object *f*. + + If *skip_disconnected* is true, all distributions that are not dependent on + any other distribution are skipped. + + .. XXX why is this not a DepGraph method? + + +Example Usage +------------- + +Depict all dependenciess in the system +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +First, we shall generate a graph of all the distributions on the system +and then create an image out of it using the tools provided by +`Graphviz `_:: + + from packaging.database import get_distributions + from packaging.depgraph import generate_graph + + dists = list(get_distributions()) + graph = generate_graph(dists) + +It would be interesting to print out the missing requirements. This can be done +as follows:: + + for dist, reqs in graph.missing.items(): + if reqs: + reqs = ' ,'.join(repr(req) for req in reqs) + print('Missing dependencies for %r: %s' % (dist.name, reqs)) + +Example output is: + +.. code-block:: none + + Missing dependencies for 'TurboCheetah': 'Cheetah' + Missing dependencies for 'TurboGears': 'ConfigObj', 'DecoratorTools', 'RuleDispatch' + Missing dependencies for 'jockey': 'PyKDE4.kdecore', 'PyKDE4.kdeui', 'PyQt4.QtCore', 'PyQt4.QtGui' + Missing dependencies for 'TurboKid': 'kid' + Missing dependencies for 'TurboJson: 'DecoratorTools', 'RuleDispatch' + +Now, we proceed with generating a graphical representation of the graph. First +we write it to a file, and then we generate a PNG image using the +:program:`dot` command-line tool:: + + from packaging.depgraph import graph_to_dot + with open('output.dot', 'w') as f: + # only show the interesting distributions, skipping the disconnected ones + graph_to_dot(graph, f, skip_disconnected=True) + +We can create the final picture using: + +.. code-block:: sh + + $ dot -Tpng output.dot > output.png + +An example result is: + +.. figure:: depgraph-output.png + :alt: Example PNG output from packaging.depgraph and dot + +If you want to include egg distributions as well, then the code requires only +one change, namely the line:: + + dists = list(packaging.database.get_distributions()) + +has to be replaced with:: + + dists = list(packaging.database.get_distributions(use_egg_info=True)) + +On many platforms, a richer graph is obtained because at the moment most +distributions are provided in the egg rather than the new standard +``.dist-info`` format. + +.. XXX missing image + + An example of a more involved graph for illustrative reasons can be seen + here: + + .. image:: depgraph_big.png + + +List all dependent distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We will list all distributions that are dependent on some given distibution. +This time, egg distributions will be considered as well:: + + import sys + from packaging.database import get_distribution, get_distributions + from packaging.depgraph import dependent_dists + + dists = list(get_distributions(use_egg_info=True)) + dist = get_distribution('bacon', use_egg_info=True) + if dist is None: + sys.exit('No such distribution in the system') + + deps = dependent_dists(dists, dist) + deps = ', '.join(repr(x.name) for x in deps) + print('Distributions depending on %r: %s' % (dist.name, deps)) + +And this is example output: + +.. with the dependency relationships as in the previous section + (depgraph_big) + +.. code-block:: none + + Distributions depending on 'bacon': 'towel-stuff', 'choxie', 'grammar' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.dist.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.dist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,108 @@ +:mod:`packaging.dist` --- The Distribution class +================================================ + +.. module:: packaging.dist + :synopsis: Core Distribution class. + + +This module provides the :class:`Distribution` class, which represents the +module distribution being built/packaged/distributed/installed. + +.. class:: Distribution(arguments) + + A :class:`Distribution` describes how to build, package, distribute and + install a Python project. + + The arguments accepted by the constructor are laid out in the following + table. Some of them will end up in a metadata object, the rest will become + data attributes of the :class:`Distribution` instance. + + .. TODO improve constructor to take a Metadata object + named params? + (i.e. Distribution(metadata, cmdclass, py_modules, etc) + .. TODO also remove obsolete(?) script_name, etc. parameters? see what + py2exe and other tools need + + +--------------------+--------------------------------+-------------------------------------------------------------+ + | argument name | value | type | + +====================+================================+=============================================================+ + | *name* | The name of the project | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *version* | The version number of the | a string | + | | release; see | | + | | :mod:`packaging.version` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *summary* | A single line describing the | a string | + | | project | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *description* | Longer description of the | a string | + | | project | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author* | The name of the project author | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author_email* | The email address of the | a string | + | | project author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer* | The name of the current | a string | + | | maintainer, if different from | | + | | the author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer_email* | The email address of the | a string | + | | current maintainer, if | | + | | different from the author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *home_page* | A URL for the proejct | a string | + | | (homepage) | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *download_url* | A URL to download the project | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *packages* | A list of Python packages that | a list of strings | + | | packaging will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *py_modules* | A list of Python modules that | a list of strings | + | | packaging will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *scripts* | A list of standalone scripts | a list of strings | + | | to be built and installed | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *ext_modules* | A list of Python extensions to | a list of instances of | + | | be built | :class:`packaging.compiler.extension.Extension` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPi | + | | distribution | `_. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *distclass* | the :class:`Distribution` | a subclass of | + | | class to use | :class:`packaging.dist.Distribution` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_name* | The name of the setup.py | a string | + | | script - defaults to | | + | | ``sys.argv[0]`` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_args* | Arguments to supply to the | a list of strings | + | | setup script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *options* | default options for the setup | a string | + | | script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *license* | The license for the | a string | + | | distribution; should be used | | + | | when there is no suitable | | + | | License classifier, or to | | + | | refine a classifier | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *keywords* | Descriptive keywords; used by | a list of strings or a comma-separated string | + | | catalogs such as PyPI | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *platforms* | Platforms compatible with this | a list of strings or a comma-separated string | + | | distribution; should be used | | + | | when there is no suitable | | + | | Platform classifier | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *cmdclass* | A mapping of command names to | a dictionary | + | | :class:`Command` subclasses | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *data_files* | A list of data files to | a list | + | | install | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *package_dir* | A mapping of Python packages | a dictionary | + | | to directory names | | + +--------------------+--------------------------------+-------------------------------------------------------------+ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.fancy_getopt.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.fancy_getopt.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,75 @@ +:mod:`packaging.fancy_getopt` --- Wrapper around the getopt module +================================================================== + +.. module:: packaging.fancy_getopt + :synopsis: Additional getopt functionality. + + +.. warning:: + This module is deprecated and will be replaced with :mod:`optparse`. + +This module provides a wrapper around the standard :mod:`getopt` module that +provides the following additional features: + +* short and long options are tied together + +* options have help strings, so :func:`fancy_getopt` could potentially create a + complete usage summary + +* options set attributes of a passed-in object + +* boolean options can have "negative aliases" --- e.g. if :option:`--quiet` is + the "negative alias" of :option:`--verbose`, then :option:`--quiet` on the + command line sets *verbose* to false. + +.. function:: fancy_getopt(options, negative_opt, object, args) + + Wrapper function. *options* is a list of ``(long_option, short_option, + help_string)`` 3-tuples as described in the constructor for + :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names + to option names, both the key and value should be in the *options* list. + *object* is an object which will be used to store values (see the :meth:`getopt` + method of the :class:`FancyGetopt` class). *args* is the argument list. Will use + ``sys.argv[1:]`` if you pass ``None`` as *args*. + + +.. class:: FancyGetopt([option_table=None]) + + The option_table is a list of 3-tuples: ``(long_option, short_option, + help_string)`` + + If an option takes an argument, its *long_option* should have ``'='`` appended; + *short_option* should just be a single character, no ``':'`` in any case. + *short_option* should be ``None`` if a *long_option* doesn't have a + corresponding *short_option*. All option tuples must have long options. + +The :class:`FancyGetopt` class provides the following methods: + + +.. method:: FancyGetopt.getopt([args=None, object=None]) + + Parse command-line options in args. Store as attributes on *object*. + + If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is + ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores + option values there, and returns a tuple ``(args, object)``. If *object* is + supplied, it is modified in place and :func:`getopt` just returns *args*; in + both cases, the returned *args* is a modified copy of the passed-in *args* list, + which is left untouched. + + .. TODO and args returned are? + + +.. method:: FancyGetopt.get_option_order() + + Returns the list of ``(option, value)`` tuples processed by the previous run of + :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called + yet. + + +.. method:: FancyGetopt.generate_help([header=None]) + + Generate help text (a list of strings, one per suggested line of output) from + the option table for this :class:`FancyGetopt` object. + + If supplied, prints the supplied *header* at the top of the help. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.install.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.install.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,112 @@ +:mod:`packaging.install` --- Installation tools +=============================================== + +.. module:: packaging.install + :synopsis: Download and installation building blocks + + +Packaging provides a set of tools to deal with downloads and installation of +distributions. Their role is to download the distribution from indexes, resolve +the dependencies, and provide a safe way to install distributions. An operation +that fails will cleanly roll back, not leave half-installed distributions on the +system. Here's the basic process followed: + +#. Move all distributions that will be removed to a temporary location. + +#. Install all the distributions that will be installed in a temporary location. + +#. If the installation fails, move the saved distributions back to their + location and delete the installed distributions. + +#. Otherwise, move the installed distributions to the right location and delete + the temporary locations. + +This is a higher-level module built on :mod:`packaging.database` and +:mod:`packaging.pypi`. + + +Public functions +---------------- + +.. function:: get_infos(requirements, index=None, installed=None, \ + prefer_final=True) + + Return information about what's going to be installed and upgraded. + *requirements* is a string containing the requirements for this + project, for example ``'FooBar 1.1'`` or ``'BarBaz (<1.2)'``. + + .. XXX are requirements comma-separated? + + If you want to use another index than the main PyPI, give its URI as *index* + argument. + + *installed* is a list of already installed distributions used to find + satisfied dependencies, obsoleted distributions and eventual conflicts. + + By default, alpha, beta and candidate versions are not picked up. Set + *prefer_final* to false to accept them too. + + The results are returned in a dictionary containing all the information + needed to perform installation of the requirements with the + :func:`install_from_infos` function: + + >>> get_install_info("FooBar (<=1.2)") + {'install': [], 'remove': [], 'conflict': []} + + .. TODO should return tuple or named tuple, not dict + .. TODO use "predicate" or "requirement" consistently in version and here + .. FIXME "info" cannot be plural in English, s/infos/info/ + + +.. function:: install(project) + + +.. function:: install_dists(dists, path, paths=None) + + Safely install all distributions provided in *dists* into *path*. *paths* is + a list of paths where already-installed distributions will be looked for to + find satisfied dependencies and conflicts (default: :data:`sys.path`). + Returns a list of installed dists. + + .. FIXME dists are instances of what? + + +.. function:: install_from_infos(install_path=None, install=[], remove=[], \ + conflicts=[], paths=None) + + Safely install and remove given distributions. This function is designed to + work with the return value of :func:`get_infos`: *install*, *remove* and + *conflicts* should be list of distributions returned by :func:`get_infos`. + If *install* is not empty, *install_path* must be given to specify the path + where the distributions should be installed. *paths* is a list of paths + where already-installed distributions will be looked for (default: + :data:`sys.path`). + + This function is a very basic installer; if *conflicts* is not empty, the + system will be in a conflicting state after the function completes. It is a + building block for more sophisticated installers with conflict resolution + systems. + + .. TODO document typical value for install_path + .. TODO document integration with default schemes, esp. user site-packages + + +.. function:: install_local_project(path) + + Install a distribution from a source directory, which must contain either a + Packaging-compliant :file:`setup.cfg` file or a legacy Distutils + :file:`setup.py` script (in which case Distutils will be used under the hood + to perform the installation). + + +.. function:: remove(project_name, paths=None, auto_confirm=True) + + Remove one distribution from the system. + + .. FIXME this is the only function using "project" instead of dist/release + +.. + Example usage + -------------- + + Get the scheme of what's gonna be installed if we install "foobar": diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.metadata.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.metadata.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,122 @@ +:mod:`packaging.metadata` --- Metadata handling +=============================================== + +.. module:: packaging.metadata + :synopsis: Class holding the metadata of a release. + + +.. TODO use sphinx-autogen to generate basic doc from the docstrings + +.. class:: Metadata + + This class can read and write metadata files complying with any of the + defined versions: 1.0 (:PEP:`241`), 1.1 (:PEP:`314`) and 1.2 (:PEP:`345`). It + implements methods to parse Metadata files and write them, and a mapping + interface to its contents. + + The :PEP:`345` implementation supports the micro-language for the environment + markers, and displays warnings when versions that are supposed to be + :PEP:`386`-compliant are violating the specification. + + +Reading metadata +---------------- + +The :class:`Metadata` class can be instantiated +with the path of the metadata file, and provides a dict-like interface to the +values:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata('PKG-INFO') + >>> metadata.keys()[:5] + ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform') + >>> metadata['Name'] + 'CLVault' + >>> metadata['Version'] + '0.5' + >>> metadata['Requires-Dist'] + ["pywin32; sys.platform == 'win32'", "Sphinx"] + + +The fields that support environment markers can be automatically ignored if +the object is instantiated using the ``platform_dependent`` option. +:class:`Metadata` will interpret in this case +the markers and will automatically remove the fields that are not compliant +with the running environment. Here's an example under Mac OS X. The win32 +dependency we saw earlier is ignored:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata('PKG-INFO', platform_dependent=True) + >>> metadata['Requires-Dist'] + ['Sphinx'] + + +If you want to provide your own execution context, let's say to test the +metadata under a particular environment that is not the current environment, +you can provide your own values in the ``execution_context`` option, which +is the dict that may contain one or more keys of the context the micro-language +expects. + +Here's an example, simulating a win32 environment:: + + >>> from packaging.metadata import Metadata + >>> context = {'sys.platform': 'win32'} + >>> metadata = Metadata('PKG-INFO', platform_dependent=True, + ... execution_context=context) + ... + >>> metadata['Requires-Dist'] = ["pywin32; sys.platform == 'win32'", + ... "Sphinx"] + ... + >>> metadata['Requires-Dist'] + ['pywin32', 'Sphinx'] + + +Writing metadata +---------------- + +Writing metadata can be done using the ``write`` method:: + + >>> metadata.write('/to/my/PKG-INFO') + +The class will pick the best version for the metadata, depending on the values +provided. If all the values provided exist in all versions, the class will +use :attr:`PKG_INFO_PREFERRED_VERSION`. It is set by default to 1.0, the most +widespread version. + + +Conflict checking and best version +---------------------------------- + +Some fields in :PEP:`345` have to comply with the version number specification +defined in :PEP:`386`. When they don't comply, a warning is emitted:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata() + >>> metadata['Requires-Dist'] = ['Funky (Groovie)'] + "Funky (Groovie)" is not a valid predicate + >>> metadata['Requires-Dist'] = ['Funky (1.2)'] + +See also :mod:`packaging.version`. + + +.. TODO talk about check() + + +:mod:`packaging.markers` --- Environment markers +================================================ + +.. module:: packaging.markers + :synopsis: Micro-language for environment markers + + +This is an implementation of environment markers `as defined in PEP 345 +`_. It is used +for some metadata fields. + +.. function:: interpret(marker, execution_context=None) + + Interpret a marker and return a boolean result depending on the environment. + Example: + + >>> interpret("python_version > '1.0'") + True diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.pypi.dist.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.pypi.dist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,114 @@ +:mod:`packaging.pypi.dist` --- Classes representing query results +================================================================= + +.. module:: packaging.pypi.dist + :synopsis: Classes representing the results of queries to indexes. + + +Information coming from the indexes is held in instances of the classes defined +in this module. + +Keep in mind that each project (eg. FooBar) can have several releases +(eg. 1.1, 1.2, 1.3), and each of these releases can be provided in multiple +distributions (eg. a source distribution, a binary one, etc). + + +ReleaseInfo +----------- + +Each release has a project name, version, metadata, and related distributions. + +This information is stored in :class:`ReleaseInfo` +objects. + +.. class:: ReleaseInfo + + +DistInfo +--------- + +:class:`DistInfo` is a simple class that contains +information related to distributions; mainly the URLs where distributions +can be found. + +.. class:: DistInfo + + +ReleasesList +------------ + +The :mod:`~packaging.pypi.dist` module provides a class which works +with lists of :class:`ReleaseInfo` classes; +used to filter and order results. + +.. class:: ReleasesList + + +Example usage +------------- + +Build a list of releases and order them +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assuming we have a list of releases:: + + >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo + >>> fb10 = ReleaseInfo("FooBar", "1.0") + >>> fb11 = ReleaseInfo("FooBar", "1.1") + >>> fb11a = ReleaseInfo("FooBar", "1.1a1") + >>> ReleasesList("FooBar", [fb11, fb11a, fb10]) + >>> releases.sort_releases() + >>> releases.get_versions() + ['1.1', '1.1a1', '1.0'] + >>> releases.add_release("1.2a1") + >>> releases.get_versions() + ['1.1', '1.1a1', '1.0', '1.2a1'] + >>> releases.sort_releases() + ['1.2a1', '1.1', '1.1a1', '1.0'] + >>> releases.sort_releases(prefer_final=True) + >>> releases.get_versions() + ['1.1', '1.0', '1.2a1', '1.1a1'] + + +Add distribution related information to releases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It's easy to add distribution information to releases:: + + >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo + >>> r = ReleaseInfo("FooBar", "1.0") + >>> r.add_distribution("sdist", url="http://example.org/foobar-1.0.tar.gz") + >>> r.dists + {'sdist': FooBar 1.0 sdist} + >>> r['sdist'].url + {'url': 'http://example.org/foobar-1.0.tar.gz', 'hashname': None, 'hashval': + None, 'is_external': True} + + +Getting attributes from the dist objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To abstract querying information returned from the indexes, attributes and +release information can be retrieved directly from dist objects. + +For instance, if you have a release instance that does not contain the metadata +attribute, it can be fetched by using the "fetch_metadata" method:: + + >>> r = Release("FooBar", "1.1") + >>> print r.metadata + None # metadata field is actually set to "None" + >>> r.fetch_metadata() + + +.. XXX add proper roles to these constructs + + +It's possible to retrieve a project's releases (`fetch_releases`), +metadata (`fetch_metadata`) and distributions (`fetch_distributions`) using +a similar work flow. + +.. XXX what is possible? + +Internally, this is possible because while retrieving information about +projects, releases or distributions, a reference to the client used is +stored which can be accessed using the objects `_index` attribute. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.pypi.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.pypi.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,74 @@ +:mod:`packaging.pypi` --- Interface to projects indexes +======================================================= + +.. module:: packaging.pypi + :synopsis: Low-level and high-level APIs to query projects indexes. + + +Packaging queries PyPI to get information about projects or download them. The +low-level facilities used internally are also part of the public API designed to +be used by other tools. + +The :mod:`packaging.pypi` package provides those facilities, which can be +used to access information about Python projects registered at indexes, the +main one being PyPI, located ad http://pypi.python.org/. + +There is two ways to retrieve data from these indexes: a screen-scraping +interface called the "simple API", and XML-RPC. The first one uses HTML pages +located under http://pypi.python.org/simple/, the second one makes XML-RPC +requests to http://pypi.python.org/pypi/. All functions and classes also work +with other indexes such as mirrors, which typically implement only the simple +interface. + +Packaging provides a class that wraps both APIs to provide full query and +download functionality: :class:`packaging.pypi.client.ClientWrapper`. If you +want more control, you can use the underlying classes +:class:`packaging.pypi.simple.Crawler` and :class:`packaging.pypi.xmlrpc.Client` +to connect to one specific interface. + + +:mod:`packaging.pypi.client` --- High-level query API +===================================================== + +.. module:: packaging.pypi.client + :synopsis: Wrapper around :mod;`packaging.pypi.xmlrpc` and + :mod:`packaging.pypi.simple` to query indexes. + + +This module provides a high-level API to query indexes and search +for releases and distributions. The aim of this module is to choose the best +way to query the API automatically, either using XML-RPC or the simple index, +with a preference toward the latter. + +.. class:: ClientWrapper + + Instances of this class will use the simple interface or XML-RPC requests to + query indexes and return :class:`packaging.pypi.dist.ReleaseInfo` and + :class:`packaging.pypi.dist.ReleasesList` objects. + + .. method:: find_projects + + .. method:: get_release + + .. method:: get_releases + + +:mod:`packaging.pypi.base` --- Base class for index crawlers +============================================================ + +.. module:: packaging.pypi.base + :synopsis: Base class used to implement crawlers. + + +.. class:: BaseClient(prefer_final, prefer_source) + + Base class containing common methods for the index crawlers or clients. One + method is currently defined: + + .. method:: download_distribution(requirements, temp_path=None, \ + prefer_source=None, prefer_final=None) + + Download a distribution from the last release according to the + requirements. If *temp_path* is provided, download to this path, + otherwise, create a temporary directory for the download. If a release is + found, the full path to the downloaded file is returned. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.pypi.simple.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.pypi.simple.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,218 @@ +:mod:`packaging.pypi.simple` --- Crawler using the PyPI "simple" interface +========================================================================== + +.. module:: packaging.pypi.simple + :synopsis: Crawler using the screen-scraping "simple" interface to fetch info + and distributions. + + +The class provided by :mod:`packaging.pypi.simple` can access project indexes +and provide useful information about distributions. PyPI, other indexes and +local indexes are supported. + +You should use this module to search distributions by name and versions, process +index external pages and download distributions. It is not suited for things +that will end up in too long index processing (like "finding all distributions +with a specific version, no matter the name"); use :mod:`packaging.pypi.xmlrpc` +for that. + + +API +--- + +.. class:: Crawler(index_url=DEFAULT_SIMPLE_INDEX_URL, \ + prefer_final=False, prefer_source=True, \ + hosts=('*',), follow_externals=False, \ + mirrors_url=None, mirrors=None, timeout=15, \ + mirrors_max_tries=0) + + *index_url* is the address of the index to use for requests. + + The first two parameters control the query results. *prefer_final* + indicates whether a final version (not alpha, beta or candidate) is to be + preferred over a newer but non-final version (for example, whether to pick + up 1.0 over 2.0a3). It is used only for queries that don't give a version + argument. Likewise, *prefer_source* tells whether to prefer a source + distribution over a binary one, if no distribution argument was prodived. + + Other parameters are related to external links (that is links that go + outside the simple index): *hosts* is a list of hosts allowed to be + processed if *follow_externals* is true (default behavior is to follow all + hosts), *follow_externals* enables or disables following external links + (default is false, meaning disabled). + + The remaining parameters are related to the mirroring infrastructure + defined in :PEP:`381`. *mirrors_url* gives a URL to look on for DNS + records giving mirror adresses; *mirrors* is a list of mirror URLs (see + the PEP). If both *mirrors* and *mirrors_url* are given, *mirrors_url* + will only be used if *mirrors* is set to ``None``. *timeout* is the time + (in seconds) to wait before considering a URL has timed out; + *mirrors_max_tries"* is the number of times to try requesting informations + on mirrors before switching. + + The following methods are defined: + + .. method:: get_distributions(project_name, version) + + Return the distributions found in the index for the given release. + + .. method:: get_metadata(project_name, version) + + Return the metadata found on the index for this project name and + version. Currently downloads and unpacks a distribution to read the + PKG-INFO file. + + .. method:: get_release(requirements, prefer_final=None) + + Return one release that fulfills the given requirements. + + .. method:: get_releases(requirements, prefer_final=None, force_update=False) + + Search for releases and return a + :class:`~packaging.pypi.dist.ReleasesList` object containing the + results. + + .. method:: search_projects(name=None) + + Search the index for projects containing the given name and return a + list of matching names. + + See also the base class :class:`packaging.pypi.base.BaseClient` for inherited + methods. + + +.. data:: DEFAULT_SIMPLE_INDEX_URL + + The address used by default by the crawler class. It is currently + ``'http://a.pypi.python.org/simple/'``, the main PyPI installation. + + + + +Usage Examples +--------------- + +To help you understand how using the `Crawler` class, here are some basic +usages. + +Request the simple index to get a specific distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Supposing you want to scan an index to get a list of distributions for +the "foobar" project. You can use the "get_releases" method for that. +The get_releases method will browse the project page, and return +:class:`ReleaseInfo` objects for each found link that rely on downloads. :: + + >>> from packaging.pypi.simple import Crawler + >>> crawler = Crawler() + >>> crawler.get_releases("FooBar") + [, ] + + +Note that you also can request the client about specific versions, using version +specifiers (described in `PEP 345 +`_):: + + >>> client.get_releases("FooBar < 1.2") + [, ] + + +`get_releases` returns a list of :class:`ReleaseInfo`, but you also can get the +best distribution that fullfil your requirements, using "get_release":: + + >>> client.get_release("FooBar < 1.2") + + + +Download distributions +^^^^^^^^^^^^^^^^^^^^^^ + +As it can get the urls of distributions provided by PyPI, the `Crawler` +client also can download the distributions and put it for you in a temporary +destination:: + + >>> client.download("foobar") + /tmp/temp_dir/foobar-1.2.tar.gz + + +You also can specify the directory you want to download to:: + + >>> client.download("foobar", "/path/to/my/dir") + /path/to/my/dir/foobar-1.2.tar.gz + + +While downloading, the md5 of the archive will be checked, if not matches, it +will try another time, then if fails again, raise `MD5HashDoesNotMatchError`. + +Internally, that's not the Crawler which download the distributions, but the +`DistributionInfo` class. Please refer to this documentation for more details. + + +Following PyPI external links +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default behavior for packaging is to *not* follow the links provided +by HTML pages in the "simple index", to find distributions related +downloads. + +It's possible to tell the PyPIClient to follow external links by setting the +`follow_externals` attribute, on instantiation or after:: + + >>> client = Crawler(follow_externals=True) + +or :: + + >>> client = Crawler() + >>> client.follow_externals = True + + +Working with external indexes, and mirrors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default `Crawler` behavior is to rely on the Python Package index stored +on PyPI (http://pypi.python.org/simple). + +As you can need to work with a local index, or private indexes, you can specify +it using the index_url parameter:: + + >>> client = Crawler(index_url="file://filesystem/path/") + +or :: + + >>> client = Crawler(index_url="http://some.specific.url/") + + +You also can specify mirrors to fallback on in case the first index_url you +provided doesnt respond, or not correctly. The default behavior for +`Crawler` is to use the list provided by Python.org DNS records, as +described in the :PEP:`381` about mirroring infrastructure. + +If you don't want to rely on these, you could specify the list of mirrors you +want to try by specifying the `mirrors` attribute. It's a simple iterable:: + + >>> mirrors = ["http://first.mirror","http://second.mirror"] + >>> client = Crawler(mirrors=mirrors) + + +Searching in the simple index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It's possible to search for projects with specific names in the package index. +Assuming you want to find all projects containing the "distutils" keyword:: + + >>> c.search_projects("distutils") + [, , , , , , ] + + +You can also search the projects starting with a specific text, or ending with +that text, using a wildcard:: + + >>> c.search_projects("distutils*") + [, , ] + + >>> c.search_projects("*distutils") + [, , , , ] diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.pypi.xmlrpc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.pypi.xmlrpc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,143 @@ +:mod:`packaging.pypi.xmlrpc` --- Crawler using the PyPI XML-RPC interface +========================================================================= + +.. module:: packaging.pypi.xmlrpc + :synopsis: Client using XML-RPC requests to fetch info and distributions. + + +Indexes can be queried using XML-RPC calls, and Packaging provides a simple +way to interface with XML-RPC. + +You should **use** XML-RPC when: + +* Searching the index for projects **on other fields than project + names**. For instance, you can search for projects based on the + author_email field. +* Searching all the versions that have existed for a project. +* you want to retrieve METADATAs information from releases or + distributions. + + +You should **avoid using** XML-RPC method calls when: + +* Retrieving the last version of a project +* Getting the projects with a specific name and version. +* The simple index can match your needs + + +When dealing with indexes, keep in mind that the index queries will always +return you :class:`packaging.pypi.dist.ReleaseInfo` and +:class:`packaging.pypi.dist.ReleasesList` objects. + +Some methods here share common APIs with the one you can find on +:class:`packaging.pypi.simple`, internally, :class:`packaging.pypi.client` +is inherited by :class:`Client` + + +API +--- + +.. class:: Client + + +Usage examples +-------------- + +Use case described here are use case that are not common to the other clients. +If you want to see all the methods, please refer to API or to usage examples +described in :class:`packaging.pypi.client.Client` + + +Finding releases +^^^^^^^^^^^^^^^^ + +It's a common use case to search for "things" within the index. We can +basically search for projects by their name, which is the most used way for +users (eg. "give me the last version of the FooBar project"). + +This can be accomplished using the following syntax:: + + >>> client = xmlrpc.Client() + >>> client.get_release("Foobar (<= 1.3)) + + >>> client.get_releases("FooBar (<= 1.3)") + [FooBar 1.1, FooBar 1.1.1, FooBar 1.2, FooBar 1.2.1] + + +And we also can find for specific fields:: + + >>> client.search_projects(field=value) + + +You could specify the operator to use, default is "or":: + + >>> client.search_projects(field=value, operator="and") + + +The specific fields you can search are: + +* name +* version +* author +* author_email +* maintainer +* maintainer_email +* home_page +* license +* summary +* description +* keywords +* platform +* download_url + + +Getting metadata information +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +XML-RPC is a preferred way to retrieve metadata information from indexes. +It's really simple to do so:: + + >>> client = xmlrpc.Client() + >>> client.get_metadata("FooBar", "1.1") + + + +Assuming we already have a :class:`packaging.pypi.ReleaseInfo` object defined, +it's possible to pass it to the xmlrpc client to retrieve and complete its +metadata:: + + >>> foobar11 = ReleaseInfo("FooBar", "1.1") + >>> client = xmlrpc.Client() + >>> returned_release = client.get_metadata(release=foobar11) + >>> returned_release + + + +Get all the releases of a project +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To retrieve all the releases for a project, you can build them using +`get_releases`:: + + >>> client = xmlrpc.Client() + >>> client.get_releases("FooBar") + [, , ] + + +Get information about distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Indexes have information about projects, releases **and** distributions. +If you're not familiar with those, please refer to the documentation of +:mod:`packaging.pypi.dist`. + +It's possible to retrieve information about distributions, e.g "what are the +existing distributions for this release ? How to retrieve them ?":: + + >>> client = xmlrpc.Client() + >>> release = client.get_distributions("FooBar", "1.1") + >>> release.dists + {'sdist': , 'bdist': } + +As you see, this does not return a list of distributions, but a release, +because a release can be used like a list of distributions. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,75 @@ +:mod:`packaging` --- Packaging support +====================================== + +.. module:: packaging + :synopsis: Packaging system and building blocks for other packaging systems. +.. sectionauthor:: Fred L. Drake, Jr. , distutils and packaging + contributors + + +The :mod:`packaging` package provides support for building, packaging, +distributing and installing additional projects into a Python installation. +Projects may include Python modules, extension modules, packages and scripts. +:mod:`packaging` also provides building blocks for other packaging systems +that are not tied to the command system. + +This manual is the reference documentation for those standalone building +blocks and for extending Packaging. If you're looking for the user-centric +guides to install a project or package your own code, head to `See also`__. + + +Building blocks +--------------- + +.. toctree:: + :maxdepth: 2 + + packaging-misc + packaging.version + packaging.metadata + packaging.database + packaging.depgraph + packaging.pypi + packaging.pypi.dist + packaging.pypi.simple + packaging.pypi.xmlrpc + packaging.install + + +The command machinery +--------------------- + +.. toctree:: + :maxdepth: 2 + + packaging.dist + packaging.command + packaging.compiler + packaging.fancy_getopt + + +Other utilities +---------------- + +.. toctree:: + :maxdepth: 2 + + packaging.util + packaging.tests.pypi_server + +.. XXX missing: compat config create (dir_util) run pypi.{base,mirrors} + + +.. __: + +.. seealso:: + + :ref:`packaging-index` + The manual for developers of Python projects who want to package and + distribute them. This describes how to use :mod:`packaging` to make + projects easily found and added to an existing Python installation. + + :ref:`packaging-install-index` + A user-centered manual which includes information on adding projects + into an existing Python installation. You do not need to be a Python + programmer to read this manual. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.tests.pypi_server.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.tests.pypi_server.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,105 @@ +:mod:`packaging.tests.pypi_server` --- PyPI mock server +======================================================= + +.. module:: packaging.tests.pypi_server + :synopsis: Mock server used to test PyPI-related modules and commands. + + +When you are testing code that works with Packaging, you might find these tools +useful. + + +The mock server +--------------- + +.. class:: PyPIServer + + PyPIServer is a class that implements an HTTP server running in a separate + thread. All it does is record the requests for further inspection. The recorded + data is available under ``requests`` attribute. The default + HTTP response can be overridden with the ``default_response_status``, + ``default_response_headers`` and ``default_response_data`` attributes. + + By default, when accessing the server with urls beginning with `/simple/`, + the server also record your requests, but will look for files under + the `/tests/pypiserver/simple/` path. + + You can tell the sever to serve static files for other paths. This could be + accomplished by using the `static_uri_paths` parameter, as below:: + + server = PyPIServer(static_uri_paths=["first_path", "second_path"]) + + + You need to create the content that will be served under the + `/tests/pypiserver/default` path. If you want to serve content from another + place, you also can specify another filesystem path (which needs to be under + `tests/pypiserver/`. This will replace the default behavior of the server, and + it will not serve content from the `default` dir :: + + server = PyPIServer(static_filesystem_paths=["path/to/your/dir"]) + + + If you just need to add some paths to the existing ones, you can do as shown, + keeping in mind that the server will always try to load paths in reverse order + (e.g here, try "another/super/path" then the default one) :: + + server = PyPIServer(test_static_path="another/super/path") + server = PyPIServer("another/super/path") + # or + server.static_filesystem_paths.append("another/super/path") + + + As a result of what, in your tests, while you need to use the PyPIServer, in + order to isolates the test cases, the best practice is to place the common files + in the `default` folder, and to create a directory for each specific test case:: + + server = PyPIServer(static_filesystem_paths = ["default", "test_pypi_server"], + static_uri_paths=["simple", "external"]) + + +Base class and decorator for tests +---------------------------------- + +.. class:: PyPIServerTestCase + + ``PyPIServerTestCase`` is a test case class with setUp and tearDown methods that + take care of a single PyPIServer instance attached as a ``pypi`` attribute on + the test class. Use it as one of the base classes in your test case:: + + + class UploadTestCase(PyPIServerTestCase): + + def test_something(self): + cmd = self.prepare_command() + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + cmd.run() + + environ, request_data = self.pypi.requests[-1] + self.assertEqual(request_data, EXPECTED_REQUEST_DATA) + + +.. decorator:: use_pypi_server + + You also can use a decorator for your tests, if you do not need the same server + instance along all you test case. So, you can specify, for each test method, + some initialisation parameters for the server. + + For this, you need to add a `server` parameter to your method, like this:: + + class SampleTestCase(TestCase): + + @use_pypi_server() + def test_something(self, server): + ... + + + The decorator will instantiate the server for you, and run and stop it just + before and after your method call. You also can pass the server initializer, + just like this:: + + class SampleTestCase(TestCase): + + @use_pypi_server("test_case_name") + def test_something(self, server): + ... diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.util.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.util.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,155 @@ +:mod:`packaging.util` --- Miscellaneous utility functions +========================================================= + +.. module:: packaging.util + :synopsis: Miscellaneous utility functions. + + +This module contains various helpers for the other modules. + +.. XXX a number of functions are missing, but the module may be split first + (it's ginormous right now, some things could go to compat for example) + +.. function:: get_platform() + + Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; e.g. for IRIX the architecture isn't particularly + important (IRIX only runs on SGI hardware), but for Linux the kernel version + isn't particularly important. + + Examples of returned values: + + * ``linux-i586`` + * ``linux-alpha`` + * ``solaris-2.6-sun4u`` + * ``irix-5.3`` + * ``irix64-6.2`` + + For non-POSIX platforms, currently just returns ``sys.platform``. + + For Mac OS X systems the OS version reflects the minimal version on which + binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` + during the build of Python), not the OS version of the current system. + + For universal binary builds on Mac OS X the architecture value reflects + the univeral binary status instead of the architecture of the current + processor. For 32-bit universal binaries the architecture is ``fat``, + for 64-bit universal binaries the architecture is ``fat64``, and + for 4-way universal binaries the architecture is ``universal``. Starting + from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for + a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for + a univeral build with the i386 and x86_64 architectures + + Examples of returned values on Mac OS X: + + * ``macosx-10.3-ppc`` + + * ``macosx-10.3-fat`` + + * ``macosx-10.5-universal`` + + * ``macosx-10.6-intel`` + + .. XXX reinvention of platform module? + + +.. function:: convert_path(pathname) + + Return 'pathname' as a name that will work on the native filesystem, i.e. + split it on '/' and put it back together again using the current directory + separator. Needed because filenames in the setup script are always supplied + in Unix style, and have to be converted to the local convention before we + can actually use them in the filesystem. Raises :exc:`ValueError` on + non-Unix-ish systems if *pathname* either starts or ends with a slash. + + +.. function:: change_root(new_root, pathname) + + Return *pathname* with *new_root* prepended. If *pathname* is relative, this + is equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires + making *pathname* relative and then joining the two, which is tricky on + DOS/Windows. + + +.. function:: check_environ() + + Ensure that 'os.environ' has all the environment variables we guarantee that + users can use in config files, command-line options, etc. Currently this + includes: + + * :envvar:`HOME` - user's home directory (Unix only) + * :envvar:`PLAT` - description of the current platform, including hardware + and OS (see :func:`get_platform`) + + +.. function:: find_executable(executable, path=None) + + Search the path for a given executable name. + + +.. function:: execute(func, args, msg=None, dry_run=False) + + Perform some action that affects the outside world (for instance, writing to + the filesystem). Such actions are special because they are disabled by the + *dry_run* flag. This method takes care of all that bureaucracy for you; + all you have to do is supply the function to call and an argument tuple for + it (to embody the "external action" being performed), and an optional message + to print. + + +.. function:: newer(source, target) + + Return true if *source* exists and is more recently modified than *target*, + or if *source* exists and *target* doesn't. Return false if both exist and + *target* is the same age or newer than *source*. Raise + :exc:`PackagingFileError` if *source* does not exist. + + +.. function:: strtobool(val) + + Convert a string representation of truth to true (1) or false (0). + + True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false + values are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises + :exc:`ValueError` if *val* is anything else. + + +.. function:: byte_compile(py_files, optimize=0, force=0, prefix=None, \ + base_dir=None, dry_run=0, direct=None) + + Byte-compile a collection of Python source files to either :file:`.pyc` or + :file:`.pyo` files in a :file:`__pycache__` subdirectory (see :pep:`3147`), + or to the same directory when using the distutils2 backport on Python + versions older than 3.2. + + *py_files* is a list of files to compile; any files that don't end in + :file:`.py` are silently skipped. *optimize* must be one of the following: + + * ``0`` - don't optimize (generate :file:`.pyc`) + * ``1`` - normal optimization (like ``python -O``) + * ``2`` - extra optimization (like ``python -OO``) + + This function is independent from the running Python's :option:`-O` or + :option:`-B` options; it is fully controlled by the parameters passed in. + + If *force* is true, all files are recompiled regardless of timestamps. + + The source filename encoded in each :term:`bytecode` file defaults to the filenames + listed in *py_files*; you can modify these with *prefix* and *basedir*. + *prefix* is a string that will be stripped off of each source filename, and + *base_dir* is a directory name that will be prepended (after *prefix* is + stripped). You can supply either or both (or neither) of *prefix* and + *base_dir*, as you wish. + + If *dry_run* is true, doesn't actually do anything that would affect the + filesystem. + + Byte-compilation is either done directly in this interpreter process with the + standard :mod:`py_compile` module, or indirectly by writing a temporary + script and executing it. Normally, you should let :func:`byte_compile` + figure out to use direct compilation or not (see the source for details). + The *direct* flag is used by the script generated in indirect mode; unless + you know what you're doing, leave it set to ``None``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/packaging.version.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/packaging.version.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,104 @@ +:mod:`packaging.version` --- Version number classes +=================================================== + +.. module:: packaging.version + :synopsis: Classes that represent project version numbers. + + +This module contains classes and functions useful to deal with version numbers. +It's an implementation of version specifiers `as defined in PEP 345 +`_. + + +Version numbers +--------------- + +.. class:: NormalizedVersion(self, s, error_on_huge_major_num=True) + + A specific version of a distribution, as described in PEP 345. *s* is a + string object containing the version number (for example ``'1.2b1'``), + *error_on_huge_major_num* a boolean specifying whether to consider an + apparent use of a year or full date as the major version number an error. + + The rationale for the second argument is that there were projects using years + or full dates as version numbers, which could cause problems with some + packaging systems sorting. + + Instances of this class can be compared and sorted:: + + >>> NormalizedVersion('1.2b1') < NormalizedVersion('1.2') + True + + :class:`NormalizedVersion` is used internally by :class:`VersionPredicate` to + do its work. + + +.. class:: IrrationalVersionError + + Exception raised when an invalid string is given to + :class:`NormalizedVersion`. + + >>> NormalizedVersion("irrational_version_number") + ... + IrrationalVersionError: irrational_version_number + + +.. function:: suggest_normalized_version(s) + + Before standardization in PEP 386, various schemes were in use. Packaging + provides a function to try to convert any string to a valid, normalized + version:: + + >>> suggest_normalized_version('2.1-rc1') + 2.1c1 + + + If :func:`suggest_normalized_version` can't make sense of the given string, + it will return ``None``:: + + >>> print(suggest_normalized_version('not a version')) + None + + +Version predicates +------------------ + +.. class:: VersionPredicate(predicate) + + This class deals with the parsing of field values like + ``ProjectName (>=version)``. + + .. method:: match(version) + + Test if a version number matches the predicate: + + >>> version = VersionPredicate("ProjectName (<1.2, >1.0)") + >>> version.match("1.2.1") + False + >>> version.match("1.1.1") + True + + +Validation helpers +------------------ + +If you want to use :term:`LBYL`-style checks instead of instantiating the +classes and catching :class:`IrrationalVersionError` and :class:`ValueError`, +you can use these functions: + +.. function:: is_valid_version(predicate) + + Check whether the given string is a valid version number. Example of valid + strings: ``'1.2'``, ``'4.2.0.dev4'``, ``'2.5.4.post2'``. + + +.. function:: is_valid_versions(predicate) + + Check whether the given string is a valid value for specifying multiple + versions, such as in the Requires-Python field. Example: ``'2.7, >=3.2'``. + + +.. function:: is_valid_predicate(predicate) + + Check whether the given string is a valid version predicate. Examples: + ``'some.project == 4.5, <= 4.7'``, ``'speciallib (> 1.0, != 1.4.2, < 2.0)'``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pathlib-inheritance.png Binary file Doc/library/pathlib-inheritance.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pathlib-inheritance.svg --- a/Doc/library/pathlib-inheritance.svg Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ - - - - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pathlib.rst --- a/Doc/library/pathlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1051 +0,0 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== - -.. module:: pathlib - :synopsis: Object-oriented filesystem paths - -.. index:: single: path; operations - -.. versionadded:: 3.4 - -This module offers classes representing filesystem paths with semantics -appropriate for different operating systems. Path classes are divided -between :ref:`pure paths `, which provide purely computational -operations without I/O, and :ref:`concrete paths `, which -inherit from pure paths but also provide I/O operations. - -.. image:: pathlib-inheritance.png - :align: center - -If you've never used this module before or just aren't sure which class is -right for your task, :class:`Path` is most likely what you need. It instantiates -a :ref:`concrete path ` for the platform the code is running on. - -Pure paths are useful in some special cases; for example: - -#. If you want to manipulate Windows paths on a Unix machine (or vice versa). - You cannot instantiate a :class:`WindowsPath` when running on Unix, but you - can instantiate :class:`PureWindowsPath`. -#. You want to make sure that your code only manipulates paths without actually - accessing the OS. In this case, instantiating one of the pure classes may be - useful since those simply don't have any OS-accessing operations. - -.. note:: - This module has been included in the standard library on a - :term:`provisional basis `. Backwards incompatible - changes (up to and including removal of the package) may occur if deemed - necessary by the core developers. - -.. seealso:: - :pep:`428`: The pathlib module -- object-oriented filesystem paths. - -.. seealso:: - For low-level path manipulation on strings, you can also use the - :mod:`os.path` module. - - -Basic use ---------- - -Importing the main class:: - - >>> from pathlib import Path - -Listing subdirectories:: - - >>> p = Path('.') - >>> [x for x in p.iterdir() if x.is_dir()] - [PosixPath('.hg'), PosixPath('docs'), PosixPath('dist'), - PosixPath('__pycache__'), PosixPath('build')] - -Listing Python source files in this directory tree:: - - >>> list(p.glob('**/*.py')) - [PosixPath('test_pathlib.py'), PosixPath('setup.py'), - PosixPath('pathlib.py'), PosixPath('docs/conf.py'), - PosixPath('build/lib/pathlib.py')] - -Navigating inside a directory tree:: - - >>> p = Path('/etc') - >>> q = p / 'init.d' / 'reboot' - >>> q - PosixPath('/etc/init.d/reboot') - >>> q.resolve() - PosixPath('/etc/rc.d/init.d/halt') - -Querying path properties:: - - >>> q.exists() - True - >>> q.is_dir() - False - -Opening a file:: - - >>> with q.open() as f: f.readline() - ... - '#!/bin/bash\n' - - -.. _pure-paths: - -Pure paths ----------- - -Pure path objects provide path-handling operations which don't actually -access a filesystem. There are three ways to access these classes, which -we also call *flavours*: - -.. class:: PurePath(*pathsegments) - - A generic class that represents the system's path flavour (instantiating - it creates either a :class:`PurePosixPath` or a :class:`PureWindowsPath`):: - - >>> PurePath('setup.py') # Running on a Unix machine - PurePosixPath('setup.py') - - Each element of *pathsegments* can be either a string representing a - path segment, or another path object:: - - >>> PurePath('foo', 'some/path', 'bar') - PurePosixPath('foo/some/path/bar') - >>> PurePath(Path('foo'), Path('bar')) - PurePosixPath('foo/bar') - - When *pathsegments* is empty, the current directory is assumed:: - - >>> PurePath() - PurePosixPath('.') - - When several absolute paths are given, the last is taken as an anchor - (mimicking :func:`os.path.join`'s behaviour):: - - >>> PurePath('/etc', '/usr', 'lib64') - PurePosixPath('/usr/lib64') - >>> PureWindowsPath('c:/Windows', 'd:bar') - PureWindowsPath('d:bar') - - However, in a Windows path, changing the local root doesn't discard the - previous drive setting:: - - >>> PureWindowsPath('c:/Windows', '/Program Files') - PureWindowsPath('c:/Program Files') - - Spurious slashes and single dots are collapsed, but double dots (``'..'``) - are not, since this would change the meaning of a path in the face of - symbolic links:: - - >>> PurePath('foo//bar') - PurePosixPath('foo/bar') - >>> PurePath('foo/./bar') - PurePosixPath('foo/bar') - >>> PurePath('foo/../bar') - PurePosixPath('foo/../bar') - - (a naïve approach would make ``PurePosixPath('foo/../bar')`` equivalent - to ``PurePosixPath('bar')``, which is wrong if ``foo`` is a symbolic link - to another directory) - -.. class:: PurePosixPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents non-Windows - filesystem paths:: - - >>> PurePosixPath('/etc') - PurePosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PureWindowsPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents Windows - filesystem paths:: - - >>> PureWindowsPath('c:/Program Files/') - PureWindowsPath('c:/Program Files') - - *pathsegments* is specified similarly to :class:`PurePath`. - -Regardless of the system you're running on, you can instantiate all of -these classes, since they don't provide any operation that does system calls. - - -General properties -^^^^^^^^^^^^^^^^^^ - -Paths are immutable and hashable. Paths of a same flavour are comparable -and orderable. These properties respect the flavour's case-folding -semantics:: - - >>> PurePosixPath('foo') == PurePosixPath('FOO') - False - >>> PureWindowsPath('foo') == PureWindowsPath('FOO') - True - >>> PureWindowsPath('FOO') in { PureWindowsPath('foo') } - True - >>> PureWindowsPath('C:') < PureWindowsPath('d:') - True - -Paths of a different flavour compare unequal and cannot be ordered:: - - >>> PureWindowsPath('foo') == PurePosixPath('foo') - False - >>> PureWindowsPath('foo') < PurePosixPath('foo') - Traceback (most recent call last): - File "", line 1, in - TypeError: '<' not supported between instances of 'PureWindowsPath' and 'PurePosixPath' - - -Operators -^^^^^^^^^ - -The slash operator helps create child paths, similarly to :func:`os.path.join`:: - - >>> p = PurePath('/etc') - >>> p - PurePosixPath('/etc') - >>> p / 'init.d' / 'apache2' - PurePosixPath('/etc/init.d/apache2') - >>> q = PurePath('bin') - >>> '/usr' / q - PurePosixPath('/usr/bin') - -The string representation of a path is the raw filesystem path itself -(in native form, e.g. with backslashes under Windows), which you can -pass to any function taking a file path as a string:: - - >>> p = PurePath('/etc') - >>> str(p) - '/etc' - >>> p = PureWindowsPath('c:/Program Files') - >>> str(p) - 'c:\\Program Files' - -Similarly, calling :class:`bytes` on a path gives the raw filesystem path as a -bytes object, as encoded by :func:`os.fsencode`:: - - >>> bytes(p) - b'/etc' - -.. note:: - Calling :class:`bytes` is only recommended under Unix. Under Windows, - the unicode form is the canonical representation of filesystem paths. - - -Accessing individual parts -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To access the individual "parts" (components) of a path, use the following -property: - -.. data:: PurePath.parts - - A tuple giving access to the path's various components:: - - >>> p = PurePath('/usr/bin/python3') - >>> p.parts - ('/', 'usr', 'bin', 'python3') - - >>> p = PureWindowsPath('c:/Program Files/PSF') - >>> p.parts - ('c:\\', 'Program Files', 'PSF') - - (note how the drive and local root are regrouped in a single part) - - -Methods and properties -^^^^^^^^^^^^^^^^^^^^^^ - -Pure paths provide the following methods and properties: - -.. data:: PurePath.drive - - A string representing the drive letter or name, if any:: - - >>> PureWindowsPath('c:/Program Files/').drive - 'c:' - >>> PureWindowsPath('/Program Files/').drive - '' - >>> PurePosixPath('/etc').drive - '' - - UNC shares are also considered drives:: - - >>> PureWindowsPath('//host/share/foo.txt').drive - '\\\\host\\share' - -.. data:: PurePath.root - - A string representing the (local or global) root, if any:: - - >>> PureWindowsPath('c:/Program Files/').root - '\\' - >>> PureWindowsPath('c:Program Files/').root - '' - >>> PurePosixPath('/etc').root - '/' - - UNC shares always have a root:: - - >>> PureWindowsPath('//host/share').root - '\\' - -.. data:: PurePath.anchor - - The concatenation of the drive and root:: - - >>> PureWindowsPath('c:/Program Files/').anchor - 'c:\\' - >>> PureWindowsPath('c:Program Files/').anchor - 'c:' - >>> PurePosixPath('/etc').anchor - '/' - >>> PureWindowsPath('//host/share').anchor - '\\\\host\\share\\' - - -.. data:: PurePath.parents - - An immutable sequence providing access to the logical ancestors of - the path:: - - >>> p = PureWindowsPath('c:/foo/bar/setup.py') - >>> p.parents[0] - PureWindowsPath('c:/foo/bar') - >>> p.parents[1] - PureWindowsPath('c:/foo') - >>> p.parents[2] - PureWindowsPath('c:/') - - -.. data:: PurePath.parent - - The logical parent of the path:: - - >>> p = PurePosixPath('/a/b/c/d') - >>> p.parent - PurePosixPath('/a/b/c') - - You cannot go past an anchor, or empty path:: - - >>> p = PurePosixPath('/') - >>> p.parent - PurePosixPath('/') - >>> p = PurePosixPath('.') - >>> p.parent - PurePosixPath('.') - - .. note:: - This is a purely lexical operation, hence the following behaviour:: - - >>> p = PurePosixPath('foo/..') - >>> p.parent - PurePosixPath('foo') - - If you want to walk an arbitrary filesystem path upwards, it is - recommended to first call :meth:`Path.resolve` so as to resolve - symlinks and eliminate `".."` components. - - -.. data:: PurePath.name - - A string representing the final path component, excluding the drive and - root, if any:: - - >>> PurePosixPath('my/library/setup.py').name - 'setup.py' - - UNC drive names are not considered:: - - >>> PureWindowsPath('//some/share/setup.py').name - 'setup.py' - >>> PureWindowsPath('//some/share').name - '' - - -.. data:: PurePath.path - - A string representing the full path:: - - >>> PurePosixPath('my/library/setup.py').path - 'my/library/setup.py' - - This always returns the same value as ``str(p)``; it is included to - serve as a one-off protocol. Code that wants to support both - strings and ``pathlib.Path`` objects as filenames can write - ``arg = getattr(arg, 'path', arg)`` to get the path as a string. - This can then be passed to various system calls or library - functions that expect a string. Unlike the alternative - ``arg = str(arg)``, this will still raise an exception if an object - of some other type is given by accident. - - A nice advantage is that this protocol is also supported by - :class:`os.DirEntry` objects returned by :func:`os.scandir`. - - .. versionadded:: 3.4.5 - .. versionadded:: 3.5.2 - -.. data:: PurePath.suffix - - The file extension of the final component, if any:: - - >>> PurePosixPath('my/library/setup.py').suffix - '.py' - >>> PurePosixPath('my/library.tar.gz').suffix - '.gz' - >>> PurePosixPath('my/library').suffix - '' - - -.. data:: PurePath.suffixes - - A list of the path's file extensions:: - - >>> PurePosixPath('my/library.tar.gar').suffixes - ['.tar', '.gar'] - >>> PurePosixPath('my/library.tar.gz').suffixes - ['.tar', '.gz'] - >>> PurePosixPath('my/library').suffixes - [] - - -.. data:: PurePath.stem - - The final path component, without its suffix:: - - >>> PurePosixPath('my/library.tar.gz').stem - 'library.tar' - >>> PurePosixPath('my/library.tar').stem - 'library' - >>> PurePosixPath('my/library').stem - 'library' - - -.. method:: PurePath.as_posix() - - Return a string representation of the path with forward slashes (``/``):: - - >>> p = PureWindowsPath('c:\\windows') - >>> str(p) - 'c:\\windows' - >>> p.as_posix() - 'c:/windows' - - -.. method:: PurePath.as_uri() - - Represent the path as a ``file`` URI. :exc:`ValueError` is raised if - the path isn't absolute. - - >>> p = PurePosixPath('/etc/passwd') - >>> p.as_uri() - 'file:///etc/passwd' - >>> p = PureWindowsPath('c:/Windows') - >>> p.as_uri() - 'file:///c:/Windows' - - -.. method:: PurePath.is_absolute() - - Return whether the path is absolute or not. A path is considered absolute - if it has both a root and (if the flavour allows) a drive:: - - >>> PurePosixPath('/a/b').is_absolute() - True - >>> PurePosixPath('a/b').is_absolute() - False - - >>> PureWindowsPath('c:/a/b').is_absolute() - True - >>> PureWindowsPath('/a/b').is_absolute() - False - >>> PureWindowsPath('c:').is_absolute() - False - >>> PureWindowsPath('//some/share').is_absolute() - True - - -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. - - -.. method:: PurePath.joinpath(*other) - - Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: - - >>> PurePosixPath('/etc').joinpath('passwd') - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath(PurePosixPath('passwd')) - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath('init.d', 'apache2') - PurePosixPath('/etc/init.d/apache2') - >>> PureWindowsPath('c:').joinpath('/Program Files') - PureWindowsPath('c:/Program Files') - - -.. method:: PurePath.match(pattern) - - Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. - - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: - - >>> PurePath('a/b.py').match('*.py') - True - >>> PurePath('/a/b/c.py').match('b/*.py') - True - >>> PurePath('/a/b/c.py').match('a/*.py') - False - - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: - - >>> PurePath('/a.py').match('/*.py') - True - >>> PurePath('a/b.py').match('/*.py') - False - - As with other methods, case-sensitivity is observed:: - - >>> PureWindowsPath('b.py').match('*.PY') - True - - -.. method:: PurePath.relative_to(*other) - - Compute a version of this path relative to the path represented by - *other*. If it's impossible, ValueError is raised:: - - >>> p = PurePosixPath('/etc/passwd') - >>> p.relative_to('/') - PurePosixPath('etc/passwd') - >>> p.relative_to('/etc') - PurePosixPath('passwd') - >>> p.relative_to('/usr') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 694, in relative_to - .format(str(self), str(formatted))) - ValueError: '/etc/passwd' does not start with '/usr' - - -.. method:: PurePath.with_name(name) - - Return a new path with the :attr:`name` changed. If the original path - doesn't have a name, ValueError is raised:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_name('setup.py') - PureWindowsPath('c:/Downloads/setup.py') - >>> p = PureWindowsPath('c:/') - >>> p.with_name('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "/home/antoine/cpython/default/Lib/pathlib.py", line 751, in with_name - raise ValueError("%r has an empty name" % (self,)) - ValueError: PureWindowsPath('c:/') has an empty name - - -.. method:: PurePath.with_suffix(suffix) - - Return a new path with the :attr:`suffix` changed. If the original path - doesn't have a suffix, the new *suffix* is appended instead:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_suffix('.bz2') - PureWindowsPath('c:/Downloads/pathlib.tar.bz2') - >>> p = PureWindowsPath('README') - >>> p.with_suffix('.txt') - PureWindowsPath('README.txt') - - -.. _concrete-paths: - - -Concrete paths --------------- - -Concrete paths are subclasses of the pure path classes. In addition to -operations provided by the latter, they also provide methods to do system -calls on path objects. There are three ways to instantiate concrete paths: - -.. class:: Path(*pathsegments) - - A subclass of :class:`PurePath`, this class represents concrete paths of - the system's path flavour (instantiating it creates either a - :class:`PosixPath` or a :class:`WindowsPath`):: - - >>> Path('setup.py') - PosixPath('setup.py') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PosixPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PurePosixPath`, this class - represents concrete non-Windows filesystem paths:: - - >>> PosixPath('/etc') - PosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: WindowsPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PureWindowsPath`, this class - represents concrete Windows filesystem paths:: - - >>> WindowsPath('c:/Program Files/') - WindowsPath('c:/Program Files') - - *pathsegments* is specified similarly to :class:`PurePath`. - -You can only instantiate the class flavour that corresponds to your system -(allowing system calls on non-compatible path flavours could lead to -bugs or failures in your application):: - - >>> import os - >>> os.name - 'posix' - >>> Path('setup.py') - PosixPath('setup.py') - >>> PosixPath('setup.py') - PosixPath('setup.py') - >>> WindowsPath('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 798, in __new__ - % (cls.__name__,)) - NotImplementedError: cannot instantiate 'WindowsPath' on your system - - -Methods -^^^^^^^ - -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist): - -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct):: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - -.. method:: Path.stat() - - Return information about this path (similarly to :func:`os.stat`). - The result is looked up at each call to this method. - - >>> p = Path('setup.py') - >>> p.stat().st_size - 956 - >>> p.stat().st_mtime - 1327883547.852554 - - -.. method:: Path.chmod(mode) - - Change the file mode and permissions, like :func:`os.chmod`:: - - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 - - -.. method:: Path.exists() - - Whether the path points to an existing file or directory:: - - >>> Path('.').exists() - True - >>> Path('setup.py').exists() - True - >>> Path('/etc').exists() - True - >>> Path('nonexistentfile').exists() - False - - .. note:: - If the path points to a symlink, :meth:`exists` returns whether the - symlink *points to* an existing file or directory. - - -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`:: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - - .. versionadded:: 3.5 - - -.. method:: Path.glob(pattern) - - Glob the given *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: - - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - - The "``**``" pattern means "this directory and all subdirectories, - recursively". In other words, it enables recursive globbing:: - - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - -.. method:: Path.group() - - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. - - -.. method:: Path.is_dir() - - Return ``True`` if the path points to a directory (or a symbolic link - pointing to a directory), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_file() - - Return ``True`` if the path points to a regular file (or a symbolic link - pointing to a regular file), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_symlink() - - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. - - -.. method:: Path.is_socket() - - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_fifo() - - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_block_device() - - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_char_device() - - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.iterdir() - - When the path points to a directory, yield path objects of the directory - contents:: - - >>> p = Path('docs') - >>> for child in p.iterdir(): child - ... - PosixPath('docs/conf.py') - PosixPath('docs/_templates') - PosixPath('docs/make.bat') - PosixPath('docs/index.rst') - PosixPath('docs/_build') - PosixPath('docs/_static') - PosixPath('docs/Makefile') - -.. method:: Path.lchmod(mode) - - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. - - -.. method:: Path.lstat() - - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. - - -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) - - Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode - and access flags. If the path already exists, :exc:`FileExistsError` - is raised. - - If *parents* is true, any missing parents of this path are created - as needed; they are created with the default permissions without taking - *mode* into account (mimicking the POSIX ``mkdir -p`` command). - - If *parents* is false (the default), a missing parent raises - :exc:`FileNotFoundError`. - - If *exist_ok* is false (the default), an :exc:`FileExistsError` is - raised if the target directory already exists. - - If *exist_ok* is true, :exc:`FileExistsError` exceptions will be - ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the - last path component is not an existing non-directory file. - - .. versionchanged:: 3.5 - The *exist_ok* parameter was added. - - -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: - - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' - - -.. method:: Path.owner() - - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. - - -.. method:: Path.read_bytes() - - Return the binary contents of the pointed-to file as a bytes object:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - .. versionadded:: 3.5 - - -.. method:: Path.read_text(encoding=None, errors=None) - - Return the decoded contents of the pointed-to file as a string:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - The optional parameters have the same meaning as in :func:`open`. - - .. versionadded:: 3.5 - - -.. method:: Path.rename(target) - - Rename this file or directory to the given *target*. *target* can be - either a string or another path object:: - - >>> p = Path('foo') - >>> p.open('w').write('some text') - 9 - >>> target = Path('bar') - >>> p.rename(target) - >>> target.open().read() - 'some text' - - -.. method:: Path.replace(target) - - Rename this file or directory to the given *target*. If *target* points - to an existing file or directory, it will be unconditionally replaced. - - -.. method:: Path.resolve() - - Make the path absolute, resolving any symlinks. A new path object is - returned:: - - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') - - `".."` components are also eliminated (this is the only method to do so):: - - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - - If the path doesn't exist, :exc:`FileNotFoundError` is raised. If an - infinite loop is encountered along the resolution path, - :exc:`RuntimeError` is raised. - - -.. method:: Path.rglob(pattern) - - This is like calling :meth:`glob` with "``**``" added in front of the - given *pattern*: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link to *target*. Under Windows, - *target_is_directory* must be true (default ``False``) if the link's target - is a directory. Under POSIX, *target_is_directory*'s value is ignored. - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. - - -.. method:: Path.touch(mode=0o777, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. - - -.. method:: Path.unlink() - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. - - -.. method:: Path.write_bytes(data) - - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - An existing file of the same name is overwritten. - - .. versionadded:: 3.5 - - -.. method:: Path.write_text(data, encoding=None, errors=None) - - Open the file pointed to in text mode, write *data* to it, and close the - file:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - .. versionadded:: 3.5 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pdb.rst --- a/Doc/library/pdb.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pdb.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,9 +6,6 @@ .. module:: pdb :synopsis: The Python debugger for interactive interpreters. -**Source code:** :source:`Lib/pdb.py` - --------------- .. index:: single: debugging @@ -44,7 +41,7 @@ .. versionchanged:: 3.3 Tab-completion via the :mod:`readline` module is available for commands and command arguments, e.g. the current global and local names are offered as - arguments of the ``p`` command. + arguments of the ``print`` command. :file:`pdb.py` can also be invoked as a script to debug other scripts. For example:: @@ -156,8 +153,8 @@ that matches one of these patterns. [1]_ By default, Pdb sets a handler for the SIGINT signal (which is sent when the - user presses :kbd:`Ctrl-C` on the console) when you give a ``continue`` command. - This allows you to break into the debugger again by pressing :kbd:`Ctrl-C`. If you + user presses Ctrl-C on the console) when you give a ``continue`` command. + This allows you to break into the debugger again by pressing Ctrl-C. If you want Pdb not to touch the SIGINT handler, set *nosigint* tot true. Example call to enable tracing with *skip*:: @@ -312,7 +309,7 @@ ``end`` to terminate the commands. An example:: (Pdb) commands 1 - (com) p some_variable + (com) print some_variable (com) end (Pdb) @@ -406,19 +403,13 @@ Print the argument list of the current function. -.. pdbcommand:: p expression +.. pdbcommand:: p(rint) expression Evaluate the *expression* in the current context and print its value. - .. note:: - - ``print()`` can also be used, but is not a debugger command --- this executes the - Python :func:`print` function. - - .. pdbcommand:: pp expression - Like the :pdbcmd:`p` command, except the value of the expression is + Like the :pdbcmd:`print` command, except the value of the expression is pretty-printed using the :mod:`pprint` module. .. pdbcommand:: whatis expression diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pickle.rst --- a/Doc/library/pickle.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pickle.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,30 +12,30 @@ .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. .. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw +.. sectionauthor:: Barry Warsaw -The :mod:`pickle` module implements binary protocols for serializing and -de-serializing a Python object structure. *"Pickling"* is the process -whereby a Python object hierarchy is converted into a byte stream, and -*"unpickling"* is the inverse operation, whereby a byte stream -(from a :term:`binary file` or :term:`bytes-like object`) is converted -back into an object hierarchy. Pickling (and unpickling) is alternatively -known as "serialization", "marshalling," [#]_ or "flattening"; however, to -avoid confusion, the terms used here are "pickling" and "unpickling". +The :mod:`pickle` module implements a fundamental, but powerful algorithm for +serializing and de-serializing a Python object structure. "Pickling" is the +process whereby a Python object hierarchy is converted into a byte stream, and +"unpickling" is the inverse operation, whereby a byte stream is converted back +into an object hierarchy. Pickling (and unpickling) is alternatively known as +"serialization", "marshalling," [#]_ or "flattening", however, to avoid +confusion, the terms used here are "pickling" and "unpickling".. .. warning:: - The :mod:`pickle` module is not secure against erroneous or maliciously - constructed data. Never unpickle data received from an untrusted or - unauthenticated source. + The :mod:`pickle` module is not intended to be secure against erroneous or + maliciously constructed data. Never unpickle data received from an untrusted + or unauthenticated source. Relationship to other Python modules ------------------------------------ -Comparison with ``marshal`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`pickle` module has an transparent optimizer (:mod:`_pickle`) written +in C. It is used whenever available. Otherwise the pure Python implementation is +used. Python has a more primitive serialization module called :mod:`marshal`, but in general :mod:`pickle` should always be the preferred way to serialize Python @@ -69,113 +69,73 @@ The :mod:`pickle` serialization format is guaranteed to be backwards compatible across Python releases. -Comparison with ``json`` -^^^^^^^^^^^^^^^^^^^^^^^^ +Note that serialization is a more primitive notion than persistence; although +:mod:`pickle` reads and writes file objects, it does not handle the issue of +naming persistent objects, nor the (even more complicated) issue of concurrent +access to persistent objects. The :mod:`pickle` module can transform a complex +object into a byte stream and it can transform the byte stream into an object +with the same internal structure. Perhaps the most obvious thing to do with +these byte streams is to write them onto a file, but it is also conceivable to +send them across a network or store them in a database. The module +:mod:`shelve` provides a simple interface to pickle and unpickle objects on +DBM-style database files. -There are fundamental differences between the pickle protocols and -`JSON (JavaScript Object Notation) `_: - -* JSON is a text serialization format (it outputs unicode text, although - most of the time it is then encoded to ``utf-8``), while pickle is - a binary serialization format; - -* JSON is human-readable, while pickle is not; - -* JSON is interoperable and widely used outside of the Python ecosystem, - while pickle is Python-specific; - -* JSON, by default, can only represent a subset of the Python built-in - types, and no custom classes; pickle can represent an extremely large - number of Python types (many of them automatically, by clever usage - of Python's introspection facilities; complex cases can be tackled by - implementing :ref:`specific object APIs `). - -.. seealso:: - The :mod:`json` module: a standard library module allowing JSON - serialization and deserialization. - - -.. _pickle-protocols: Data stream format ------------------ .. index:: + single: XDR single: External Data Representation The data format used by :mod:`pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as -JSON or XDR (which can't represent pointer sharing); however it means that -non-Python programs may not be able to reconstruct pickled Python objects. +XDR (which can't represent pointer sharing); however it means that non-Python +programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary -representation. If you need optimal size characteristics, you can efficiently -:doc:`compress ` pickled data. +By default, the :mod:`pickle` data format uses a compact binary representation. +The module :mod:`pickletools` contains tools for analyzing data streams +generated by :mod:`pickle`. -The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive -comments about opcodes used by pickle protocols. +There are currently 4 different protocols which can be used for pickling. -There are currently 5 different protocols which can be used for pickling. -The higher the protocol used, the more recent the version of Python needed -to read the pickle produced. - -* Protocol version 0 is the original "human-readable" protocol and is +* Protocol version 0 is the original human-readable protocol and is backwards compatible with earlier versions of Python. -* Protocol version 1 is an old binary format which is also compatible with +* Protocol version 1 is the old binary format which is also compatible with earlier versions of Python. * Protocol version 2 was introduced in Python 2.3. It provides much more - efficient pickling of :term:`new-style class`\es. Refer to :pep:`307` for - information about improvements brought by protocol 2. + efficient pickling of :term:`new-style class`\es. * Protocol version 3 was added in Python 3.0. It has explicit support for - :class:`bytes` objects and cannot be unpickled by Python 2.x. This is - the default protocol, and the recommended protocol when compatibility with - other Python 3 versions is required. + bytes and cannot be unpickled by Python 2.x pickle modules. This is + the current recommended protocol, use it whenever it is possible. -* Protocol version 4 was added in Python 3.4. It adds support for very large - objects, pickling more kinds of objects, and some data format - optimizations. Refer to :pep:`3154` for information about improvements - brought by protocol 4. - -.. note:: - Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of - naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex - object into a byte stream and it can transform the byte stream into an object - with the same internal structure. Perhaps the most obvious thing to do with - these byte streams is to write them onto a file, but it is also conceivable to - send them across a network or store them in a database. The :mod:`shelve` - module provides a simple interface to pickle and unpickle objects on - DBM-style database files. +Refer to :pep:`307` for information about improvements brought by +protocol 2. See :mod:`pickletools`'s source code for extensive +comments about opcodes used by pickle protocols. Module Interface ---------------- -To serialize an object hierarchy, you simply call the :func:`dumps` function. -Similarly, to de-serialize a data stream, you call the :func:`loads` function. -However, if you want more control over serialization and de-serialization, -you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. - -The :mod:`pickle` module provides the following constants: +To serialize an object hierarchy, you first create a pickler, then you call the +pickler's :meth:`dump` method. To de-serialize a data stream, you first create +an unpickler, then you call the unpickler's :meth:`load` method. The +:mod:`pickle` module provides the following constant: .. data:: HIGHEST_PROTOCOL - An integer, the highest :ref:`protocol version ` - available. This value can be passed as a *protocol* value to functions - :func:`dump` and :func:`dumps` as well as the :class:`Pickler` - constructor. + The highest protocol version available. This value can be passed as a + *protocol* value. .. data:: DEFAULT_PROTOCOL - An integer, the default :ref:`protocol version ` used - for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 3, a new protocol designed for Python 3. + The default protocol used for pickling. May be less than HIGHEST_PROTOCOL. + Currently the default protocol is 3; a backward-incompatible protocol + designed for Python 3.0. The :mod:`pickle` module provides the following functions to make the pickling @@ -186,68 +146,78 @@ Write a pickled representation of *obj* to the open :term:`file object` *file*. This is equivalent to ``Pickler(file, protocol).dump(obj)``. - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. + The optional *protocol* argument tells the pickler to use the given protocol; + supported protocols are 0, 1, 2, 3. The default protocol is 3; a + backward-incompatible protocol designed for Python 3.0. + + Specifying a negative protocol version selects the highest protocol version + supported. The higher the protocol used, the more recent the version of + Python needed to read the pickle produced. The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an + argument. It can thus be an on-disk file opened for binary writing, a :class:`io.BytesIO` instance, or any other custom object that meets this interface. - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. + If *fix_imports* is True and *protocol* is less than 3, pickle will try to + map the new Python 3.x names to the old module names used in Python 2.x, + so that the pickle data stream is readable with Python 2.x. .. function:: dumps(obj, protocol=None, \*, fix_imports=True) - Return the pickled representation of the object as a :class:`bytes` object, - instead of writing it to a file. + Return the pickled representation of the object as a :class:`bytes` + object, instead of writing it to a file. - Arguments *protocol* and *fix_imports* have the same meaning as in - :func:`dump`. + The optional *protocol* argument tells the pickler to use the given protocol; + supported protocols are 0, 1, 2, 3. The default protocol is 3; a + backward-incompatible protocol designed for Python 3.0. + + Specifying a negative protocol version selects the highest protocol version + supported. The higher the protocol used, the more recent the version of + Python needed to read the pickle produced. + + If *fix_imports* is True and *protocol* is less than 3, pickle will try to + map the new Python 3.x names to the old module names used in Python 2.x, + so that the pickle data stream is readable with Python 2.x. .. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict") - Read a pickled object representation from the open :term:`file object` - *file* and return the reconstituted object hierarchy specified therein. - This is equivalent to ``Unpickler(file).load()``. + Read a pickled object representation from the open :term:`file object` *file* + and return the reconstituted object hierarchy specified therein. This is + equivalent to ``Unpickler(file).load()``. - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. + The protocol version of the pickle is detected automatically, so no protocol + argument is needed. Bytes past the pickled object's representation are + ignored. The argument *file* must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file opened for - binary reading, an :class:`io.BytesIO` object, or any other custom object + methods should return bytes. Thus *file* can be an on-disk file opened + for binary reading, a :class:`io.BytesIO` object, or any other custom object that meets this interface. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and + by Python 2.x. If *fix_imports* is True, pickle will try to map the old + Python 2.x names to the new names used in Python 3.x. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. + 2.x; these default to 'ASCII' and 'strict', respectively. .. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict") Read a pickled object hierarchy from a :class:`bytes` object and return the - reconstituted object hierarchy specified therein. + reconstituted object hierarchy specified therein - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled object's - representation are ignored. + The protocol version of the pickle is detected automatically, so no protocol + argument is needed. Bytes past the pickled object's representation are + ignored. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and + by Python 2.x. If *fix_imports* is True, pickle will try to map the old + Python 2.x names to the new names used in Python 3.x. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. + 2.x; these default to 'ASCII' and 'strict', respectively. The :mod:`pickle` module defines three exceptions: @@ -282,19 +252,21 @@ This takes a binary file for writing a pickle data stream. - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. + The optional *protocol* argument tells the pickler to use the given protocol; + supported protocols are 0, 1, 2, 3. The default protocol is 3; a + backward-incompatible protocol designed for Python 3.0. + + Specifying a negative protocol version selects the highest protocol version + supported. The higher the protocol used, the more recent the version of + Python needed to read the pickle produced. The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. + argument. It can thus be an on-disk file opened for binary writing, a + :class:`io.BytesIO` instance, or any other custom object that meets this interface. - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. + If *fix_imports* is True and *protocol* is less than 3, pickle will try to + map the new Python 3.x names to the old module names used in Python 2.x, + so that the pickle data stream is readable with Python 2.x. .. method:: dump(obj) @@ -320,7 +292,7 @@ :func:`copyreg.pickle`. It is a mapping whose keys are classes and whose values are reduction functions. A reduction function takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` + conform to the same interface as a :meth:`~object.__reduce__` method. By default, a pickler object will not have a @@ -356,17 +328,16 @@ The argument *file* must have two methods, a read() method that takes an integer argument, and a readline() method that requires no arguments. Both - methods should return bytes. Thus *file* can be an on-disk file object - opened for binary reading, an :class:`io.BytesIO` object, or any other - custom object that meets this interface. + methods should return bytes. Thus *file* can be an on-disk file object opened + for binary reading, a :class:`io.BytesIO` object, or any other custom object + that meets this interface. Optional keyword arguments are *fix_imports*, *encoding* and *errors*, which are used to control compatibility support for pickle stream generated - by Python 2. If *fix_imports* is true, pickle will try to map the old - Python 2 names to the new names used in Python 3. The *encoding* and + by Python 2.x. If *fix_imports* is True, pickle will try to map the old + Python 2.x names to the new names used in Python 3.x. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python - 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these ß8-bit string instances as bytes objects. + 2.x; these default to 'ASCII' and 'strict', respectively. .. method:: load() @@ -411,28 +382,26 @@ * tuples, lists, sets, and dictionaries containing only picklable objects -* functions defined at the top level of a module (using :keyword:`def`, not - :keyword:`lambda`) +* functions defined at the top level of a module * built-in functions defined at the top level of a module * classes that are defined at the top level of a module -* instances of such classes whose :attr:`~object.__dict__` or the result of - calling :meth:`__getstate__` is picklable (see section :ref:`pickle-inst` for - details). +* instances of such classes whose :attr:`__dict__` or :meth:`__setstate__` is + picklable (see section :ref:`pickle-inst` for details) Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` exception; when this happens, an unspecified number of bytes may have already been written to the underlying file. Trying to pickle a highly recursive data -structure may exceed the maximum recursion depth, a :exc:`RecursionError` will be +structure may exceed the maximum recursion depth, a :exc:`RuntimeError` will be raised in this case. You can carefully raise this limit with :func:`sys.setrecursionlimit`. Note that functions (built-in and user-defined) are pickled by "fully qualified" -name reference, not by value. [#]_ This means that only the function name is -pickled, along with the name of the module the function is defined in. Neither -the function's code, nor any of its function attributes are pickled. Thus the +name reference, not by value. This means that only the function name is +pickled, along with the name of the module the function is defined in. Neither the +function's code, nor any of its function attributes are pickled. Thus the defining module must be importable in the unpickling environment, and the module must contain the named object, otherwise an exception will be raised. [#]_ @@ -463,8 +432,6 @@ Pickling Class Instances ------------------------ -.. currentmodule:: None - In this section, we describe the general mechanisms available to you to define, customize, and control how class instances are pickled and unpickled. @@ -486,36 +453,12 @@ Classes can alter the default behaviour by providing one or several special methods: -.. method:: object.__getnewargs_ex__() - - In protocols 2 and newer, classes that implements the - :meth:`__getnewargs_ex__` method can dictate the values passed to the - :meth:`__new__` method upon unpickling. The method must return a pair - ``(args, kwargs)`` where *args* is a tuple of positional arguments - and *kwargs* a dictionary of named arguments for constructing the - object. Those will be passed to the :meth:`__new__` method upon - unpickling. - - You should implement this method if the :meth:`__new__` method of your - class requires keyword-only arguments. Otherwise, it is recommended for - compatibility to implement :meth:`__getnewargs__`. - - .. versionchanged:: 3.6 - :meth:`__getnewargs_ex__` is now used in protocols 2 and 3. - - .. method:: object.__getnewargs__() - This method serve a similar purpose as :meth:`__getnewargs_ex__`, but - supports only positional arguments. It must return a tuple of arguments - ``args`` which will be passed to the :meth:`__new__` method upon unpickling. - - :meth:`__getnewargs__` will not be called if :meth:`__getnewargs_ex__` is - defined. - - .. versionchanged:: 3.6 - Before Python 3.6, :meth:`__getnewargs__` was called instead of - :meth:`__getnewargs_ex__` in protocols 2 and 3. + In protocol 2 and newer, classes that implements the :meth:`__getnewargs__` + method can dictate the values passed to the :meth:`__new__` method upon + unpickling. This is often needed for classes whose :meth:`__new__` method + requires arguments. .. method:: object.__getstate__() @@ -524,7 +467,7 @@ defines the method :meth:`__getstate__`, it is called and the returned object is pickled as the contents for the instance, instead of the contents of the instance's dictionary. If the :meth:`__getstate__` method is absent, the - instance's :attr:`~object.__dict__` is pickled as usual. + instance's :attr:`__dict__` is pickled as usual. .. method:: object.__setstate__(state) @@ -547,10 +490,10 @@ At unpickling time, some methods like :meth:`__getattr__`, :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the - instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__getnewargs__` or - :meth:`__getnewargs_ex__` to establish such an invariant; otherwise, - neither :meth:`__new__` nor :meth:`__init__` will be called. + instance. In case those methods rely on some internal invariant being true, + the type should implement :meth:`__getnewargs__` to establish such an + invariant; otherwise, neither :meth:`__new__` nor :meth:`__init__` will be + called. .. index:: pair: copy; protocol @@ -562,7 +505,7 @@ Although powerful, implementing :meth:`__reduce__` directly in your classes is error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and +interface (i.e., :meth:`__getnewargs__`, :meth:`__getstate__` and :meth:`__setstate__`) whenever possible. We will show, however, cases where using :meth:`__reduce__` is the only option or leads to more efficient pickling or both. @@ -593,7 +536,7 @@ * Optionally, the object's state, which will be passed to the object's :meth:`__setstate__` method as previously described. If the object has no such method then, the value must be a dictionary and it will be added to - the object's :attr:`~object.__dict__` attribute. + the object's :attr:`__dict__` attribute. * Optionally, an iterator (and not a sequence) yielding successive items. These items will be appended to the object either using @@ -619,8 +562,6 @@ the extended version. The main use for this method is to provide backwards-compatible reduce values for older Python releases. -.. currentmodule:: pickle - .. _pickle-persistent: Persistence of External Objects @@ -638,19 +579,19 @@ The resolution of such persistent IDs is not defined by the :mod:`pickle` module; it will delegate this resolution to the user defined methods on the -pickler and unpickler, :meth:`~Pickler.persistent_id` and -:meth:`~Unpickler.persistent_load` respectively. +pickler and unpickler, :meth:`persistent_id` and :meth:`persistent_load` +respectively. To pickle objects that have an external persistent id, the pickler must have a -custom :meth:`~Pickler.persistent_id` method that takes an object as an -argument and returns either ``None`` or the persistent id for that object. -When ``None`` is returned, the pickler simply pickles the object as normal. -When a persistent ID string is returned, the pickler will pickle that object, -along with a marker so that the unpickler will recognize it as a persistent ID. +custom :meth:`persistent_id` method that takes an object as an argument and +returns either ``None`` or the persistent id for that object. When ``None`` is +returned, the pickler simply pickles the object as normal. When a persistent ID +string is returned, the pickler will pickle that object, along with a marker so +that the unpickler will recognize it as a persistent ID. To unpickle external objects, the unpickler must have a custom -:meth:`~Unpickler.persistent_load` method that takes a persistent ID object and -returns the referenced object. +:meth:`persistent_load` method that takes a persistent ID object and returns the +referenced object. Here is a comprehensive example presenting how persistent ID can be used to pickle external objects by reference. @@ -707,7 +648,7 @@ Here's an example that shows how to modify pickling behavior for a class. The :class:`TextReader` class opens a text file, and returns the line number and -line contents each time its :meth:`!readline` method is called. If a +line contents each time its :meth:`readline` method is called. If a :class:`TextReader` instance is pickled, all attributes *except* the file object member are saved. When the instance is unpickled, the file is reopened, and reading resumes from the last location. The :meth:`__setstate__` and @@ -786,10 +727,9 @@ inoffensive, it is not difficult to imagine one that could damage your system. For this reason, you may want to control what gets unpickled by customizing -:meth:`Unpickler.find_class`. Unlike its name suggests, -:meth:`Unpickler.find_class` is called whenever a global (i.e., a class or -a function) is requested. Thus it is possible to either completely forbid -globals or restrict them to a safe subset. +:meth:`Unpickler.find_class`. Unlike its name suggests, :meth:`find_class` is +called whenever a global (i.e., a class or a function) is requested. Thus it is +possible to either completely forbid globals or restrict them to a safe subset. Here is an example of an unpickler allowing only few safe classes from the :mod:`builtins` module to be loaded:: @@ -845,14 +785,6 @@ third-party solutions. -Performance ------------ - -Recent versions of the pickle protocol (from protocol 2 and upwards) feature -efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. - - .. _pickle-example: Examples @@ -866,7 +798,7 @@ data = { 'a': [1, 2.0, 3, 4+6j], 'b': ("character string", b"byte string"), - 'c': {None, True, False} + 'c': set([None, True, False]) } with open('data.pickle', 'wb') as f: @@ -910,9 +842,6 @@ .. [#] Don't confuse this with the :mod:`marshal` module -.. [#] This is why :keyword:`lambda` functions cannot be pickled: all - :keyword:`lambda` functions share the same name: ````. - .. [#] The exception raised will likely be an :exc:`ImportError` or an :exc:`AttributeError` but it could be something else. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pipes.rst --- a/Doc/library/pipes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pipes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,12 +26,12 @@ Example:: >>> import pipes - >>> t = pipes.Template() + >>> t=pipes.Template() >>> t.append('tr a-z A-Z', '--') - >>> f = t.open('pipefile', 'w') + >>> f=t.open('/tmp/1', 'w') >>> f.write('hello world') >>> f.close() - >>> open('pipefile').read() + >>> open('/tmp/1').read() 'HELLO WORLD' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pkgutil.rst --- a/Doc/library/pkgutil.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pkgutil.rst Mon Jan 25 17:05:13 2016 +0100 @@ -56,35 +56,22 @@ Note that :class:`ImpImporter` does not currently support being used by placement on :data:`sys.meta_path`. - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully PEP 302 compliant and available in :mod:`importlib`. - .. class:: ImpLoader(fullname, file, filename, etc) :pep:`302` Loader that wraps Python's "classic" import algorithm. - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully PEP 302 compliant and available in :mod:`importlib`. - .. function:: find_loader(fullname) - Retrieve a :pep:`302` module loader for the given *fullname*. + Find a :pep:`302` "loader" object for *fullname*. - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`ModuleSpec`. + If *fullname* contains dots, path must be the containing package's + ``__path__``. Returns ``None`` if the module cannot be found or imported. + This function uses :func:`iter_importers`, and is thus subject to the same + limitations regarding platform-specific special import locations such as the + Windows registry. - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` .. function:: get_importer(path_item) @@ -93,13 +80,13 @@ The returned importer is cached in :data:`sys.path_importer_cache` if it was newly created by a path hook. + If there is no importer, a wrapper around the basic import machinery is + returned. This wrapper is never inserted into the importer cache (``None`` + is inserted instead). + The cache (or part of it) can be cleared manually if a rescan of :data:`sys.path_hooks` is necessary. - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - .. function:: get_loader(module_or_name) @@ -111,56 +98,50 @@ not already imported, its containing package (if any) is imported, in order to establish the package ``__path__``. - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` + This function uses :func:`iter_importers`, and is thus subject to the same + limitations regarding platform-specific special import locations such as the + Windows registry. .. function:: iter_importers(fullname='') Yield :pep:`302` importers for the given module name. - If fullname contains a '.', the importers will be for the package - containing fullname, otherwise they will be all registered top level - importers (i.e. those on both sys.meta_path and sys.path_hooks). + If fullname contains a '.', the importers will be for the package containing + fullname, otherwise they will be importers for :data:`sys.meta_path`, + :data:`sys.path`, and Python's "classic" import machinery, in that order. If + the named module is in a package, that package is imported as a side effect + of invoking this function. - If the named module is in a package, that package is imported as a side - effect of invoking this function. + Non-:pep:`302` mechanisms (e.g. the Windows registry) used by the standard + import machinery to find files in alternative locations are partially + supported, but are searched *after* :data:`sys.path`. Normally, these + locations are searched *before* :data:`sys.path`, preventing :data:`sys.path` + entries from shadowing them. - If no module name is specified, all top level importers are produced. + For this to cause a visible difference in behaviour, there must be a module + or package name that is accessible via both :data:`sys.path` and one of the + non-:pep:`302` file system mechanisms. In this case, the emulation will find + the former version, while the builtin import mechanism will find the latter. - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. + Items of the following types can be affected by this discrepancy: + ``imp.C_EXTENSION``, ``imp.PY_SOURCE``, ``imp.PY_COMPILED``, + ``imp.PKG_DIRECTORY``. .. function:: iter_modules(path=None, prefix='') - Yields ``(module_finder, name, ispkg)`` for all submodules on *path*, or, if + Yields ``(module_loader, name, ispkg)`` for all submodules on *path*, or, if path is ``None``, all top-level modules on ``sys.path``. *path* should be either ``None`` or a list of paths to look for modules in. *prefix* is a string to output on the front of every module name on output. - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - .. function:: walk_packages(path=None, prefix='', onerror=None) - Yields ``(module_finder, name, ispkg)`` for all modules recursively on + Yields ``(module_loader, name, ispkg)`` for all modules recursively on *path*, or, if path is ``None``, all accessible modules. *path* should be either ``None`` or a list of paths to look for modules in. @@ -185,17 +166,6 @@ # list all submodules of ctypes walk_packages(ctypes.__path__, ctypes.__name__ + '.') - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal PEP 302 import emulation. - .. function:: get_data(package, resource) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/platform.rst --- a/Doc/library/platform.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/platform.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,7 @@ .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. -.. moduleauthor:: Marc-André Lemburg +.. moduleauthor:: Marc-Andre Lemburg .. sectionauthor:: Bjorn Pettersen **Source code:** :source:`Lib/platform.py` @@ -30,8 +30,8 @@ returned as strings. Values that cannot be determined are returned as given by the parameter presets. - If bits is given as ``''``, the ``sizeof(pointer)`` (or - ``sizeof(long)`` on Python version < 1.5.2) is used as indicator for the + If bits is given as ``''``, the :c:func:`sizeof(pointer)` (or + :c:func:`sizeof(long)` on Python version < 1.5.2) is used as indicator for the supported pointer size. The function relies on the system's :file:`file` command to do the actual work. @@ -117,7 +117,7 @@ .. function:: python_version() - Returns the Python version as string ``'major.minor.patchlevel'``. + Returns the Python version as string ``'major.minor.patchlevel'`` Note that unlike the Python ``sys.version``, the returned value will always include the patchlevel (it defaults to 0). @@ -158,20 +158,14 @@ .. function:: uname() - Fairly portable uname interface. Returns a :func:`~collections.namedtuple` - containing six attributes: :attr:`system`, :attr:`node`, :attr:`release`, - :attr:`version`, :attr:`machine`, and :attr:`processor`. + Fairly portable uname interface. Returns a tuple of strings ``(system, node, + release, version, machine, processor)`` identifying the underlying platform. - Note that this adds a sixth attribute (:attr:`processor`) not present - in the :func:`os.uname` result. Also, the attribute names are different - for the first two attributes; :func:`os.uname` names them - :attr:`sysname` and :attr:`nodename`. + Note that unlike the :func:`os.uname` function this also returns possible + processor information as additional tuple entry. Entries which cannot be determined are set to ``''``. - .. versionchanged:: 3.3 - Result changed from a tuple to a namedtuple. - Java Platform ------------- @@ -194,8 +188,8 @@ .. function:: win32_ver(release='', version='', csd='', ptype='') Get additional version information from the Windows Registry and return a tuple - ``(release, version, csd, ptype)`` referring to OS release, version number, - CSD level (service pack) and OS type (multi/single processor). + ``(version, csd, ptype)`` referring to version number, CSD level + (service pack) and OS type (multi/single processor). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers @@ -247,8 +241,6 @@ This is another name for :func:`linux_distribution`. - .. deprecated-removed:: 3.5 3.7 - .. function:: linux_distribution(distname='', version='', id='', supported_dists=('SuSE','debian','redhat','mandrake',...), full_distribution_name=1) Tries to determine the name of the Linux OS distribution name. @@ -265,8 +257,6 @@ parameters. ``id`` is the item in parentheses after the version number. It is usually the version codename. - .. deprecated-removed:: 3.5 3.7 - .. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=2048) Tries to determine the libc version against which the file executable (defaults diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/plistlib.rst --- a/Doc/library/plistlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/plistlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,174 +16,66 @@ -------------- This module provides an interface for reading and writing the "property list" -files used mainly by Mac OS X and supports both binary and XML plist files. +XML files used mainly by Mac OS X. -The property list (``.plist``) file format is a simple serialization supporting +The property list (``.plist``) file format is a simple XML pickle supporting basic object types, like dictionaries, lists, numbers and strings. Usually the top level object is a dictionary. -To write out and to parse a plist file, use the :func:`dump` and -:func:`load` functions. +To write out and to parse a plist file, use the :func:`writePlist` and +:func:`readPlist` functions. -To work with plist data in bytes objects, use :func:`dumps` -and :func:`loads`. +To work with plist data in bytes objects, use :func:`writePlistToBytes` +and :func:`readPlistFromBytes`. Values can be strings, integers, floats, booleans, tuples, lists, dictionaries -(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray` -or :class:`datetime.datetime` objects. +(but only with string keys), :class:`Data` or :class:`datetime.datetime` +objects. String values (including dictionary keys) have to be unicode strings -- +they will be written out as UTF-8. -.. versionchanged:: 3.4 - New API, old API deprecated. Support for binary format plists added. +The ```` plist type is supported through the :class:`Data` class. This is +a thin wrapper around a Python bytes object. Use :class:`Data` if your strings +contain control characters. .. seealso:: - `PList manual page `_ + `PList manual page `_ Apple's documentation of the file format. This module defines the following functions: -.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict) +.. function:: readPlist(pathOrFile) - Read a plist file. *fp* should be a readable and binary file object. - Return the unpacked root object (which usually is a + Read a plist file. *pathOrFile* may either be a file name or a (readable) + file object. Return the unpacked root object (which usually is a dictionary). - The *fmt* is the format of the file and the following values are valid: - - * :data:`None`: Autodetect the file format - - * :data:`FMT_XML`: XML file format - - * :data:`FMT_BINARY`: Binary plist format - - If *use_builtin_types* is true (the default) binary data will be returned - as instances of :class:`bytes`, otherwise it is returned as instances of - :class:`Data`. - - The *dict_type* is the type used for dictionaries that are read from the - plist file. The exact structure of the plist can be recovered by using - :class:`collections.OrderedDict` (although the order of keys shouldn't be - important in plist files). - - XML data for the :data:`FMT_XML` format is parsed using the Expat parser - from :mod:`xml.parsers.expat` -- see its documentation for possible - exceptions on ill-formed XML. Unknown elements will simply be ignored - by the plist parser. - - The parser for the binary format raises :exc:`InvalidFileException` - when the file cannot be parsed. - - .. versionadded:: 3.4 - - -.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict) - - Load a plist from a bytes object. See :func:`load` for an explanation of - the keyword arguments. - - .. versionadded:: 3.4 - - -.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Write *value* to a plist file. *Fp* should be a writable, binary - file object. - - The *fmt* argument specifies the format of the plist file and can be - one of the following values: - - * :data:`FMT_XML`: XML formatted plist file - - * :data:`FMT_BINARY`: Binary formatted plist file - - When *sort_keys* is true (the default) the keys for dictionaries will be - written to the plist in sorted order, otherwise they will be written in - the iteration order of the dictionary. - - When *skipkeys* is false (the default) the function raises :exc:`TypeError` - when a key of a dictionary is not a string, otherwise such keys are skipped. - - A :exc:`TypeError` will be raised if the object is of an unsupported type or - a container that contains objects of unsupported types. - - An :exc:`OverflowError` will be raised for integer values that cannot - be represented in (binary) plist files. - - .. versionadded:: 3.4 - - -.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Return *value* as a plist-formatted bytes object. See - the documentation for :func:`dump` for an explanation of the keyword - arguments of this function. - - .. versionadded:: 3.4 - -The following functions are deprecated: - -.. function:: readPlist(pathOrFile) - - Read a plist file. *pathOrFile* may be either a file name or a (readable - and binary) file object. Returns the unpacked root object (which usually - is a dictionary). - - This function calls :func:`load` to do the actual work, see the documentation - of :func:`that function ` for an explanation of the keyword arguments. - - .. note:: - - Dict values in the result have a ``__getattr__`` method that defers - to ``__getitem_``. This means that you can use attribute access to - access items of these dictionaries. - - .. deprecated:: 3.4 Use :func:`load` instead. + The XML data is parsed using the Expat parser from :mod:`xml.parsers.expat` + -- see its documentation for possible exceptions on ill-formed XML. + Unknown elements will simply be ignored by the plist parser. .. function:: writePlist(rootObject, pathOrFile) - Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name - or a (writable and binary) file object + Write *rootObject* to a plist file. *pathOrFile* may either be a file name + or a (writable) file object. - .. deprecated:: 3.4 Use :func:`dump` instead. + A :exc:`TypeError` will be raised if the object is of an unsupported type or + a container that contains objects of unsupported types. .. function:: readPlistFromBytes(data) Read a plist data from a bytes object. Return the root object. - See :func:`load` for a description of the keyword arguments. - - .. note:: - - Dict values in the result have a ``__getattr__`` method that defers - to ``__getitem_``. This means that you can use attribute access to - access items of these dictionaries. - - .. deprecated:: 3.4 Use :func:`loads` instead. - .. function:: writePlistToBytes(rootObject) - Return *rootObject* as an XML plist-formatted bytes object. + Return *rootObject* as a plist-formatted bytes object. - .. deprecated:: 3.4 Use :func:`dumps` instead. - -The following classes are available: - -.. class:: Dict([dict]): - - Return an extended mapping object with the same value as dictionary - *dict*. - - This class is a subclass of :class:`dict` where attribute access can - be used to access items. That is, ``aDict.key`` is the same as - ``aDict['key']`` for getting, setting and deleting items in the mapping. - - .. deprecated:: 3.0 - +The following class is available: .. class:: Data(data) @@ -194,24 +86,6 @@ It has one attribute, :attr:`data`, that can be used to retrieve the Python bytes object stored in it. - .. deprecated:: 3.4 Use a :class:`bytes` object instead. - - -The following constants are available: - -.. data:: FMT_XML - - The XML format for plist files. - - .. versionadded:: 3.4 - - -.. data:: FMT_BINARY - - The binary format for plist files - - .. versionadded:: 3.4 - Examples -------- @@ -229,15 +103,13 @@ aTrueValue = True, aFalseValue = False, ), - someData = b"", - someMoreData = b"" * 10, + someData = Data(b""), + someMoreData = Data(b"" * 10), aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), ) - with open(fileName, 'wb') as fp: - dump(pl, fp) + writePlist(pl, fileName) Parsing a plist:: - with open(fileName, 'rb') as fp: - pl = load(fp) + pl = readPlist(pathOrFile) print(pl["aKey"]) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/poplib.rst --- a/Doc/library/poplib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/poplib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,11 +13,8 @@ -------------- This module defines a class, :class:`POP3`, which encapsulates a connection to a -POP3 server and implements the protocol as defined in :rfc:`1939`. The -:class:`POP3` class supports both the minimal and optional command sets from -:rfc:`1939`. The :class:`POP3` class also supports the ``STLS`` command introduced -in :rfc:`2595` to enable encrypted communication on an already established connection. - +POP3 server and implements the protocol as defined in :rfc:`1725`. The +:class:`POP3` class supports both the minimal and optional command sets. Additionally, this module provides a class :class:`POP3_SSL`, which provides support for connecting to POP3 servers that use SSL as an underlying protocol layer. @@ -27,7 +24,7 @@ mailserver supports IMAP, you would be better off using the :class:`imaplib.IMAP4` class, as IMAP servers tend to be better implemented. -The :mod:`poplib` module provides two classes: +A single class is provided by the :mod:`poplib` module: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -43,23 +40,16 @@ This is a subclass of :class:`POP3` that connects to the server over an SSL encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL - port is used. *timeout* works as in the :class:`POP3` constructor. - *context* is an optional :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into a - single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* - they can - point to PEM-formatted private key and certificate chain files, - respectively, for the SSL connection. + port is used. *keyfile* and *certfile* are also optional - they can contain a + PEM formatted private key and certificate chain file for the SSL connection. + *timeout* works as in the :class:`POP3` constructor. *context* parameter is a + :class:`ssl.SSLContext` object which allows bundling SSL configuration + options, certificates and private keys into a single (potentially long-lived) + structure. .. versionchanged:: 3.2 *context* parameter added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). One exception is defined as an attribute of the :mod:`poplib` module: @@ -107,14 +97,6 @@ Returns the greeting string sent by the POP3 server. -.. method:: POP3.capa() - - Query the server's capabilities as specified in :rfc:`2449`. - Returns a dictionary in the form ``{'name': ['param'...]}``. - - .. versionadded:: 3.4 - - .. method:: POP3.user(username) Send user command, response should indicate that a password is required. @@ -123,7 +105,7 @@ .. method:: POP3.pass_(password) Send password, response includes message count and mailbox size. Note: the - mailbox on the server is locked until :meth:`~poplib.quit` is called. + mailbox on the server is locked until :meth:`quit` is called. .. method:: POP3.apop(user, secret) @@ -194,32 +176,6 @@ the unique id for that message in the form ``'response mesgnum uid``, otherwise result is list ``(response, ['mesgnum uid', ...], octets)``. - -.. method:: POP3.utf8() - - Try to switch to UTF-8 mode. Returns the server response if successful, - raises :class:`error_proto` if not. Specified in :RFC:`6856`. - - .. versionadded:: 3.5 - - -.. method:: POP3.stls(context=None) - - Start a TLS session on the active connection as specified in :rfc:`2595`. - This is only allowed before user authentication - - *context* parameter is a :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into - a single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - This method supports hostname checking via - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. versionadded:: 3.4 - - Instances of :class:`POP3_SSL` have no additional methods. The interface of this subclass is identical to its parent. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/posix.rst --- a/Doc/library/posix.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/posix.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,7 +19,7 @@ available through the :mod:`os` interface. Once :mod:`os` is imported, there is *no* performance penalty in using it instead of :mod:`posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling -:func:`~os.putenv` when an entry in ``os.environ`` is changed. +:func:`putenv` when an entry in ``os.environ`` is changed. Errors are reported as exceptions; the usual exceptions are given for type errors, while errors reported by the system calls raise :exc:`OSError`. @@ -37,7 +37,7 @@ .. sectionauthor:: Steve Clift Several operating systems (including AIX, HP-UX, Irix and Solaris) provide -support for files that are larger than 2 GiB from a C programming model where +support for files that are larger than 2 GB from a C programming model where :c:type:`int` and :c:type:`long` are 32-bit values. This is typically accomplished by defining the relevant size and offset types as 64-bit values. Such files are sometimes referred to as :dfn:`large files`. @@ -74,10 +74,9 @@ pathname of your home directory, equivalent to ``getenv("HOME")`` in C. Modifying this dictionary does not affect the string environment passed on by - :func:`~os.execv`, :func:`~os.popen` or :func:`~os.system`; if you need to - change the environment, pass ``environ`` to :func:`~os.execve` or add - variable assignments and export statements to the command string for - :func:`~os.system` or :func:`~os.popen`. + :func:`execv`, :func:`popen` or :func:`system`; if you need to change the + environment, pass ``environ`` to :func:`execve` or add variable assignments and + export statements to the command string for :func:`system` or :func:`popen`. .. versionchanged:: 3.2 On Unix, keys and values are bytes. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pprint.rst --- a/Doc/library/pprint.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pprint.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,8 +14,8 @@ Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects -such as files, sockets or classes are included, as well as many other -objects which are not representable as Python literals. +such as files, sockets, classes, or instances are included, as well as many +other built-in objects which are not representable as Python constants. The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width. @@ -29,14 +29,14 @@ .. First the implementation class: -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False) +.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None) Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. An output stream may be set using the *stream* keyword; the only method used on the stream object is the file protocol's :meth:`write` method. If not specified, the :class:`PrettyPrinter` adopts - ``sys.stdout``. The + ``sys.stdout``. Three additional parameters may be used to control the + formatted representation. The keywords are *indent*, *depth*, and *width*. The amount of indentation added for each recursive level is specified by *indent*; the default is one. Other values can cause output to look a little odd, but can make nesting easier to spot. The number of levels which may be printed is @@ -45,12 +45,7 @@ the depth of the objects being formatted. The desired output width is constrained using the *width* parameter; the default is 80 characters. If a structure cannot be formatted within the constrained width, a best effort will - be made. If *compact* is false (the default) each item of a long sequence - will be formatted on a separate line. If *compact* is true, as many items - as will fit within the *width* will be formatted on each output line. - - .. versionchanged:: 3.4 - Added the *compact* parameter. + be made. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -63,12 +58,6 @@ 'lumberjack', 'knights', 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', ... ('parrot', ('fresh fruit',)))))))) >>> pp = pprint.PrettyPrinter(depth=6) @@ -76,30 +65,23 @@ ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) -The :mod:`pprint` module also provides several shortcut functions: +The :class:`PrettyPrinter` class supports several derivative functions: -.. function:: pformat(object, indent=1, width=80, depth=None, *, compact=False) +.. function:: pformat(object, indent=1, width=80, depth=None) - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth* and *compact* will be passed to the :class:`PrettyPrinter` - constructor as formatting parameters. + Return the formatted representation of *object* as a string. *indent*, *width* + and *depth* will be passed to the :class:`PrettyPrinter` constructor as + formatting parameters. - .. versionchanged:: 3.4 - Added the *compact* parameter. - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False) +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None) Prints the formatted representation of *object* on *stream*, followed by a newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used in the interactive interpreter instead of the :func:`print` function for inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). *indent*, *width*, *depth* and *compact* will be passed - to the :class:`PrettyPrinter` constructor as formatting parameters. - - .. versionchanged:: 3.4 - Added the *compact* parameter. + within a scope). *indent*, *width* and *depth* will be passed to the + :class:`PrettyPrinter` constructor as formatting parameters. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -211,140 +193,101 @@ ------- To demonstrate several uses of the :func:`pprint` function and its parameters, -let's fetch information about a project from `PyPI `_:: +let's fetch information about a project from PyPI:: >>> import json >>> import pprint >>> from urllib.request import urlopen - >>> with urlopen('http://pypi.python.org/pypi/Twisted/json') as url: + >>> with urlopen('http://pypi.python.org/pypi/configparser/json') as url: ... http_info = url.info() ... raw_data = url.read().decode(http_info.get_content_charset()) >>> project_info = json.loads(raw_data) + >>> result = {'headers': http_info.items(), 'body': project_info} In its basic form, :func:`pprint` shows the whole object:: - >>> pprint.pprint(project_info) - {'info': {'_pypi_hidden': False, - '_pypi_ordering': 125, - 'author': 'Glyph Lefkowitz', - 'author_email': 'glyph@twistedmatrix.com', - 'bugtrack_url': '', - 'cheesecake_code_kwalitee_id': None, - 'cheesecake_documentation_id': None, - 'cheesecake_installability_id': None, - 'classifiers': ['Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 2 :: Only'], - 'description': 'An extensible framework for Python programming, with ' - 'special focus\r\n' - 'on event-based network programming and multiprotocol ' - 'integration.', - 'docs_url': '', - 'download_url': 'UNKNOWN', - 'home_page': 'http://twistedmatrix.com/', - 'keywords': '', - 'license': 'MIT', - 'maintainer': '', - 'maintainer_email': '', - 'name': 'Twisted', - 'package_url': 'http://pypi.python.org/pypi/Twisted', - 'platform': 'UNKNOWN', - 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', - 'requires_python': None, - 'stable_version': None, - 'summary': 'An asynchronous networking framework written in Python', - 'version': '12.3.0'}, - 'urls': [{'comment_text': '', - 'downloads': 71844, - 'filename': 'Twisted-12.3.0.tar.bz2', - 'has_sig': False, - 'md5_digest': '6e289825f3bf5591cfd670874cc0862d', - 'packagetype': 'sdist', - 'python_version': 'source', - 'size': 2615733, - 'upload_time': '2012-12-26T12:47:03', - 'url': 'https://pypi.python.org/packages/source/T/Twisted/Twisted-12.3.0.tar.bz2'}, - {'comment_text': '', - 'downloads': 5224, - 'filename': 'Twisted-12.3.0.win32-py2.7.msi', - 'has_sig': False, - 'md5_digest': '6b778f5201b622a5519a2aca1a2fe512', - 'packagetype': 'bdist_msi', - 'python_version': '2.7', - 'size': 2916352, - 'upload_time': '2012-12-26T12:48:15', - 'url': 'https://pypi.python.org/packages/2.7/T/Twisted/Twisted-12.3.0.win32-py2.7.msi'}]} + >>> pprint.pprint(result) + {'body': {'info': {'_pypi_hidden': False, + '_pypi_ordering': 12, + 'classifiers': ['Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules'], + 'download_url': 'UNKNOWN', + 'home_page': 'http://docs.python.org/py3k/library/configparser.html', + 'keywords': 'configparser ini parsing conf cfg configuration file', + 'license': 'MIT', + 'name': 'configparser', + 'package_url': 'http://pypi.python.org/pypi/configparser', + 'platform': 'any', + 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3', + 'requires_python': None, + 'stable_version': None, + 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.', + 'version': '3.2.0r3'}, + 'urls': [{'comment_text': '', + 'downloads': 47, + 'filename': 'configparser-3.2.0r3.tar.gz', + 'has_sig': False, + 'md5_digest': '8500fd87c61ac0de328fc996fce69b96', + 'packagetype': 'sdist', + 'python_version': 'source', + 'size': 32281, + 'upload_time': '2011-05-10T16:28:50', + 'url': 'http://pypi.python.org/packages/source/c/configparser/configparser-3.2.0r3.tar.gz'}]}, + 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'), + ('Server', 'Apache/2.2.16 (Debian)'), + ('Content-Disposition', 'inline'), + ('Connection', 'close'), + ('Transfer-Encoding', 'chunked'), + ('Content-Type', 'application/json; charset="UTF-8"')]} The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: - >>> pprint.pprint(project_info, depth=2) - {'info': {'_pypi_hidden': False, - '_pypi_ordering': 125, - 'author': 'Glyph Lefkowitz', - 'author_email': 'glyph@twistedmatrix.com', - 'bugtrack_url': '', - 'cheesecake_code_kwalitee_id': None, - 'cheesecake_documentation_id': None, - 'cheesecake_installability_id': None, - 'classifiers': [...], - 'description': 'An extensible framework for Python programming, with ' - 'special focus\r\n' - 'on event-based network programming and multiprotocol ' - 'integration.', - 'docs_url': '', - 'download_url': 'UNKNOWN', - 'home_page': 'http://twistedmatrix.com/', - 'keywords': '', - 'license': 'MIT', - 'maintainer': '', - 'maintainer_email': '', - 'name': 'Twisted', - 'package_url': 'http://pypi.python.org/pypi/Twisted', - 'platform': 'UNKNOWN', - 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', - 'requires_python': None, - 'stable_version': None, - 'summary': 'An asynchronous networking framework written in Python', - 'version': '12.3.0'}, - 'urls': [{...}, {...}]} + >>> pprint.pprint(result, depth=3) + {'body': {'info': {'_pypi_hidden': False, + '_pypi_ordering': 12, + 'classifiers': [...], + 'download_url': 'UNKNOWN', + 'home_page': 'http://docs.python.org/py3k/library/configparser.html', + 'keywords': 'configparser ini parsing conf cfg configuration file', + 'license': 'MIT', + 'name': 'configparser', + 'package_url': 'http://pypi.python.org/pypi/configparser', + 'platform': 'any', + 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3', + 'requires_python': None, + 'stable_version': None, + 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.', + 'version': '3.2.0r3'}, + 'urls': [{...}]}, + 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'), + ('Server', 'Apache/2.2.16 (Debian)'), + ('Content-Disposition', 'inline'), + ('Connection', 'close'), + ('Transfer-Encoding', 'chunked'), + ('Content-Type', 'application/json; charset="UTF-8"')]} -Additionally, maximum character *width* can be suggested. If a long object -cannot be split, the specified width will be exceeded:: +Additionally, maximum *width* can be suggested. If a long object cannot be +split, the specified width will be exceeded:: - >>> pprint.pprint(project_info, depth=2, width=50) - {'info': {'_pypi_hidden': False, - '_pypi_ordering': 125, - 'author': 'Glyph Lefkowitz', - 'author_email': 'glyph@twistedmatrix.com', - 'bugtrack_url': '', - 'cheesecake_code_kwalitee_id': None, - 'cheesecake_documentation_id': None, - 'cheesecake_installability_id': None, - 'classifiers': [...], - 'description': 'An extensible ' - 'framework for Python ' - 'programming, with ' - 'special focus\r\n' - 'on event-based network ' - 'programming and ' - 'multiprotocol ' - 'integration.', - 'docs_url': '', - 'download_url': 'UNKNOWN', - 'home_page': 'http://twistedmatrix.com/', - 'keywords': '', - 'license': 'MIT', - 'maintainer': '', - 'maintainer_email': '', - 'name': 'Twisted', - 'package_url': 'http://pypi.python.org/pypi/Twisted', - 'platform': 'UNKNOWN', - 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0', - 'requires_python': None, - 'stable_version': None, - 'summary': 'An asynchronous networking ' - 'framework written in ' - 'Python', - 'version': '12.3.0'}, - 'urls': [{...}, {...}]} + >>> pprint.pprint(result['headers'], width=30) + [('Date', + 'Sat, 14 May 2011 12:48:52 GMT'), + ('Server', + 'Apache/2.2.16 (Debian)'), + ('Content-Disposition', + 'inline'), + ('Connection', 'close'), + ('Transfer-Encoding', + 'chunked'), + ('Content-Type', + 'application/json; charset="UTF-8"')] diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/profile.rst --- a/Doc/library/profile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/profile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,11 @@ The Python Profilers ******************** +.. sectionauthor:: James Roskind + +.. module:: profile + :synopsis: Python source profiler. + **Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` -------------- @@ -17,13 +22,14 @@ single: deterministic profiling single: profiling, deterministic -:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. +A :dfn:`profiler` is a program that describes the run time performance of a +program, providing a variety of statistics. This documentation describes the +profiler functionality provided in the modules :mod:`cProfile`, :mod:`profile` +and :mod:`pstats`. This profiler provides :dfn:`deterministic profiling` of +Python programs. It also provides a series of report generation tools to allow +users to rapidly examine the results of a profile operation. -The Python standard library provides two different implementations of the same -profiling interface: +The Python standard library provides two different profilers: 1. :mod:`cProfile` is recommended for most users; it's a C extension with reasonable overhead that makes it suitable for profiling long-running @@ -31,9 +37,14 @@ Czotter. 2. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. + :mod:`cProfile`. Adds significant overhead to profiled programs. If you're + trying to extend the profiler in some way, the task might be easier with this + module. + +The :mod:`profile` and :mod:`cProfile` modules export the same interface, so +they are mostly interchangeable; :mod:`cProfile` has a much lower overhead but +is newer and might not be available on all systems. :mod:`cProfile` is really a +compatibility layer on top of the internal :mod:`_lsprof` module. .. note:: @@ -54,94 +65,57 @@ provides a very brief overview, and allows a user to rapidly perform profiling on an existing application. -To profile a function that takes a single argument, you can do:: +To profile an application with a main entry point of :func:`foo`, you would add +the following to your module:: import cProfile - import re - cProfile.run('re.compile("foo|bar")') + cProfile.run('foo()') (Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on your system.) -The above action would run :func:`re.compile` and print profile results like -the following:: - - 197 function calls (192 primitive calls) in 0.002 seconds - - Ordered by: standard name - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 re.py:212(compile) - 1 0.000 0.000 0.001 0.001 re.py:268(_compile) - 1 0.000 0.000 0.000 0.000 sre_compile.py:172(_compile_charset) - 1 0.000 0.000 0.000 0.000 sre_compile.py:201(_optimize_charset) - 4 0.000 0.000 0.000 0.000 sre_compile.py:25(_identityfunction) - 3/1 0.000 0.000 0.000 0.000 sre_compile.py:33(_compile) - -The first line indicates that 197 calls were monitored. Of those calls, 192 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: standard name``, indicates that the text string in the -far right column was used to sort the output. The column headings include: - -ncalls - for the number of calls, - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: +The above action would cause :func:`foo` to be run, and a series of informative +lines (the profile) to be printed. The above approach is most useful when +working with the interpreter. If you would like to save the results of a +profile into a file for later examination, you can supply a file name as the +second argument to the :func:`run` function:: import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') + cProfile.run('foo()', 'fooprof') -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -The file :mod:`cProfile` can also be invoked as a script to profile another +The file :file:`cProfile.py` can also be invoked as a script to profile another script. For example:: - python -m cProfile [-o output_file] [-s sort_order] myscript.py + python -m cProfile myscript.py -``-o`` writes the profile results to a file instead of to stdout +:file:`cProfile.py` accepts two optional arguments on the command line:: -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. + cProfile.py [-o output_file] [-s sort_order] -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: +``-s`` only applies to standard output (``-o`` is not supplied). +Look in the :class:`Stats` documentation for valid sort values. + +When you wish to review the profile, you should use the methods in the +:mod:`pstats` module. Typically you would load the statistics data as follows:: import pstats - p = pstats.Stats('restats') + p = pstats.Stats('fooprof') + +The class :class:`Stats` (the above code just created an instance of this class) +has a variety of methods for manipulating and printing the data that was just +read into ``p``. When you ran :func:`cProfile.run` above, what was printed was +the result of three method calls:: + p.strip_dirs().sort_stats(-1).print_stats() -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: +The first method removed the extraneous path from all the module names. The +second method sorted all the entries according to the standard module/line/name +string that is printed. The third method printed out all the statistics. You +might try the following sort calls: + +.. (this is to comply with the semantics of the old profiler). + +:: p.sort_stats('name') p.print_stats() @@ -172,7 +146,7 @@ for only the class init methods (since they are spelled with ``__init__`` in them). As one final example, you could try:: - p.sort_stats('time', 'cumulative').print_stats(.5, 'init') + p.sort_stats('time', 'cum').print_stats(.5, 'init') This line sorts statistics with a primary key of time, and a secondary key of cumulative time, and then prints out some of the statistics. To be specific, the @@ -190,303 +164,12 @@ guess what the following functions do:: p.print_callees() - p.add('restats') + p.add('fooprof') Invoked as a script, the :mod:`pstats` module is a statistics browser for reading and examining profile dumps. It has a simple line-oriented interface (implemented using :mod:`cmd`) and interactive help. -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: - -.. function:: run(command, filename=None, sort=-1) - - This function takes a single argument that can be passed to the :func:`exec` - function, and an optional file name. In all cases this routine executes:: - - exec(command, __main__.__dict__, __main__.__dict__) - - and gathers profiling statistics from the execution. If no file name is - present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified - it is passed to this :class:`~pstats.Stats` instance to control how the - results are sorted. - -.. function:: runctx(command, globals, locals, filename=None) - - This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine - executes:: - - exec(command, globals, locals) - - and gathers profiling statistics as in the :func:`run` function above. - -.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) - - This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. - - A custom timer can be supplied for measuring how long code takes to run via - the *timer* argument. This must be a function that returns a single number - representing the current time. If the number is an integer, the *timeunit* - specifies a multiplier that specifies the duration of each unit of time. For - example, if the timer returns times measured in thousands of seconds, the - time unit would be ``.001``. - - Directly using the :class:`Profile` class allows formatting profile results - without writing the profile data to a file:: - - import cProfile, pstats, io - pr = cProfile.Profile() - pr.enable() - # ... do something ... - pr.disable() - s = io.StringIO() - sortby = 'cumulative' - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - ps.print_stats() - print(s.getvalue()) - - .. method:: enable() - - Start collecting profiling data. - - .. method:: disable() - - Stop collecting profiling data. - - .. method:: create_stats() - - Stop collecting profiling data and record the results internally - as the current profile. - - .. method:: print_stats(sort=-1) - - Create a :class:`~pstats.Stats` object based on the current - profile and print the results to stdout. - - .. method:: dump_stats(filename) - - Write the results of the current profile to *filename*. - - .. method:: run(cmd) - - Profile the cmd via :func:`exec`. - - .. method:: runctx(cmd, globals, locals) - - Profile the cmd via :func:`exec` with the specified global and - local environment. - - .. method:: runcall(func, *args, **kwargs) - - Profile ``func(*args, **kwargs)`` - -.. _profile-stats: - -The :class:`Stats` Class -======================== - -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. - -.. module:: pstats - :synopsis: Statistics object for use with the profiler. - -.. class:: Stats(*filenames or profile, stream=sys.stdout) - - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. - - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers. If several files are provided, all the statistics for identical - functions will be coalesced, so that an overall view of several processes can - be considered in a single report. If additional files need to be combined - with data in an existing :class:`~pstats.Stats` object, the - :meth:`~pstats.Stats.add` method can be used. - - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument is typically a string identifying the - basis of a sort (example: ``'time'`` or ``'name'``). - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats('name', 'file')`` will sort all the - entries according to their function name, and resolve all ties (identical - function names) by sorting by file name. - - Abbreviations can be used for any key names, as long as the abbreviation - is unambiguous. The following are the keys currently defined: - - +------------------+----------------------+ - | Valid Arg | Meaning | - +==================+======================+ - | ``'calls'`` | call count | - +------------------+----------------------+ - | ``'cumulative'`` | cumulative time | - +------------------+----------------------+ - | ``'cumtime'`` | cumulative time | - +------------------+----------------------+ - | ``'file'`` | file name | - +------------------+----------------------+ - | ``'filename'`` | file name | - +------------------+----------------------+ - | ``'module'`` | file name | - +------------------+----------------------+ - | ``'ncalls'`` | call count | - +------------------+----------------------+ - | ``'pcalls'`` | primitive call count | - +------------------+----------------------+ - | ``'line'`` | line number | - +------------------+----------------------+ - | ``'name'`` | function name | - +------------------+----------------------+ - | ``'nfl'`` | name/file/line | - +------------------+----------------------+ - | ``'stdname'`` | standard name | - +------------------+----------------------+ - | ``'time'`` | internal time | - +------------------+----------------------+ - | ``'tottime'`` | internal time | - +------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``'nfl'`` and ``'stdname'`` is that the standard name is a sort of the - name as printed, which means that the embedded line numbers get compared - in an odd way. For example, lines 3, 20, and 40 would (if the file names - were the same) appear in the string order 20, 3 and 40. In contrast, - ``'nfl'`` does a numeric compare of the line numbers. In fact, - ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', - 'line')``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a regular expression (to pattern match - the standard name that is printed. If several restrictions are provided, - then they are applied sequentially. For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - .. _deterministic-profiling: @@ -521,7 +204,271 @@ implementations. -.. _profile-limitations: +Reference Manual -- :mod:`profile` and :mod:`cProfile` +====================================================== + +.. module:: cProfile + :synopsis: Python profiler + + +The primary entry point for the profiler is the global function +:func:`profile.run` (resp. :func:`cProfile.run`). It is typically used to create +any profile information. The reports are formatted and printed using methods of +the class :class:`pstats.Stats`. The following is a description of all of these +standard entry points and functions. For a more in-depth view of some of the +code, consider reading the later section on Profiler Extensions, which includes +discussion of how to derive "better" profilers from the classes presented, or +reading the source code for these modules. + + +.. function:: run(command, filename=None, sort=-1) + + This function takes a single argument that can be passed to the :func:`exec` + function, and an optional file name. In all cases this routine attempts to + :func:`exec` its first argument, and gather profiling statistics from the + execution. If no file name is present, then this function automatically + prints a simple profiling report, sorted by the standard name string + (file/line/function-name) that is presented in each line. The following is a + typical output from such a call:: + + 2706 function calls (2004 primitive calls) in 4.504 CPU seconds + + Ordered by: standard name + + ncalls tottime percall cumtime percall filename:lineno(function) + 2 0.006 0.003 0.953 0.477 pobject.py:75(save_objects) + 43/3 0.533 0.012 0.749 0.250 pobject.py:99(evaluate) + ... + + The first line indicates that 2706 calls were monitored. Of those calls, 2004 + were :dfn:`primitive`. We define :dfn:`primitive` to mean that the call was not + induced via recursion. The next line: ``Ordered by: standard name``, indicates + that the text string in the far right column was used to sort the output. The + column headings include: + + ncalls + for the number of calls, + + tottime + for the total time spent in the given function (and excluding time made in + calls to sub-functions), + + percall + is the quotient of ``tottime`` divided by ``ncalls`` + + cumtime + is the total time spent in this and all subfunctions (from invocation till + exit). This figure is accurate *even* for recursive functions. + + percall + is the quotient of ``cumtime`` divided by primitive calls + + filename:lineno(function) + provides the respective data of each function + + When there are two numbers in the first column (for example, ``43/3``), then the + latter is the number of primitive calls, and the former is the actual number of + calls. Note that when the function does not recurse, these two values are the + same, and only the single figure is printed. + + If *sort* is given, it can be one of ``'stdname'`` (sort by filename:lineno), + ``'calls'`` (sort by number of calls), ``'time'`` (sort by total time) or + ``'cumulative'`` (sort by cumulative time). The default is ``'stdname'``. + + +.. function:: runctx(command, globals, locals, filename=None) + + This function is similar to :func:`run`, with added arguments to supply the + globals and locals dictionaries for the *command* string. + + +Analysis of the profiler data is done using the :class:`pstats.Stats` class. + + +.. module:: pstats + :synopsis: Statistics object for use with the profiler. + + +.. class:: Stats(*filenames, stream=sys.stdout) + + This class constructor creates an instance of a "statistics object" from a + *filename* (or set of filenames). :class:`Stats` objects are manipulated by + methods, in order to print useful reports. You may specify an alternate output + stream by giving the keyword argument, ``stream``. + + The file selected by the above constructor must have been created by the + corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, + there is *no* file compatibility guaranteed with future versions of this + profiler, and there is no compatibility with files produced by other profilers. + If several files are provided, all the statistics for identical functions will + be coalesced, so that an overall view of several processes can be considered in + a single report. If additional files need to be combined with data in an + existing :class:`Stats` object, the :meth:`add` method can be used. + + .. (such as the old system profiler). + + +.. _profile-stats: + +The :class:`Stats` Class +------------------------ + +:class:`Stats` objects have the following methods: + + +.. method:: Stats.strip_dirs() + + This method for the :class:`Stats` class removes all leading path information + from file names. It is very useful in reducing the size of the printout to fit + within (close to) 80 columns. This method modifies the object, and the stripped + information is lost. After performing a strip operation, the object is + considered to have its entries in a "random" order, as it was just after object + initialization and loading. If :meth:`strip_dirs` causes two function names to + be indistinguishable (they are on the same line of the same filename, and have + the same function name), then the statistics for these two entries are + accumulated into a single entry. + + +.. method:: Stats.add(*filenames) + + This method of the :class:`Stats` class accumulates additional profiling + information into the current profiling object. Its arguments should refer to + filenames created by the corresponding version of :func:`profile.run` or + :func:`cProfile.run`. Statistics for identically named (re: file, line, name) + functions are automatically accumulated into single function statistics. + + +.. method:: Stats.dump_stats(filename) + + Save the data loaded into the :class:`Stats` object to a file named *filename*. + The file is created if it does not exist, and is overwritten if it already + exists. This is equivalent to the method of the same name on the + :class:`profile.Profile` and :class:`cProfile.Profile` classes. + + +.. method:: Stats.sort_stats(*keys) + + This method modifies the :class:`Stats` object by sorting it according to the + supplied criteria. The argument is typically a string identifying the basis of + a sort (example: ``'time'`` or ``'name'``). + + When more than one key is provided, then additional keys are used as secondary + criteria when there is equality in all keys selected before them. For example, + ``sort_stats('name', 'file')`` will sort all the entries according to their + function name, and resolve all ties (identical function names) by sorting by + file name. + + Abbreviations can be used for any key names, as long as the abbreviation is + unambiguous. The following are the keys currently defined: + + +------------------+----------------------+ + | Valid Arg | Meaning | + +==================+======================+ + | ``'calls'`` | call count | + +------------------+----------------------+ + | ``'cumulative'`` | cumulative time | + +------------------+----------------------+ + | ``'file'`` | file name | + +------------------+----------------------+ + | ``'module'`` | file name | + +------------------+----------------------+ + | ``'pcalls'`` | primitive call count | + +------------------+----------------------+ + | ``'line'`` | line number | + +------------------+----------------------+ + | ``'name'`` | function name | + +------------------+----------------------+ + | ``'nfl'`` | name/file/line | + +------------------+----------------------+ + | ``'stdname'`` | standard name | + +------------------+----------------------+ + | ``'time'`` | internal time | + +------------------+----------------------+ + + Note that all sorts on statistics are in descending order (placing most time + consuming items first), where as name, file, and line number searches are in + ascending order (alphabetical). The subtle distinction between ``'nfl'`` and + ``'stdname'`` is that the standard name is a sort of the name as printed, which + means that the embedded line numbers get compared in an odd way. For example, + lines 3, 20, and 40 would (if the file names were the same) appear in the string + order 20, 3 and 40. In contrast, ``'nfl'`` does a numeric compare of the line + numbers. In fact, ``sort_stats('nfl')`` is the same as ``sort_stats('name', + 'file', 'line')``. + + For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, ``1``, + and ``2`` are permitted. They are interpreted as ``'stdname'``, ``'calls'``, + ``'time'``, and ``'cumulative'`` respectively. If this old style format + (numeric) is used, only one sort key (the numeric key) will be used, and + additional arguments will be silently ignored. + + .. For compatibility with the old profiler, + + +.. method:: Stats.reverse_order() + + This method for the :class:`Stats` class reverses the ordering of the basic list + within the object. Note that by default ascending vs descending order is + properly selected based on the sort key of choice. + + .. This method is provided primarily for compatibility with the old profiler. + + +.. method:: Stats.print_stats(*restrictions) + + This method for the :class:`Stats` class prints out a report as described in the + :func:`profile.run` definition. + + The order of the printing is based on the last :meth:`sort_stats` operation done + on the object (subject to caveats in :meth:`add` and :meth:`strip_dirs`). + + The arguments provided (if any) can be used to limit the list down to the + significant entries. Initially, the list is taken to be the complete set of + profiled functions. Each restriction is either an integer (to select a count of + lines), or a decimal fraction between 0.0 and 1.0 inclusive (to select a + percentage of lines), or a regular expression (to pattern match the standard + name that is printed; as of Python 1.5b1, this uses the Perl-style regular + expression syntax defined by the :mod:`re` module). If several restrictions are + provided, then they are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + would first limit the printing to first 10% of list, and then only print + functions that were part of filename :file:`.\*foo:`. In contrast, the + command:: + + print_stats('foo:', .1) + + would limit the list to all functions having file names :file:`.\*foo:`, and + then proceed to only print the first 10% of them. + + +.. method:: Stats.print_callers(*restrictions) + + This method for the :class:`Stats` class prints a list of all functions that + called each function in the profiled database. The ordering is identical to + that provided by :meth:`print_stats`, and the definition of the restricting + argument is also identical. Each caller is reported on its own line. The + format differs slightly depending on the profiler that produced the stats: + + * With :mod:`profile`, a number is shown in parentheses after each caller to + show how many times this specific call was made. For convenience, a second + non-parenthesized number repeats the cumulative time spent in the function + at the right. + + * With :mod:`cProfile`, each caller is preceded by three numbers: the number of + times this specific call was made, and the total and cumulative times spent in + the current function while it was invoked by this specific caller. + + +.. method:: Stats.print_callees(*restrictions) + + This method for the :class:`Stats` class prints a list of all function that were + called by the indicated function. Aside from this reversal of direction of + calls (re: called vs was called by), the arguments and ordering are identical to + the :meth:`print_callers` method. + + +.. _profile-limits: Limitations =========== @@ -564,7 +511,7 @@ handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see -:ref:`profile-limitations`). :: +discussion in section Limitations above). :: import profile pr = profile.Profile() @@ -574,8 +521,8 @@ The method executes the number of Python calls given by the argument, directly and again under the profiler, measuring the time for both. It then computes the hidden overhead per profiler event, and returns that as a float. For example, -on a 1.8Ghz Intel Core i5 running Mac OS X, and using Python's time.clock() as -the timer, the magical number is about 4.04e-6. +on an 800 MHz Pentium running Windows 2000, and using Python's time.clock() as +the timer, the magical number is about 12.5e-6. The object of this exercise is to get a fairly consistent result. If your computer is *very* fast, or your timer function has poor resolution, you might @@ -598,51 +545,50 @@ If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. -.. _profile-timers: -Using a custom timer -==================== +.. _profiler-extensions: -If you want to change how current time is determined (for example, to force use -of wall-clock time or elapsed process time), pass the timing function you want -to the :class:`Profile` class constructor:: +Extensions --- Deriving Better Profilers +======================================== - pr = profile.Profile(your_time_func) +The :class:`Profile` class of both modules, :mod:`profile` and :mod:`cProfile`, +were written so that derived classes could be developed to extend the profiler. +The details are not described here, as doing this successfully requires an +expert understanding of how the :class:`Profile` class works internally. Study +the source code of the module carefully if you want to pursue this. -The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, -``your_time_func``'s return value will be interpreted differently: +If all you want to do is change how current time is determined (for example, to +force use of wall-clock time or elapsed process time), pass the timing function +you want to the :class:`Profile` class constructor:: + + pr = profile.Profile(your_time_func) + +The resulting profiler will then call :func:`your_time_func`. :class:`profile.Profile` - ``your_time_func`` should return a single number, or a list of numbers whose - sum is the current time (like what :func:`os.times` returns). If the - function returns a single time number, or the list of returned numbers has - length 2, then you will get an especially fast version of the dispatch - routine. + :func:`your_time_func` should return a single number, or a list of numbers whose + sum is the current time (like what :func:`os.times` returns). If the function + returns a single time number, or the list of returned numbers has length 2, then + you will get an especially fast version of the dispatch routine. Be warned that you should calibrate the profiler class for the timer function - that you choose (see :ref:`profile-calibration`). For most machines, a timer - that returns a lone integer value will provide the best results in terms of - low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a - better timer in the cleanest fashion, derive a class and hardwire a - replacement dispatch method that best handles your timer call, along with the - appropriate calibration constant. + that you choose. For most machines, a timer that returns a lone integer value + will provide the best results in terms of low overhead during profiling. + (:func:`os.times` is *pretty* bad, as it returns a tuple of floating point + values). If you want to substitute a better timer in the cleanest fashion, + derive a class and hardwire a replacement dispatch method that best handles your + timer call, along with the appropriate calibration constant. :class:`cProfile.Profile` - ``your_time_func`` should return a single number. If it returns integers, - you can also invoke the class constructor with a second argument specifying - the real duration of one unit of time. For example, if - ``your_integer_time_func`` returns times measured in thousands of seconds, + :func:`your_time_func` should return a single number. If it returns + integers, you can also invoke the class constructor with a second argument + specifying the real duration of one unit of time. For example, if + :func:`your_integer_time_func` returns times measured in thousands of seconds, you would construct the :class:`Profile` instance as follows:: - pr = cProfile.Profile(your_integer_time_func, 0.001) + pr = profile.Profile(your_integer_time_func, 0.001) - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. - -Python 3.3 adds several new functions in :mod:`time` that can be used to make -precise measurements of process or wall-clock time. For example, see -:func:`time.perf_counter`. + As the :mod:`cProfile.Profile` class cannot be calibrated, custom timer + functions should be used with care and should be as fast as possible. For the + best results with a custom timer, it might be necessary to hard-code it in the C + source of the internal :mod:`_lsprof` module. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pty.rst --- a/Doc/library/pty.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pty.rst Mon Jan 25 17:05:13 2016 +0100 @@ -45,9 +45,6 @@ a file descriptor. The defaults try to read 1024 bytes each time they are called. - .. versionchanged:: 3.4 - :func:`spawn` now returns the status value from :func:`os.waitpid` - on the child process. Example ------- @@ -58,32 +55,40 @@ pseudo-terminal to record all input and output of a terminal session in a "typescript". :: - import argparse - import os - import pty - import sys - import time + import sys, os, time, getopt + import pty - parser = argparse.ArgumentParser() - parser.add_argument('-a', dest='append', action='store_true') - parser.add_argument('-p', dest='use_python', action='store_true') - parser.add_argument('filename', nargs='?', default='typescript') - options = parser.parse_args() + mode = 'wb' + shell = 'sh' + filename = 'typescript' + if 'SHELL' in os.environ: + shell = os.environ['SHELL'] - shell = sys.executable if options.use_python else os.environ.get('SHELL', 'sh') - filename = options.filename - mode = 'ab' if options.append else 'wb' + try: + opts, args = getopt.getopt(sys.argv[1:], 'ap') + except getopt.error as msg: + print('%s: %s' % (sys.argv[0], msg)) + sys.exit(2) - with open(filename, mode) as script: - def read(fd): - data = os.read(fd, 1024) - script.write(data) - return data + for opt, arg in opts: + # option -a: append to typescript file + if opt == '-a': + mode = 'ab' + # option -p: use a Python shell as the terminal command + elif opt == '-p': + shell = sys.executable + if args: + filename = args[0] - print('Script started, file is', filename) - script.write(('Script started on %s\n' % time.asctime()).encode()) + script = open(filename, mode) - pty.spawn(shell, read) + def read(fd): + data = os.read(fd, 1024) + script.write(data) + return data - script.write(('Script done on %s\n' % time.asctime()).encode()) - print('Script done, file is', filename) + sys.stdout.write('Script started, file is %s\n' % filename) + script.write(('Script started on %s\n' % time.asctime()).encode()) + pty.spawn(shell, read) + script.write(('Script done on %s\n' % time.asctime()).encode()) + sys.stdout.write('Script done, file is %s\n' % filename) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/py_compile.rst --- a/Doc/library/py_compile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/py_compile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -28,10 +28,10 @@ .. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) - Compile a source file to byte-code and write out the byte-code cache file. - The source code is loaded from the file name *file*. The byte-code is - written to *cfile*, which defaults to the :pep:`3147`/:pep:`488` path, ending - in ``.pyc``. + Compile a source file to byte-code and write out the byte-code cache file. + The source code is loaded from the file name *file*. The byte-code is + written to *cfile*, which defaults to the :PEP:`3147` path, ending in + ``.pyc`` (``.pyo`` if optimization is enabled in the current interpreter). For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is specified, it is used as the name of the source file in error messages when @@ -41,13 +41,6 @@ is raised. This function returns the path to byte-compiled file, i.e. whatever *cfile* value was used. - If the path that *cfile* becomes (either explicitly specified or computed) - is a symlink or non-regular file, :exc:`FileExistsError` will be raised. - This is to act as a warning that import will turn those paths into regular - files if it is allowed to write byte-compiled files to those paths. This is - a side-effect of import using file renaming to place the final byte-compiled - file into place to prevent concurrent file writing issues. - *optimize* controls the optimization level and is passed to the built-in :func:`compile` function. The default of ``-1`` selects the optimization level of the current interpreter. @@ -57,18 +50,11 @@ default was *file* + ``'c'`` (``'o'`` if optimization was enabled). Also added the *optimize* parameter. - .. versionchanged:: 3.4 - Changed code to use :mod:`importlib` for the byte-code cache file writing. - This means file creation/writing semantics now match what :mod:`importlib` - does, e.g. permissions, write-and-move semantics, etc. Also added the - caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or - non-regular file. - .. function:: main(args=None) Compile several source files. The files named in *args* (or on the command - line, if *args* is ``None``) are compiled and the resulting byte-code is + line, if *args* is ``None``) are compiled and the resulting bytecode is cached in the normal manner. This function does not search a directory structure to locate source files; it only compiles files named explicitly. If ``'-'`` is the only parameter in args, the list of files is taken from @@ -86,3 +72,4 @@ Module :mod:`compileall` Utilities to compile all Python source files in a directory tree. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pydoc.rst --- a/Doc/library/pydoc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pydoc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -20,13 +20,6 @@ modules. The documentation can be presented as pages of text on the console, served to a Web browser, or saved to HTML files. -For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, -and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just -above the definition of the class, function or method in the source file, or at -the top of the module (see :func:`inspect.getcomments`). - The built-in function :func:`help` invokes the online help system in the interactive interpreter, which uses :mod:`pydoc` to generate its documentation as text on the console. The same text documentation can also be viewed from @@ -51,10 +44,6 @@ executed on that occasion. Use an ``if __name__ == '__main__':`` guard to only execute code when a file is invoked as a script and not just imported. -When printing output to the console, :program:`pydoc` attempts to paginate the -output for easier reading. If the :envvar:`PAGER` environment variable is set, -:program:`pydoc` will use its value as a pagination program. - Specifying a ``-w`` flag before the argument will cause HTML documentation to be written out to a file in the current directory, instead of displaying text on the console. @@ -70,6 +59,11 @@ documentation at ``http://localhost:1234/`` in your preferred Web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +:program:`pydoc -g` will start the server and additionally bring up a +small :mod:`tkinter`\ -based graphical interface to help you search for +documentation pages. The ``-g`` option is deprecated, since the server can +now be controlled directly from HTTP clients. + :program:`pydoc -b` will start the server and additionally open a web browser to a module index page. Each served page has a navigation bar at the top where you can *Get* help on an individual item, *Search* all modules with a @@ -82,19 +76,11 @@ Python interpreter and typed ``import spam``. Module docs for core modules are assumed to reside in -``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the +``http://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the major and minor version numbers of the Python interpreter. This can be overridden by setting the :envvar:`PYTHONDOCS` environment variable to a different URL or to a local directory containing the Library Reference Manual pages. .. versionchanged:: 3.2 - Added the ``-b`` option. - -.. versionchanged:: 3.3 - The ``-g`` command line option was removed. - -.. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than - :func:`inspect.getfullargspec` to extract signature information from - callables. + Added the ``-b`` option, deprecated the ``-g`` option. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/pyexpat.rst --- a/Doc/library/pyexpat.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/pyexpat.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,13 +14,6 @@ references to these attributes should be marked using the :member: role. -.. warning:: - - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - .. index:: single: Expat The :mod:`xml.parsers.expat` module is a Python interface to the Expat @@ -100,11 +93,6 @@ http://www.python.org/ns/ elem1 elem2 - Due to limitations in the ``Expat`` library used by :mod:`pyexpat`, - the :class:`xmlparser` instance returned can only be used to parse a single - XML document. Call ``ParserCreate`` for each document to provide unique - parser instances. - .. seealso:: @@ -124,9 +112,7 @@ Parses the contents of the string *data*, calling the appropriate handler functions to process the parsed data. *isfinal* must be true on the final call - to this method; it allows the parsing of a single file in fragments, - not the submission of multiple files. - *data* can be the empty string at any time. + to this method. *data* can be the empty string at any time. .. method:: xmlparser.ParseFile(file) @@ -346,10 +332,8 @@ .. method:: xmlparser.StartElementHandler(name, attributes) Called for the start of every element. *name* is a string containing the - element name, and *attributes* is the element attributes. If - :attr:`ordered_attributes` is true, this is a list (see - :attr:`ordered_attributes` for a full description). Otherwise it's a - dictionary mapping names to values. + element name, and *attributes* is a dictionary mapping attribute names to their + values. .. method:: xmlparser.EndElementHandler(name) @@ -418,7 +402,7 @@ .. method:: xmlparser.CommentHandler(data) Called for comments. *data* is the text of the comment, excluding the leading - ``''``. + '````'. .. method:: xmlparser.StartCdataSectionHandler() @@ -491,8 +475,8 @@ .. attribute:: ExpatError.code Expat's internal error number for the specific error. The - :data:`errors.messages ` dictionary maps - these error numbers to Expat's error messages. For example:: + :data:`errors.messages` dictionary maps these error numbers to Expat's error + messages. For example:: from xml.parsers.expat import ParserCreate, ExpatError, errors @@ -502,9 +486,9 @@ except ExpatError as err: print("Error:", errors.messages[err.code]) - The :mod:`~xml.parsers.expat.errors` module also provides error message - constants and a dictionary :data:`~xml.parsers.expat.errors.codes` mapping - these messages back to the error codes, see below. + The :mod:`errors` module also provides error message constants and a + dictionary :data:`~errors.codes` mapping these messages back to the error + codes, see below. .. attribute:: ExpatError.lineno @@ -763,7 +747,7 @@ .. data:: XML_ERROR_UNDEFINED_ENTITY - A reference was made to an entity which was not defined. + A reference was made to a entity which was not defined. .. data:: XML_ERROR_UNKNOWN_ENCODING @@ -868,5 +852,5 @@ .. [#] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and http://www.iana.org/assignments/character-sets/character-sets.xhtml. + and http://www.iana.org/assignments/character-sets . diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/python.rst --- a/Doc/library/python.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/python.rst Mon Jan 25 17:05:13 2016 +0100 @@ -25,3 +25,5 @@ inspect.rst site.rst fpectl.rst + packaging.rst + distutils.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/queue.rst --- a/Doc/library/queue.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/queue.rst Mon Jan 25 17:05:13 2016 +0100 @@ -15,8 +15,8 @@ availability of thread support in Python; see the :mod:`threading` module. -The module implements three types of queue, which differ only in the order in -which the entries are retrieved. In a FIFO queue, the first tasks added are +Implements three types of queue whose only difference is the order that +the entries are retrieved. In a FIFO queue, the first tasks added are the first retrieved. In a LIFO queue, the most recently added entry is the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the @@ -54,15 +54,13 @@ .. exception:: Empty - Exception raised when non-blocking :meth:`~Queue.get` (or - :meth:`~Queue.get_nowait`) is called + Exception raised when non-blocking :meth:`get` (or :meth:`get_nowait`) is called on a :class:`Queue` object which is empty. .. exception:: Full - Exception raised when non-blocking :meth:`~Queue.put` (or - :meth:`~Queue.put_nowait`) is called + Exception raised when non-blocking :meth:`put` (or :meth:`put_nowait`) is called on a :class:`Queue` object which is full. @@ -158,32 +156,22 @@ Example of how to wait for enqueued tasks to be completed:: - def worker(): - while True: - item = q.get() - if item is None: - break - do_work(item) - q.task_done() + def worker(): + while True: + item = q.get() + do_work(item) + q.task_done() - q = queue.Queue() - threads = [] - for i in range(num_worker_threads): - t = threading.Thread(target=worker) + q = Queue() + for i in range(num_worker_threads): + t = Thread(target=worker) + t.daemon = True t.start() - threads.append(t) - for item in source(): - q.put(item) + for item in source(): + q.put(item) - # block until all tasks are done - q.join() - - # stop workers - for i in range(num_worker_threads): - q.put(None) - for t in threads: - t.join() + q.join() # block until all tasks are done .. seealso:: @@ -193,6 +181,6 @@ context. :class:`collections.deque` is an alternative implementation of unbounded - queues with fast atomic :meth:`~collections.deque.append` and - :meth:`~collections.deque.popleft` operations that do not require locking. + queues with fast atomic :func:`append` and :func:`popleft` operations that + do not require locking. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/quopri.rst --- a/Doc/library/quopri.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/quopri.rst Mon Jan 25 17:05:13 2016 +0100 @@ -24,8 +24,9 @@ .. function:: decode(input, output, header=False) Decode the contents of the *input* file and write the resulting decoded binary - data to the *output* file. *input* and *output* must be :term:`binary file objects - `. If the optional argument *header* is present and true, underscore + data to the *output* file. *input* and *output* must be :term:`file objects + `. *input* will be read until ``input.readline()`` returns an + empty string. If the optional argument *header* is present and true, underscore will be decoded as space. This is used to decode "Q"-encoded headers as described in :rfc:`1522`: "MIME (Multipurpose Internet Mail Extensions) Part Two: Message Header Extensions for Non-ASCII Text". @@ -33,28 +34,27 @@ .. function:: encode(input, output, quotetabs, header=False) - Encode the contents of the *input* file and write the resulting quoted- - printable data to the *output* file. *input* and *output* must be - :term:`binary file objects `. *quotetabs*, a flag which controls - whether to encode embedded spaces and tabs must be provideda and when true it - encodes such embedded whitespace, and when false it leaves them unencoded. - Note that spaces and tabs appearing at the end of lines are always encoded, - as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded - as underscores as per :rfc:`1522`. + Encode the contents of the *input* file and write the resulting quoted-printable + data to the *output* file. *input* and *output* must be :term:`file objects + `. *input* will be read until ``input.readline()`` returns an + empty string. *quotetabs* is a flag which controls whether to encode embedded + spaces and tabs; when true it encodes such embedded whitespace, and when + false it leaves them unencoded. Note that spaces and tabs appearing at the + end of lines are always encoded, as per :rfc:`1521`. *header* is a flag + which controls if spaces are encoded as underscores as per :rfc:`1522`. .. function:: decodestring(s, header=False) - Like :func:`decode`, except that it accepts a source :class:`bytes` and - returns the corresponding decoded :class:`bytes`. + Like :func:`decode`, except that it accepts a source string and returns the + corresponding decoded string. .. function:: encodestring(s, quotetabs=False, header=False) - Like :func:`encode`, except that it accepts a source :class:`bytes` and - returns the corresponding encoded :class:`bytes`. By default, it sends a - False value to *quotetabs* parameter of the :func:`encode` function. - + Like :func:`encode`, except that it accepts a source string and returns the + corresponding encoded string. *quotetabs* and *header* are optional + (defaulting to ``False``), and are passed straight through to :func:`encode`. .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/random.rst --- a/Doc/library/random.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/random.rst Mon Jan 25 17:05:13 2016 +0100 @@ -45,26 +45,27 @@ .. warning:: - The pseudo-random generators of this module should not be used for - security purposes. + The generators of the :mod:`random` module should not be used for security + purposes. Use :func:`ssl.RAND_bytes` if you require a cryptographically + secure pseudorandom number generator. Bookkeeping functions: -.. function:: seed(a=None, version=2) +.. function:: seed([x], version=2) Initialize the random number generator. - If *a* is omitted or ``None``, the current system time is used. If + If *x* is omitted or ``None``, the current system time is used. If randomness sources are provided by the operating system, they are used instead of the system time (see the :func:`os.urandom` function for details on availability). - If *a* is an int, it is used directly. + If *x* is an int, it is used directly. With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` object gets converted to an :class:`int` and all of its bits are used. With version 1, - the :func:`hash` of *a* is used instead. + the :func:`hash` of *x* is used instead. .. versionchanged:: 3.2 Moved to the version 2 scheme which uses all of the bits in a string seed. @@ -79,7 +80,7 @@ *state* should have been obtained from a previous call to :func:`getstate`, and :func:`setstate` restores the internal state of the generator to what it was at - the time :func:`getstate` was called. + the time :func:`setstate` was called. .. function:: getrandbits(k) @@ -92,8 +93,7 @@ Functions for integers: -.. function:: randrange(stop) - randrange(start, stop[, step]) +.. function:: randrange([start,] stop[, step]) Return a randomly selected element from ``range(start, stop, step)``. This is equivalent to ``choice(range(start, stop, step))``, but doesn't actually build a @@ -150,9 +150,6 @@ argument. This is especially fast and space efficient for sampling from a large population: ``sample(range(10000000), 60)``. - If the sample size is larger than the population size, a :exc:`ValueError` - is raised. - The following functions generate specific real-valued distributions. Function parameters are named after the corresponding variables in the distribution's equation, as used in common mathematical practice; most of these equations can @@ -318,7 +315,7 @@ >>> random.sample([1, 2, 3, 4, 5], 3) # Three samples without replacement [4, 1, 5] -A common task is to make a :func:`random.choice` with weighted probabilities. +A common task is to make a :func:`random.choice` with weighted probababilites. If the weights are small integer ratios, a simple technique is to build a sample population with repeats:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/re.rst --- a/Doc/library/re.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/re.rst Mon Jan 25 17:05:13 2016 +0100 @@ -38,6 +38,13 @@ that don't require you to compile a regex object first, but miss some fine-tuning parameters. +.. seealso:: + + Mastering Regular Expressions + Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The + second edition of the book no longer covers Python at all, but the first + edition covered writing good regular expression patterns in great detail. + .. _re-syntax: @@ -235,32 +242,21 @@ ``(?P...)`` Similar to regular parentheses, but the substring matched by the group is - accessible via the symbolic group name *name*. Group names must be valid - Python identifiers, and each group name must be defined only once within a - regular expression. A symbolic group is also a numbered group, just as if - the group were not named. + accessible within the rest of the regular expression via the symbolic group + name *name*. Group names must be valid Python identifiers, and each group + name must be defined only once within a regular expression. A symbolic group + is also a numbered group, just as if the group were not named. So the group + named ``id`` in the example below can also be referenced as the numbered group + ``1``. - Named groups can be referenced in three contexts. If the pattern is - ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either - single or double quotes): - - +---------------------------------------+----------------------------------+ - | Context of reference to group "quote" | Ways to reference it | - +=======================================+==================================+ - | in the same pattern itself | * ``(?P=quote)`` (as shown) | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ - | when processing match object ``m`` | * ``m.group('quote')`` | - | | * ``m.end('quote')`` (etc.) | - +---------------------------------------+----------------------------------+ - | in a string passed to the ``repl`` | * ``\g`` | - | argument of ``re.sub()`` | * ``\g<1>`` | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ + For example, if the pattern is ``(?P[a-zA-Z_]\w*)``, the group can be + referenced by its name in arguments to methods of match objects, such as + ``m.group('id')`` or ``m.end('id')``, and also by name in the regular + expression itself (using ``(?P=id)``) and replacement text given to + ``.sub()`` (using ``\g``). ``(?P=name)`` - A backreference to a named group; it matches whatever text was matched by the - earlier group named *name*. + Matches whatever text was matched by the earlier group named *name*. ``(?#...)`` A comment; the contents of the parentheses are simply ignored. @@ -282,7 +278,7 @@ lookbehind will back up 3 characters and check if the contained pattern matches. The contained pattern must only match strings of some fixed length, meaning that ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Note that - patterns which start with positive lookbehind assertions will not match at the + patterns which start with positive lookbehind assertions will never match at the beginning of the string being searched; you will most likely want to use the :func:`search` function rather than the :func:`match` function: @@ -297,9 +293,6 @@ >>> m.group(0) 'egg' - .. versionchanged: 3.5 - Added support for group references of fixed length. - ``(?|$)`` is a poor email matching pattern, which will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. + not with ``''`` . The special sequences consist of ``'\'`` and a character from the list below. @@ -323,7 +316,7 @@ ``\number`` Matches the contents of the group of the same number. Groups are numbered starting from 1. For example, ``(.+) \1`` matches ``'the the'`` or ``'55 55'``, - but not ``'thethe'`` (note the space after the group). This special sequence + but not ``'the end'`` (note the space after the group). This special sequence can only be used to match one of the first 99 groups. If the first digit of *number* is 0, or *number* is 3 octal digits long, it will not be interpreted as a group match, but as the character with octal value *number*. Inside the @@ -421,36 +414,14 @@ accepted by the regular expression parser:: \a \b \f \n - \r \t \u \U - \v \x \\ - -(Note that ``\b`` is used to represent word boundaries, and means "backspace" -only inside character classes.) - -``'\u'`` and ``'\U'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are not treated specially. + \r \t \v \x + \\ Octal escapes are included in a limited form. If the first digit is a 0, or if there are three octal digits, it is considered an octal escape. Otherwise, it is a group reference. As for string literals, octal escapes are always at most three digits in length. -.. versionchanged:: 3.3 - The ``'\u'`` and ``'\U'`` escape sequences have been added. - -.. deprecated-removed:: 3.5 3.6 - Unknown escapes consist of ``'\'`` and ASCII letter now raise a - deprecation warning and will be forbidden in Python 3.6. - - -.. seealso:: - - Mastering Regular Expressions - Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The - second edition of the book no longer covers Python at all, but the first - edition covered writing good regular expression patterns in great detail. - - .. _contents-of-module-re: @@ -466,8 +437,8 @@ .. function:: compile(pattern, flags=0) Compile a regular expression pattern into a regular expression object, which - can be used for matching using its :func:`~regex.match` and - :func:`~regex.search` methods, described below. + can be used for matching using its :func:`match` and :func:`search` methods, + described below. The expression's behaviour can be modified by specifying a *flags* value. Values can be any of the following variables, combined using bitwise OR (the @@ -489,7 +460,7 @@ .. note:: The compiled versions of the most recent patterns passed to - :func:`re.compile` and the module-level matching functions are cached, so + :func:`re.match`, :func:`re.search` or :func:`re.compile` are cached, so programs that use only a few regular expressions at a time needn't worry about compiling regular expressions. @@ -528,11 +499,7 @@ current locale. The use of this flag is discouraged as the locale mechanism is very unreliable, and it only handles one "culture" at a time anyway; you should use Unicode matching instead, which is the default in Python 3 - for Unicode (str) patterns. This flag makes sense only with bytes patterns. - - .. deprecated-removed:: 3.5 3.6 - Deprecated the use of :const:`re.LOCALE` with string patterns or - :const:`re.ASCII`. + for Unicode (str) patterns. .. data:: M @@ -556,15 +523,13 @@ .. data:: X VERBOSE - This flag allows you to write regular expressions that look nicer and are - more readable by allowing you to visually separate logical sections of the - pattern and add comments. Whitespace within the pattern is ignored, except - when in a character class or when preceded by an unescaped backslash. - When a line contains a ``#`` that is not in a character class and is not - preceded by an unescaped backslash, all characters from the leftmost such - ``#`` through the end of the line are ignored. + This flag allows you to write regular expressions that look nicer. Whitespace + within the pattern is ignored, except when in a character class or preceded by + an unescaped backslash, and, when a line contains a ``'#'`` neither in a + character class or preceded by an unescaped backslash, all characters from the + leftmost such ``'#'`` through the end of the line are ignored. - This means that the two following regular expression objects that match a + That means that the two following regular expression objects that match a decimal number are functionally equal:: a = re.compile(r"""\d + # the integral part @@ -577,7 +542,7 @@ .. function:: search(pattern, string, flags=0) - Scan through *string* looking for the first location where the regular expression + Scan through *string* looking for a location where the regular expression *pattern* produces a match, and return a corresponding :ref:`match object `. Return ``None`` if no position in the string matches the pattern; note that this is different from finding a zero-length match at some @@ -598,16 +563,6 @@ instead (see also :ref:`search-vs-match`). -.. function:: fullmatch(pattern, string, flags=0) - - If the whole *string* matches the regular expression *pattern*, return a - corresponding :ref:`match object `. Return ``None`` if the - string does not match the pattern; note that this is different from a - zero-length match. - - .. versionadded:: 3.4 - - .. function:: split(pattern, string, maxsplit=0, flags=0) Split *string* by the occurrences of *pattern*. If capturing parentheses are @@ -635,37 +590,17 @@ That way, separator components are always found at the same relative indices within the result list. - .. note:: + Note that *split* will never split a string on an empty pattern match. + For example: - :func:`split` doesn't currently split a string on an empty pattern match. - For example: - - >>> re.split('x*', 'axbc') - ['a', 'bc'] - - Even though ``'x*'`` also matches 0 'x' before 'a', between 'b' and 'c', - and after 'c', currently these matches are ignored. The correct behavior - (i.e. splitting on empty matches too and returning ``['', 'a', 'b', 'c', - '']``) will be implemented in future versions of Python, but since this - is a backward incompatible change, a :exc:`FutureWarning` will be raised - in the meanwhile. - - Patterns that can only match empty strings currently never split the - string. Since this doesn't match the expected behavior, a - :exc:`ValueError` will be raised starting from Python 3.5:: - - >>> re.split("^$", "foo\n\nbar\n", flags=re.M) - Traceback (most recent call last): - File "", line 1, in - ... - ValueError: split() requires a non-empty pattern match. + >>> re.split('x*', 'foo') + ['foo'] + >>> re.split("(?m)^$", "foo\n\nbar\n") + ['foo\n\nbar\n'] .. versionchanged:: 3.1 Added the optional flags argument. - .. versionchanged:: 3.5 - Splitting on a pattern that could match an empty string now raises - a warning. Patterns that can only match empty strings are now rejected. .. function:: findall(pattern, string, flags=0) @@ -693,7 +628,7 @@ *string* is returned unchanged. *repl* can be a string or a function; if it is a string, any backslash escapes in it are processed. That is, ``\n`` is converted to a single newline character, ``\r`` is converted to a carriage return, and - so forth. Unknown escapes such as ``\&`` are left alone. Backreferences, such + so forth. Unknown escapes such as ``\j`` are left alone. Backreferences, such as ``\6``, are replaced with the substring matched by group 6 in the pattern. For example: @@ -722,8 +657,7 @@ when not adjacent to a previous match, so ``sub('x*', '-', 'abc')`` returns ``'-a-b-c-'``. - In string-type *repl* arguments, in addition to the character escapes and - backreferences described above, + In addition to character escapes and backreferences as described above, ``\g`` will use the substring matched by the group named ``name``, as defined by the ``(?P...)`` syntax. ``\g`` uses the corresponding group number; ``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous @@ -735,13 +669,6 @@ .. versionchanged:: 3.1 Added the optional flags argument. - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - - .. deprecated-removed:: 3.5 3.6 - Unknown escapes consist of ``'\'`` and ASCII letter now raise a - deprecation warning and will be forbidden in Python 3.6. - .. function:: subn(pattern, repl, string, count=0, flags=0) @@ -751,9 +678,6 @@ .. versionchanged:: 3.1 Added the optional flags argument. - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - .. function:: escape(string) @@ -770,36 +694,13 @@ Clear the regular expression cache. -.. exception:: error(msg, pattern=None, pos=None) +.. exception:: error Exception raised when a string passed to one of the functions here is not a valid regular expression (for example, it might contain unmatched parentheses) or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has - the following additional attributes: + error if a string contains no match for a pattern. - .. attribute:: msg - - The unformatted error message. - - .. attribute:: pattern - - The regular expression pattern. - - .. attribute:: pos - - The index of *pattern* where compilation failed. - - .. attribute:: lineno - - The line corresponding to *pos*. - - .. attribute:: colno - - The column corresponding to *pos*. - - .. versionchanged:: 3.5 - Added additional attributes. .. _re-objects: @@ -832,7 +733,7 @@ >>> pattern = re.compile("d") >>> pattern.search("dog") # Match at index 0 - <_sre.SRE_Match object; span=(0, 1), match='d'> + <_sre.SRE_Match object at ...> >>> pattern.search("dog", 1) # No match; search doesn't include the "d" @@ -849,30 +750,12 @@ >>> pattern = re.compile("o") >>> pattern.match("dog") # No match as "o" is not at the start of "dog". >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". - <_sre.SRE_Match object; span=(1, 2), match='o'> + <_sre.SRE_Match object at ...> If you want to locate a match anywhere in *string*, use :meth:`~regex.search` instead (see also :ref:`search-vs-match`). -.. method:: regex.fullmatch(string[, pos[, endpos]]) - - If the whole *string* matches this regular expression, return a corresponding - :ref:`match object `. Return ``None`` if the string does not - match the pattern; note that this is different from a zero-length match. - - The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~regex.search` method. - - >>> pattern = re.compile("o[gh]") - >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". - >>> pattern.fullmatch("ogre") # No match as not the full string matches. - >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. - <_sre.SRE_Match object; span=(1, 3), match='og'> - - .. versionadded:: 3.4 - - .. method:: regex.split(string, maxsplit=0) Identical to the :func:`split` function, using the compiled pattern. @@ -931,16 +814,9 @@ Match Objects ------------- -Match objects always have a boolean value of ``True``. -Since :meth:`~regex.match` and :meth:`~regex.search` return ``None`` -when there is no match, you can test whether there was a match with a simple -``if`` statement:: - - match = re.search(pattern, string) - if match: - process(match) - -Match objects support the following methods and attributes: +Match objects always have a boolean value of :const:`True`. This lets you +use a simple if-statement to test whether a match was found. Match objects +support the following methods and attributes: .. method:: match.expand(template) @@ -952,8 +828,6 @@ (``\g<1>``, ``\g``) are replaced by the contents of the corresponding group. - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. .. method:: match.group([group1, ...]) @@ -1198,13 +1072,13 @@ +--------------------------------+---------------------------------------------+ | ``%i`` | ``[-+]?(0[xX][\dA-Fa-f]+|0[0-7]*|\d+)`` | +--------------------------------+---------------------------------------------+ -| ``%o`` | ``[-+]?[0-7]+`` | +| ``%o`` | ``0[0-7]*`` | +--------------------------------+---------------------------------------------+ | ``%s`` | ``\S+`` | +--------------------------------+---------------------------------------------+ | ``%u`` | ``\d+`` | +--------------------------------+---------------------------------------------+ -| ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | +| ``%x``, ``%X`` | ``0[xX][\dA-Fa-f]+`` | +--------------------------------+---------------------------------------------+ To extract the filename and numbers from a string like :: @@ -1236,7 +1110,7 @@ >>> re.match("c", "abcdef") # No match >>> re.search("c", "abcdef") # Match - <_sre.SRE_Match object; span=(2, 3), match='c'> + <_sre.SRE_Match object at ...> Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: @@ -1244,7 +1118,7 @@ >>> re.match("c", "abcdef") # No match >>> re.search("^c", "abcdef") # No match >>> re.search("^a", "abcdef") # Match - <_sre.SRE_Match object; span=(0, 1), match='a'> + <_sre.SRE_Match object at ...> Note however that in :const:`MULTILINE` mode :func:`match` only matches at the beginning of the string, whereas using :func:`search` with a regular expression @@ -1252,7 +1126,7 @@ >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match - <_sre.SRE_Match object; span=(4, 5), match='X'> + <_sre.SRE_Match object at ...> Making a Phonebook @@ -1371,9 +1245,9 @@ functionally identical: >>> re.match(r"\W(.)\1\W", " ff ") - <_sre.SRE_Match object; span=(0, 4), match=' ff '> + <_sre.SRE_Match object at ...> >>> re.match("\\W(.)\\1\\W", " ff ") - <_sre.SRE_Match object; span=(0, 4), match=' ff '> + <_sre.SRE_Match object at ...> When one wants to match a literal backslash, it must be escaped in the regular expression. With raw string notation, this means ``r"\\"``. Without raw string @@ -1381,9 +1255,9 @@ functionally identical: >>> re.match(r"\\", r"\\") - <_sre.SRE_Match object; span=(0, 1), match='\\'> + <_sre.SRE_Match object at ...> >>> re.match("\\\\", r"\\") - <_sre.SRE_Match object; span=(0, 1), match='\\'> + <_sre.SRE_Match object at ...> Writing a Tokenizer @@ -1402,36 +1276,36 @@ Token = collections.namedtuple('Token', ['typ', 'value', 'line', 'column']) - def tokenize(code): + def tokenize(s): keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} token_specification = [ ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number ('ASSIGN', r':='), # Assignment operator ('END', r';'), # Statement terminator ('ID', r'[A-Za-z]+'), # Identifiers - ('OP', r'[+\-*/]'), # Arithmetic operators + ('OP', r'[+*\/\-]'), # Arithmetic operators ('NEWLINE', r'\n'), # Line endings - ('SKIP', r'[ \t]+'), # Skip over spaces and tabs - ('MISMATCH',r'.'), # Any other character + ('SKIP', r'[ \t]'), # Skip over spaces and tabs ] tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification) - line_num = 1 - line_start = 0 - for mo in re.finditer(tok_regex, code): - kind = mo.lastgroup - value = mo.group(kind) - if kind == 'NEWLINE': - line_start = mo.end() - line_num += 1 - elif kind == 'SKIP': - pass - elif kind == 'MISMATCH': - raise RuntimeError('%r unexpected on line %d' % (value, line_num)) - else: - if kind == 'ID' and value in keywords: - kind = value - column = mo.start() - line_start - yield Token(kind, value, line_num, column) + get_token = re.compile(tok_regex).match + line = 1 + pos = line_start = 0 + mo = get_token(s) + while mo is not None: + typ = mo.lastgroup + if typ == 'NEWLINE': + line_start = pos + line += 1 + elif typ != 'SKIP': + val = mo.group(typ) + if typ == 'ID' and val in keywords: + typ = val + yield Token(typ, val, line, mo.start()-line_start) + pos = mo.end() + mo = get_token(s, pos) + if pos != len(s): + raise RuntimeError('Unexpected character %r on line %d' %(s[pos], line)) statements = ''' IF quantity THEN @@ -1445,22 +1319,22 @@ The tokenizer produces the following output:: - Token(typ='IF', value='IF', line=2, column=4) - Token(typ='ID', value='quantity', line=2, column=7) - Token(typ='THEN', value='THEN', line=2, column=16) - Token(typ='ID', value='total', line=3, column=8) - Token(typ='ASSIGN', value=':=', line=3, column=14) - Token(typ='ID', value='total', line=3, column=17) - Token(typ='OP', value='+', line=3, column=23) - Token(typ='ID', value='price', line=3, column=25) - Token(typ='OP', value='*', line=3, column=31) - Token(typ='ID', value='quantity', line=3, column=33) - Token(typ='END', value=';', line=3, column=41) - Token(typ='ID', value='tax', line=4, column=8) - Token(typ='ASSIGN', value=':=', line=4, column=12) - Token(typ='ID', value='price', line=4, column=15) - Token(typ='OP', value='*', line=4, column=21) - Token(typ='NUMBER', value='0.05', line=4, column=23) - Token(typ='END', value=';', line=4, column=27) - Token(typ='ENDIF', value='ENDIF', line=5, column=4) - Token(typ='END', value=';', line=5, column=9) + Token(typ='IF', value='IF', line=2, column=5) + Token(typ='ID', value='quantity', line=2, column=8) + Token(typ='THEN', value='THEN', line=2, column=17) + Token(typ='ID', value='total', line=3, column=9) + Token(typ='ASSIGN', value=':=', line=3, column=15) + Token(typ='ID', value='total', line=3, column=18) + Token(typ='OP', value='+', line=3, column=24) + Token(typ='ID', value='price', line=3, column=26) + Token(typ='OP', value='*', line=3, column=32) + Token(typ='ID', value='quantity', line=3, column=34) + Token(typ='END', value=';', line=3, column=42) + Token(typ='ID', value='tax', line=4, column=9) + Token(typ='ASSIGN', value=':=', line=4, column=13) + Token(typ='ID', value='price', line=4, column=16) + Token(typ='OP', value='*', line=4, column=22) + Token(typ='NUMBER', value='0.05', line=4, column=24) + Token(typ='END', value=';', line=4, column=28) + Token(typ='ENDIF', value='ENDIF', line=5, column=5) + Token(typ='END', value=';', line=5, column=10) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/readline.rst --- a/Doc/library/readline.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/readline.rst Mon Jan 25 17:05:13 2016 +0100 @@ -59,14 +59,6 @@ Save a readline history file. The default filename is :file:`~/.history`. -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* of history to a file. The default filename is - :file:`~/.history`. The file must already exist. - - .. versionadded:: 3.5 - - .. function:: clear_history() Clear the current history. (Note: this function is not available if the @@ -198,55 +190,28 @@ The following example demonstrates how to use the :mod:`readline` module's history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: +file named :file:`.pyhist` from the user's home directory. The code below would +normally be executed automatically during interactive sessions from the user's +:envvar:`PYTHONSTARTUP` file. :: - import atexit import os import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") + histfile = os.path.join(os.path.expanduser("~"), ".pyhist") try: readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) except FileNotFoundError: pass - + import atexit atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) + del os, histfile The following example extends the :class:`code.InteractiveConsole` class to support history save/restore. :: + import code + import readline import atexit - import code import os - import readline class HistoryConsole(code.InteractiveConsole): def __init__(self, locals=None, filename="", @@ -264,5 +229,5 @@ atexit.register(self.save_history, histfile) def save_history(self, histfile): - readline.set_history_length(1000) readline.write_history_file(histfile) + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/reprlib.rst --- a/Doc/library/reprlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/reprlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -49,8 +49,8 @@ >>> class MyList(list): ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' + ... def __repr__(self): + ... return '<' + '|'.join(map(repr, self)) + '>' ... >>> m = MyList('abc') >>> m.append(m) @@ -129,9 +129,9 @@ Formatting methods for specific types are implemented as methods with a name based on the type name. In the method name, **TYPE** is replaced by - ``'_'.join(type(obj).__name__.split())``. Dispatch to these methods is - handled by :meth:`repr1`. Type-specific methods which need to recursively - format a value should call ``self.repr1(subobj, level - 1)``. + ``string.join(string.split(type(obj).__name__, '_'))``. Dispatch to these + methods is handled by :meth:`repr1`. Type-specific methods which need to + recursively format a value should call ``self.repr1(subobj, level - 1)``. .. _subclassing-reprs: @@ -148,11 +148,12 @@ import sys class MyRepr(reprlib.Repr): - - def repr_TextIOWrapper(self, obj, level): - if obj.name in {'', '', ''}: + def repr_file(self, obj, level): + if obj.name in ['', '', '']: return obj.name - return repr(obj) + else: + return repr(obj) aRepr = MyRepr() print(aRepr.repr(sys.stdin)) # prints '' + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/resource.rst --- a/Doc/library/resource.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/resource.rst Mon Jan 25 17:05:13 2016 +0100 @@ -43,11 +43,6 @@ this module for those platforms. -.. data:: RLIM_INFINITY - - Constant used to represent the limit for an unlimited resource. - - .. function:: getrlimit(resource) Returns a tuple ``(soft, hard)`` with the current soft and hard limits of @@ -59,41 +54,12 @@ Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is - unlimited. + ``-1`` can be used to specify the maximum possible upper limit. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft - limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or - system limit for that resource is not unlimited will result in a - :exc:`ValueError`. A process with the effective UID of super-user can - request any valid limit value, including unlimited, but :exc:`ValueError` - will still be raised if the requested limit exceeds the system imposed - limit. - - ``setrlimit`` may also raise :exc:`error` if the underlying system call - fails. - -.. function:: prlimit(pid, resource[, limits]) - - Combines :func:`setrlimit` and :func:`getrlimit` in one function and - supports to get and set the resources limits of an arbitrary process. If - *pid* is 0, then the call applies to the current process. *resource* and - *limits* have the same meaning as in :func:`setrlimit`, except that - *limits* is optional. - - When *limits* is not given the function returns the *resource* limit of the - process *pid*. When *limits* is given the *resource* limit of the process is - set and the former resource limit is returned. - - Raises :exc:`ProcessLookupError` when *pid* can't be found and - :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for - the process. - - Availability: Linux 2.6.36 or later with glibc 2.13 or later - - .. versionadded:: 3.4 - + limit exceeds the hard limit, or if a process tries to raise its hard limit + (unless the process has an effective UID of super-user). Can also raise + :exc:`error` if the underlying system call fails. These symbols define resources whose consumption can be controlled using the :func:`setrlimit` and :func:`getrlimit` functions described below. The values of @@ -172,80 +138,6 @@ The maximum area (in bytes) of address space which may be taken by the process. -.. data:: RLIMIT_MSGQUEUE - - The number of bytes that can be allocated for POSIX message queues. - - Availability: Linux 2.6.8 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_NICE - - The ceiling for the process's nice level (calculated as 20 - rlim_cur). - - Availability: Linux 2.6.12 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTPRIO - - The ceiling of the real-time priority. - - Availability: Linux 2.6.12 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTTIME - - The time limit (in microseconds) on CPU time that a process can spend - under real-time scheduling without making a blocking syscall. - - Availability: Linux 2.6.25 or later. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_SIGPENDING - - The number of signals which the process may queue. - - Availability: Linux 2.6.8 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SBSIZE - - The maximum size (in bytes) of socket buffer usage for this user. - This limits the amount of network memory, and hence the amount of mbufs, - that this user may hold at any time. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SWAP - - The maximum size (in bytes) of the swap space that may be reserved or - used by all of this user id's processes. - This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. - Please see :manpage:`tuning(7)` for a complete description of this sysctl. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_NPTS - - The maximum number of pseudo-terminals created by this user id. - - Availability: FreeBSD 9 or later. - - .. versionadded:: 3.4 - Resource Usage -------------- @@ -316,7 +208,10 @@ .. function:: getpagesize() Returns the number of bytes in a system page. (This need not be the same as the - hardware page size.) + hardware page size.) This function is useful for determining the number of bytes + of memory a process is using. The third element of the tuple returned by + :func:`getrusage` describes memory usage in pages; multiplying by page size + produces number of bytes. The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` function to specify which processes information should be provided for. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/rlcompleter.rst --- a/Doc/library/rlcompleter.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/rlcompleter.rst Mon Jan 25 17:05:13 2016 +0100 @@ -27,10 +27,18 @@ readline.__name__ readline.parse_and_bind( >>> readline. -The :mod:`rlcompleter` module is designed for use with Python's -:ref:`interactive mode `. Unless Python is run with the -:option:`-S` option, the module is automatically imported and configured -(see :ref:`rlcompleter-config`). +The :mod:`rlcompleter` module is designed for use with Python's interactive +mode. A user can add the following lines to his or her initialization file +(identified by the :envvar:`PYTHONSTARTUP` environment variable) to get +automatic :kbd:`Tab` completion:: + + try: + import readline + except ImportError: + print("Module readline not available.") + else: + import rlcompleter + readline.parse_and_bind("tab: complete") On platforms without :mod:`readline`, the :class:`Completer` class defined by this module can still be used for custom purposes. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/runpy.rst --- a/Doc/library/runpy.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/runpy.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,30 +14,17 @@ line switch that allows scripts to be located using the Python module namespace rather than the filesystem. -Note that this is *not* a sandbox module - all code is executed in the -current process, and any side effects (such as cached imports of other -modules) will remain in place after the functions have returned. - -Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. -If that limitation is not acceptable for a given use case, :mod:`importlib` -is likely to be a more suitable choice than this module. - The :mod:`runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) - .. index:: - module: __main__ - Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using the standard import mechanism (refer to :pep:`302` for details) and then executed in a fresh module namespace. - The *mod_name* argument should be an absolute module name. - If the module name refers to a package rather than a normal + If the supplied module name refers to a package rather than a normal module, then that package is imported and the ``__main__`` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -48,22 +35,28 @@ below are defined in the supplied dictionary, those definitions are overridden by :func:`run_module`. - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + The special global variables ``__name__``, ``__file__``, ``__cached__``, + ``__loader__`` + and ``__package__`` are set in the globals dictionary before the module + code is executed (Note that this is a minimal set of variables - other + variables may be set implicitly as an interpreter implementation detail). ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a package and to the *mod_name* argument otherwise. - ``__spec__`` will be set appropriately for the *actually* imported - module (that is, ``__spec__.name`` will always be *mod_name* or - ``mod_name + '.__main__``, never *run_name*). + ``__file__`` is set to the name provided by the module loader. If the + loader does not make filename information available, this variable is set + to :const:`None`. - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are - :ref:`set as normal ` based on the module spec. + ``__cached__`` will be set to ``None``. + + ``__loader__`` is set to the :pep:`302` module loader used to retrieve the + code for the module (This loader may be a wrapper around the standard + import mechanism). + + ``__package__`` is set to *mod_name* if the named module is a package and + to ``mod_name.rpartition('.')[0]`` otherwise. If the argument *alter_sys* is supplied and evaluates to :const:`True`, then ``sys.argv[0]`` is updated with the value of ``__file__`` and @@ -76,27 +69,16 @@ arguments. It is recommended that the :mod:`sys` module be left alone when invoking this function from threaded code. - .. seealso:: - The :option:`-m` option offering equivalent functionality from the - command line. .. versionchanged:: 3.1 Added ability to execute packages by looking for a ``__main__`` submodule. .. versionchanged:: 3.2 - Added ``__cached__`` global variable (see :pep:`3147`). + Added ``__cached__`` global variable (see :PEP:`3147`). - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly for modules - run this way, as well as ensuring the real module name is always - accessible as ``__spec__.name``. .. function:: run_path(file_path, init_globals=None, run_name=None) - .. index:: - module: __main__ - Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a @@ -117,25 +99,23 @@ below are defined in the supplied dictionary, those definitions are overridden by :func:`run_path`. - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). + The special global variables ``__name__``, ``__file__``, ``__loader__`` + and ``__package__`` are set in the globals dictionary before the module + code is executed (Note that this is a minimal set of variables - other + variables may be set implicitly as an interpreter implementation detail). ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. - If the supplied path directly references a script file (whether as source - or as precompiled byte code), then ``__file__`` will be set to the - supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and - ``__package__`` will all be set to :const:`None`. + ``__file__`` is set to the name provided by the module loader. If the + loader does not make filename information available, this variable is set + to :const:`None`. For a simple script, this will be set to ``file_path``. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` - module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be - :ref:`set as normal ` based on the module spec. + ``__loader__`` is set to the :pep:`302` module loader used to retrieve the + code for the module (This loader may be a wrapper around the standard + import mechanism). For a simple script, this will be set to :const:`None`. + + ``__package__`` is set to ``__name__.rpartition('.')[0]``. A number of alterations are also made to the :mod:`sys` module. Firstly, ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated @@ -150,29 +130,14 @@ limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. - .. seealso:: - :ref:`using-on-interface-options` for equivalent functionality on the - command line (``python path/to/script``). - .. versionadded:: 3.2 - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather - than being executed directly. - .. seealso:: - :pep:`338` -- Executing modules as scripts + :pep:`338` - Executing modules as scripts PEP written and implemented by Nick Coghlan. - :pep:`366` -- Main module explicit relative imports + :pep:`366` - Main module explicit relative imports PEP written and implemented by Nick Coghlan. - :pep:`451` -- A ModuleSpec Type for the Import System - PEP written and implemented by Eric Snow - :ref:`using-on-general` - CPython command line details - - The :func:`importlib.import_module` function diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sched.rst --- a/Doc/library/sched.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sched.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,13 +14,12 @@ The :mod:`sched` module defines a class which implements a general purpose event scheduler: -.. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) +.. class:: scheduler(timefunc=time.time, delayfunc=time.sleep) The :class:`scheduler` class defines a generic interface to scheduling events. It needs two functions to actually deal with the "outside world" --- *timefunc* should be callable without arguments, and return a number (the "time", in any - units whatsoever). If time.monotonic is not available, the *timefunc* default - is time.time instead. The *delayfunc* function should be callable with one + units whatsoever). The *delayfunc* function should be callable with one argument, compatible with the output of *timefunc*, and should delay that many time units. *delayfunc* will also be called with the argument ``0`` after each event is run to allow other threads an opportunity to run in multi-threaded @@ -28,7 +27,6 @@ .. versionchanged:: 3.3 *timefunc* and *delayfunc* parameters are optional. - .. versionchanged:: 3.3 :class:`scheduler` class can be safely used in multi-threaded environments. @@ -37,22 +35,19 @@ >>> import sched, time >>> s = sched.scheduler(time.time, time.sleep) - >>> def print_time(a='default'): - ... print("From print_time", time.time(), a) + >>> def print_time(): print("From print_time", time.time()) ... >>> def print_some_times(): ... print(time.time()) - ... s.enter(10, 1, print_time) - ... s.enter(5, 2, print_time, argument=('positional',)) - ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) + ... s.enter(5, 1, print_time, ()) + ... s.enter(10, 1, print_time, ()) ... s.run() ... print(time.time()) ... >>> print_some_times() 930343690.257 - From print_time 930343695.274 positional - From print_time 930343695.275 keyword - From print_time 930343700.273 default + From print_time 930343695.274 + From print_time 930343700.273 930343700.276 .. _scheduler-objects: @@ -63,7 +58,7 @@ :class:`scheduler` instances have the following methods and attributes: -.. method:: scheduler.enterabs(time, priority, action, argument=(), kwargs={}) +.. method:: scheduler.enterabs(time, priority, action, argument=[], kwargs={}) Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. @@ -71,8 +66,8 @@ *priority*. Executing the event means executing ``action(*argument, **kwargs)``. - *argument* is a sequence holding the positional arguments for *action*. - *kwargs* is a dictionary holding the keyword arguments for *action*. + *argument* must be a sequence holding the parameters for *action*. + *kwargs* must be a dictionary holding the keyword parameters for *action*. Return value is an event which may be used for later cancellation of the event (see :meth:`cancel`). @@ -84,7 +79,7 @@ *kwargs* parameter was added. -.. method:: scheduler.enter(delay, priority, action, argument=(), kwargs={}) +.. method:: scheduler.enter(delay, priority, action, argument=[], kwargs={}) Schedule an event for *delay* more time units. Other than the relative time, the other arguments, the effect and the return value are the same as those for @@ -113,7 +108,7 @@ function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest + If *blocking* is False executes the scheduled events due to expire soonest (if any) and then return the deadline of the next scheduled call in the scheduler (if any). @@ -134,4 +129,4 @@ Read-only attribute returning a list of upcoming events in the order they will be run. Each event is shown as a :term:`named tuple` with the - following fields: time, priority, action, argument, kwargs. + following fields: time, priority, action, argument. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/select.rst --- a/Doc/library/select.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/select.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,14 +14,6 @@ It cannot be used on regular files to determine whether a file has grown since it was last read. -.. note:: - - The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are - encouraged to use the :mod:`selectors` module instead, unless they want - precise control over the OS-level primitives used. - - The module defines the following: @@ -45,37 +37,22 @@ increases this value, :c:func:`devpoll` may return an incomplete list of active file descriptors. - The new file descriptor is :ref:`non-inheritable `. - .. versionadded:: 3.3 - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - .. function:: epoll(sizehint=-1, flags=0) (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. *sizehint* is deprecated and completely ignored. *flags* can be set to :const:`EPOLL_CLOEXEC`, which causes the epoll descriptor to be closed - automatically when :func:`os.execve` is called. + automatically when :func:`os.execve` is called. See section + :ref:`epoll-objects` below for the methods supported by epolling objects. - See the :ref:`epoll-objects` section below for the methods supported by - epolling objects. - - ``epoll`` objects support the context management protocol: when used in a - :keyword:`with` statement, the new file descriptor is automatically closed - at the end of the block. - - The new file descriptor is :ref:`non-inheritable `. .. versionchanged:: 3.3 + Added the *flags* parameter. - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - The new file descriptor is now non-inheritable. - .. function:: poll() @@ -90,11 +67,6 @@ (Only supported on BSD.) Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. - The new file descriptor is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) @@ -107,7 +79,7 @@ This is a straightforward interface to the Unix :c:func:`select` system call. The first three arguments are sequences of 'waitable objects': either integers representing file descriptors or objects with a parameterless method - named :meth:`~io.IOBase.fileno` returning such an integer: + named :meth:`fileno` returning such an integer: * *rlist*: wait until ready for reading * *wlist*: wait until ready for writing @@ -133,8 +105,8 @@ objects ` (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, - as long as it has an appropriate :meth:`~io.IOBase.fileno` method (that - really returns a file descriptor, not just a random integer). + as long as it has an appropriate :meth:`fileno` method (that really returns + a file descriptor, not just a random integer). .. note:: @@ -145,17 +117,10 @@ library, and does not handle file descriptors that don't originate from WinSock. - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - .. attribute:: PIPE_BUF The minimum number of bytes which can be written without blocking to a pipe - when the pipe has been reported as ready for writing by :func:`~select.select`, + when the pipe has been reported as ready for writing by :func:`select`, :func:`poll` or another interface in this module. This doesn't apply to other kind of file-like objects such as sockets. @@ -167,7 +132,10 @@ .. _devpoll-objects: ``/dev/poll`` Polling Objects ------------------------------ +---------------------------------------------- + + http://developers.sun.com/solaris/articles/using_devpoll.html + http://developers.sun.com/solaris/articles/polling_efficient.html Solaris and derivatives have ``/dev/poll``. While :c:func:`select` is O(highest file descriptor) and :c:func:`poll` is O(number of file @@ -177,34 +145,13 @@ object. -.. method:: devpoll.close() - - Close the file descriptor of the polling object. - - .. versionadded:: 3.4 - - -.. attribute:: devpoll.closed - - ``True`` if the polling object is closed. - - .. versionadded:: 3.4 - - -.. method:: devpoll.fileno() - - Return the file descriptor number of the polling object. - - .. versionadded:: 3.4 - - .. method:: devpoll.register(fd[, eventmask]) Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. + :meth:`poll` method will then check whether the file descriptor has any pending + I/O events. *fd* can be either an integer, or an object with a :meth:`fileno` + method that returns an integer. File objects implement :meth:`fileno`, so they + can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to check for. The constants are the same that with :c:func:`poll` @@ -214,7 +161,7 @@ .. warning:: Registering a file descriptor that's already registered is not an - error, but the result is undefined. The appropriate action is to + error, but the result is undefined. The appropiate action is to unregister or modify it first. This is an important difference compared with :c:func:`poll`. @@ -230,7 +177,7 @@ Remove a file descriptor being tracked by a polling object. Just like the :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. + :meth:`fileno` method that returns an integer. Attempting to remove a file descriptor that was never registered is safely ignored. @@ -249,12 +196,6 @@ returning. If *timeout* is omitted, -1, or :const:`None`, the call will block until there is an event for this poll object. - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - .. _epoll-objects: @@ -301,11 +242,6 @@ Close the control file descriptor of the epoll object. -.. attribute:: epoll.closed - - ``True`` if the epoll object is closed. - - .. method:: epoll.fileno() Return the file descriptor number of the control fd. @@ -323,7 +259,7 @@ .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a register file descriptor. .. method:: epoll.unregister(fd) @@ -331,16 +267,10 @@ Remove a registered file descriptor from the epoll object. -.. method:: epoll.poll(timeout=-1, maxevents=-1) +.. method:: epoll.poll([timeout=-1[, maxevents=-1]]) Wait for events. timeout in seconds (float) - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - .. _poll-objects: @@ -359,10 +289,10 @@ .. method:: poll.register(fd[, eventmask]) Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. + :meth:`poll` method will then check whether the file descriptor has any pending + I/O events. *fd* can be either an integer, or an object with a :meth:`fileno` + method that returns an integer. File objects implement :meth:`fileno`, so they + can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to check for, and can be a combination of the constants :const:`POLLIN`, @@ -393,7 +323,7 @@ Modifies an already registered fd. This has the same effect as ``register(fd, eventmask)``. Attempting to modify a file descriptor - that was never registered causes an :exc:`OSError` exception with errno + that was never registered causes an :exc:`IOError` exception with errno :const:`ENOENT` to be raised. @@ -401,7 +331,7 @@ Remove a file descriptor being tracked by a polling object. Just like the :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. + :meth:`fileno` method that returns an integer. Attempting to remove a file descriptor that was never registered causes a :exc:`KeyError` exception to be raised. @@ -420,12 +350,6 @@ returning. If *timeout* is omitted, negative, or :const:`None`, the call will block until there is an event for this poll object. - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - .. _kqueue-objects: @@ -437,11 +361,6 @@ Close the control file descriptor of the kqueue object. -.. attribute:: kqueue.closed - - ``True`` if the kqueue object is closed. - - .. method:: kqueue.fileno() Return the file descriptor number of the control fd. @@ -460,12 +379,6 @@ - max_events must be 0 or a positive integer - timeout in seconds (floats possible) - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - .. _kevent-objects: @@ -478,8 +391,8 @@ Value used to identify the event. The interpretation depends on the filter but it's usually the file descriptor. In the constructor ident can either - be an int or an object with a :meth:`~io.IOBase.fileno` method. kevent - stores the integer internally. + be an int or an object with a fileno() function. kevent stores the integer + internally. .. attribute:: kevent.filter diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/selectors.rst --- a/Doc/library/selectors.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,274 +0,0 @@ -:mod:`selectors` -- High-level I/O multiplexing -=============================================== - -.. module:: selectors - :synopsis: High-level I/O multiplexing. - -.. versionadded:: 3.4 - - -Introduction ------------- - -This module allows high-level and efficient I/O multiplexing, built upon the -:mod:`select` module primitives. Users are encouraged to use this module -instead, unless they want precise control over the OS-level primitives used. - -It defines a :class:`BaseSelector` abstract base class, along with several -concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), -that can be used to wait for I/O readiness notification on multiple file -objects. In the following, "file object" refers to any object with a -:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`. - -:class:`DefaultSelector` is an alias to the most efficient implementation -available on the current platform: this should be the default choice for most -users. - -.. note:: - The type of file objects supported depends on the platform: on Windows, - sockets are supported, but not pipes, whereas on Unix, both are supported - (some other types may be supported as well, such as fifos or special file - devices). - -.. seealso:: - - :mod:`select` - Low-level I/O multiplexing module. - - -Classes -------- - -Classes hierarchy:: - - BaseSelector - +-- SelectSelector - +-- PollSelector - +-- EpollSelector - +-- DevpollSelector - +-- KqueueSelector - - -In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules -constants below: - - +-----------------------+-----------------------------------------------+ - | Constant | Meaning | - +=======================+===============================================+ - | :const:`EVENT_READ` | Available for read | - +-----------------------+-----------------------------------------------+ - | :const:`EVENT_WRITE` | Available for write | - +-----------------------+-----------------------------------------------+ - - -.. class:: SelectorKey - - A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file decriptor, selected event - mask and attached data. It is returned by several :class:`BaseSelector` - methods. - - .. attribute:: fileobj - - File object registered. - - .. attribute:: fd - - Underlying file descriptor. - - .. attribute:: events - - Events that must be waited for on this file object. - - .. attribute:: data - - Optional opaque data associated to this file object: for example, this - could be used to store a per-client session ID. - - -.. class:: BaseSelector - - A :class:`BaseSelector` is used to wait for I/O event readiness on multiple - file objects. It supports file stream registration, unregistration, and a - method to wait for I/O events on those streams, with an optional timeout. - It's an abstract base class, so cannot be instantiated. Use - :class:`DefaultSelector` instead, or one of :class:`SelectSelector`, - :class:`KqueueSelector` etc. if you want to specifically use an - implementation, and your platform supports it. - :class:`BaseSelector` and its concrete implementations support the - :term:`context manager` protocol. - - .. abstractmethod:: register(fileobj, events, data=None) - - Register a file object for selection, monitoring it for I/O events. - - *fileobj* is the file object to monitor. It may either be an integer - file descriptor or an object with a ``fileno()`` method. - *events* is a bitwise mask of events to monitor. - *data* is an opaque object. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is already registered. - - .. abstractmethod:: unregister(fileobj) - - Unregister a file object from selection, removing it from monitoring. A - file object shall be unregistered prior to being closed. - - *fileobj* must be a file object previously registered. - - This returns the associated :class:`SelectorKey` instance, or raises a - :exc:`KeyError` if *fileobj* is not registered. It will raise - :exc:`ValueError` if *fileobj* is invalid (e.g. it has no ``fileno()`` - method or its ``fileno()`` method has an invalid return value). - - .. method:: modify(fileobj, events, data=None) - - Change a registered file object's monitored events or attached data. - - This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed - by :meth:`BaseSelector.register(fileobj, events, data)`, except that it - can be implemented more efficiently. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: select(timeout=None) - - Wait until some registered file objects become ready, or the timeout - expires. - - If ``timeout > 0``, this specifies the maximum wait time, in seconds. - If ``timeout <= 0``, the call won't block, and will report the currently - ready file objects. - If *timeout* is ``None``, the call will block until a monitored file object - becomes ready. - - This returns a list of ``(key, events)`` tuples, one for each ready file - object. - - *key* is the :class:`SelectorKey` instance corresponding to a ready file - object. - *events* is a bitmask of events ready on this file object. - - .. note:: - This method can return before any file object becomes ready or the - timeout has elapsed if the current process receives a signal: in this - case, an empty list will be returned. - - .. versionchanged:: 3.5 - The selector is now retried with a recomputed timeout when interrupted - by a signal if the signal handler did not raise an exception (see - :pep:`475` for the rationale), instead of returning an empty list - of events before the timeout. - - .. method:: close() - - Close the selector. - - This must be called to make sure that any underlying resource is freed. - The selector shall not be used once it has been closed. - - .. method:: get_key(fileobj) - - Return the key associated with a registered file object. - - This returns the :class:`SelectorKey` instance associated to this file - object, or raises :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: get_map() - - Return a mapping of file objects to selector keys. - - This returns a :class:`~collections.abc.Mapping` instance mapping - registered file objects to their associated :class:`SelectorKey` - instance. - - -.. class:: DefaultSelector() - - The default selector class, using the most efficient implementation - available on the current platform. This should be the default choice for - most users. - - -.. class:: SelectSelector() - - :func:`select.select`-based selector. - - -.. class:: PollSelector() - - :func:`select.poll`-based selector. - - -.. class:: EpollSelector() - - :func:`select.epoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.epoll` object. - -.. class:: DevpollSelector() - - :func:`select.devpoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.devpoll` object. - - .. versionadded:: 3.5 - -.. class:: KqueueSelector() - - :func:`select.kqueue`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.kqueue` object. - - -Examples --------- - -Here is a simple echo server implementation:: - - import selectors - import socket - - sel = selectors.DefaultSelector() - - def accept(sock, mask): - conn, addr = sock.accept() # Should be ready - print('accepted', conn, 'from', addr) - conn.setblocking(False) - sel.register(conn, selectors.EVENT_READ, read) - - def read(conn, mask): - data = conn.recv(1000) # Should be ready - if data: - print('echoing', repr(data), 'to', conn) - conn.send(data) # Hope it won't block - else: - print('closing', conn) - sel.unregister(conn) - conn.close() - - sock = socket.socket() - sock.bind(('localhost', 1234)) - sock.listen(100) - sock.setblocking(False) - sel.register(sock, selectors.EVENT_READ, accept) - - while True: - events = sel.select() - for key, mask in events: - callback = key.data - callback(key.fileobj, mask) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/shelve.rst --- a/Doc/library/shelve.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/shelve.rst Mon Jan 25 17:05:13 2016 +0100 @@ -44,11 +44,8 @@ .. note:: Do not rely on the shelf being closed automatically; always call - :meth:`~Shelf.close` explicitly when you don't need it any more, or - use :func:`shelve.open` as a context manager:: - - with shelve.open('spam') as db: - db['eggs'] = 'eggs' + :meth:`close` explicitly when you don't need it any more, or use a + :keyword:`with` statement with :func:`contextlib.closing`. .. warning:: @@ -106,8 +103,8 @@ .. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - A subclass of :class:`collections.abc.MutableMapping` which stores pickled - values in the *dict* object. + A subclass of :class:`collections.MutableMapping` which stores pickled values + in the *dict* object. By default, version 0 pickles are used to serialize values. The version of the pickle protocol can be specified with the *protocol* parameter. See the @@ -121,16 +118,10 @@ The *keyencoding* parameter is the encoding used to encode keys before they are used with the underlying dict. - A :class:`Shelf` object can also be used as a context manager, in which - case it will be automatically closed when the :keyword:`with` block ends. - - .. versionchanged:: 3.2 - Added the *keyencoding* parameter; previously, keys were always encoded in + .. versionadded:: 3.2 + The *keyencoding* parameter; previously, keys were always encoded in UTF-8. - .. versionchanged:: 3.4 - Added context manager support. - .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/shlex.rst --- a/Doc/library/shlex.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/shlex.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,9 +12,9 @@ -------------- -The :class:`~shlex.shlex` class makes it easy to write lexical analyzers for -simple syntaxes resembling that of the Unix shell. This will often be useful -for writing minilanguages, (for example, in run control files for Python +The :class:`shlex` class makes it easy to write lexical analyzers for simple +syntaxes resembling that of the Unix shell. This will often be useful for +writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. The :mod:`shlex` module defines the following functions: @@ -24,16 +24,15 @@ Split the string *s* using shell-like syntax. If *comments* is :const:`False` (the default), the parsing of comments in the given string will be disabled - (setting the :attr:`~shlex.commenters` attribute of the - :class:`~shlex.shlex` instance to the empty string). This function operates - in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is - false. + (setting the :attr:`commenters` attribute of the :class:`shlex` instance to + the empty string). This function operates in POSIX mode by default, but uses + non-POSIX mode if the *posix* argument is false. .. note:: - Since the :func:`split` function instantiates a :class:`~shlex.shlex` - instance, passing ``None`` for *s* will read the string to split from - standard input. + Since the :func:`split` function instantiates a :class:`shlex` instance, + passing ``None`` for *s* will read the string to split from standard + input. .. function:: quote(s) @@ -74,19 +73,17 @@ .. class:: shlex(instream=None, infile=None, posix=False) - A :class:`~shlex.shlex` instance or subclass instance is a lexical analyzer - object. The initialization argument, if present, specifies where to read - characters from. It must be a file-/stream-like object with - :meth:`~io.TextIOBase.read` and :meth:`~io.TextIOBase.readline` methods, or - a string. If no argument is given, input will be taken from ``sys.stdin``. - The second optional argument is a filename string, which sets the initial - value of the :attr:`~shlex.infile` attribute. If the *instream* - argument is omitted or equal to ``sys.stdin``, this second argument - defaults to "stdin". The *posix* argument defines the operational mode: - when *posix* is not true (default), the :class:`~shlex.shlex` instance will - operate in compatibility mode. When operating in POSIX mode, - :class:`~shlex.shlex` will try to be as close as possible to the POSIX shell - parsing rules. + A :class:`shlex` instance or subclass instance is a lexical analyzer object. + The initialization argument, if present, specifies where to read characters + from. It must be a file-/stream-like object with :meth:`read` and + :meth:`readline` methods, or a string. If no argument is given, input will + be taken from ``sys.stdin``. The second optional argument is a filename + string, which sets the initial value of the :attr:`infile` attribute. If the + *instream* argument is omitted or equal to ``sys.stdin``, this second + argument defaults to "stdin". The *posix* argument defines the operational + mode: when *posix* is not true (default), the :class:`shlex` instance will + operate in compatibility mode. When operating in POSIX mode, :class:`shlex` + will try to be as close as possible to the POSIX shell parsing rules. .. seealso:: @@ -100,14 +97,14 @@ shlex Objects ------------- -A :class:`~shlex.shlex` instance has the following methods: +A :class:`shlex` instance has the following methods: .. method:: shlex.get_token() Return a token. If tokens have been stacked using :meth:`push_token`, pop a token off the stack. Otherwise, read one from the input stream. If reading - encounters an immediate end-of-file, :attr:`eof` is returned (the empty + encounters an immediate end-of-file, :attr:`self.eof` is returned (the empty string (``''``) in non-POSIX mode, and ``None`` in POSIX mode). @@ -125,9 +122,9 @@ .. method:: shlex.sourcehook(filename) - When :class:`~shlex.shlex` detects a source request (see :attr:`source` - below) this method is given the following token as argument, and expected - to return a tuple consisting of a filename and an open file-like object. + When :class:`shlex` detects a source request (see :attr:`source` below) this + method is given the following token as argument, and expected to return a tuple + consisting of a filename and an open file-like object. Normally, this method first strips any quotes off the argument. If the result is an absolute pathname, or there was no previous source request in effect, or @@ -144,9 +141,8 @@ This hook is exposed so that you can use it to implement directory search paths, addition of file extensions, and other namespace hacks. There is no - corresponding 'close' hook, but a shlex instance will call the - :meth:`~io.IOBase.close` method of the sourced input stream when it returns - EOF. + corresponding 'close' hook, but a shlex instance will call the :meth:`close` + method of the sourced input stream when it returns EOF. For more explicit control of source stacking, use the :meth:`push_source` and :meth:`pop_source` methods. @@ -176,8 +172,8 @@ messages in the standard, parseable format understood by Emacs and other Unix tools. -Instances of :class:`~shlex.shlex` subclasses have some public instance -variables which either control lexical analysis or can be used for debugging: +Instances of :class:`shlex` subclasses have some public instance variables which +either control lexical analysis or can be used for debugging: .. attribute:: shlex.commenters @@ -222,8 +218,8 @@ .. attribute:: shlex.whitespace_split If ``True``, tokens will only be split in whitespaces. This is useful, for - example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. + example, for parsing command lines with :class:`shlex`, getting tokens in a + similar way to shell arguments. .. attribute:: shlex.infile @@ -235,8 +231,7 @@ .. attribute:: shlex.instream - The input stream from which this :class:`~shlex.shlex` instance is reading - characters. + The input stream from which this :class:`shlex` instance is reading characters. .. attribute:: shlex.source @@ -245,16 +240,16 @@ string will be recognized as a lexical-level inclusion request similar to the ``source`` keyword in various shells. That is, the immediately following token will opened as a filename and input taken from that stream until EOF, at which - point the :meth:`~io.IOBase.close` method of that stream will be called and - the input source will again become the original input stream. Source - requests may be stacked any number of levels deep. + point the :meth:`close` method of that stream will be called and the input + source will again become the original input stream. Source requests may be + stacked any number of levels deep. .. attribute:: shlex.debug - If this attribute is numeric and ``1`` or more, a :class:`~shlex.shlex` - instance will print verbose progress output on its behavior. If you need - to use this, you can read the module source code to learn the details. + If this attribute is numeric and ``1`` or more, a :class:`shlex` instance will + print verbose progress output on its behavior. If you need to use this, you can + read the module source code to learn the details. .. attribute:: shlex.lineno @@ -278,7 +273,7 @@ Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`shlex` will try to obey to the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -292,17 +287,16 @@ * Closing quotes separate words (``"Do"Separate`` is parsed as ``"Do"`` and ``Separate``); -* If :attr:`~shlex.whitespace_split` is ``False``, any character not - declared to be a word character, whitespace, or a quote will be returned as - a single-character token. If it is ``True``, :class:`~shlex.shlex` will only - split words in whitespaces; +* If :attr:`whitespace_split` is ``False``, any character not declared to be a + word character, whitespace, or a quote will be returned as a single-character + token. If it is ``True``, :class:`shlex` will only split words in whitespaces; * EOF is signaled with an empty string (``''``); * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the -following parsing rules. +When operating in POSIX mode, :class:`shlex` will try to obey to the following +parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is parsed as the single word ``DoNotSeparate``); @@ -310,16 +304,14 @@ * Non-quoted escape characters (e.g. ``'\'``) preserve the literal value of the next character that follows; -* Enclosing characters in quotes which are not part of - :attr:`~shlex.escapedquotes` (e.g. ``"'"``) preserve the literal value - of all characters within the quotes; +* Enclosing characters in quotes which are not part of :attr:`escapedquotes` + (e.g. ``"'"``) preserve the literal value of all characters within the quotes; -* Enclosing characters in quotes which are part of - :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value - of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its - special meaning only when followed by the quote in use, or the escape - character itself. Otherwise the escape character will be considered a +* Enclosing characters in quotes which are part of :attr:`escapedquotes` (e.g. + ``'"'``) preserves the literal value of all characters within the quotes, with + the exception of the characters mentioned in :attr:`escape`. The escape + characters retain its special meaning only when followed by the quote in use, or + the escape character itself. Otherwise the escape character will be considered a normal character. * EOF is signaled with a :const:`None` value; diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/shutil.rst --- a/Doc/library/shutil.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/shutil.rst Mon Jan 25 17:05:13 2016 +0100 @@ -47,142 +47,69 @@ be copied. -.. function:: copyfile(src, dst, *, follow_symlinks=True) +.. function:: copyfile(src, dst[, symlinks=False]) Copy the contents (no metadata) of the file named *src* to a file named - *dst* and return *dst*. *src* and *dst* are path names given as strings. - *dst* must be the complete target file name; look at :func:`shutil.copy` - for a copy that accepts a target directory path. If *src* and *dst* - specify the same file, :exc:`SameFileError` is raised. + *dst*. *dst* must be the complete target file name; look at + :func:`shutil.copy` for a copy that accepts a target directory path. If + *src* and *dst* are the same files, :exc:`Error` is raised. - The destination location must be writable; otherwise, an :exc:`OSError` - exception will be raised. If *dst* already exists, it will be replaced. - Special files such as character or block devices and pipes cannot be - copied with this function. + The destination location must be writable; otherwise, an :exc:`OSError` exception + will be raised. If *dst* already exists, it will be replaced. Special files + such as character or block devices and pipes cannot be copied with this + function. *src* and *dst* are path names given as strings. - If *follow_symlinks* is false and *src* is a symbolic link, - a new symbolic link will be created instead of copying the - file *src* points to. + If *symlinks* is true and *src* is a symbolic link, a new symbolic link will + be created instead of copying the file *src* points to. .. versionchanged:: 3.3 :exc:`IOError` used to be raised instead of :exc:`OSError`. - Added *follow_symlinks* argument. - Now returns *dst*. + Added *symlinks* argument. - .. versionchanged:: 3.4 - Raise :exc:`SameFileError` instead of :exc:`Error`. Since the former is - a subclass of the latter, this change is backward compatible. - -.. exception:: SameFileError - - This exception is raised if source and destination in :func:`copyfile` - are the same file. - - .. versionadded:: 3.4 - - -.. function:: copymode(src, dst, *, follow_symlinks=True) +.. function:: copymode(src, dst[, symlinks=False]) Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. - If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, - :func:`copymode` will attempt to modify the mode of *dst* itself (rather - than the file it points to). This functionality is not available on every - platform; please see :func:`copystat` for more information. If - :func:`copymode` cannot modify symbolic links on the local platform, and it - is asked to do so, it will do nothing and return. + group are unaffected. *src* and *dst* are path names given as strings. If + *symlinks* is true, *src* a symbolic link and the operating system supports + modes for symbolic links (for example BSD-based ones), the mode of the link + will be copied. .. versionchanged:: 3.3 - Added *follow_symlinks* argument. + Added *symlinks* argument. -.. function:: copystat(src, dst, *, follow_symlinks=True) +.. function:: copystat(src, dst[, symlinks=False]) - Copy the permission bits, last access time, last modification time, and - flags from *src* to *dst*. On Linux, :func:`copystat` also copies the - "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path names given as strings. - - If *follow_symlinks* is false, and *src* and *dst* both - refer to symbolic links, :func:`copystat` will operate on - the symbolic links themselves rather than the files the - symbolic links refer to--reading the information from the - *src* symbolic link, and writing the information to the - *dst* symbolic link. - - .. note:: - - Not all platforms provide the ability to examine and - modify symbolic links. Python itself can tell you what - functionality is locally available. - - * If ``os.chmod in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the permission - bits of a symbolic link. - - * If ``os.utime in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the last access - and modification times of a symbolic link. - - * If ``os.chflags in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the flags of - a symbolic link. (``os.chflags`` is not available on - all platforms.) - - On platforms where some or all of this functionality - is unavailable, when asked to modify a symbolic link, - :func:`copystat` will copy everything it can. - :func:`copystat` never returns failure. - - Please see :data:`os.supports_follow_symlinks` - for more information. + Copy the permission bits, last access time, last modification time, and flags + from *src* to *dst*. The file contents, owner, and group are unaffected. *src* + and *dst* are path names given as strings. If *src* and *dst* are both + symbolic links and *symlinks* true, the stats of the link will be copied as + far as the platform allows. .. versionchanged:: 3.3 - Added *follow_symlinks* argument and support for Linux extended attributes. + Added *symlinks* argument. -.. function:: copy(src, dst, *, follow_symlinks=True) +.. function:: copy(src, dst[, symlinks=False])) - Copies the file *src* to the file or directory *dst*. *src* and *dst* - should be strings. If *dst* specifies a directory, the file will be - copied into *dst* using the base filename from *src*. Returns the - path to the newly created file. - - If *follow_symlinks* is false, and *src* is a symbolic link, - *dst* will be created as a symbolic link. If *follow_symlinks* - is true and *src* is a symbolic link, *dst* will be a copy of - the file *src* refers to. - - :func:`copy` copies the file data and the file's permission - mode (see :func:`os.chmod`). Other metadata, like the - file's creation and modification times, is not preserved. - To preserve all file metadata from the original, use - :func:`~shutil.copy2` instead. + Copy the file *src* to the file or directory *dst*. If *dst* is a directory, a + file with the same basename as *src* is created (or overwritten) in the + directory specified. Permission bits are copied. *src* and *dst* are path + names given as strings. If *symlinks* is true, symbolic links won't be + followed but recreated instead -- this resembles GNU's :program:`cp -P`. .. versionchanged:: 3.3 - Added *follow_symlinks* argument. - Now returns path to the newly created file. + Added *symlinks* argument. -.. function:: copy2(src, dst, *, follow_symlinks=True) +.. function:: copy2(src, dst[, symlinks=False]) - Identical to :func:`~shutil.copy` except that :func:`copy2` - also attempts to preserve all file metadata. - - When *follow_symlinks* is false, and *src* is a symbolic - link, :func:`copy2` attempts to copy all metadata from the - *src* symbolic link to the newly-created *dst* symbolic link. - However, this functionality is not available on all platforms. - On platforms where some or all of this functionality is - unavailable, :func:`copy2` will preserve all the metadata - it can; :func:`copy2` never returns failure. - - :func:`copy2` uses :func:`copystat` to copy the file metadata. - Please see :func:`copystat` for more information - about platform support for modifying symbolic link metadata. + Similar to :func:`shutil.copy`, but metadata is copied as well -- in fact, + this is just :func:`shutil.copy` followed by :func:`copystat`. This is + similar to the Unix command :program:`cp -p`. If *symlinks* is true, + symbolic links won't be followed but recreated instead -- this resembles + GNU's :program:`cp -P`. .. versionchanged:: 3.3 - Added *follow_symlinks* argument, try to copy extended - file system attributes too (currently Linux only). - Now returns path to the newly created file. + Added *symlinks* argument. .. function:: ignore_patterns(\*patterns) @@ -191,11 +118,9 @@ match one of the glob-style *patterns* provided. See the example below. -.. function:: copytree(src, dst, symlinks=False, ignore=None, \ - copy_function=copy2, ignore_dangling_symlinks=False) +.. function:: copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False) - Recursively copy an entire directory tree rooted at *src*, returning the - destination directory. The destination + Recursively copy an entire directory tree rooted at *src*. The destination directory, named by *dst*, must not already exist; it will be created as well as missing parent directories. Permissions and times of directories are copied with :func:`copystat`, individual files are copied using @@ -207,8 +132,8 @@ and metadata of the linked files are copied to the new tree. When *symlinks* is false, if the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an :exc:`Error` exception at the end of the copy process. + exist, a exception will be added in the list of errors raised in + a :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you want to silence this exception. Notice that this option has no effect on platforms that don't support :func:`os.symlink`. @@ -230,16 +155,17 @@ as arguments. By default, :func:`shutil.copy2` is used, but any function that supports the same signature (like :func:`shutil.copy`) can be used. - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - .. versionchanged:: 3.2 Added the *copy_function* argument to be able to provide a custom copy function. + + .. versionchanged:: 3.2 Added the *ignore_dangling_symlinks* argument to silent dangling symlinks errors when *symlinks* is false. + .. versionchanged:: 3.3 + Copy metadata when *symlinks* is false. + .. function:: rmtree(path, ignore_errors=False, onerror=None) @@ -251,67 +177,35 @@ handled by calling a handler specified by *onerror* or, if that is omitted, they raise an exception. - .. note:: + If *onerror* is provided, it must be a callable that accepts three + parameters: *function*, *path*, and *excinfo*. The first parameter, + *function*, is the function which raised the exception; it will be + :func:`os.path.islink`, :func:`os.listdir`, :func:`os.remove` or + :func:`os.rmdir`. The second parameter, *path*, will be the path name passed + to *function*. The third parameter, *excinfo*, will be the exception + information return by :func:`sys.exc_info`. Exceptions raised by *onerror* + will not be caught. - On platforms that support the necessary fd-based functions a symlink - attack resistant version of :func:`rmtree` is used by default. On other - platforms, the :func:`rmtree` implementation is susceptible to a symlink - attack: given proper timing and circumstances, attackers can manipulate - symlinks on the filesystem to delete files they wouldn't be able to access - otherwise. Applications can use the :data:`rmtree.avoids_symlink_attacks` - function attribute to determine which case applies. - If *onerror* is provided, it must be a callable that accepts three - parameters: *function*, *path*, and *excinfo*. +.. function:: move(src, dst) - The first parameter, *function*, is the function which raised the exception; - it depends on the platform and implementation. The second parameter, - *path*, will be the path name passed to *function*. The third parameter, - *excinfo*, will be the exception information returned by - :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. + Recursively move a file or directory (*src*) to another location (*dst*). - .. versionchanged:: 3.3 - Added a symlink attack resistant version that is used automatically - if platform supports fd-based functions. + If the destination is a directory or a symlink to a directory, then *src* is + moved inside that directory. - .. attribute:: rmtree.avoids_symlink_attacks - - Indicates whether the current platform and implementation provides a - symlink attack resistant version of :func:`rmtree`. Currently this is - only true for platforms supporting fd-based directory access functions. - - .. versionadded:: 3.3 - - -.. function:: move(src, dst, copy_function=copy2) - - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. - - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. + The destination directory must not already exist. If the destination already + exists but is not a directory, it may be overwritten depending on + :func:`os.rename` semantics. If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. - - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dest* if - :func:`os.rename` cannot be used. If the source is a directory, - :func:`copytree` is called, passing it the :func:`copy_function`. The - default *copy_function* is :func:`copy2`. Using :func:`copy` as the - *copy_function* allows the move to succeed when it is not possible to also - copy the metadata, at the expense of not copying any of the metadata. + used. Otherwise, *src* is copied (using :func:`shutil.copy2`) to *dst* and + then removed. In case of symlinks, a new symlink pointing to the target of + *src* will be created in or as *dst* and *src* will be removed. .. versionchanged:: 3.3 Added explicit symlink handling for foreign filesystems, thus adapting it to the behavior of GNU's :program:`mv`. - Now returns *dst*. - - .. versionchanged:: 3.5 - Added the *copy_function* keyword argument. .. function:: disk_usage(path) @@ -337,31 +231,6 @@ .. versionadded:: 3.3 -.. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) - - Return the path to an executable which would be run if the given *cmd* was - called. If no *cmd* would be called, return ``None``. - - *mode* is a permission mask passed a to :func:`os.access`, by default - determining if the file exists and executable. - - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :attr:`os.defpath`. - - On Windows, the current directory is always prepended to the *path* whether - or not you use the default or provide your own, which is the behavior the - command shell uses when finding executables. Additionally, when finding the - *cmd* in the *path*, the ``PATHEXT`` environment variable is checked. For - example, if you call ``shutil.which("python")``, :func:`which` will search - ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* - directories. For example, on Windows:: - - >>> shutil.which("python") - 'C:\\Python33\\python.EXE' - - .. versionadded:: 3.3 - - .. exception:: Error This exception collects exceptions that are raised during a multi-file @@ -372,7 +241,7 @@ .. _shutil-copytree-example: copytree example -~~~~~~~~~~~~~~~~ +:::::::::::::::: This example is the implementation of the :func:`copytree` function, described above, with the docstring omitted. It demonstrates many of the other functions @@ -394,7 +263,7 @@ else: copy2(srcname, dstname) # XXX What about devices, sockets etc.? - except OSError as why: + except (IOError, os.error) as why: errors.append((srcname, dstname, str(why))) # catch the Error from the recursive copytree so that we can # continue with other files @@ -402,10 +271,11 @@ errors.extend(err.args[0]) try: copystat(src, dst) + except WindowsError: + # can't copy file access times on Windows + pass except OSError as why: - # can't copy file access times on Windows - if why.winerror is None: - errors.extend((src, dst, str(why))) + errors.extend((src, dst, str(why))) if errors: raise Error(errors) @@ -430,33 +300,11 @@ copytree(source, destination, ignore=_logpath) -.. _shutil-rmtree-example: - -rmtree example -~~~~~~~~~~~~~~ - -This example shows how to remove a directory tree on Windows where some -of the files have their read-only bit set. It uses the onerror callback -to clear the readonly bit and reattempt the remove. Any subsequent failure -will propagate. :: - - import os, stat - import shutil - - def remove_readonly(func, path, _): - "Clear the readonly bit and reattempt the removal" - os.chmod(path, stat.S_IWRITE) - func(path) - - shutil.rmtree(directory, onerror=remove_readonly) - .. _archiving-operations: Archiving operations -------------------- -.. versionadded:: 3.2 - High-level utilities to create and read compressed and archived files are also provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. @@ -466,8 +314,7 @@ *base_name* is the name of the file to create, including the path, minus any format-specific extension. *format* is the archive format: one of - "zip", "tar", "bztar" (if the :mod:`bz2` module is available), "xztar" - (if the :mod:`lzma` module is available) or "gztar". + "zip", "tar", "bztar" (if the :mod:`bz2` module is available) or "gztar". *root_dir* is a directory that will be the root directory of the archive; for example, we typically chdir into *root_dir* before creating the @@ -479,59 +326,53 @@ *root_dir* and *base_dir* both default to the current directory. - If *dry_run* is true, no archive is created, but the operations that would be - executed are logged to *logger*. - *owner* and *group* are used when creating a tar archive. By default, uses the current owner and group. *logger* must be an object compatible with :pep:`282`, usually an instance of :class:`logging.Logger`. - The *verbose* argument is unused and deprecated. - - .. versionchanged:: 3.5 - Added support for the *xztar* format. + .. versionadded:: 3.2 .. function:: get_archive_formats() Return a list of supported formats for archiving. - Each element of the returned sequence is a tuple ``(name, description)``. + Each element of the returned sequence is a tuple ``(name, description)`` By default :mod:`shutil` provides these formats: - *gztar*: gzip'ed tar-file - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available.) - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available.) - *tar*: uncompressed tar file - *zip*: ZIP file You can register new formats or provide your own archiver for any existing formats, by using :func:`register_archive_format`. + .. versionadded:: 3.2 + .. function:: register_archive_format(name, function, [extra_args, [description]]) - Register an archiver for the format *name*. - - *function* is the callable that will be used to unpack archives. The callable - will receive the *base_name* of the file to create, followed by the - *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. - Further arguments are passed as keyword arguments: *owner*, *group*, - *dry_run* and *logger* (as passed in :func:`make_archive`). + Register an archiver for the format *name*. *function* is a callable that + will be used to invoke the archiver. If given, *extra_args* is a sequence of ``(name, value)`` pairs that will be used as extra keywords arguments when the archiver callable is used. *description* is used by :func:`get_archive_formats` which returns the - list of archivers. Defaults to an empty string. + list of archivers. Defaults to an empty list. + + .. versionadded:: 3.2 .. function:: unregister_archive_format(name) Remove the archive format *name* from the list of supported formats. + .. versionadded:: 3.2 + .. function:: unpack_archive(filename[, extract_dir[, format]]) @@ -546,6 +387,8 @@ and see if an unpacker was registered for that extension. In case none is found, a :exc:`ValueError` is raised. + .. versionadded:: 3.2 + .. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) @@ -563,11 +406,15 @@ *description* can be provided to describe the format, and will be returned by the :func:`get_unpack_formats` function. + .. versionadded:: 3.2 + .. function:: unregister_unpack_format(name) Unregister an unpack format. *name* is the name of the format. + .. versionadded:: 3.2 + .. function:: get_unpack_formats() @@ -579,18 +426,19 @@ - *gztar*: gzip'ed tar-file - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available.) - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available.) - *tar*: uncompressed tar file - *zip*: ZIP file You can register new formats or provide your own unpacker for any existing formats, by using :func:`register_unpack_format`. + .. versionadded:: 3.2 + .. _shutil-archiving-example: Archiving example -~~~~~~~~~~~~~~~~~ +::::::::::::::::: In this example, we create a gzip'ed tar-file archive containing all files found in the :file:`.ssh` directory of the user:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/signal.rst --- a/Doc/library/signal.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/signal.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,75 +5,43 @@ :synopsis: Set handlers for asynchronous events. -This module provides mechanisms to use signal handlers in Python. +This module provides mechanisms to use signal handlers in Python. Some general +rules for working with signals and their handlers: +* A handler for a particular signal, once set, remains installed until it is + explicitly reset (Python emulates the BSD style interface regardless of the + underlying implementation), with the exception of the handler for + :const:`SIGCHLD`, which follows the underlying implementation. -General rules -------------- +* Although Python signal handlers are called asynchronously as far as the Python + user is concerned, they can only occur between the "atomic" instructions of the + Python interpreter. This means that signals arriving during long calculations + implemented purely in C (such as regular expression matches on large bodies of + text) may be delayed for an arbitrary amount of time. -The :func:`signal.signal` function allows to define custom handlers to be -executed when a signal is received. A small number of default handlers are -installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets -can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception. +* When a signal arrives during an I/O operation, it is possible that the I/O + operation raises an exception after the signal handler returns. This is + dependent on the underlying Unix system's semantics regarding interrupted system + calls. -A handler for a particular signal, once set, remains installed until it is -explicitly reset (Python emulates the BSD style interface regardless of the -underlying implementation), with the exception of the handler for -:const:`SIGCHLD`, which follows the underlying implementation. +* Because the C signal handler always returns, it makes little sense to catch + synchronous errors like :const:`SIGFPE` or :const:`SIGSEGV`. -There is no way to "block" signals temporarily from critical sections (since -this is not supported by all Unix flavors). +* Python installs a small number of signal handlers by default: :const:`SIGPIPE` + is ignored (so write errors on pipes and sockets can be reported as ordinary + Python exceptions) and :const:`SIGINT` is translated into a + :exc:`KeyboardInterrupt` exception. All of these can be overridden. - -Execution of Python signal handlers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A Python signal handler does not get executed inside the low-level (C) signal -handler. Instead, the low-level signal handler sets a flag which tells the -:term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). -This has consequences: - -* It makes little sense to catch synchronous errors like :const:`SIGFPE` or - :const:`SIGSEGV` that are caused by an invalid operation in C code. Python - will return from the signal handler to the C code, which is likely to raise - the same signal again, causing Python to apparently hang. From Python 3.3 - onwards, you can use the :mod:`faulthandler` module to report on synchronous - errors. - -* A long-running calculation implemented purely in C (such as regular - expression matching on a large body of text) may run uninterrupted for an - arbitrary amount of time, regardless of any signals received. The Python - signal handlers will be called when the calculation finishes. - - -.. _signals-and-threads: - - -Signals and threads -^^^^^^^^^^^^^^^^^^^ - -Python signal handlers are always executed in the main Python thread, -even if the signal was received in another thread. This means that signals -can't be used as a means of inter-thread communication. You can use -the synchronization primitives from the :mod:`threading` module instead. - -Besides, only the main thread is allowed to set a new signal handler. - - -Module contents ---------------- - -.. versionchanged:: 3.5 - signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask - (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`) - related constants listed below were turned into - :class:`enums `. - :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and - :func:`sigwait` functions return human-readable - :class:`enums `. - +* Some care must be taken if both signals and threads are used in the same + program. The fundamental thing to remember in using signals and threads + simultaneously is: always perform :func:`signal` operations in the main thread + of execution. Any thread can perform an :func:`alarm`, :func:`getsignal`, + :func:`pause`, :func:`setitimer` or :func:`getitimer`; only the main thread + can set a new signal handler, and the main thread will be the only one to + receive signals (this is enforced by the Python :mod:`signal` module, even + if the underlying thread implementation supports sending signals to + individual threads). This means that signals can't be used as a means of + inter-thread communication. Use locks instead. The variables defined in the :mod:`signal` module are: @@ -105,7 +73,7 @@ .. data:: CTRL_C_EVENT - The signal corresponding to the :kbd:`Ctrl+C` keystroke event. This signal can + The signal corresponding to the CTRL+C keystroke event. This signal can only be used with :func:`os.kill`. Availability: Windows. @@ -115,7 +83,7 @@ .. data:: CTRL_BREAK_EVENT - The signal corresponding to the :kbd:`Ctrl+Break` keystroke event. This signal can + The signal corresponding to the CTRL+BREAK keystroke event. This signal can only be used with :func:`os.kill`. Availability: Windows. @@ -219,22 +187,17 @@ :func:`sigpending`. -.. function:: pthread_kill(thread_id, signalnum) +.. function:: pthread_kill(thread_id, signum) - Send the signal *signalnum* to the thread *thread_id*, another thread in the - same process as the caller. The target thread can be executing any code - (Python or not). However, if the target thread is executing the Python - interpreter, the Python signal handlers will be :ref:`executed by the main - thread `. Therefore, the only point of sending a - signal to a particular Python thread would be to force a running system call - to fail with :exc:`InterruptedError`. + Send the signal *signum* to the thread *thread_id*, another thread in the same + process as the caller. The signal is asynchronously directed to thread. Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` - attribute of :class:`threading.Thread` objects to get a suitable value - for *thread_id*. + attribute of :attr:`threading.Thread` to get a 'thread identifier' for + *thread_id*. - If *signalnum* is 0, then no signal is sent, but error checking is still - performed; this can be used to check if the target thread is still running. + If *signum* is 0, then no signal is sent, but error checking is still + performed; this can be used to check if a thread is still running. Availability: Unix (see the man page :manpage:`pthread_kill(3)` for further information). @@ -252,13 +215,13 @@ The behavior of the call is dependent on the value of *how*, as follows. - * :data:`SIG_BLOCK`: The set of blocked signals is the union of the current - set and the *mask* argument. - * :data:`SIG_UNBLOCK`: The signals in *mask* are removed from the current - set of blocked signals. It is permissible to attempt to unblock a - signal which is not blocked. - * :data:`SIG_SETMASK`: The set of blocked signals is set to the *mask* - argument. + * :data:`SIG_BLOCK`: The set of blocked signals is the union of the current + set and the *mask* argument. + * :data:`SIG_UNBLOCK`: The signals in *mask* are removed from the current + set of blocked signals. It is permissible to attempt to unblock a + signal which is not blocked. + * :data:`SIG_SETMASK`: The set of blocked signals is set to the *mask* + argument. *mask* is a set of signal numbers (e.g. {:const:`signal.SIGINT`, :const:`signal.SIGTERM`}). Use ``range(1, signal.NSIG)`` for a full mask @@ -318,9 +281,6 @@ attempting to call it from other threads will cause a :exc:`ValueError` exception to be raised. - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - .. function:: siginterrupt(signalnum, flag) @@ -408,11 +368,6 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.5 - The function is now retried if interrupted by a signal not in *sigset* - and the signal handler does not raise an exception (see :pep:`475` for - the rationale). - .. function:: sigtimedwait(sigset, timeout) @@ -427,11 +382,6 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.5 - The function is now retried with the recomputed *timeout* if interrupted - by a signal not in *sigset* and the signal handler does not raise an - exception (see :pep:`475` for the rationale). - .. _signal-example: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/site.rst --- a/Doc/library/site.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/site.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,28 +26,17 @@ :option:`-S`. .. index:: + pair: site-python; directory pair: site-packages; directory It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh). For each -of the distinct head-tail combinations, it sees if it refers to an existing -directory, and if so, adds it to ``sys.path`` and also inspects the newly -added path for configuration files. - -.. versionchanged:: 3.5 - Support for the "site-python" directory has been removed. - -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "false" -(case-insensitive), the system-level prefixes will still also be -searched for site-packages; otherwise they won't. +:file:`lib/python|version|/site-packages` and then :file:`lib/site-python` (on +Unix and Macintosh). For each of the distinct head-tail combinations, it sees +if it refers to an existing directory, and if so, adds it to ``sys.path`` and +also inspects the newly added path for configuration files. A path configuration file is a file whose name has the form :file:`{name}.pth` and exists in one of the four directories mentioned above; its contents are @@ -98,11 +87,7 @@ :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. If Python is started without output streams available, as -with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any exception other -than :exc:`ImportError` causes a silent and perhaps mysterious failure of the -process. +silently ignored. .. index:: module: usercustomize @@ -117,27 +102,6 @@ :mod:`sitecustomize` and :mod:`usercustomize` is still attempted. -.. _rlcompleter-config: - -Readline configuration ----------------------- - -On systems that support :mod:`readline`, this module will also import and -configure the :mod:`rlcompleter` module, if Python is started in -:ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use -:file:`~/.python_history` as the history save file. To disable it, delete (or -override) the :data:`sys.__interactivehook__` attribute in your -:mod:`sitecustomize` or :mod:`usercustomize` module or your -:envvar:`PYTHONSTARTUP` file. - -.. versionchanged:: 3.4 - Activation of rlcompleter and history was made automatic. - - -Module contents ---------------- - .. data:: PREFIXES A list of prefixes for site-packages directories. @@ -170,9 +134,9 @@ :func:`getuserbase` hasn't been called yet. Default value is :file:`~/.local` for UNIX and Mac OS X non-framework builds, :file:`~/Library/Python/{X.Y}` for Mac framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to + :file:`{%APPDATA%}\\Python` for Windows. This value is used by Packaging to compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. + etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. @@ -180,10 +144,11 @@ Adds all the standard site-specific directories to the module search path. This function is called automatically when this module is imported, - unless the Python interpreter was started with the :option:`-S` flag. + unless the :program:`python` interpreter was started with the :option:`-S` + flag. .. versionchanged:: 3.3 - This function used to be called unconditionally. + This function used to be called unconditionnally. .. function:: addsitedir(sitedir, known_paths=None) @@ -194,7 +159,8 @@ .. function:: getsitepackages() - Return a list containing all global site-packages directories. + Return a list containing all global site-packages directories (and possibly + site-python). .. versionadded:: 3.2 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/smtpd.rst --- a/Doc/library/smtpd.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/smtpd.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,7 @@ .. module:: smtpd :synopsis: A SMTP server implementation in Python. -.. moduleauthor:: Barry Warsaw +.. moduleauthor:: Barry Warsaw .. sectionauthor:: Moshe Zadka **Source code:** :source:`Lib/smtpd.py` @@ -20,98 +20,32 @@ Additionally the SMTPChannel may be extended to implement very specific interaction behaviour with SMTP clients. -The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531` -SMTPUTF8 extensions. - - SMTPServer Objects ------------------ -.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=True) +.. class:: SMTPServer(localaddr, remoteaddr) Create a new :class:`SMTPServer` object, which binds to local address *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It inherits from :class:`asyncore.dispatcher`, and so will insert itself into :mod:`asyncore`'s event loop on instantiation. - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. + .. method:: process_message(peer, mailfrom, rcpttos, data) - *map* is the socket map to use for connections (an initially empty - dictionary is a suitable value). If not specified the :mod:`asyncore` - global socket map is used. - - *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. If set to - ``True``, *decode_data* must be ``False`` (otherwise an error is raised). - When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` - command and when present is passed to :meth:`process_message` in the - ``kwargs['mail_options']`` list. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``True`` for backward - compatibility reasons, but will change to ``False`` in Python 3.6; specify - the keyword value explicitly to avoid the :exc:`DeprecationWarning`. When - *decode_data* is set to ``False`` the server advertises the ``8BITMIME`` - extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to - the ``MAIL`` command, and when present passes it to :meth:`process_message` - in the ``kwargs['mail_options']`` list. - - .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) - - Raise a :exc:`NotImplementedError` exception. Override this in subclasses to + Raise :exc:`NotImplementedError` exception. Override this in subclasses to do something useful with this message. Whatever was passed in the constructor as *remoteaddr* will be available as the :attr:`_remoteaddr` attribute. *peer* is the remote host's address, *mailfrom* is the envelope originator, *rcpttos* are the envelope recipients and *data* is a string - containing the contents of the e-mail (which should be in :rfc:`5321` + containing the contents of the e-mail (which should be in :rfc:`2822` format). - If the *decode_data* constructor keyword is set to ``True``, the *data* - argument will be a unicode string. If it is set to ``False``, it - will be a bytes object. - - *kwargs* is a dictionary containing additional information. It is empty - unless at least one of ``decode_data=False`` or ``enable_SMTPUTF8=True`` - was given as an init parameter, in which case it contains the following - keys: - - *mail_options*: - a list of all received parameters to the ``MAIL`` - command (the elements are uppercase strings; example: - ``['BODY=8BITMIME', 'SMTPUTF8']``). - - *rcpt_options*: - same as *mail_options* but for the ``RCPT`` command. - Currently no ``RCPT TO`` options are supported, so for now - this will always be an empty list. - - Implementations of ``process_message`` should use the ``**kwargs`` - signature to accept arbitrary keyword arguments, since future feature - enhancements may add keys to the kwargs dictionary. - - Return ``None`` to request a normal ``250 Ok`` response; otherwise - return the desired response string in :RFC:`5321` format. - .. attribute:: channel_class Override this in subclasses to use a custom :class:`SMTPChannel` for managing SMTP clients. - .. versionadded:: 3.4 - The *map* constructor argument. - - .. versionchanged:: 3.5 - *localaddr* and *remoteaddr* may now contain IPv6 addresses. - - .. versionadded:: 3.5 - the *decode_data* and *enable_SMTPUTF8* constructor arguments, and the - *kwargs* argument to :meth:`process_message` when one or more of these is - specified. - DebuggingServer Objects ----------------------- @@ -149,36 +83,14 @@ SMTPChannel Objects ------------------- -.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=True) +.. class:: SMTPChannel(server, conn, addr) Create a new :class:`SMTPChannel` object which manages the communication between the server and a single SMTP client. - *conn* and *addr* are as per the instance variables described below. - - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. - - *enable_SMTPUTF8* determins whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. A - :exc:`ValueError` is raised if both *enable_SMTPUTF8* and *decode_data* are - set to ``True`` at the same time. - - A dictionary can be specified in *map* to avoid using a global socket map. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``True`` for backward - compatibility reasons, but will change to ``False`` in Python 3.6. Specify - the keyword value explicitly to avoid the :exc:`DeprecationWarning`. - To use a custom SMTPChannel implementation you need to override the :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. - .. versionchanged:: 3.5 - the *decode_data* and *enable_SMTPUTF8* arguments were added. - The :class:`SMTPChannel` has the following instance variables: .. attribute:: smtp_server @@ -192,13 +104,12 @@ .. attribute:: addr Holds the address of the client, the second value returned by - :func:`socket.accept ` + socket.accept() .. attribute:: received_lines Holds a list of the line strings (decoded using UTF-8) received from - the client. The lines have their ``"\r\n"`` line ending translated to - ``"\n"``. + the client. The lines have their "\r\n" line ending translated to "\n". .. attribute:: smtp_state @@ -223,12 +134,12 @@ .. attribute:: received_data Holds a string containing all of the data sent by the client during the - DATA state, up to but not including the terminating ``"\r\n.\r\n"``. + DATA state, up to but not including the terminating "\r\n.\r\n". .. attribute:: fqdn Holds the fully-qualified domain name of the server as returned by - :func:`socket.getfqdn`. + ``socket.getfqdn()``. .. attribute:: peer @@ -244,23 +155,16 @@ Command Action taken ======== =================================================================== HELO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to base command mode. - EHLO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to extended command mode. + :attr:`seen_greeting`. NOOP Takes no action. QUIT Closes the connection cleanly. MAIL Accepts the "MAIL FROM:" syntax and stores the supplied address as - :attr:`mailfrom`. In extended command mode, accepts the - :rfc:`1870` SIZE attribute and responds appropriately based on the - value of *data_size_limit*. + :attr:`mailfrom`. RCPT Accepts the "RCPT TO:" syntax and stores the supplied addresses in the :attr:`rcpttos` list. RSET Resets the :attr:`mailfrom`, :attr:`rcpttos`, and :attr:`received_data`, but not the greeting. DATA Sets the internal state to :attr:`DATA` and stores remaining lines from the client in :attr:`received_data` until the terminator - ``"\r\n.\r\n"`` is received. - HELP Returns minimal information on command syntax - VRFY Returns code 252 (the server doesn't know if the address is valid) - EXPN Reports that the command is not implemented. + "\r\n.\r\n" is received. ======== =================================================================== diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/smtplib.rst --- a/Doc/library/smtplib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/smtplib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -22,27 +22,22 @@ .. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) - An :class:`SMTP` instance encapsulates an SMTP connection. It has methods + A :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. If the optional - host and port parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO - command. Otherwise, the local hostname is found using - :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other - than a success code, an :exc:`SMTPConnectError` is raised. The optional + host and port parameters are given, the SMTP :meth:`connect` method is called + with those parameters during initialization. An :exc:`SMTPConnectError` is + raised if the specified host doesn't respond correctly. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout - setting will be used). If the timeout expires, :exc:`socket.timeout` is - raised. The optional source_address parameter allows to bind - to some specific source address in a machine with multiple network - interfaces, and/or to some specific source TCP port. It takes a 2-tuple - (host, port), for the socket to bind to as its source address before - connecting. If omitted (or if host or port are ``''`` and/or 0 respectively) - the OS default behavior will be used. + setting will be used). The optional source_address parameter allows to bind to some + specific source address in a machine with multiple network interfaces, + and/or to some specific source TCP port. It takes a 2-tuple (host, port), + for the socket to bind to as its source address before connecting. If + omitted (or if host or port are ``''`` and/or 0 respectively) the OS default + behavior will be used. For normal use, you should only require the initialization/connect, - :meth:`sendmail`, and :meth:`~smtplib.quit` methods. - An example is included below. + :meth:`sendmail`, and :meth:`quit` methods. An example is included below. The :class:`SMTP` class supports the :keyword:`with` statement. When used like this, the SMTP ``QUIT`` command is issued automatically when the @@ -61,28 +56,25 @@ .. versionchanged:: 3.3 source_address argument was added. - .. versionadded:: 3.5 - The SMTPUTF8 extension (:rfc:`6531`) is now supported. +.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, certfile=None[, timeout], context=None, source_address=None) - -.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ - certfile=None [, timeout], context=None, \ - source_address=None) - - An :class:`SMTP_SSL` instance behaves exactly the same as instances of + A :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is required from the beginning of the connection and using :meth:`starttls` is not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same - meaning as they do in the :class:`SMTP` class. *context*, also optional, - can contain a :class:`~ssl.SSLContext` and allows to configure various - aspects of the secure connection. Please read :ref:`ssl-security` for - best practices. - - *keyfile* and *certfile* are a legacy alternative to *context*, and can - point to a PEM formatted private key and certificate chain file for the - SSL connection. + *port* is zero, the standard SMTP-over-SSL port (465) is used. *keyfile* + and *certfile* are also optional, and can contain a PEM formatted private key + and certificate chain file for the SSL connection. *context* also optional, can contain + a SSLContext, and is an alternative to keyfile and certfile; If it is specified both + keyfile and certfile must be None. The optional *timeout* + parameter specifies a timeout in seconds for blocking operations like the + connection attempt (if not specified, the global default timeout setting + will be used). The optional source_address parameter allows to bind to some + specific source address in a machine with multiple network interfaces, + and/or to some specific source tcp port. It takes a 2-tuple (host, port), + for the socket to bind to as its source address before connecting. If + omitted (or if host or port are ``''`` and/or 0 respectively) the OS default + behavior will be used. .. versionchanged:: 3.3 *context* was added. @@ -90,10 +82,6 @@ .. versionchanged:: 3.3 source_address argument was added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None) @@ -101,12 +89,12 @@ standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` method must support that as well as a regular host:port server. The optional arguments local_hostname and source_address have the - same meaning as they do in the :class:`SMTP` class. To specify a Unix - socket, you must use an absolute path for *host*, starting with a '/'. + same meaning as that of SMTP client. To specify a Unix socket, you must use + an absolute path for *host*, starting with a '/'. - Authentication is supported, using the regular SMTP mechanism. When using a - Unix socket, LMTP generally don't support or require any authentication, but - your mileage might vary. + Authentication is supported, using the regular SMTP mechanism. When using a Unix + socket, LMTP generally don't support or require any authentication, but your + mileage might vary. A nice selection of exceptions is defined as well: @@ -114,11 +102,7 @@ .. exception:: SMTPException - Subclass of :exc:`OSError` that is the base exception class for all - the other exceptions provided by this module. - - .. versionchanged:: 3.4 - SMTPException became subclass of :exc:`OSError` + Base exception class for all exceptions raised by this module. .. exception:: SMTPServerDisconnected @@ -165,13 +149,6 @@ The server refused our ``HELO`` message. -.. exception:: SMTPNotSupportedError - - The command or option attempted is not supported by the server. - - .. versionadded:: 3.5 - - .. exception:: SMTPAuthenticationError SMTP authentication went wrong. Most probably the server didn't accept the @@ -200,12 +177,17 @@ .. method:: SMTP.set_debuglevel(level) - Set the debug output level. A value of 1 or ``True`` for *level* results in - debug messages for connection and for all messages sent to and received from - the server. A value of 2 for *level* results in these messages being - timestamped. + Set the debug output level. A true value for *level* results in debug messages + for connection and for all messages sent to and received from the server. - .. versionchanged:: 3.5 Added debuglevel 2. + +.. method:: SMTP.connect(host='localhost', port=0) + + Connect to a host on a given port. The defaults are to connect to the local + host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) + followed by a number, that suffix will be stripped off and the number + interpreted as the port number to use. This method is automatically invoked by + the constructor if a host is specified during instantiation. .. method:: SMTP.docmd(cmd, args='') @@ -224,17 +206,6 @@ :exc:`SMTPServerDisconnected` will be raised. -.. method:: SMTP.connect(host='localhost', port=0) - - Connect to a host on a given port. The defaults are to connect to the local - host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) - followed by a number, that suffix will be stripped off and the number - interpreted as the port number to use. This method is automatically invoked by - the constructor if a host is specified during instantiation. Returns a - 2-tuple of the response code and message sent by the server in its - connection response. - - .. method:: SMTP.helo(name='') Identify yourself to the SMTP server using ``HELO``. The hostname argument @@ -255,7 +226,8 @@ the server is stored as the :attr:`ehlo_resp` attribute, :attr:`does_esmtp` is set to true or false depending on whether the server supports ESMTP, and :attr:`esmtp_features` will be a dictionary containing the names of the - SMTP service extensions this server supports, and their parameters (if any). + SMTP service extensions this server supports, and their + parameters (if any). Unless you wish to use :meth:`has_extn` before sending mail, it should not be necessary to call this method explicitly. It will be implicitly called by @@ -288,7 +260,7 @@ Many sites disable SMTP ``VRFY`` in order to foil spammers. -.. method:: SMTP.login(user, password, *, initial_response_ok=True) +.. method:: SMTP.login(user, password) Log in on an SMTP server that requires authentication. The arguments are the username and the password to authenticate with. If there has been no previous @@ -302,68 +274,9 @@ :exc:`SMTPAuthenticationError` The server didn't accept the username/password combination. - :exc:`SMTPNotSupportedError` - The ``AUTH`` command is not supported by the server. - :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in - turn if they are advertised as supported by the server. See :meth:`auth` - for a list of supported authentication methods. *initial_response_ok* is - passed through to :meth:`auth`. - - Optional keyword argument *initial_response_ok* specifies whether, for - authentication methods that support it, an "initial response" as specified - in :rfc:`4954` can be sent along with the ``AUTH`` command, rather than - requiring a challenge/response. - - .. versionchanged:: 3.5 - :exc:`SMTPNotSupportedError` may be raised, and the - *initial_response_ok* parameter was added. - - -.. method:: SMTP.auth(mechanism, authobject, *, initial_response_ok=True) - - Issue an ``SMTP`` ``AUTH`` command for the specified authentication - *mechanism*, and handle the challenge response via *authobject*. - - *mechanism* specifies which authentication mechanism is to - be used as argument to the ``AUTH`` command; the valid values are - those listed in the ``auth`` element of :attr:`esmtp_features`. - - *authobject* must be a callable object taking an optional single argument: - - data = authobject(challenge=None) - - If optional keyword argument *initial_response_ok* is true, - ``authobject()`` will be called first with no argument. It can return the - :rfc:`4954` "initial response" bytes which will be encoded and sent with - the ``AUTH`` command as below. If the ``authobject()`` does not support an - initial response (e.g. because it requires a challenge), it should return - None when called with ``challenge=None``. If *initial_response_ok* is - false, then ``authobject()`` will not be called first with None. - - If the initial response check returns None, or if *initial_response_ok* is - false, ``authobject()`` will be called to process the server's challenge - response; the *challenge* argument it is passed will be a ``bytes``. It - should return ``bytes`` *data* that will be base64 encoded and sent to the - server. - - The ``SMTP`` class provides ``authobjects`` for the ``CRAM-MD5``, ``PLAIN``, - and ``LOGIN`` mechanisms; they are named ``SMTP.auth_cram_md5``, - ``SMTP.auth_plain``, and ``SMTP.auth_login`` respectively. They all require - that the ``user`` and ``password`` properties of the ``SMTP`` instance are - set to appropriate values. - - User code does not normally need to call ``auth`` directly, but can instead - call the :meth:`login` method, which will try each of the above mechanisms - in turn, in the order listed. ``auth`` is exposed to facilitate the - implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. - - .. versionadded:: 3.5 - .. method:: SMTP.starttls(keyfile=None, certfile=None, context=None) @@ -383,7 +296,7 @@ :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. - :exc:`SMTPNotSupportedError` + :exc:`SMTPException` The server does not support the STARTTLS extension. :exc:`RuntimeError` @@ -392,16 +305,6 @@ .. versionchanged:: 3.3 *context* was added. - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). - - .. versionchanged:: 3.5 - The error raised for lack of STARTTLS support is now the - :exc:`SMTPNotSupportedError` subclass instead of the base - :exc:`SMTPException`. - .. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[]) @@ -438,9 +341,6 @@ recipient that was refused. Each entry contains a tuple of the SMTP error code and the accompanying error message sent by the server. - If ``SMTPUTF8`` is included in *mail_options*, and the server supports it, - *from_addr* and *to_addr* may contain non-ASCII characters. - This method may raise the following exceptions: :exc:`SMTPRecipientsRefused` @@ -459,19 +359,10 @@ The server replied with an unexpected error code (other than a refusal of a recipient). - :exc:`SMTPNotSupportedError` - ``SMTPUTF8`` was given in the *mail_options* but is not supported by the - server. - Unless otherwise noted, the connection will be open even after an exception is raised. - .. versionchanged:: 3.2 - *msg* may be a byte string. - - .. versionchanged:: 3.5 - ``SMTPUTF8`` support added, and :exc:`SMTPNotSupportedError` may be - raised if ``SMTPUTF8`` is specified but the server does not support it. + .. versionchanged:: 3.2 *msg* may be a byte string. .. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \ @@ -484,7 +375,7 @@ If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills those arguments with addresses extracted from the headers of *msg* as - specified in :rfc:`5322`\: *from_addr* is set to the :mailheader:`Sender` + specified in :rfc:`2822`\: *from_addr* is set to the :mailheader:`Sender` field if it is present, and otherwise to the :mailheader:`From` field. *to_adresses* combines the values (if any) of the :mailheader:`To`, :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*. If exactly one @@ -499,18 +390,10 @@ calls :meth:`sendmail` to transmit the resulting message. Regardless of the values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear - in *msg*. If any of the addresses in *from_addr* and *to_addrs* contain - non-ASCII characters and the server does not advertise ``SMTPUTF8`` support, - an :exc:`SMTPNotSupported` error is raised. Otherwise the ``Message`` is - serialized with a clone of its :mod:`~email.policy` with the - :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and - ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*. + in *msg*. .. versionadded:: 3.2 - .. versionadded:: 3.5 - Support for internationalized addresses (``SMTPUTF8``). - .. method:: SMTP.quit() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sndhdr.rst --- a/Doc/library/sndhdr.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sndhdr.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,9 +16,8 @@ The :mod:`sndhdr` provides utility functions which attempt to determine the type of sound data which is in a file. When these functions are able to determine -what type of sound data is stored in a file, they return a -:func:`~collections.namedtuple`, containing five attributes: (``filetype``, -``framerate``, ``nchannels``, ``nframes``, ``sampwidth``). The value for *type* +what type of sound data is stored in a file, they return a tuple ``(type, +sampling_rate, channels, frames, bits_per_sample)``. The value for *type* indicates the data type and will be one of the strings ``'aifc'``, ``'aiff'``, ``'au'``, ``'hcom'``, ``'sndr'``, ``'sndt'``, ``'voc'``, ``'wav'``, ``'8svx'``, ``'sb'``, ``'ub'``, or ``'ul'``. The *sampling_rate* will be either the actual @@ -32,19 +31,13 @@ .. function:: what(filename) Determines the type of sound data stored in the file *filename* using - :func:`whathdr`. If it succeeds, returns a namedtuple as described above, otherwise + :func:`whathdr`. If it succeeds, returns a tuple as described above, otherwise ``None`` is returned. - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - .. function:: whathdr(filename) Determines the type of sound data stored in a file based on the file header. - The name of the file is given by *filename*. This function returns a namedtuple as + The name of the file is given by *filename*. This function returns a tuple as described above on success, or ``None``. - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/socket.rst --- a/Doc/library/socket.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/socket.rst Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,8 @@ This module provides access to the BSD *socket* interface. It is available on -all modern Unix systems, Windows, MacOS, and probably additional platforms. +all modern Unix systems, Windows, MacOS, OS/2, and probably additional +platforms. .. note:: @@ -17,7 +18,7 @@ The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement +:func:`socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -46,24 +47,21 @@ - The address of an :const:`AF_UNIX` socket bound to a file system node is represented as a string, using the file system encoding and the ``'surrogateescape'`` error handler (see :pep:`383`). An address in - Linux's abstract namespace is returned as a :term:`bytes-like object` with + Linux's abstract namespace is returned as a :class:`bytes` object with an initial null byte; note that sockets in this namespace can communicate with normal file system sockets, so programs intended to run on Linux may need to deal with both types of address. A string or - bytes-like object can be used for either type of address when + :class:`bytes` object can be used for either type of address when passing it as an argument. .. versionchanged:: 3.3 Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 encoding. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - - A pair ``(host, port)`` is used for the :const:`AF_INET` address family, where *host* is a string representing either a hostname in Internet domain notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integer. + and *port* is an integral port number. - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scopeid)`` is used, where *flowinfo* and *scopeid* represent the ``sin6_flowinfo`` @@ -93,6 +91,9 @@ If *addr_type* is :const:`TIPC_ADDR_ID`, then *v1* is the node, *v2* is the reference, and *v3* should be set to 0. + If *addr_type* is :const:`TIPC_ADDR_ID`, then *v1* is the node, *v2* is the + reference, and *v3* should be set to 0. + - A tuple ``(interface, )`` is used for the :const:`AF_CAN` address family, where *interface* is a string representing a network interface name like ``'can0'``. The network interface name ``''`` can be used to receive packets @@ -106,29 +107,7 @@ .. versionadded:: 3.3 -- :const:`AF_BLUETOOTH` supports the following protocols and address - formats: - - - :const:`BTPROTO_L2CAP` accepts ``(bdaddr, psm)`` where ``bdaddr`` is - the Bluetooth address as a string and ``psm`` is an integer. - - - :const:`BTPROTO_RFCOMM` accepts ``(bdaddr, channel)`` where ``bdaddr`` - is the Bluetooth address as a string and ``channel`` is an integer. - - - :const:`BTPROTO_HCI` accepts ``(device_id,)`` where ``device_id`` is - either an integer or a string with the Bluetooth address of the - interface. (This depends on your OS; NetBSD and DragonFlyBSD expect - a Bluetooth address while everything else expects an integer.) - - .. versionchanged:: 3.2 - NetBSD and DragonFlyBSD support added. - - - :const:`BTPROTO_SCO` accepts ``bdaddr`` where ``bdaddr`` is a - :class:`bytes` object containing the Bluetooth address in a - string format. (ex. ``b'12:23:34:45:56:67'``) This protocol is not - supported under FreeBSD. - -- Certain other address families (:const:`AF_PACKET`, :const:`AF_CAN`) +- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`) support specific representations. .. XXX document them! @@ -159,12 +138,9 @@ Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`socket` exports the following constants and functions: -Exceptions -^^^^^^^^^^ - .. exception:: error A deprecated alias of :exc:`OSError`. @@ -210,21 +186,12 @@ .. versionchanged:: 3.3 This class was made a subclass of :exc:`OSError`. - -Constants -^^^^^^^^^ - - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. - - .. versionadded:: 3.4 - .. data:: AF_UNIX AF_INET AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. More constants may be available depending on the system. @@ -236,7 +203,7 @@ SOCK_SEQPACKET These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. + :func:`socket`. More constants may be available depending on the system. (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally useful.) @@ -290,28 +257,6 @@ .. versionadded:: 3.3 -.. data:: CAN_BCM - CAN_BCM_* - - CAN_BCM, in the CAN protocol family, is the broadcast manager (BCM) protocol. - Broadcast manager constants, documented in the Linux documentation, are also - defined in the socket module. - - Availability: Linux >= 2.6.25. - - .. versionadded:: 3.4 - -.. data:: CAN_RAW_FD_FRAMES - - Enables CAN FD support in a CAN_RAW socket. This is disabled by default. - This allows your application to send both CAN and CAN FD frames; however, - you one must accept both CAN and CAN FD frames when reading from the socket. - - This constant is documented in the Linux documentation. - - Availability: Linux >= 3.6. - - .. versionadded:: 3.5 .. data:: AF_RDS PF_RDS @@ -330,7 +275,7 @@ RCVALL_* Constants for Windows' WSAIoctl(). The constants are used as arguments to the - :meth:`~socket.socket.ioctl` method of socket objects. + :meth:`ioctl` method of socket objects. .. data:: TIPC_* @@ -338,90 +283,12 @@ TIPC related constants, matching the ones exported by the C socket API. See the TIPC documentation for more information. -.. data:: AF_LINK - - Availability: BSD, OSX. - - .. versionadded:: 3.4 .. data:: has_ipv6 This constant contains a boolean value which indicates if IPv6 is supported on this platform. -.. data:: BDADDR_ANY - BDADDR_LOCAL - - These are string constants containing Bluetooth addresses with special - meanings. For example, :const:`BDADDR_ANY` can be used to indicate - any address when specifying the binding socket with - :const:`BTPROTO_RFCOMM`. - -.. data:: HCI_FILTER - HCI_TIME_STAMP - HCI_DATA_DIR - - For use with :const:`BTPROTO_HCI`. :const:`HCI_FILTER` is not - available for NetBSD or DragonFlyBSD. :const:`HCI_TIME_STAMP` and - :const:`HCI_DATA_DIR` are not available for FreeBSD, NetBSD, or - DragonFlyBSD. - -Functions -^^^^^^^^^ - -Creating sockets -'''''''''''''''' - -The following functions all create :ref:`socket objects `. - - -.. function:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN` or :const:`AF_RDS`. The - socket type should be :const:`SOCK_STREAM` (the default), - :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` - constants. The protocol number is usually zero and may be omitted or in the - case where the address family is :const:`AF_CAN` the protocol should be one - of :const:`CAN_RAW` or :const:`CAN_BCM`. If *fileno* is specified, the other - arguments are ignored, causing the socket with the specified file descriptor - to return. Unlike :func:`socket.fromfd`, *fileno* will return the same - socket and not a duplicate. This may help close a detached socket using - :meth:`socket.close()`. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - -.. function:: socketpair([family[, type[, proto]]]) - - Build a pair of connected socket objects using the given address family, socket - type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` - if defined on the platform; otherwise, the default is :const:`AF_INET`. - - The newly created sockets are :ref:`non-inheritable `. - - .. versionchanged:: 3.2 - The returned socket objects now support the whole socket API, rather - than a subset. - - .. versionchanged:: 3.4 - The returned sockets are now non-inheritable. - - .. versionchanged:: 3.5 - Windows support added. - .. function:: create_connection(address[, timeout[, source_address]]) @@ -449,45 +316,6 @@ support for the :keyword:`with` statement was added. -.. function:: fromfd(fd, family, type, proto=0) - - Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- - subsequent operations on the object may fail if the file descriptor is invalid. - This function is rarely needed, but can be used to get or set socket options on - a socket passed to a program as standard input or output (such as a server - started by the Unix inet daemon). The socket is assumed to be in blocking mode. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - -.. function:: fromshare(data) - - Instantiate a socket from data obtained from the :meth:`socket.share` - method. The socket is assumed to be in blocking mode. - - Availability: Windows. - - .. versionadded:: 3.3 - - -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - -Other functions -''''''''''''''' - -The :mod:`socket` module also offers various network-related services: - - .. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) Translate the *host*/*port* argument into a sequence of 5-tuples that contain @@ -510,7 +338,7 @@ ``(family, type, proto, canonname, sockaddr)`` In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be + meant to be passed to the :func:`socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -520,17 +348,15 @@ method. The following example fetches address information for a hypothetical TCP - connection to ``example.org`` on port 80 (results may differ on your + connection to ``www.python.org`` on port 80 (results may differ on your system if IPv6 isn't enabled):: - >>> socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP) - [(, , - 6, '', ('2606:2800:220:1:248:1893:25c8:1946', 80, 0, 0)), - (, , - 6, '', ('93.184.216.34', 80))] + >>> socket.getaddrinfo("www.python.org", 80, proto=socket.SOL_TCP) + [(2, 1, 6, '', ('82.94.164.162', 80)), + (10, 1, 6, '', ('2001:888:2000:d::a2', 80, 0, 0))] .. versionchanged:: 3.2 - parameters can now be passed using keyword arguments. + parameters can now be passed as single keyword arguments. .. function:: getfqdn([name]) @@ -568,8 +394,13 @@ Return a string containing the hostname of the machine where the Python interpreter is currently executing. + If you want to know the current machine's IP address, you may want to use + ``gethostbyname(gethostname())``. This operation assumes that there is a + valid address-to-host mapping for the host, and the assumption does not + always hold. + Note: :func:`gethostname` doesn't always return the fully qualified domain - name; use :func:`getfqdn` for that. + name; use ``getfqdn()`` (see above). .. function:: gethostbyaddr(ip_address) @@ -594,7 +425,7 @@ .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` + suitable for passing as the (optional) third argument to the :func:`socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -614,6 +445,46 @@ ``'udp'``, otherwise any protocol will match. +.. function:: socket([family[, type[, proto]]]) + + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN` or :const:`AF_RDS`. The + socket type should be :const:`SOCK_STREAM` (the default), + :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` + constants. The protocol number is usually zero and may be omitted in that + case or :const:`CAN_RAW` in case the address family is :const:`AF_CAN`. + + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. + + +.. function:: socketpair([family[, type[, proto]]]) + + Build a pair of connected socket objects using the given address family, socket + type, and protocol number. Address family, socket type, and protocol number are + as for the :func:`socket` function above. The default family is :const:`AF_UNIX` + if defined on the platform; otherwise, the default is :const:`AF_INET`. + Availability: Unix. + + .. versionchanged:: 3.2 + The returned socket objects now support the whole socket API, rather + than a subset. + + +.. function:: fromfd(fd, family, type[, proto]) + + Duplicate the file descriptor *fd* (an integer as returned by a file object's + :meth:`fileno` method) and build a socket object from the result. Address + family, socket type and protocol number are as for the :func:`socket` function + above. The file descriptor should refer to a socket, but this is not checked --- + subsequent operations on the object may fail if the file descriptor is invalid. + This function is rarely needed, but can be used to get or set socket options on + a socket passed to a program as standard input or output (such as a server + started by the Unix inet daemon). The socket is assumed to be in blocking mode. + + .. function:: ntohl(x) Convert 32-bit positive integers from network to host byte order. On machines @@ -663,8 +534,8 @@ .. function:: inet_ntoa(packed_ip) - Convert a 32-bit packed IPv4 address (a :term:`bytes-like object` four - bytes in length) to its standard dotted-quad string representation (for example, + Convert a 32-bit packed IPv4 address (a bytes object four characters in + length) to its standard dotted-quad string representation (for example, '123.45.67.89'). This is useful when conversing with a program that uses the standard C library and needs objects of type :c:type:`struct in_addr`, which is the C type for the 32-bit packed binary data this function takes as an @@ -675,9 +546,6 @@ support IPv6, and :func:`inet_ntop` should be used instead for IPv4/v6 dual stack support. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. function:: inet_pton(address_family, ip_string) @@ -692,33 +560,23 @@ both the value of *address_family* and the underlying implementation of :c:func:`inet_pton`. - Availability: Unix (maybe not all platforms), Windows. - - .. versionchanged:: 3.4 - Windows support added + Availability: Unix (maybe not all platforms). .. function:: inet_ntop(address_family, packed_ip) - Convert a packed IP address (a :term:`bytes-like object` of some number of - bytes) to its standard, family-specific string representation (for - example, ``'7.10.0.5'`` or ``'5aef:2b::8'``). - :func:`inet_ntop` is useful when a library or network protocol returns an - object of type :c:type:`struct in_addr` (similar to :func:`inet_ntoa`) or - :c:type:`struct in6_addr`. + Convert a packed IP address (a bytes object of some number of characters) to its + standard, family-specific string representation (for example, ``'7.10.0.5'`` or + ``'5aef:2b::8'``). :func:`inet_ntop` is useful when a library or network protocol + returns an object of type :c:type:`struct in_addr` (similar to :func:`inet_ntoa`) + or :c:type:`struct in6_addr`. Supported values for *address_family* are currently :const:`AF_INET` and - :const:`AF_INET6`. If the bytes object *packed_ip* is not the correct - length for the specified address family, :exc:`ValueError` will be raised. + :const:`AF_INET6`. If the string *packed_ip* is not the correct length for the + specified address family, :exc:`ValueError` will be raised. A :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`. - Availability: Unix (maybe not all platforms), Windows. - - .. versionchanged:: 3.4 - Windows support added - - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. + Availability: Unix (maybe not all platforms). .. @@ -781,7 +639,7 @@ .. function:: sethostname(name) - Set the machine's hostname to *name*. This will raise an + Set the machine's hostname to *name*. This will raise a :exc:`OSError` if you don't have enough rights. Availability: Unix. @@ -813,7 +671,7 @@ .. function:: if_indextoname(if_index) - Return a network interface name corresponding to an + Return a network interface name corresponding to a interface index number. :exc:`OSError` if no interface with the given index exists. @@ -822,14 +680,19 @@ .. versionadded:: 3.3 +.. data:: SocketType + + This is a Python type object that represents the socket object type. It is the + same as ``type(socket(...))``. + + .. _socket-objects: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +Socket objects have the following methods. Except for :meth:`makefile` these +correspond to Unix system calls applicable to sockets. .. method:: socket.accept() @@ -839,16 +702,6 @@ *new* socket object usable to send and receive data on the connection, and *address* is the address bound to the socket on the other end of the connection. - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.bind(address) @@ -858,18 +711,11 @@ .. method:: socket.close() - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile()` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). - - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. + Close the socket. All future operations on the socket object will fail. The + remote end will receive no more data (after queued data is flushed). Sockets are + automatically closed when they are garbage-collected. .. note:: - :meth:`close()` releases the resource associated with a connection but does not necessarily close the connection immediately. If you want to close the connection in a timely fashion, call :meth:`shutdown()` @@ -881,19 +727,6 @@ Connect to a remote socket at *address*. (The format of *address* depends on the address family --- see above.) - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`socket.timeout` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). - .. method:: socket.connect_ex(address) @@ -914,16 +747,6 @@ .. versionadded:: 3.2 -.. method:: socket.dup() - - Duplicate the socket. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - .. method:: socket.fileno() Return the socket's file descriptor (a small integer). This is useful with @@ -934,15 +757,6 @@ this limitation. -.. method:: socket.get_inheritable() - - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. - - .. versionadded:: 3.4 - - .. method:: socket.getpeername() Return the remote address to which the socket is connected. This is useful to @@ -989,15 +803,12 @@ On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` functions may be used; they accept a socket object as their first argument. -.. method:: socket.listen([backlog]) +.. method:: socket.listen(backlog) - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. + Listen for connections made to the socket. The *backlog* argument specifies the + maximum number of queued connections and should be at least 0; the maximum value + is system-dependent (usually 5), the minimum value is forced to 0. - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. .. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ errors=None, newline=None) @@ -1008,13 +819,10 @@ type depends on the arguments given to :meth:`makefile`. These arguments are interpreted the same way as by the built-in :func:`open` function. - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. - - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Closing the file object won't close the socket unless there are no remaining + references to the socket. The socket must be in blocking mode; it can have + a timeout, but the file object's internal buffer may end up in a inconsistent + state if a timeout occurs. .. note:: @@ -1035,11 +843,6 @@ For best match with hardware and network realities, the value of *bufsize* should be a relatively small power of 2, for example, 4096. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.recvfrom(bufsize[, flags]) @@ -1049,11 +852,6 @@ :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults to zero. (The format of *address* depends on the address family --- see above.) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) @@ -1120,11 +918,6 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) @@ -1191,11 +984,6 @@ application needs to attempt delivery of the remaining data. For further information on this topic, consult the :ref:`socket-howto`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.sendall(bytes[, flags]) @@ -1206,18 +994,8 @@ success. On error, an exception is raised, and there is no way to determine how much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no more reset each time data is sent successfuly. - The socket timeout is now the maximum total duration to send all data. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) +.. method:: socket.sendto(bytes[, flags], address) Send data to the socket. The socket should not be connected to a remote socket, since the destination socket is specified by *address*. The optional *flags* @@ -1225,19 +1003,13 @@ bytes sent. (The format of *address* depends on the address family --- see above.) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) Send normal and ancillary data to the socket, gathering the non-ancillary data from a series of buffers and concatenating it into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` + non-ancillary data as an iterable of buffer-compatible objects (e.g. :class:`bytes` objects); the operating system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers that can be used. The *ancdata* argument specifies the ancillary @@ -1245,7 +1017,7 @@ ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and *cmsg_type* are integers specifying the protocol level and protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that + buffer-compatible object holding the associated data. Note that some systems (in particular, systems without :func:`CMSG_SPACE`) might support sending only one control message per call. The *flags* argument defaults to 0 and has the same meaning as for @@ -1266,34 +1038,6 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - -.. method:: socket.sendfile(file, offset=0, count=None) - - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. Non- - blocking sockets are not supported. - - .. versionadded:: 3.5 - -.. method:: socket.set_inheritable(inheritable) - - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - - .. versionadded:: 3.4 - .. method:: socket.setblocking(flag) @@ -1325,15 +1069,11 @@ Set the value of the given socket option (see the Unix manual page :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer or - a :term:`bytes-like object` representing a buffer. In the latter case it is - up to the caller to + :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer or a + bytes object representing a buffer. In the latter case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to encode C structures as bytestrings). - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - .. method:: socket.shutdown(how) @@ -1342,21 +1082,6 @@ are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are disallowed. - -.. method:: socket.share(process_id) - - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. - - Availability: Windows. - - .. versionadded:: 3.3 - - Note that there are no methods :meth:`read` or :meth:`write`; use :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. @@ -1441,10 +1166,10 @@ Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, +using it. Note that a server must perform the sequence :func:`socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. @@ -1580,16 +1305,7 @@ s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF) The last example shows how to use the socket interface to communicate to a CAN -network using the raw socket protocol. To use CAN with the broadcast -manager protocol instead, open a socket with:: - - socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) - -After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you -can use the :meth:`socket.send`, and the :meth:`socket.recv` operations (and -their counterparts) on the socket object as usual. - -This example might require special privileges:: +network. This example might require special priviledge:: import socket import struct @@ -1663,3 +1379,4 @@ details of socket semantics. For Unix, refer to the manual pages; for Windows, see the WinSock (or Winsock 2) specification. For IPv6-ready APIs, readers may want to refer to :rfc:`3493` titled Basic Socket Interface Extensions for IPv6. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/socketserver.rst --- a/Doc/library/socketserver.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/socketserver.rst Mon Jan 25 17:05:13 2016 +0100 @@ -33,10 +33,9 @@ handler class by subclassing the :class:`BaseRequestHandler` class and overriding its :meth:`handle` method; this method will process incoming requests. Second, you must instantiate one of the server classes, passing it -the server's address and the request handler class. Then call the +the server's address and the request handler class. Finally, call the :meth:`handle_request` or :meth:`serve_forever` method of the server object to -process one or many requests. Finally, call :meth:`~BaseServer.server_close` -to close the socket. +process one or many requests. When inheriting from :class:`ThreadingMixIn` for threaded connection behavior, you should explicitly declare how you want your threads to behave on an abrupt @@ -112,13 +111,13 @@ the request handler class :meth:`handle` method. Another approach to handling multiple simultaneous requests in an environment -that supports neither threads nor :func:`~os.fork` (or where these are too -expensive or inappropriate for the service) is to maintain an explicit table of -partially finished requests and to use :mod:`selectors` to decide which -request to work on next (or whether to handle a new incoming request). This is -particularly important for stream services where each client can potentially be -connected for a long time (if threads or subprocesses cannot be used). See -:mod:`asyncore` for another way to manage this. +that supports neither threads nor :func:`fork` (or where these are too expensive +or inappropriate for the service) is to maintain an explicit table of partially +finished requests and to use :func:`select` to decide which request to work on +next (or whether to handle a new incoming request). This is particularly +important for stream services where each client can potentially be connected for +a long time (if threads or subprocesses cannot be used). See :mod:`asyncore` +for another way to manage this. .. XXX should data and methods be intermingled, or separate? how should the distinction between class and instance variables be drawn? @@ -137,7 +136,7 @@ .. method:: BaseServer.fileno() Return an integer file descriptor for the socket on which the server is - listening. This function is most commonly passed to :mod:`selectors`, to + listening. This function is most commonly passed to :func:`select.select`, to allow monitoring multiple servers in the same process. @@ -154,22 +153,20 @@ .. method:: BaseServer.serve_forever(poll_interval=0.5) - Handle requests until an explicit :meth:`shutdown` request. Poll for - shutdown every *poll_interval* seconds. Ignores :attr:`self.timeout`. It - also calls :meth:`service_actions`, which may be used by a subclass or mixin - to provide actions specific to a given service. For example, the - :class:`ForkingMixIn` class uses :meth:`service_actions` to clean up zombie - child processes. + Handle requests until an explicit :meth:`shutdown` request. + Poll for shutdown every *poll_interval* seconds. Ignores :attr:`self.timeout`. It also calls + :meth:`service_actions` which may be used by a subclass or Mixin to provide + various cleanup actions. For e.g. ForkingMixin class uses + :meth:`service_actions` to cleanup the zombie child processes. .. versionchanged:: 3.3 - Added ``service_actions`` call to the ``serve_forever`` method. + Added service_actions call to the serve_forever method. .. method:: BaseServer.service_actions() - This is called in the :meth:`serve_forever` loop. This method can be - overridden by subclasses or mixin classes to perform actions specific to - a given service, such as cleanup actions. + This is called by the serve_forever loop. This method is can be overridden + by Mixin's to add cleanup or service specific actions. .. versionadded:: 3.3 @@ -178,13 +175,6 @@ Tell the :meth:`serve_forever` loop to stop and wait until it does. -.. method:: BaseServer.server_close() - - Clean up the server. May be overridden. - - .. versionadded:: 2.6 - - .. attribute:: BaseServer.address_family The family of protocols to which the server's socket belongs. @@ -321,8 +311,8 @@ .. method:: RequestHandler.finish() Called after the :meth:`handle` method to perform any clean-up actions - required. The default implementation does nothing. If :meth:`setup` - raises an exception, this function will not be called. + required. The default implementation does nothing. If :meth:`setup` or + :meth:`handle` raise an exception, this function will not be called. .. method:: RequestHandler.handle() @@ -555,7 +545,6 @@ client(ip, port, "Hello World 3") server.shutdown() - server.server_close() The output of the example should look something like this:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/someos.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/someos.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,24 @@ +.. _someos: + +********************************** +Optional Operating System Services +********************************** + +The modules described in this chapter provide interfaces to operating system +features that are available on selected operating systems only. The interfaces +are generally modeled after the Unix or C interfaces but they are available on +some other systems as well (e.g. Windows). Here's an overview: + + +.. toctree:: + + select.rst + threading.rst + multiprocessing.rst + concurrent.futures.rst + mmap.rst + readline.rst + rlcompleter.rst + dummy_threading.rst + _thread.rst + _dummy_thread.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/spwd.rst --- a/Doc/library/spwd.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/spwd.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,9 +19,9 @@ +-------+---------------+---------------------------------+ | Index | Attribute | Meaning | +=======+===============+=================================+ -| 0 | ``sp_namp`` | Login name | +| 0 | ``sp_nam`` | Login name | +-------+---------------+---------------------------------+ -| 1 | ``sp_pwdp`` | Encrypted password | +| 1 | ``sp_pwd`` | Encrypted password | +-------+---------------+---------------------------------+ | 2 | ``sp_lstchg`` | Date of last change | +-------+---------------+---------------------------------+ @@ -36,15 +36,15 @@ +-------+---------------+---------------------------------+ | 6 | ``sp_inact`` | Number of days after password | | | | expires until account is | -| | | disabled | +| | | blocked | +-------+---------------+---------------------------------+ | 7 | ``sp_expire`` | Number of days since 1970-01-01 | -| | | when account expires | +| | | until account is disabled | +-------+---------------+---------------------------------+ | 8 | ``sp_flag`` | Reserved | +-------+---------------+---------------------------------+ -The sp_namp and sp_pwdp items are strings, all others are integers. +The sp_nam and sp_pwd items are strings, all others are integers. :exc:`KeyError` is raised if the entry asked for cannot be found. The following functions are defined: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sqlite3.rst --- a/Doc/library/sqlite3.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sqlite3.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,15 +13,15 @@ application using SQLite and then port the code to a larger database such as PostgreSQL or Oracle. -The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`. +sqlite3 was written by Gerhard Häring and provides a SQL interface compliant +with the DB-API 2.0 specification described by :pep:`249`. To use the module, you must first create a :class:`Connection` object that represents the database. Here the data will be stored in the -:file:`example.db` file:: +:file:`/tmp/example` file:: import sqlite3 - conn = sqlite3.connect('example.db') + conn = sqlite3.connect('/tmp/example') You can also supply the special name ``:memory:`` to create a database in RAM. @@ -31,29 +31,23 @@ c = conn.cursor() # Create table - c.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') + c.execute('''create table stocks + (date text, trans text, symbol text, + qty real, price real)''') # Insert a row of data - c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + c.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") # Save (commit) the changes conn.commit() - # We can also close the connection if we are done with it. - # Just be sure any changes have been committed or they will be lost. - conn.close() - -The data you've saved is persistent and is available in subsequent sessions:: - - import sqlite3 - conn = sqlite3.connect('example.db') - c = conn.cursor() + # We can also close the cursor if we are done with it + c.close() Usually your SQL operations will need to use values from Python variables. You shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see http://xkcd.com/327/ for humorous example of what can go wrong). +is insecure; it makes your program vulnerable to an SQL injection attack. Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder wherever you want to use a value, and then provide a tuple of values as the @@ -62,20 +56,19 @@ example:: # Never do this -- insecure! - symbol = 'RHAT' - c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + symbol = 'IBM' + c.execute("select * from stocks where symbol = '%s'" % symbol) # Do this instead - t = ('RHAT',) - c.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(c.fetchone()) + t = (symbol,) + c.execute('select * from stocks where symbol=?', t) - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + # Larger example + for t in [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), + ('2006-04-06', 'SELL', 'IBM', 500, 53.00), + ]: + c.execute('insert into stocks values (?,?,?,?,?)', t) To retrieve data after executing a SELECT statement, you can either treat the cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to @@ -84,18 +77,21 @@ This example uses the iterator form:: - >>> for row in c.execute('SELECT * FROM stocks ORDER BY price'): - print(row) - + >>> c = conn.cursor() + >>> c.execute('select * from stocks order by price') + >>> for row in c: + ... print(row) + ... ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) + ('2006-04-05', 'BUY', 'MSOFT', 1000, 72.0) + >>> .. seealso:: - https://github.com/ghaering/pysqlite + http://code.google.com/p/pysqlite/ The pysqlite web page -- sqlite3 is developed externally under the name "pysqlite". @@ -103,9 +99,6 @@ The SQLite web page; the documentation describes the syntax and the available data types for the supported SQL dialect. - http://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - :pep:`249` - Database API Specification 2.0 PEP written by Marc-André Lemburg. @@ -116,28 +109,6 @@ ------------------------------ -.. data:: version - - The version number of this module, as a string. This is not the version of - the SQLite library. - - -.. data:: version_info - - The version number of this module, as a tuple of integers. This is not the - version of the SQLite library. - - -.. data:: sqlite_version - - The version number of the run-time SQLite library, as a string. - - -.. data:: sqlite_version_info - - The version number of the run-time SQLite library, as a tuple of integers. - - .. data:: PARSE_DECLTYPES This constant is meant to be used with the *detect_types* parameter of the @@ -166,7 +137,7 @@ first blank for the column name: the column name would simply be "x". -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) +.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements]) Opens a connection to the SQLite database file *database*. You can use ``":memory:"`` to open a database connection to a database that resides in RAM @@ -181,7 +152,7 @@ For the *isolation_level* parameter, please see the :attr:`Connection.isolation_level` property of :class:`Connection` objects. - SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If + SQLite natively supports only the types TEXT, INTEGER, FLOAT, BLOB and NULL. If you want to use other types you must add support for them yourself. The *detect_types* parameter and the using custom **converters** registered with the module-level :func:`register_converter` function allow you to easily do that. @@ -202,18 +173,6 @@ for the connection, you can set the *cached_statements* parameter. The currently implemented default is to cache 100 statements. - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. - - .. versionchanged:: 3.4 - Added the *uri* parameter. - .. function:: register_converter(typename, callable) @@ -248,10 +207,10 @@ .. function:: enable_callback_tracebacks(flag) By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. + aggregates, converters, authorizer callbacks etc. If you want to debug them, you + can call this function with *flag* as True. Afterwards, you will get tracebacks + from callbacks on ``sys.stderr``. Use :const:`False` to disable the feature + again. .. _sqlite3-connection-objects: @@ -263,254 +222,250 @@ A SQLite database connection has the following attributes and methods: - .. attribute:: isolation_level +.. attribute:: Connection.isolation_level - Get or set the current isolation level. :const:`None` for autocommit mode or - one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section - :ref:`sqlite3-controlling-transactions` for a more detailed explanation. + Get or set the current isolation level. :const:`None` for autocommit mode or + one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section + :ref:`sqlite3-controlling-transactions` for a more detailed explanation. - .. attribute:: in_transaction +.. attribute:: Connection.in_transaction - :const:`True` if a transaction is active (there are uncommitted changes), - :const:`False` otherwise. Read-only attribute. + :const:`True` if a transaction is active (there are uncommitted changes), + :const:`False` otherwise. Read-only attribute. - .. versionadded:: 3.2 + .. versionadded:: 3.2 - .. method:: cursor([cursorClass]) +.. method:: Connection.cursor([cursorClass]) - The cursor method accepts a single optional parameter *cursorClass*. If - supplied, this must be a custom cursor class that extends - :class:`sqlite3.Cursor`. + The cursor method accepts a single optional parameter *cursorClass*. If + supplied, this must be a custom cursor class that extends + :class:`sqlite3.Cursor`. - .. method:: commit() +.. method:: Connection.commit() - This method commits the current transaction. If you don't call this method, - anything you did since the last call to ``commit()`` is not visible from - other database connections. If you wonder why you don't see the data you've - written to the database, please check you didn't forget to call this method. + This method commits the current transaction. If you don't call this method, + anything you did since the last call to ``commit()`` is not visible from + other database connections. If you wonder why you don't see the data you've + written to the database, please check you didn't forget to call this method. - .. method:: rollback() +.. method:: Connection.rollback() - This method rolls back any changes to the database since the last call to - :meth:`commit`. + This method rolls back any changes to the database since the last call to + :meth:`commit`. - .. method:: close() +.. method:: Connection.close() - This closes the database connection. Note that this does not automatically - call :meth:`commit`. If you just close your database connection without - calling :meth:`commit` first, your changes will be lost! + This closes the database connection. Note that this does not automatically + call :meth:`commit`. If you just close your database connection without + calling :meth:`commit` first, your changes will be lost! - .. method:: execute(sql, [parameters]) +.. method:: Connection.execute(sql, [parameters]) - This is a nonstandard shortcut that creates an intermediate cursor object by - calling the cursor method, then calls the cursor's :meth:`execute - ` method with the parameters given. + This is a nonstandard shortcut that creates an intermediate cursor object by + calling the cursor method, then calls the cursor's :meth:`execute + ` method with the parameters given. - .. method:: executemany(sql, [parameters]) +.. method:: Connection.executemany(sql, [parameters]) - This is a nonstandard shortcut that creates an intermediate cursor object by - calling the cursor method, then calls the cursor's :meth:`executemany - ` method with the parameters given. + This is a nonstandard shortcut that creates an intermediate cursor object by + calling the cursor method, then calls the cursor's :meth:`executemany + ` method with the parameters given. - .. method:: executescript(sql_script) +.. method:: Connection.executescript(sql_script) - This is a nonstandard shortcut that creates an intermediate cursor object by - calling the cursor method, then calls the cursor's :meth:`executescript - ` method with the parameters given. + This is a nonstandard shortcut that creates an intermediate cursor object by + calling the cursor method, then calls the cursor's :meth:`executescript + ` method with the parameters given. - .. method:: create_function(name, num_params, func) +.. method:: Connection.create_function(name, num_params, func) - Creates a user-defined function that you can later use from within SQL - statements under the function name *name*. *num_params* is the number of - parameters the function accepts, and *func* is a Python callable that is called - as the SQL function. + Creates a user-defined function that you can later use from within SQL + statements under the function name *name*. *num_params* is the number of + parameters the function accepts, and *func* is a Python callable that is called + as the SQL function. - The function can return any of the types supported by SQLite: bytes, str, int, - float and None. + The function can return any of the types supported by SQLite: bytes, str, int, + float and None. - Example: + Example: - .. literalinclude:: ../includes/sqlite3/md5func.py + .. literalinclude:: ../includes/sqlite3/md5func.py - .. method:: create_aggregate(name, num_params, aggregate_class) +.. method:: Connection.create_aggregate(name, num_params, aggregate_class) - Creates a user-defined aggregate function. + Creates a user-defined aggregate function. - The aggregate class must implement a ``step`` method, which accepts the number - of parameters *num_params*, and a ``finalize`` method which will return the - final result of the aggregate. + The aggregate class must implement a ``step`` method, which accepts the number + of parameters *num_params*, and a ``finalize`` method which will return the + final result of the aggregate. - The ``finalize`` method can return any of the types supported by SQLite: - bytes, str, int, float and None. + The ``finalize`` method can return any of the types supported by SQLite: + bytes, str, int, float and None. - Example: + Example: - .. literalinclude:: ../includes/sqlite3/mysumaggr.py + .. literalinclude:: ../includes/sqlite3/mysumaggr.py - .. method:: create_collation(name, callable) +.. method:: Connection.create_collation(name, callable) - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. + Creates a collation with the specified *name* and *callable*. The callable will + be passed two string arguments. It should return -1 if the first is ordered + lower than the second, 0 if they are ordered equal and 1 if the first is ordered + higher than the second. Note that this controls sorting (ORDER BY in SQL) so + your comparisons don't affect other SQL operations. - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. + Note that the callable will get its parameters as Python bytestrings, which will + normally be encoded in UTF-8. - The following example shows a custom collation that sorts "the wrong way": + The following example shows a custom collation that sorts "the wrong way": - .. literalinclude:: ../includes/sqlite3/collation_reverse.py + .. literalinclude:: ../includes/sqlite3/collation_reverse.py - To remove a collation, call ``create_collation`` with None as callable:: + To remove a collation, call ``create_collation`` with None as callable:: - con.create_collation("reverse", None) + con.create_collation("reverse", None) - .. method:: interrupt() +.. method:: Connection.interrupt() - You can call this method from a different thread to abort any queries that might - be executing on the connection. The query will then abort and the caller will - get an exception. + You can call this method from a different thread to abort any queries that might + be executing on the connection. The query will then abort and the caller will + get an exception. - .. method:: set_authorizer(authorizer_callback) +.. method:: Connection.set_authorizer(authorizer_callback) - This routine registers a callback. The callback is invoked for each attempt to - access a column of a table in the database. The callback should return - :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL - statement should be aborted with an error and :const:`SQLITE_IGNORE` if the - column should be treated as a NULL value. These constants are available in the - :mod:`sqlite3` module. + This routine registers a callback. The callback is invoked for each attempt to + access a column of a table in the database. The callback should return + :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL + statement should be aborted with an error and :const:`SQLITE_IGNORE` if the + column should be treated as a NULL value. These constants are available in the + :mod:`sqlite3` module. - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or :const:`None` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - :const:`None` if this access attempt is directly from input SQL code. + The first argument to the callback signifies what kind of operation is to be + authorized. The second and third argument will be arguments or :const:`None` + depending on the first argument. The 4th argument is the name of the database + ("main", "temp", etc.) if applicable. The 5th argument is the name of the + inner-most trigger or view that is responsible for the access attempt or + :const:`None` if this access attempt is directly from input SQL code. - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`sqlite3` module. + Please consult the SQLite documentation about the possible values for the first + argument and the meaning of the second and third argument depending on the first + one. All necessary constants are available in the :mod:`sqlite3` module. - .. method:: set_progress_handler(handler, n) +.. method:: Connection.set_progress_handler(handler, n) - This routine registers a callback. The callback is invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. + This routine registers a callback. The callback is invoked for every *n* + instructions of the SQLite virtual machine. This is useful if you want to + get called from SQLite during long-running operations, for example to update + a GUI. - If you want to clear any previously installed progress handler, call the - method with :const:`None` for *handler*. + If you want to clear any previously installed progress handler, call the + method with :const:`None` for *handler*. - .. method:: set_trace_callback(trace_callback) +.. method:: Connection.set_trace_callback(trace_callback) - Registers *trace_callback* to be called for each SQL statement that is - actually executed by the SQLite backend. + Registers *trace_callback* to be called for each SQL statement that is + actually executed by the SQLite backend. - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. + The only argument passed to the callback is the statement (as string) that + is being executed. The return value of the callback is ignored. Note that + the backend does not only run statements passed to the :meth:`Cursor.execute` + methods. Other sources include the transaction management of the Python + module and the execution of triggers defined in the current database. - Passing :const:`None` as *trace_callback* will disable the trace callback. + Passing :const:`None` as *trace_callback* will disable the trace callback. - .. versionadded:: 3.3 + .. versionadded:: 3.3 - .. method:: enable_load_extension(enabled) +.. method:: Connection.enable_load_extension(enabled) - This routine allows/disallows the SQLite engine to load SQLite extensions - from shared libraries. SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. + This routine allows/disallows the SQLite engine to load SQLite extensions + from shared libraries. SQLite extensions can define new functions, + aggregates or whole new virtual table implementations. One well-known + extension is the fulltext-search extension distributed with SQLite. - Loadable extensions are disabled by default. See [#f1]_. + Loadable extensions are disabled by default. See [#f1]_. - .. versionadded:: 3.2 + .. versionadded:: 3.2 - .. literalinclude:: ../includes/sqlite3/load_extension.py + .. literalinclude:: ../includes/sqlite3/load_extension.py - .. method:: load_extension(path) +.. method:: Connection.load_extension(path) - This routine loads a SQLite extension from a shared library. You have to - enable extension loading with :meth:`enable_load_extension` before you can - use this routine. + This routine loads a SQLite extension from a shared library. You have to + enable extension loading with :meth:`enable_load_extension` before you can + use this routine. - Loadable extensions are disabled by default. See [#f1]_. + Loadable extensions are disabled by default. See [#f1]_. - .. versionadded:: 3.2 + .. versionadded:: 3.2 - .. attribute:: row_factory +.. attribute:: Connection.row_factory - You can change this attribute to a callable that accepts the cursor and the - original row as a tuple and will return the real result row. This way, you can - implement more advanced ways of returning results, such as returning an object - that can also access columns by name. + You can change this attribute to a callable that accepts the cursor and the + original row as a tuple and will return the real result row. This way, you can + implement more advanced ways of returning results, such as returning an object + that can also access columns by name. - Example: + Example: - .. literalinclude:: ../includes/sqlite3/row_factory.py + .. literalinclude:: ../includes/sqlite3/row_factory.py - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. + If returning a tuple doesn't suffice and you want name-based access to + columns, you should consider setting :attr:`row_factory` to the + highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both + index-based and case-insensitive name-based access to columns with almost no + memory overhead. It will probably be better than your own custom + dictionary-based approach or even a db_row based solution. - .. XXX what's a db_row-based solution? + .. XXX what's a db_row-based solution? - .. attribute:: text_factory +.. attribute:: Connection.text_factory - Using this attribute you can control what objects are returned for the ``TEXT`` - data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. + Using this attribute you can control what objects are returned for the ``TEXT`` + data type. By default, this attribute is set to :class:`str` and the + :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to + return bytestrings instead, you can set it to :class:`bytes`. - For efficiency reasons, there's also a way to return :class:`str` objects - only for non-ASCII data, and :class:`bytes` otherwise. To activate it, set - this attribute to :const:`sqlite3.OptimizedUnicode`. + You can also set it to any other callable that accepts a single bytestring + parameter and returns the resulting object. - You can also set it to any other callable that accepts a single bytestring - parameter and returns the resulting object. + See the following example code for illustration: - See the following example code for illustration: + .. literalinclude:: ../includes/sqlite3/text_factory.py - .. literalinclude:: ../includes/sqlite3/text_factory.py +.. attribute:: Connection.total_changes - .. attribute:: total_changes + Returns the total number of database rows that have been modified, inserted, or + deleted since the database connection was opened. - Returns the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. +.. attribute:: Connection.iterdump - .. attribute:: iterdump + Returns an iterator to dump the database in an SQL text format. Useful when + saving an in-memory database for later restoration. This function provides + the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` + shell. - Returns an iterator to dump the database in an SQL text format. Useful when - saving an in-memory database for later restoration. This function provides - the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` - shell. + Example:: - Example:: + # Convert file existing_db.db to SQL dump file dump.sql + import sqlite3, os - # Convert file existing_db.db to SQL dump file dump.sql - import sqlite3, os - - con = sqlite3.connect('existing_db.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) + con = sqlite3.connect('existing_db.db') + with open('dump.sql', 'w') as f: + for line in con.iterdump(): + f.write('%s\n' % line) .. _sqlite3-cursor-objects: @@ -522,110 +477,110 @@ A :class:`Cursor` instance has the following attributes and methods. - .. method:: execute(sql, [parameters]) +.. method:: Cursor.execute(sql, [parameters]) - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). + Executes an SQL statement. The SQL statement may be parametrized (i. e. + placeholders instead of SQL literals). The :mod:`sqlite3` module supports two + kinds of placeholders: question marks (qmark style) and named placeholders + (named style). - Here's an example of both styles: + Here's an example of both styles: - .. literalinclude:: ../includes/sqlite3/execute_1.py + .. literalinclude:: ../includes/sqlite3/execute_1.py - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a Warning. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. + :meth:`execute` will only execute a single SQL statement. If you try to execute + more than one statement with it, it will raise a Warning. Use + :meth:`executescript` if you want to execute multiple SQL statements with one + call. - .. method:: executemany(sql, seq_of_parameters) +.. method:: Cursor.executemany(sql, seq_of_parameters) - Executes an SQL command against all parameter sequences or mappings found in - the sequence *sql*. The :mod:`sqlite3` module also allows using an - :term:`iterator` yielding parameters instead of a sequence. + Executes an SQL command against all parameter sequences or mappings found in + the sequence *sql*. The :mod:`sqlite3` module also allows using an + :term:`iterator` yielding parameters instead of a sequence. - .. literalinclude:: ../includes/sqlite3/executemany_1.py + .. literalinclude:: ../includes/sqlite3/executemany_1.py - Here's a shorter example using a :term:`generator`: + Here's a shorter example using a :term:`generator`: - .. literalinclude:: ../includes/sqlite3/executemany_2.py + .. literalinclude:: ../includes/sqlite3/executemany_2.py - .. method:: executescript(sql_script) +.. method:: Cursor.executescript(sql_script) - This is a nonstandard convenience method for executing multiple SQL statements - at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. + This is a nonstandard convenience method for executing multiple SQL statements + at once. It issues a ``COMMIT`` statement first, then executes the SQL script it + gets as a parameter. - *sql_script* can be an instance of :class:`str` or :class:`bytes`. + *sql_script* can be an instance of :class:`str` or :class:`bytes`. - Example: + Example: - .. literalinclude:: ../includes/sqlite3/executescript.py + .. literalinclude:: ../includes/sqlite3/executescript.py - .. method:: fetchone() +.. method:: Cursor.fetchone() - Fetches the next row of a query result set, returning a single sequence, - or :const:`None` when no more data is available. + Fetches the next row of a query result set, returning a single sequence, + or :const:`None` when no more data is available. - .. method:: fetchmany(size=cursor.arraysize) +.. method:: Cursor.fetchmany([size=cursor.arraysize]) - Fetches the next set of rows of a query result, returning a list. An empty - list is returned when no more rows are available. + Fetches the next set of rows of a query result, returning a list. An empty + list is returned when no more rows are available. - The number of rows to fetch per call is specified by the *size* parameter. - If it is not given, the cursor's arraysize determines the number of rows - to be fetched. The method should try to fetch as many rows as indicated by - the size parameter. If this is not possible due to the specified number of - rows not being available, fewer rows may be returned. + The number of rows to fetch per call is specified by the *size* parameter. + If it is not given, the cursor's arraysize determines the number of rows + to be fetched. The method should try to fetch as many rows as indicated by + the size parameter. If this is not possible due to the specified number of + rows not being available, fewer rows may be returned. - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. + Note there are performance considerations involved with the *size* parameter. + For optimal performance, it is usually best to use the arraysize attribute. + If the *size* parameter is used, then it is best for it to retain the same + value from one :meth:`fetchmany` call to the next. - .. method:: fetchall() +.. method:: Cursor.fetchall() - Fetches all (remaining) rows of a query result, returning a list. Note that - the cursor's arraysize attribute can affect the performance of this operation. - An empty list is returned when no rows are available. + Fetches all (remaining) rows of a query result, returning a list. Note that + the cursor's arraysize attribute can affect the performance of this operation. + An empty list is returned when no rows are available. - .. attribute:: rowcount +.. attribute:: Cursor.rowcount - Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this - attribute, the database engine's own support for the determination of "rows - affected"/"rows selected" is quirky. + Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this + attribute, the database engine's own support for the determination of "rows + affected"/"rows selected" is quirky. - For :meth:`executemany` statements, the number of modifications are summed up - into :attr:`rowcount`. + For :meth:`executemany` statements, the number of modifications are summed up + into :attr:`rowcount`. - As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in - case no ``executeXX()`` has been performed on the cursor or the rowcount of the - last operation is not determinable by the interface". This includes ``SELECT`` - statements because we cannot determine the number of rows a query produced - until all rows were fetched. + As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in + case no ``executeXX()`` has been performed on the cursor or the rowcount of the + last operation is not determinable by the interface". This includes ``SELECT`` + statements because we cannot determine the number of rows a query produced + until all rows were fetched. - With SQLite versions before 3.6.5, :attr:`rowcount` is set to 0 if - you make a ``DELETE FROM table`` without any condition. + With SQLite versions before 3.6.5, :attr:`rowcount` is set to 0 if + you make a ``DELETE FROM table`` without any condition. - .. attribute:: lastrowid +.. attribute:: Cursor.lastrowid - This read-only attribute provides the rowid of the last modified row. It is - only set if you issued an ``INSERT`` statement using the :meth:`execute` - method. For operations other than ``INSERT`` or when :meth:`executemany` is - called, :attr:`lastrowid` is set to :const:`None`. + This read-only attribute provides the rowid of the last modified row. It is + only set if you issued a ``INSERT`` statement using the :meth:`execute` + method. For operations other than ``INSERT`` or when :meth:`executemany` is + called, :attr:`lastrowid` is set to :const:`None`. - .. attribute:: description +.. attribute:: Cursor.description - This read-only attribute provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are :const:`None`. + This read-only attribute provides the column names of the last query. To + remain compatible with the Python DB API, it returns a 7-tuple for each + column where the last six items of each tuple are :const:`None`. - It is set for ``SELECT`` statements without any matching rows as well. + It is set for ``SELECT`` statements without any matching rows as well. .. _sqlite3-row-objects: @@ -646,12 +601,9 @@ .. method:: keys - This method returns a list of column names. Immediately after a query, + This method returns a tuple of column names. Immediately after a query, it is the first member of each tuple in :attr:`Cursor.description`. - .. versionchanged:: 3.5 - Added support of slicing. - Let's assume we initialize a table as in the example given above:: conn = sqlite3.connect(":memory:") @@ -724,20 +676,19 @@ This is how SQLite types are converted to Python types by default: -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | :const:`None` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ ++-------------+---------------------------------------------+ +| SQLite type | Python type | ++=============+=============================================+ +| ``NULL`` | :const:`None` | ++-------------+---------------------------------------------+ +| ``INTEGER`` | :class:`int` | ++-------------+---------------------------------------------+ +| ``REAL`` | :class:`float` | ++-------------+---------------------------------------------+ +| ``TEXT`` | depends on text_factory, str by default | ++-------------+---------------------------------------------+ +| ``BLOB`` | :class:`bytes` | ++-------------+---------------------------------------------+ The type system of the :mod:`sqlite3` module is extensible in two ways: you can store additional Python types in a SQLite database via object adaptation, and @@ -753,6 +704,9 @@ sqlite3 module's supported types for SQLite: one of NoneType, int, float, str, bytes. +The :mod:`sqlite3` module uses Python object adaptation, as described in +:pep:`246` for this. The protocol to use is :class:`PrepareProtocol`. + There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python type to one of the supported ones. @@ -808,8 +762,8 @@ .. note:: - Converter functions **always** get called with a :class:`bytes` object, no - matter under which data type you sent the value to SQLite. + Converter functions **always** get called with a string, no matter under which + data type you sent the value to SQLite. :: @@ -850,10 +804,6 @@ .. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - .. _sqlite3-controlling-transactions: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/ssl.rst --- a/Doc/library/ssl.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/ssl.rst Mon Jan 25 17:05:13 2016 +0100 @@ -26,14 +26,7 @@ Some behavior may be platform dependent, since calls are made to the operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.1 and TLSv1.2 come with - openssl version 1.0.1. - -.. warning:: - Don't use this module without reading the :ref:`ssl-security`. Doing so - may lead to a false sense of security, as the default settings of the - ssl module are not necessarily appropriate for your application. - + cause variations in behavior. This section documents the objects and functions in the ``ssl`` module; for more general information about TLS, SSL, and certificates, the reader is referred to @@ -66,22 +59,6 @@ .. versionchanged:: 3.3 :exc:`SSLError` used to be a subtype of :exc:`socket.error`. - .. attribute:: library - - A string mnemonic designating the OpenSSL submodule in which the error - occurred, such as ``SSL``, ``PEM`` or ``X509``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - - .. attribute:: reason - - A string mnemonic designating the reason this error occurred, for - example ``CERTIFICATE_VERIFY_FAILED``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - .. exception:: SSLZeroReturnError A subclass of :exc:`SSLError` raised when trying to read or write and @@ -142,16 +119,13 @@ Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps - the underlying socket in an SSL context. ``sock`` must be a - :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported. - - For client-side sockets, the context construction is lazy; if the - underlying socket isn't connected yet, the context construction will be - performed after :meth:`connect` is called on the socket. For - server-side sockets, if the socket has no remote peer, it is assumed - to be a listening socket, and the server-side SSL wrapping is - automatically performed on client connections accepted via the - :meth:`accept` method. :func:`wrap_socket` may raise :exc:`SSLError`. + the underlying socket in an SSL context. For client-side sockets, the + context construction is lazy; if the underlying socket isn't connected yet, + the context construction will be performed after :meth:`connect` is called on + the socket. For server-side sockets, if the socket has no remote peer, it is + assumed to be a listening socket, and the server-side SSL wrapping is + automatically performed on client connections accepted via the :meth:`accept` + method. :func:`wrap_socket` may raise :exc:`SSLError`. The ``keyfile`` and ``certfile`` parameters specify optional files which contain a certificate to be used to identify the local side of the @@ -187,22 +161,25 @@ .. table:: - ======================== ========= ========= ========== ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1** **TLSv1.1** **TLSv1.2** - ------------------------ --------- --------- ---------- --------- ----------- ----------- - *SSLv2* yes no yes no no no - *SSLv3* no yes yes no no no - *SSLv23* no yes yes yes yes yes - *TLSv1* no no yes yes no no - *TLSv1.1* no no yes no yes no - *TLSv1.2* no no yes no no yes - ======================== ========= ========= ========== ========= =========== =========== + ======================== ========= ========= ========== ========= + *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1** + ------------------------ --------- --------- ---------- --------- + *SSLv2* yes no yes no + *SSLv3* no yes yes no + *SSLv23* yes no yes no + *TLSv1* no no yes yes + ======================== ========= ========= ========== ========= .. note:: Which connections succeed will vary depending on the version of - OpenSSL. For example, before OpenSSL 1.0.0, an SSLv23 client - would always attempt SSLv2 connections. + OpenSSL. For instance, in some older versions of OpenSSL (such + as 0.9.7l on OS X 10.4), an SSLv2 client could not connect to an + SSLv23 server. Another example: beginning with OpenSSL 1.0.0, + an SSLv23 client will not actually attempt SSLv2 connections + unless you explicitly enable SSLv2 ciphers; for example, you + might specify ``"ALL"`` or ``"SSLv2"`` as the *ciphers* parameter + to enable them. The *ciphers* parameter sets the available ciphers for this SSL object. It should be a string in the `OpenSSL cipher list format @@ -225,75 +202,17 @@ .. versionchanged:: 3.2 New optional argument *ciphers*. - -Context creation -^^^^^^^^^^^^^^^^ - -A convenience function helps create :class:`SSLContext` objects for common -purposes. - -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) - - Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, - and usually represent a higher security level than when calling the - :class:`SSLContext` constructor directly. - - *cafile*, *capath*, *cadata* represent optional CA certificates to - trust for certificate verification, as in - :meth:`SSLContext.load_verify_locations`. If all three are - :const:`None`, this function can choose to trust the system's default - CA certificates instead. - - The settings are: :data:`PROTOCOL_SSLv23`, :data:`OP_NO_SSLv2`, and - :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and - without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` - as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` - and either loads CA certificates (when at least one of *cafile*, *capath* or - *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load - default CA certificates. - - .. note:: - The protocol, options, cipher and other settings may change to more - restrictive values anytime without prior deprecation. The values - represent a fair balance between compatibility and security. - - If your application needs specific settings, you should create a - :class:`SSLContext` and apply the settings yourself. - - .. note:: - If you find that when certain older clients or servers attempt to connect - with a :class:`SSLContext` created by this function that they get an error - stating "Protocol or cipher suite mismatch", it may be that they only - support SSL3.0 which this function excludes using the - :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken - `_. If you still wish to continue to - use this function but still allow SSL 3.0 connections you can re-enable - them using:: - - ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) - ctx.options &= ~ssl.OP_NO_SSLv3 - - .. versionadded:: 3.4 - - .. versionchanged:: 3.4.4 - - RC4 was dropped from the default cipher string. - - Random generation ^^^^^^^^^^^^^^^^^ .. function:: RAND_bytes(num) - Return *num* cryptographically strong pseudo-random bytes. Raises an + Returns *num* cryptographically strong pseudo-random bytes. Raises an :class:`SSLError` if the PRNG has not been seeded with enough data or if the operation is not supported by the current RAND method. :func:`RAND_status` can be used to check the status of the PRNG and :func:`RAND_add` can be used to seed the PRNG. - For almost all applications :func:`os.urandom` is preferable. - Read the Wikipedia article, `Cryptographically secure pseudorandom number generator (CSPRNG) `_, @@ -303,8 +222,8 @@ .. function:: RAND_pseudo_bytes(num) - Return (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, - is_cryptographic is ``True`` if the bytes generated are cryptographically + Returns (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, + is_cryptographic is True if the bytes generated are cryptographically strong. Raises an :class:`SSLError` if the operation is not supported by the current RAND method. @@ -313,16 +232,14 @@ for non-cryptographic purposes and for certain purposes in cryptographic protocols, but usually not for key generation etc. - For almost all applications :func:`os.urandom` is preferable. - .. versionadded:: 3.3 .. function:: RAND_status() - Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. + Returns True if the SSL pseudo-random number generator has been seeded with + 'enough' randomness, and False otherwise. You can use :func:`ssl.RAND_egd` + and :func:`ssl.RAND_add` to increase the randomness of the pseudo-random + number generator. .. function:: RAND_egd(path) @@ -335,18 +252,13 @@ See http://egd.sourceforge.net/ or http://prngd.sourceforge.net/ for sources of entropy-gathering daemons. - Availability: not available with LibreSSL. - .. function:: RAND_add(bytes, entropy) - Mix the given *bytes* into the SSL pseudo-random number generator. The + Mixes the given *bytes* into the SSL pseudo-random number generator. The parameter *entropy* (a float) is a lower bound on the entropy contained in string (so you can always use :const:`0.0`). See :rfc:`1750` for more information on sources of entropy. - .. versionchanged: 3.5 - Writable :term:`bytes-like object` is now accepted. - Certificate handling ^^^^^^^^^^^^^^^^^^^^ @@ -355,9 +267,10 @@ Verify that *cert* (in decoded format as returned by :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818` and :rfc:`6125`. In addition to HTTPS, this function - should be suitable for checking the identity of servers in various - SSL-based protocols such as FTPS, IMAPS, POPS and others. + in :rfc:`2818`, except that IP addresses are not currently supported. + In addition to HTTPS, this function should be suitable for checking the + identity of servers in various SSL-based protocols such as FTPS, IMAPS, + POPS and others. :exc:`CertificateError` is raised on failure. On success, the function returns nothing:: @@ -372,45 +285,22 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.3.3 - The function now follows :rfc:`6125`, section 6.4.3 and does neither - match multiple wildcards (e.g. ``*.*.com`` or ``*a*.example.org``) nor - a wildcard inside an internationalized domain names (IDN) fragment. - IDN A-labels such as ``www*.xn--pthon-kva.org`` are still supported, - but ``x*.python.org`` no longer matches ``xn--tda.python.org``. +.. function:: cert_time_to_seconds(timestring) - .. versionchanged:: 3.5 - Matching of IP addresses, when present in the subjectAltName field - of the certificate, is now supported. + Returns a floating-point value containing a normal seconds-after-the-epoch + time value, given the time-string representing the "notBefore" or "notAfter" + date from a certificate. -.. function:: cert_time_to_seconds(cert_time) + Here's an example:: - Return the time in seconds since the Epoch, given the ``cert_time`` - string representing the "notBefore" or "notAfter" date from a - certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C - locale). + >>> import ssl + >>> ssl.cert_time_to_seconds("May 9 00:00:00 2007 GMT") + 1178694000.0 + >>> import time + >>> time.ctime(ssl.cert_time_to_seconds("May 9 00:00:00 2007 GMT")) + 'Wed May 9 00:00:00 2007' - Here's an example: - - .. doctest:: newcontext - - >>> import ssl - >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") - >>> timestamp - 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) - 2018-01-05 09:34:43 - - "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). - - .. versionchanged:: 3.5 - Interpret the input time as a time in UTC as specified by 'GMT' - timezone in the input string. Local timezone was used - previously. Return an integer (no fractions of a second in the - input format) - -.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None) +.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None) Given the address ``addr`` of an SSL-protected server, as a (*hostname*, *port-number*) pair, fetches the server's certificate, and returns it as a @@ -424,10 +314,6 @@ .. versionchanged:: 3.3 This function is now IPv6-compatible. - .. versionchanged:: 3.5 - The default *ssl_version* is changed from :data:`PROTOCOL_SSLv3` to - :data:`PROTOCOL_SSLv23` for maximum compatibility with modern servers. - .. function:: DER_cert_to_PEM_cert(DER_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded @@ -438,61 +324,6 @@ Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of bytes for that same certificate. -.. function:: get_default_verify_paths() - - Returns a named tuple with paths to OpenSSL's default cafile and capath. - The paths are the same as used by - :meth:`SSLContext.set_default_verify_paths`. The return value is a - :term:`named tuple` ``DefaultVerifyPaths``: - - * :attr:`cafile` - resolved path to cafile or None if the file doesn't exist, - * :attr:`capath` - resolved path to capath or None if the directory doesn't exist, - * :attr:`openssl_cafile_env` - OpenSSL's environment key that points to a cafile, - * :attr:`openssl_cafile` - hard coded path to a cafile, - * :attr:`openssl_capath_env` - OpenSSL's environment key that points to a capath, - * :attr:`openssl_capath` - hard coded path to a capath directory - - .. versionadded:: 3.4 - -.. function:: enum_certificates(store_name) - - Retrieve certificates from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. Trust specifies the purpose of the certificate as a set - of OIDS or exactly ``True`` if the certificate is trustworthy for all - purposes. - - Example:: - - >>> ssl.enum_certificates("CA") - [(b'data...', 'x509_asn', {'1.3.6.1.5.5.7.3.1', '1.3.6.1.5.5.7.3.2'}), - (b'data...', 'x509_asn', True)] - - Availability: Windows. - - .. versionadded:: 3.4 - -.. function:: enum_crls(store_name) - - Retrieve CRLs from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. - - Availability: Windows. - - .. versionadded:: 3.4 - - Constants ^^^^^^^^^ @@ -529,91 +360,34 @@ be passed, either to :meth:`SSLContext.load_verify_locations` or as a value of the ``ca_certs`` parameter to :func:`wrap_socket`. -.. data:: VERIFY_DEFAULT - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, certificate - revocation lists (CRLs) are not checked. By default OpenSSL does neither - require nor verify CRLs. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_LEAF - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the - peer cert is check but non of the intermediate CA certificates. The mode - requires a valid CRL that is signed by the peer cert's issuer (its direct - ancestor CA). If no proper has been loaded - :attr:`SSLContext.load_verify_locations`, validation will fail. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_CHAIN - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, CRLs of - all certificates in the peer cert chain are checked. - - .. versionadded:: 3.4 - -.. data:: VERIFY_X509_STRICT - - Possible value for :attr:`SSLContext.verify_flags` to disable workarounds - for broken X.509 certificates. - - .. versionadded:: 3.4 - -.. data:: VERIFY_X509_TRUSTED_FIRST - - Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to - prefer trusted certificates when building the trust chain to validate a - certificate. This flag is enabled by default. - - .. versionadded:: 3.4.4 - -.. data:: PROTOCOL_SSLv23 - - Selects the highest protocol version that both the client and server support. - Despite the name, this option can select "TLS" protocols as well as "SSL". - .. data:: PROTOCOL_SSLv2 Selects SSL version 2 as the channel encryption protocol. - This protocol is not available if OpenSSL is compiled with the - ``OPENSSL_NO_SSL2`` flag. + This protocol is not available if OpenSSL is compiled with OPENSSL_NO_SSL2 + flag. .. warning:: SSL version 2 is insecure. Its use is highly discouraged. +.. data:: PROTOCOL_SSLv23 + + Selects SSL version 2 or 3 as the channel encryption protocol. This is a + setting to use with servers for maximum compatibility with the other end of + an SSL connection, but it may cause the specific ciphers chosen for the + encryption to be of fairly low quality. + .. data:: PROTOCOL_SSLv3 - Selects SSL version 3 as the channel encryption protocol. - - This protocol is not be available if OpenSSL is compiled with the - ``OPENSSL_NO_SSLv3`` flag. - - .. warning:: - - SSL version 3 is insecure. Its use is highly discouraged. + Selects SSL version 3 as the channel encryption protocol. For clients, this + is the maximally compatible SSL variant. .. data:: PROTOCOL_TLSv1 - Selects TLS version 1.0 as the channel encryption protocol. - -.. data:: PROTOCOL_TLSv1_1 - - Selects TLS version 1.1 as the channel encryption protocol. - Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - -.. data:: PROTOCOL_TLSv1_2 - - Selects TLS version 1.2 as the channel encryption protocol. This is the - most modern version, and probably the best choice for maximum protection, - if both sides can speak it. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 + Selects TLS version 1 as the channel encryption protocol. This is the most + modern version, and probably the best choice for maximum protection, if both + sides can speak it. .. data:: OP_ALL @@ -647,22 +421,6 @@ .. versionadded:: 3.2 -.. data:: OP_NO_TLSv1_1 - - Prevents a TLSv1.1 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.1 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - -.. data:: OP_NO_TLSv1_2 - - Prevents a TLSv1.2 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.2 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - .. data:: OP_CIPHER_SERVER_PREFERENCE Use the server's cipher ordering preference, rather than the client's. @@ -695,13 +453,6 @@ .. versionadded:: 3.3 -.. data:: HAS_ALPN - - Whether the OpenSSL library has built-in support for the *Application-Layer - Protocol Negotiation* TLS extension as described in :rfc:`7301`. - - .. versionadded:: 3.5 - .. data:: HAS_ECDH Whether the OpenSSL library has built-in support for Elliptic Curve-based @@ -713,7 +464,9 @@ .. data:: HAS_SNI Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`4366`). + Indication* extension to the SSLv3 and TLSv1 protocols (as defined in + :rfc:`4366`). When true, you can use the *server_hostname* argument to + :meth:`SSLContext.wrap_socket`. .. versionadded:: 3.2 @@ -764,225 +517,85 @@ .. versionadded:: 3.2 -.. data:: ALERT_DESCRIPTION_HANDSHAKE_FAILURE - ALERT_DESCRIPTION_INTERNAL_ERROR - ALERT_DESCRIPTION_* - - Alert Descriptions from :rfc:`5246` and others. The `IANA TLS Alert Registry - `_ - contains this list and references to the RFCs where their meaning is defined. - - Used as the return value of the callback function in - :meth:`SSLContext.set_servername_callback`. - - .. versionadded:: 3.4 - -.. data:: Purpose.SERVER_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate Web servers (therefore, it will - be used to create client-side sockets). - - .. versionadded:: 3.4 - -.. data:: Purpose.CLIENT_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate Web clients (therefore, it will - be used to create server-side sockets). - - .. versionadded:: 3.4 - SSL Sockets ----------- -.. class:: SSLSocket(socket.socket) +SSL sockets provide the following methods of :ref:`socket-objects`: - SSL sockets provide the following methods of :ref:`socket-objects`: +- :meth:`~socket.socket.accept()` +- :meth:`~socket.socket.bind()` +- :meth:`~socket.socket.close()` +- :meth:`~socket.socket.connect()` +- :meth:`~socket.socket.detach()` +- :meth:`~socket.socket.fileno()` +- :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` +- :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` +- :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, + :meth:`~socket.socket.setblocking()` +- :meth:`~socket.socket.listen()` +- :meth:`~socket.socket.makefile()` +- :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + (but passing a non-zero ``flags`` argument is not allowed) +- :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + the same limitation) +- :meth:`~socket.socket.shutdown()` - - :meth:`~socket.socket.accept()` - - :meth:`~socket.socket.bind()` - - :meth:`~socket.socket.close()` - - :meth:`~socket.socket.connect()` - - :meth:`~socket.socket.detach()` - - :meth:`~socket.socket.fileno()` - - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` - - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` - - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` - - :meth:`~socket.socket.listen()` - - :meth:`~socket.socket.makefile()` - - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` - (but passing a non-zero ``flags`` argument is not allowed) - - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with - the same limitation) - - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send()` will be used) - - :meth:`~socket.socket.shutdown()` - - However, since the SSL (and TLS) protocol has its own framing atop - of TCP, the SSL sockets abstraction can, in certain respects, diverge from - the specification of normal, OS-level sockets. See especially the - :ref:`notes on non-blocking sockets `. - - Usually, :class:`SSLSocket` are not created directly, but using the - :func:`wrap_socket` function or the :meth:`SSLContext.wrap_socket` method. - - .. versionchanged:: 3.5 - The :meth:`sendfile` method was added. - - .. versionchanged:: 3.5 - The :meth:`shutdown` does not reset the socket timeout each time bytes - are received or sent. The socket timeout is now to maximum total duration - of the shutdown. - +However, since the SSL (and TLS) protocol has its own framing atop +of TCP, the SSL sockets abstraction can, in certain respects, diverge from +the specification of normal, OS-level sockets. See especially the +:ref:`notes on non-blocking sockets `. SSL sockets also have the following additional methods and attributes: -.. method:: SSLSocket.read(len=0, buffer=None) - - Read up to *len* bytes of data from the SSL socket and return the result as - a ``bytes`` instance. If *buffer* is specified, then read into the buffer - instead, and return the number of bytes read. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the read would block. - - As at any time a re-negotiation is possible, a call to :meth:`read` can also - cause write operations. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration to read up to *len* - bytes. - -.. method:: SSLSocket.write(buf) - - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the write would block. - - As at any time a re-negotiation is possible, a call to :meth:`write` can - also cause read operations. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration to write *buf*. - -.. note:: - - The :meth:`~SSLSocket.read` and :meth:`~SSLSocket.write` methods are the - low-level methods that read and write unencrypted, application-level data - and decrypt/encrypt it to encrypted, wire-level data. These methods - require an active SSL connection, i.e. the handshake was completed and - :meth:`SSLSocket.unwrap` was not called. - - Normally you should use the socket API methods like - :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these - methods. - .. method:: SSLSocket.do_handshake() Perform the SSL setup handshake. - .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the - :attr:`~SSLContext.check_hostname` attribute of the socket's - :attr:`~SSLSocket.context` is true. - - .. versionchanged:: 3.5 - The socket timeout is no more reset each time bytes are received or sent. - The socket timeout is now to maximum total duration of the handshake. - .. method:: SSLSocket.getpeercert(binary_form=False) If there is no certificate for the peer on the other end of the connection, - return ``None``. If the SSL handshake hasn't been done yet, raise - :exc:`ValueError`. + returns ``None``. - If the ``binary_form`` parameter is :const:`False`, and a certificate was + If the parameter ``binary_form`` is :const:`False`, and a certificate was received from the peer, this method returns a :class:`dict` instance. If the certificate was not validated, the dict is empty. If the certificate was - validated, it returns a dict with several keys, amongst them ``subject`` - (the principal for which the certificate was issued) and ``issuer`` - (the principal issuing the certificate). If a certificate contains an - instance of the *Subject Alternative Name* extension (see :rfc:`3280`), - there will also be a ``subjectAltName`` key in the dictionary. + validated, it returns a dict with the keys ``subject`` (the principal for + which the certificate was issued), and ``notAfter`` (the time after which the + certificate should not be trusted). If a certificate contains an instance + of the *Subject Alternative Name* extension (see :rfc:`3280`), there will + also be a ``subjectAltName`` key in the dictionary. - The ``subject`` and ``issuer`` fields are tuples containing the sequence - of relative distinguished names (RDNs) given in the certificate's data - structure for the respective fields, and each RDN is a sequence of - name-value pairs. Here is a real-world example:: + The "subject" field is a tuple containing the sequence of relative + distinguished names (RDNs) given in the certificate's data structure for the + principal, and each RDN is a sequence of name-value pairs:: - {'issuer': ((('countryName', 'IL'),), - (('organizationName', 'StartCom Ltd.'),), - (('organizationalUnitName', - 'Secure Digital Certificate Signing'),), - (('commonName', - 'StartCom Class 2 Primary Intermediate Server CA'),)), - 'notAfter': 'Nov 22 08:15:19 2013 GMT', - 'notBefore': 'Nov 21 03:09:52 2011 GMT', - 'serialNumber': '95F0', - 'subject': ((('description', '571208-SLe257oHY9fVQ07Z'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'California'),), - (('localityName', 'San Francisco'),), - (('organizationName', 'Electronic Frontier Foundation, Inc.'),), - (('commonName', '*.eff.org'),), - (('emailAddress', 'hostmaster@eff.org'),)), - 'subjectAltName': (('DNS', '*.eff.org'), ('DNS', 'eff.org')), - 'version': 3} - - .. note:: - - To validate a certificate for a particular service, you can use the - :func:`match_hostname` function. + {'notAfter': 'Feb 16 16:54:50 2013 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'Delaware'),), + (('localityName', 'Wilmington'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'SSL'),), + (('commonName', 'somemachine.python.org'),))} If the ``binary_form`` parameter is :const:`True`, and a certificate was provided, this method returns the DER-encoded form of the entire certificate as a sequence of bytes, or :const:`None` if the peer did not provide a - certificate. Whether the peer provides a certificate depends on the SSL - socket's role: - - * for a client SSL socket, the server will always provide a certificate, - regardless of whether validation was required; - - * for a server SSL socket, the client will only provide a certificate - when requested by the server; therefore :meth:`getpeercert` will return - :const:`None` if you used :const:`CERT_NONE` (rather than - :const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`). + certificate. This return value is independent of validation; if validation + was required (:const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`), it will have + been validated, but if :const:`CERT_NONE` was used to establish the + connection, the certificate, if present, will not have been validated. .. versionchanged:: 3.2 The returned dictionary includes additional items such as ``issuer`` and ``notBefore``. - .. versionchanged:: 3.4 - :exc:`ValueError` is raised when the handshake isn't done. - The returned dictionary includes additional X509v3 extension items - such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs. - .. method:: SSLSocket.cipher() Returns a three-value tuple containing the name of the cipher being used, the version of the SSL protocol that defines its use, and the number of secret bits being used. If no connection has been established, returns ``None``. -.. method:: SSLSocket.shared_ciphers() - - Return the list of ciphers shared by the client during the handshake. Each - entry of the returned list is a three-value tuple containing the name of the - cipher, the version of the SSL protocol that defines its use, and the number - of secret bits the cipher uses. :meth:`~SSLSocket.shared_ciphers` returns - ``None`` if no connection has been established or the socket is a client - socket. - - .. versionadded:: 3.5 - .. method:: SSLSocket.compression() Return the compression algorithm being used as a string, or ``None`` @@ -1006,22 +619,12 @@ .. versionadded:: 3.3 -.. method:: SSLSocket.selected_alpn_protocol() - - Return the protocol that was selected during the TLS handshake. If - :meth:`SSLContext.set_alpn_protocols` was not called, if the other party does - not support ALPN, if this socket does not support any of the client's - proposed protocols, or if the handshake has not happened yet, ``None`` is - returned. - - .. versionadded:: 3.5 - .. method:: SSLSocket.selected_npn_protocol() - Return the higher-level protocol that was selected during the TLS/SSL - handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or - if the other party does not support NPN, or if the handshake has not yet - happened, this will return ``None``. + Returns the protocol that was selected during the TLS/SSL handshake. If + :meth:`SSLContext.set_npn_protocols` was not called, or if the other party + does not support NPN, or if the handshake has not yet happened, this will + return ``None``. .. versionadded:: 3.3 @@ -1033,21 +636,6 @@ returned socket should always be used for further communication with the other side of the connection, rather than the original socket. -.. method:: SSLSocket.version() - - Return the actual SSL protocol version negotiated by the connection - as a string, or ``None`` is no secure connection is established. - As of this writing, possible return values include ``"SSLv2"``, - ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. - Recent OpenSSL versions may define more return values. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.pending() - - Returns the number of already decrypted bytes available for read, pending on - the connection. - .. attribute:: SSLSocket.context The :class:`SSLContext` object this SSL socket is tied to. If the SSL @@ -1057,20 +645,6 @@ .. versionadded:: 3.2 -.. attribute:: SSLSocket.server_side - - A boolean which is ``True`` for server-side sockets and ``False`` for - client-side sockets. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.server_hostname - - Hostname of the server: :class:`str` type, or ``None`` for server-side - socket or if the hostname was not specified in the constructor. - - .. versionadded:: 3.2 - SSL Contexts ------------ @@ -1086,30 +660,11 @@ Create a new SSL context. You must pass *protocol* which must be one of the ``PROTOCOL_*`` constants defined in this module. - :data:`PROTOCOL_SSLv23` is currently recommended for maximum - interoperability. - - .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose - security settings for a given purpose. + :data:`PROTOCOL_SSLv23` is recommended for maximum interoperability. :class:`SSLContext` objects have the following methods and attributes: -.. method:: SSLContext.cert_store_stats() - - Get statistics about quantities of loaded X.509 certificates, count of - X.509 certificates flagged as CA certificates and certificate revocation - lists as dictionary. - - Example for a context with one CA cert and one other cert:: - - >>> context.cert_store_stats() - {'crl': 0, 'x509_ca': 1, 'x509': 2} - - .. versionadded:: 3.4 - - .. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None) Load a private key and the corresponding certificate. The *certfile* @@ -1140,32 +695,12 @@ .. versionchanged:: 3.3 New optional argument *password*. -.. method:: SSLContext.load_default_certs(purpose=Purpose.SERVER_AUTH) - - Load a set of default "certification authority" (CA) certificates from - default locations. On Windows it loads CA certs from the ``CA`` and - ``ROOT`` system stores. On other systems it calls - :meth:`SSLContext.set_default_verify_paths`. In the future the method may - load CA certificates from other locations, too. - - The *purpose* flag specifies what kind of CA certificates are loaded. The - default settings :data:`Purpose.SERVER_AUTH` loads certificates, that are - flagged and trusted for TLS web server authentication (client side - sockets). :data:`Purpose.CLIENT_AUTH` loads CA certificates for client - certificate verification on the server side. - - .. versionadded:: 3.4 - -.. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None) +.. method:: SSLContext.load_verify_locations(cafile=None, capath=None) Load a set of "certification authority" (CA) certificates used to validate other peers' certificates when :data:`verify_mode` is other than :data:`CERT_NONE`. At least one of *cafile* or *capath* must be specified. - This method can also load certification revocation lists (CRLs) in PEM or - DER format. In order to make use of CRLs, :attr:`SSLContext.verify_flags` - must be configured properly. - The *cafile* string, if present, is the path to a file of concatenated CA certificates in PEM format. See the discussion of :ref:`ssl-certificates` for more information about how to arrange the @@ -1176,29 +711,6 @@ following an `OpenSSL specific layout `_. - The *cadata* object, if present, is either an ASCII string of one or more - PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded - certificates. Like with *capath* extra lines around PEM-encoded - certificates are ignored but at least one certificate must be present. - - .. versionchanged:: 3.4 - New optional argument *cadata* - -.. method:: SSLContext.get_ca_certs(binary_form=False) - - Get a list of loaded "certification authority" (CA) certificates. If the - ``binary_form`` parameter is :const:`False` each list - entry is a dict like the output of :meth:`SSLSocket.getpeercert`. Otherwise - the method returns a list of DER-encoded certificates. The returned list - does not contain certificates from *capath* unless a certificate was - requested and loaded by a SSL connection. - - .. note:: - Certificates in a capath directory aren't loaded unless they have - been used at least once. - - .. versionadded:: 3.4 - .. method:: SSLContext.set_default_verify_paths() Load a set of default "certification authority" (CA) certificates from @@ -1221,23 +733,9 @@ when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will give the currently selected cipher. -.. method:: SSLContext.set_alpn_protocols(protocols) - - Specify which protocols the socket should advertise during the SSL/TLS - handshake. It should be a list of ASCII strings, like ``['http/1.1', - 'spdy/2']``, ordered by preference. The selection of a protocol will happen - during the handshake, and will play out according to :rfc:`7301`. After a - successful handshake, the :meth:`SSLSocket.selected_alpn_protocol` method will - return the agreed-upon protocol. - - This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is - False. - - .. versionadded:: 3.5 - .. method:: SSLContext.set_npn_protocols(protocols) - Specify which protocols the socket should advertise during the SSL/TLS + Specify which protocols the socket should avertise during the SSL/TLS handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, ordered by preference. The selection of a protocol will happen during the handshake, and will play out according to the `NPN draft specification @@ -1250,56 +748,6 @@ .. versionadded:: 3.3 -.. method:: SSLContext.set_servername_callback(server_name_callback) - - Register a callback function that will be called after the TLS Client Hello - handshake message has been received by the SSL/TLS server when the TLS client - specifies a server name indication. The server name indication mechanism - is specified in :rfc:`6066` section 3 - Server Name Indication. - - Only one callback can be set per ``SSLContext``. If *server_name_callback* - is ``None`` then the callback is disabled. Calling this function a - subsequent time will disable the previously registered callback. - - The callback function, *server_name_callback*, will be called with three - arguments; the first being the :class:`ssl.SSLSocket`, the second is a string - that represents the server name that the client is intending to communicate - (or :const:`None` if the TLS Client Hello does not contain a server name) - and the third argument is the original :class:`SSLContext`. The server name - argument is the IDNA decoded server name. - - A typical use of this callback is to change the :class:`ssl.SSLSocket`'s - :attr:`SSLSocket.context` attribute to a new object of type - :class:`SSLContext` representing a certificate chain that matches the server - name. - - Due to the early negotiation phase of the TLS connection, only limited - methods and attributes are usable like - :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`. - :meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.getpeercert`, - :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compress` methods require that - the TLS connection has progressed beyond the TLS Client Hello and therefore - will not contain return meaningful values nor can they be called safely. - - The *server_name_callback* function must return ``None`` to allow the - TLS negotiation to continue. If a TLS failure is required, a constant - :const:`ALERT_DESCRIPTION_* ` can be - returned. Other return values will result in a TLS fatal error with - :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`. - - If there is an IDNA decoding error on the server name, the TLS connection - will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS - alert message to the client. - - If an exception is raised from the *server_name_callback* function the TLS - connection will terminate with a fatal TLS alert message - :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`. - - This method will raise :exc:`NotImplementedError` if the OpenSSL library - had OPENSSL_NO_TLSEXT defined when it was built. - - .. versionadded:: 3.4 - .. method:: SSLContext.load_dh_params(dhfile) Load the key generation parameters for Diffie-Helman (DH) key exchange. @@ -1337,10 +785,7 @@ server_hostname=None) Wrap an existing Python socket *sock* and return an :class:`SSLSocket` - object. *sock* must be a :data:`~socket.SOCK_STREAM` socket; other socket - types are unsupported. - - The returned SSL socket is tied to the context, its settings and + object. The SSL socket is tied to the context, its settings and certificates. The parameters *server_side*, *do_handshake_on_connect* and *suppress_ragged_eofs* have the same meaning as in the top-level :func:`wrap_socket` function. @@ -1348,22 +793,11 @@ On client connections, the optional parameter *server_hostname* specifies the hostname of the service which we are connecting to. This allows a single server to host multiple SSL-based services with distinct certificates, - quite similarly to HTTP virtual hosts. Specifying *server_hostname* will - raise a :exc:`ValueError` if *server_side* is true. - - .. versionchanged:: 3.5 - Always allow a server_hostname to be passed, even if OpenSSL does not - have SNI. - -.. method:: SSLContext.wrap_bio(incoming, outgoing, server_side=False, \ - server_hostname=None) - - Create a new :class:`SSLObject` instance by wrapping the BIO objects - *incoming* and *outgoing*. The SSL routines will read input data from the - incoming BIO and write data to the outgoing BIO. - - The *server_side* and *server_hostname* parameters have the same meaning as - in :meth:`SSLContext.wrap_socket`. + quite similarly to HTTP virtual hosts. Specifying *server_hostname* + will raise a :exc:`ValueError` if the OpenSSL library doesn't have support + for it (that is, if :data:`HAS_SNI` is :const:`False`). Specifying + *server_hostname* will also raise a :exc:`ValueError` if *server_side* + is true. .. method:: SSLContext.session_stats() @@ -1377,33 +811,6 @@ >>> stats['hits'], stats['misses'] (0, 0) -.. attribute:: SSLContext.check_hostname - - Whether to match the peer cert's hostname with :func:`match_hostname` in - :meth:`SSLSocket.do_handshake`. The context's - :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or - :data:`CERT_REQUIRED`, and you must pass *server_hostname* to - :meth:`~SSLContext.wrap_socket` in order to match the hostname. - - Example:: - - import socket, ssl - - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True - context.load_default_certs() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ssl_sock = context.wrap_socket(s, server_hostname='www.verisign.com') - ssl_sock.connect(('www.verisign.com', 443)) - - .. versionadded:: 3.4 - - .. note:: - - This features requires OpenSSL 0.9.8f or newer. - .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. @@ -1420,15 +827,6 @@ The protocol version chosen when constructing the context. This attribute is read-only. -.. attribute:: SSLContext.verify_flags - - The flags for certificate verification operations. You can set flags like - :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL - does neither require nor verify certificate revocation lists (CRLs). - Available only with openssl version 0.9.8+. - - .. versionadded:: 3.4 - .. attribute:: SSLContext.verify_mode Whether to try to verify other peers' certificates and how to behave @@ -1514,9 +912,20 @@ certificate, you need to provide a "CA certs" file, filled with the certificate chains for each issuer you are willing to trust. Again, this file just contains these chains concatenated together. For validation, Python will use the first -chain it finds in the file which matches. The platform's certificates file can -be used by calling :meth:`SSLContext.load_default_certs`, this is done -automatically with :func:`.create_default_context`. +chain it finds in the file which matches. Some "standard" root certificates are +available from various certification authorities: `CACert.org +`_, `Thawte +`_, `Verisign +`_, `Positive SSL +`_ +(used by python.org), `Equifax and GeoTrust +`_. + +In general, if you are using SSL3 or TLS1, you don't need to put the full chain +in your "CA certs" file; you only need the root certificates, and the remote +peer is supposed to furnish the other certificates necessary to chain from its +certificate to a root certificate. See :rfc:`4158` for more discussion of the +way in which certification chains can be built. Combined key and certificate ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1590,100 +999,118 @@ Client-side operation ^^^^^^^^^^^^^^^^^^^^^ -This example creates a SSL context with the recommended security settings -for client sockets, including automatic certificate verification:: +This example connects to an SSL server and prints the server's certificate:: - >>> context = ssl.create_default_context() + import socket, ssl, pprint -If you prefer to tune security settings yourself, you might create -a context from scratch (but beware that you might not get the settings -right):: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # require a certificate from the server + ssl_sock = ssl.wrap_socket(s, + ca_certs="/etc/ca_certs_file", + cert_reqs=ssl.CERT_REQUIRED) + ssl_sock.connect(('www.verisign.com', 443)) + + pprint.pprint(ssl_sock.getpeercert()) + # note that closing the SSLSocket will also close the underlying socket + ssl_sock.close() + +As of January 6, 2012, the certificate printed by this program looks like +this:: + + {'issuer': ((('countryName', 'US'),), + (('organizationName', 'VeriSign, Inc.'),), + (('organizationalUnitName', 'VeriSign Trust Network'),), + (('organizationalUnitName', + 'Terms of use at https://www.verisign.com/rpa (c)06'),), + (('commonName', + 'VeriSign Class 3 Extended Validation SSL SGC CA'),)), + 'notAfter': 'May 25 23:59:59 2012 GMT', + 'notBefore': 'May 26 00:00:00 2010 GMT', + 'serialNumber': '53D2BEF924A7245E83CA01E46CAA2477', + 'subject': ((('1.3.6.1.4.1.311.60.2.1.3', 'US'),), + (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), + (('businessCategory', 'V1.0, Clause 5.(b)'),), + (('serialNumber', '2497886'),), + (('countryName', 'US'),), + (('postalCode', '94043'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('streetAddress', '487 East Middlefield Road'),), + (('organizationName', 'VeriSign, Inc.'),), + (('organizationalUnitName', ' Production Security Services'),), + (('commonName', 'www.verisign.com'),)), + 'subjectAltName': (('DNS', 'www.verisign.com'), + ('DNS', 'verisign.com'), + ('DNS', 'www.verisign.net'), + ('DNS', 'verisign.net'), + ('DNS', 'www.verisign.mobi'), + ('DNS', 'verisign.mobi'), + ('DNS', 'www.verisign.eu'), + ('DNS', 'verisign.eu')), + 'version': 3} + +This other example first creates an SSL context, instructs it to verify +certificates sent by peers, and feeds it a set of recognized certificate +authorities (CA):: >>> context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) >>> context.verify_mode = ssl.CERT_REQUIRED - >>> context.check_hostname = True >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt") -(this snippet assumes your operating system places a bundle of all CA -certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an -error and have to adjust the location) +(it is assumed your operating system places a bundle of all CA certificates +in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an error and have +to adjust the location) When you use the context to connect to a server, :const:`CERT_REQUIRED` validates the server certificate: it ensures that the server certificate was signed with one of the CA certificates, and checks the signature for correctness:: - >>> conn = context.wrap_socket(socket.socket(socket.AF_INET), - ... server_hostname="www.python.org") - >>> conn.connect(("www.python.org", 443)) + >>> conn = context.wrap_socket(socket.socket(socket.AF_INET)) + >>> conn.connect(("linuxfr.org", 443)) -You may then fetch the certificate:: +You should then fetch the certificate and check its fields for conformity:: >>> cert = conn.getpeercert() + >>> ssl.match_hostname(cert, "linuxfr.org") Visual inspection shows that the certificate does identify the desired service -(that is, the HTTPS host ``www.python.org``):: +(that is, the HTTPS host ``linuxfr.org``):: >>> pprint.pprint(cert) - {'OCSP': ('http://ocsp.digicert.com',), - 'caIssuers': ('http://cacerts.digicert.com/DigiCertSHA2ExtendedValidationServerCA.crt',), - 'crlDistributionPoints': ('http://crl3.digicert.com/sha2-ev-server-g1.crl', - 'http://crl4.digicert.com/sha2-ev-server-g1.crl'), - 'issuer': ((('countryName', 'US'),), - (('organizationName', 'DigiCert Inc'),), - (('organizationalUnitName', 'www.digicert.com'),), - (('commonName', 'DigiCert SHA2 Extended Validation Server CA'),)), - 'notAfter': 'Sep 9 12:00:00 2016 GMT', - 'notBefore': 'Sep 5 00:00:00 2014 GMT', - 'serialNumber': '01BB6F00122B177F36CAB49CEA8B6B26', - 'subject': ((('businessCategory', 'Private Organization'),), - (('1.3.6.1.4.1.311.60.2.1.3', 'US'),), - (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), - (('serialNumber', '3359300'),), - (('streetAddress', '16 Allen Rd'),), - (('postalCode', '03894-4801'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'NH'),), - (('localityName', 'Wolfeboro,'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'www.python.org'),)), - 'subjectAltName': (('DNS', 'www.python.org'), - ('DNS', 'python.org'), - ('DNS', 'pypi.python.org'), - ('DNS', 'docs.python.org'), - ('DNS', 'testpypi.python.org'), - ('DNS', 'bugs.python.org'), - ('DNS', 'wiki.python.org'), - ('DNS', 'hg.python.org'), - ('DNS', 'mail.python.org'), - ('DNS', 'packaging.python.org'), - ('DNS', 'pythonhosted.org'), - ('DNS', 'www.pythonhosted.org'), - ('DNS', 'test.pythonhosted.org'), - ('DNS', 'us.pycon.org'), - ('DNS', 'id.python.org')), + {'issuer': ((('organizationName', 'CAcert Inc.'),), + (('organizationalUnitName', 'http://www.CAcert.org'),), + (('commonName', 'CAcert Class 3 Root'),)), + 'notAfter': 'Jun 7 21:02:24 2013 GMT', + 'notBefore': 'Jun 8 21:02:24 2011 GMT', + 'serialNumber': 'D3E9', + 'subject': ((('commonName', 'linuxfr.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('othername', ''), + ('DNS', 'linuxfr.org'), + ('othername', ''), + ('DNS', 'dev.linuxfr.org'), + ('othername', ''), + ('DNS', 'prod.linuxfr.org'), + ('othername', ''), + ('DNS', 'alpha.linuxfr.org'), + ('othername', ''), + ('DNS', '*.linuxfr.org'), + ('othername', '')), 'version': 3} -Now the SSL channel is established and the certificate verified, you can -proceed to talk with the server:: +Now that you are assured of its authenticity, you can proceed to talk with +the server:: >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: linuxfr.org\r\n\r\n") >>> pprint.pprint(conn.recv(1024).split(b"\r\n")) - [b'HTTP/1.1 200 OK', - b'Date: Sat, 18 Oct 2014 18:27:20 GMT', - b'Server: nginx', - b'Content-Type: text/html; charset=utf-8', - b'X-Frame-Options: SAMEORIGIN', - b'Content-Length: 45679', - b'Accept-Ranges: bytes', - b'Via: 1.1 varnish', - b'Age: 2188', - b'X-Served-By: cache-lcy1134-LCY', - b'X-Cache: HIT', - b'X-Cache-Hits: 11', - b'Vary: Cookie', - b'Strict-Transport-Security: max-age=63072000; includeSubDomains', + [b'HTTP/1.1 302 Found', + b'Date: Sun, 16 May 2010 13:43:28 GMT', + b'Server: Apache/2.2', + b'Location: https://linuxfr.org/pub/', + b'Vary: Accept-Encoding', b'Connection: close', + b'Content-Type: text/html; charset=iso-8859-1', b'', b''] @@ -1701,7 +1128,7 @@ import socket, ssl - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.load_cert_chain(certfile="mycertfile", keyfile="mykeyfile") bindsocket = socket.socket() @@ -1737,7 +1164,7 @@ And go back to listening for new client connections (of course, a real server would probably handle each client connection in a separate thread, or put -the sockets in :ref:`non-blocking mode ` and use an event loop). +the sockets in non-blocking mode and use an event loop). .. _ssl-nonblocking: @@ -1745,25 +1172,8 @@ Notes on non-blocking sockets ----------------------------- -SSL sockets behave slightly different than regular sockets in -non-blocking mode. When working with non-blocking sockets, there are -thus several things you need to be aware of: - -- Most :class:`SSLSocket` methods will raise either - :exc:`SSLWantWriteError` or :exc:`SSLWantReadError` instead of - :exc:`BlockingIOError` if an I/O operation would - block. :exc:`SSLWantReadError` will be raised if a read operation on - the underlying socket is necessary, and :exc:`SSLWantWriteError` for - a write operation on the underlying socket. Note that attempts to - *write* to an SSL socket may require *reading* from the underlying - socket first, and attempts to *read* from the SSL socket may require - a prior *write* to the underlying socket. - - .. versionchanged:: 3.5 - - In earlier Python versions, the :meth:`!SSLSocket.send` method - returned zero instead of raising :exc:`SSLWantWriteError` or - :exc:`SSLWantReadError`. +When working with non-blocking sockets, there are several things you need +to be aware of: - Calling :func:`~select.select` tells you that the OS-level socket can be read from (or written to), but it does not imply that there is sufficient @@ -1772,14 +1182,8 @@ and :meth:`SSLSocket.send` failures, and retry after another call to :func:`~select.select`. -- Conversely, since the SSL layer has its own framing, a SSL socket may - still have data available for reading without :func:`~select.select` - being aware of it. Therefore, you should first call - :meth:`SSLSocket.recv` to drain any potentially available data, and then - only block on a :func:`~select.select` call if still necessary. - (of course, similar provisions apply when using other primitives such as - :func:`~select.poll`, or those in the :mod:`selectors` module) + :func:`~select.poll`) - The SSL handshake itself will be non-blocking: the :meth:`SSLSocket.do_handshake` method has to be retried until it returns @@ -1795,184 +1199,15 @@ except ssl.SSLWantWriteError: select.select([], [sock], []) -.. seealso:: - - The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and - handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and - :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously - as well. - - -Memory BIO Support ------------------- - -.. versionadded:: 3.5 - -Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket` -class has provided two related but distinct areas of functionality: - -- SSL protocol handling -- Network IO - -The network IO API is identical to that provided by :class:`socket.socket`, -from which :class:`SSLSocket` also inherits. This allows an SSL socket to be -used as a drop-in replacement for a regular socket, making it very easy to add -SSL support to an existing application. - -Combining SSL protocol handling and network IO usually works well, but there -are some cases where it doesn't. An example is async IO frameworks that want to -use a different IO multiplexing model than the "select/poll on a file -descriptor" (readiness based) model that is assumed by :class:`socket.socket` -and by the internal OpenSSL socket IO routines. This is mostly relevant for -platforms like Windows where this model is not efficient. For this purpose, a -reduced scope variant of :class:`SSLSocket` called :class:`SSLObject` is -provided. - -.. class:: SSLObject - - A reduced-scope variant of :class:`SSLSocket` representing an SSL protocol - instance that does not contain any network IO methods. This class is - typically used by framework authors that want to implement asynchronous IO - for SSL through memory buffers. - - This class implements an interface on top of a low-level SSL object as - implemented by OpenSSL. This object captures the state of an SSL connection - but does not provide any network IO itself. IO needs to be performed through - separate "BIO" objects which are OpenSSL's IO abstraction layer. - - An :class:`SSLObject` instance can be created using the - :meth:`~SSLContext.wrap_bio` method. This method will create the - :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming* - BIO is used to pass data from Python to the SSL protocol instance, while the - *outgoing* BIO is used to pass data the other way around. - - The following methods are available: - - - :attr:`~SSLSocket.context` - - :attr:`~SSLSocket.server_side` - - :attr:`~SSLSocket.server_hostname` - - :meth:`~SSLSocket.read` - - :meth:`~SSLSocket.write` - - :meth:`~SSLSocket.getpeercert` - - :meth:`~SSLSocket.selected_npn_protocol` - - :meth:`~SSLSocket.cipher` - - :meth:`~SSLSocket.shared_ciphers` - - :meth:`~SSLSocket.compression` - - :meth:`~SSLSocket.pending` - - :meth:`~SSLSocket.do_handshake` - - :meth:`~SSLSocket.unwrap` - - :meth:`~SSLSocket.get_channel_binding` - - When compared to :class:`SSLSocket`, this object lacks the following - features: - - - Any form of network IO incluging methods such as ``recv()`` and - ``send()``. - - - There is no *do_handshake_on_connect* machinery. You must always manually - call :meth:`~SSLSocket.do_handshake` to start the handshake. - - - There is no handling of *suppress_ragged_eofs*. All end-of-file conditions - that are in violation of the protocol are reported via the - :exc:`SSLEOFError` exception. - - - The method :meth:`~SSLSocket.unwrap` call does not return anything, - unlike for an SSL socket where it returns the underlying socket. - - - The *server_name_callback* callback passed to - :meth:`SSLContext.set_servername_callback` will get an :class:`SSLObject` - instance instead of a :class:`SSLSocket` instance as its first parameter. - - Some notes related to the use of :class:`SSLObject`: - - - All IO on an :class:`SSLObject` is :ref:`non-blocking `. - This means that for example :meth:`~SSLSocket.read` will raise an - :exc:`SSLWantReadError` if it needs more data than the incoming BIO has - available. - - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - -An SSLObject communicates with the outside world using memory buffers. The -class :class:`MemoryBIO` provides a memory buffer that can be used for this -purpose. It wraps an OpenSSL memory BIO (Basic IO) object: - -.. class:: MemoryBIO - - A memory buffer that can be used to pass data between Python and an SSL - protocol instance. - - .. attribute:: MemoryBIO.pending - - Return the number of bytes currently in the memory buffer. - - .. attribute:: MemoryBIO.eof - - A boolean indicating whether the memory BIO is current at the end-of-file - position. - - .. method:: MemoryBIO.read(n=-1) - - Read up to *n* bytes from the memory buffer. If *n* is not specified or - negative, all bytes are returned. - - .. method:: MemoryBIO.write(buf) - - Write the bytes from *buf* to the memory BIO. The *buf* argument must be an - object supporting the buffer protocol. - - The return value is the number of bytes written, which is always equal to - the length of *buf*. - - .. method:: MemoryBIO.write_eof() - - Write an EOF marker to the memory BIO. After this method has been called, it - is illegal to call :meth:`~MemoryBIO.write`. The attribute :attr:`eof` will - become true after all data currently in the buffer has been read. - .. _ssl-security: Security considerations ----------------------- -Best defaults -^^^^^^^^^^^^^ +Verifying certificates +^^^^^^^^^^^^^^^^^^^^^^ -For **client use**, if you don't have any special requirements for your -security policy, it is highly recommended that you use the -:func:`create_default_context` function to create your SSL context. -It will load the system's trusted CA certificates, enable certificate -validation and hostname checking, and try to choose reasonably secure -protocol and cipher settings. - -For example, here is how you would use the :class:`smtplib.SMTP` class to -create a trusted, secure connection to a SMTP server:: - - >>> import ssl, smtplib - >>> smtp = smtplib.SMTP("mail.python.org", port=587) - >>> context = ssl.create_default_context() - >>> smtp.starttls(context=context) - (220, b'2.0.0 Ready to start TLS') - -If a client certificate is needed for the connection, it can be added with -:meth:`SSLContext.load_cert_chain`. - -By contrast, if you create the SSL context by calling the :class:`SSLContext` -constructor yourself, it will not have certificate validation nor hostname -checking enabled by default. If you do so, please read the paragraphs below -to achieve a good security level. - -Manual settings -^^^^^^^^^^^^^^^ - -Verifying certificates -'''''''''''''''''''''' - -When calling the :class:`SSLContext` constructor directly, :const:`CERT_NONE` is the default. Since it does not authenticate the other peer, it can be insecure, especially in client mode where most of time you would like to ensure the authenticity of the server you're talking to. @@ -1981,9 +1216,7 @@ have to check that the server certificate, which can be obtained by calling :meth:`SSLSocket.getpeercert`, matches the desired service. For many protocols and applications, the service can be identified by the hostname; -in this case, the :func:`match_hostname` function can be used. This common -check is automatically performed when :attr:`SSLContext.check_hostname` is -enabled. +in this case, the :func:`match_hostname` function can be used. In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have @@ -1996,52 +1229,46 @@ by default). Protocol versions -''''''''''''''''' +^^^^^^^^^^^^^^^^^ -SSL versions 2 and 3 are considered insecure and are therefore dangerous to -use. If you want maximum compatibility between clients and servers, it is -recommended to use :const:`PROTOCOL_SSLv23` as the protocol version and then -disable SSLv2 and SSLv3 explicitly using the :data:`SSLContext.options` -attribute:: +SSL version 2 is considered insecure and is therefore dangerous to use. If +you want maximum compatibility between clients and servers, it is recommended +to use :const:`PROTOCOL_SSLv23` as the protocol version and then disable +SSLv2 explicitly using the :data:`SSLContext.options` attribute:: context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 - context.options |= ssl.OP_NO_SSLv3 -The SSL context created above will only allow TLSv1 and later (if -supported by your system) connections. +The SSL context created above will allow SSLv3 and TLSv1 connections, but +not SSLv2. Cipher selection -'''''''''''''''' +^^^^^^^^^^^^^^^^ If you have advanced security requirements, fine-tuning of the ciphers enabled when negotiating a SSL session is possible through the :meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the ssl module disables certain weak ciphers by default, but you may want -to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. -If you want to check which ciphers are enabled by a given cipher list, use the -``openssl ciphers`` command on your system. +to further restrict the cipher choice. For example:: -Multi-processing -^^^^^^^^^^^^^^^^ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.set_ciphers('HIGH:!aNULL:!eNULL') -If using this module as part of a multi-processed application (using, -for example the :mod:`multiprocessing` or :mod:`concurrent.futures` modules), -be aware that OpenSSL's internal random number generator does not properly -handle forked processes. Applications must change the PRNG state of the -parent process if they use any SSL feature with :func:`os.fork`. Any -successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or -:func:`~ssl.RAND_pseudo_bytes` is sufficient. +The ``!aNULL:!eNULL`` part of the cipher spec is necessary to disable ciphers +which don't provide both encryption and authentication. Be sure to read +OpenSSL's documentation about the `cipher list +format `_. +If you want to check which ciphers are enabled by a given cipher list, +use the ``openssl ciphers`` command on your system. .. seealso:: Class :class:`socket.socket` - Documentation of underlying :mod:`socket` class + Documentation of underlying :mod:`socket` class - `SSL/TLS Strong Encryption: An Introduction `_ - Intro from the Apache webserver documentation + `TLS (Transport Layer Security) and SSL (Secure Socket Layer) `_ + Debby Koren `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent @@ -2054,12 +1281,3 @@ `RFC 4366: Transport Layer Security (TLS) Extensions `_ Blake-Wilson et. al. - - `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_ - T. Dierks et. al. - - `RFC 6066: Transport Layer Security (TLS) Extensions `_ - D. Eastlake - - `IANA TLS: Transport Layer Security (TLS) Parameters `_ - IANA diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/stat.rst --- a/Doc/library/stat.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/stat.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,13 +1,12 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== +:mod:`stat` --- Interpreting :func:`stat` results +================================================= .. module:: stat :synopsis: Utilities for interpreting the results of os.stat(), os.lstat() and os.fstat(). .. sectionauthor:: Skip Montanaro -**Source code:** :source:`Modules/_stat.c` - :source:`Lib/stat.py` +**Source code:** :source:`Lib/stat.py` -------------- @@ -16,9 +15,6 @@ exist). For complete details about the :c:func:`stat`, :c:func:`fstat` and :c:func:`lstat` calls, consult the documentation for your system. -.. versionchanged:: 3.4 - The stat module is backed by a C implementation. - The :mod:`stat` module defines the following functions to test for specific file types: @@ -57,24 +53,6 @@ Return non-zero if the mode is from a socket. -.. function:: S_ISDOOR(mode) - - Return non-zero if the mode is from a door. - - .. versionadded:: 3.4 - -.. function:: S_ISPORT(mode) - - Return non-zero if the mode is from an event port. - - .. versionadded:: 3.4 - -.. function:: S_ISWHT(mode) - - Return non-zero if the mode is from a whiteout. - - .. versionadded:: 3.4 - Two additional functions are defined for more general manipulation of the file's mode: @@ -126,20 +104,6 @@ if __name__ == '__main__': walktree(sys.argv[1], visitfile) -An additional utility function is provided to convert a file's mode in a human -readable string: - -.. function:: filemode(mode) - - Convert a file's mode to a string of the form '-rwxrwxrwx'. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The function supports :data:`S_IFDOOR`, :data:`S_IFPORT` and - :data:`S_IFWHT`. - - All the variables below are simply symbolic indexes into the 10-tuple returned by :func:`os.stat`, :func:`os.fstat` or :func:`os.lstat`. @@ -208,6 +172,10 @@ Use of the functions above is more portable than use of the first set of flags: +.. data:: S_IFMT + + Bit mask for the file type bit fields. + .. data:: S_IFSOCK Socket. @@ -236,29 +204,6 @@ FIFO. -.. data:: S_IFDOOR - - Door. - - .. versionadded:: 3.4 - -.. data:: S_IFPORT - - Event port. - - .. versionadded:: 3.4 - -.. data:: S_IFWHT - - Whiteout. - - .. versionadded:: 3.4 - -.. note:: - - :data:`S_IFDOOR`, :data:`S_IFPORT` or :data:`S_IFWHT` are defined as - 0 when the platform does not have support for the file types. - The following flags can also be used in the *mode* argument of :func:`os.chmod`: .. data:: S_ISUID @@ -400,28 +345,3 @@ See the \*BSD or Mac OS systems man page :manpage:`chflags(2)` for more information. -On Windows, the following file attribute constants are available for use when -testing bits in the ``st_file_attributes`` member returned by :func:`os.stat`. -See the `Windows API documentation -`_ -for more detail on the meaning of these constants. - -.. data:: FILE_ATTRIBUTE_ARCHIVE - FILE_ATTRIBUTE_COMPRESSED - FILE_ATTRIBUTE_DEVICE - FILE_ATTRIBUTE_DIRECTORY - FILE_ATTRIBUTE_ENCRYPTED - FILE_ATTRIBUTE_HIDDEN - FILE_ATTRIBUTE_INTEGRITY_STREAM - FILE_ATTRIBUTE_NORMAL - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED - FILE_ATTRIBUTE_NO_SCRUB_DATA - FILE_ATTRIBUTE_OFFLINE - FILE_ATTRIBUTE_READONLY - FILE_ATTRIBUTE_REPARSE_POINT - FILE_ATTRIBUTE_SPARSE_FILE - FILE_ATTRIBUTE_SYSTEM - FILE_ATTRIBUTE_TEMPORARY - FILE_ATTRIBUTE_VIRTUAL - - .. versionadded:: 3.5 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/statistics.rst --- a/Doc/library/statistics.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,418 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: mathematical statistics functions -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -.. testsetup:: * - - from statistics import * - __name__ = '' - -**Source code:** :source:`Lib/statistics.py` - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`Real`-valued) data. - -.. note:: - - Unless explicitly noted otherwise, these functions support :class:`int`, - :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`. - Behaviour with other types (whether in the numeric tower or not) is - currently unsupported. Mixed types are also undefined and - implementation-dependent. If your input data consists of mixed types, - you may be able to use :func:`map` to ensure a consistent result, e.g. - ``map(float, input_data)``. - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= ============================================= -:func:`mean` Arithmetic mean ("average") of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Mode (most common value) of discrete data. -======================= ============================================= - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data*, a sequence or iterator of - real-valued numbers. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of the - data points. For more robust, although less efficient, measures of - central location, see :func:`median` and :func:`mode`. (In this case, - "efficient" refers to statistical efficiency rather than computational - efficiency.) - - The sample mean gives an unbiased estimate of the true population mean, - which means that, taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - - The median is a robust measure of central location, and is less affected by - the presence of outliers in your data. When the number of data points is - odd, the middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped` - - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5-1.5, 2 - is the midpoint of 1.5-2.5, 3 is the midpoint of 2.5-3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5-4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * Calculating the `median `_. - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the most common data point from discrete or nominal *data*. The mode - (when it exists) is the most typical value, and is a robust measure of - central location. - - If *data* is empty, or if there is not exactly one most common value, - :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data, and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic which also applies - to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty iterable of real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *mu* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - This function does not attempt to verify that you have passed the actual mean - as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible - results. - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this function - to calculate the variance of a sample, giving the known population mean as - the second argument. Provided the data points are representative - (e.g. independent and identically distributed), the result will be an - unbiased estimate of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/stdtypes.rst --- a/Doc/library/stdtypes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/stdtypes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -269,8 +269,8 @@ :func:`complex` can be used to produce numbers of a specific type. All numeric types (except complex) support the following operations, sorted by -ascending priority (all numeric operations have a higher priority than -comparison operations): +ascending priority (operations in the same box have the same priority; all +numeric operations have a higher priority than comparison operations): +---------------------+---------------------------------+---------+--------------------+ | Operation | Result | Notes | Full documentation | @@ -339,8 +339,8 @@ pair: C; language Conversion from floating point to integer may round or truncate - as in C; see functions :func:`math.floor` and :func:`math.ceil` for - well-defined conversions. + as in C; see functions :func:`floor` and :func:`ceil` in the :mod:`math` module + for well-defined conversions. (4) float also accepts the strings "nan" and "inf" with an optional prefix "+" @@ -354,7 +354,7 @@ The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedNumericType.txt + See http://www.unicode.org/Public/6.0.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. @@ -404,7 +404,8 @@ operations and higher than the comparisons; the unary operation ``~`` has the same priority as the other unary numeric operations (``+`` and ``-``). -This table lists the bitwise operations sorted in ascending priority: +This table lists the bitwise operations sorted in ascending priority +(operations in the same box have the same priority): +------------+--------------------------------+----------+ | Operation | Result | Notes | @@ -443,7 +444,7 @@ ----------------------------------- The int type implements the :class:`numbers.Integral` :term:`abstract base -class`. In addition, it provides a few more methods: +class`. In addition, it provides one more method: .. method:: int.bit_length() @@ -518,8 +519,9 @@ >>> int.from_bytes([255, 0, 0], byteorder='big') 16711680 - The argument *bytes* must either be a :term:`bytes-like object` or an - iterable producing bytes. + The argument *bytes* must either support the buffer protocol or be an + iterable producing bytes. :class:`bytes` and :class:`bytearray` are + examples of built-in objects that support the buffer protocol. The *byteorder* argument determines the byte order used to represent the integer. If *byteorder* is ``"big"``, the most significant byte is at the @@ -630,7 +632,7 @@ :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`) Python's hash for numeric types is based on a single mathematical function that's defined for any rational number, and hence applies to all instances of -:class:`int` and :class:`fractions.Fraction`, and all finite instances of +:class:`int` and :class:`fraction.Fraction`, and all finite instances of :class:`float` and :class:`decimal.Decimal`. Essentially, this function is given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is made available to Python as the :attr:`modulus` attribute of @@ -670,7 +672,7 @@ To clarify the above rules, here's some example Python code, -equivalent to the built-in hash, for computing the hash of a rational +equivalent to the builtin hash, for computing the hash of a rational number, :class:`float`, or :class:`complex`:: @@ -750,7 +752,7 @@ iterators for those iteration types. (An example of an object supporting multiple forms of iteration would be a tree structure which supports both breadth-first and depth-first traversal.) This method corresponds to the - :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C + :attr:`tp_iter` slot of the type structure for Python objects in the Python/C API. The iterator objects themselves are required to support the following two @@ -761,7 +763,7 @@ Return the iterator object itself. This is required to allow both containers and iterators to be used with the :keyword:`for` and :keyword:`in` statements. - This method corresponds to the :c:member:`~PyTypeObject.tp_iter` slot of the type structure for + This method corresponds to the :attr:`tp_iter` slot of the type structure for Python objects in the Python/C API. @@ -769,7 +771,7 @@ Return the next item from the container. If there are no further items, raise the :exc:`StopIteration` exception. This method corresponds to the - :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for Python objects in the + :attr:`tp_iternext` slot of the type structure for Python objects in the Python/C API. Python defines several iterator objects to support iteration over general and @@ -777,9 +779,9 @@ specific types are not important beyond their implementation of the iterator protocol. -Once an iterator's :meth:`~iterator.__next__` method raises -:exc:`StopIteration`, it must continue to do so on subsequent calls. -Implementations that do not obey this property are deemed broken. +Once an iterator's :meth:`__next__` method raises :exc:`StopIteration`, it must +continue to do so on subsequent calls. Implementations that do not obey this +property are deemed broken. .. _generator-types: @@ -790,43 +792,116 @@ Python's :term:`generator`\s provide a convenient way to implement the iterator protocol. If a container object's :meth:`__iter__` method is implemented as a generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` -methods. +generator object) supplying the :meth:`__iter__` and :meth:`__next__` methods. More information about generators can be found in :ref:`the documentation for the yield expression `. .. _typesseq: -Sequence Types --- :class:`list`, :class:`tuple`, :class:`range` -================================================================ - -There are three basic sequence types: lists, tuples, and range objects. -Additional sequence types tailored for processing of -:ref:`binary data ` and :ref:`text strings ` are -described in dedicated sections. - - -.. _typesseq-common: - -Common Sequence Operations --------------------------- - -.. index:: object: sequence - -The operations in the following table are supported by most sequence types, -both mutable and immutable. The :class:`collections.abc.Sequence` ABC is -provided to make it easier to correctly implement these operations on -custom sequence types. - -This table lists the sequence operations sorted in ascending priority. In the -table, *s* and *t* are sequences of the same type, *n*, *i*, *j* and *k* are -integers and *x* is an arbitrary object that meets any type and value -restrictions imposed by *s*. - -The ``in`` and ``not in`` operations have the same priorities as the -comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. +Sequence Types --- :class:`str`, :class:`bytes`, :class:`bytearray`, :class:`list`, :class:`tuple`, :class:`range` +================================================================================================================== + +There are six sequence types: strings, byte sequences (:class:`bytes` objects), +byte arrays (:class:`bytearray` objects), lists, tuples, and range objects. For +other containers see the built in :class:`dict` and :class:`set` classes, and +the :mod:`collections` module. + + +.. index:: + object: sequence + object: string + object: bytes + object: bytearray + object: tuple + object: list + object: range + +Strings contain Unicode characters. Their literals are written in single or +double quotes: ``'xyzzy'``, ``"frobozz"``. See :ref:`strings` for more about +string literals. In addition to the functionality described here, there are +also string-specific methods described in the :ref:`string-methods` section. + +Bytes and bytearray objects contain single bytes -- the former is immutable +while the latter is a mutable sequence. Bytes objects can be constructed the +constructor, :func:`bytes`, and from literals; use a ``b`` prefix with normal +string syntax: ``b'xyzzy'``. To construct byte arrays, use the +:func:`bytearray` function. + +While string objects are sequences of characters (represented by strings of +length 1), bytes and bytearray objects are sequences of *integers* (between 0 +and 255), representing the ASCII value of single bytes. That means that for +a bytes or bytearray object *b*, ``b[0]`` will be an integer, while +``b[0:1]`` will be a bytes or bytearray object of length 1. The +representation of bytes objects uses the literal format (``b'...'``) since it +is generally more useful than e.g. ``bytes([50, 19, 100])``. You can always +convert a bytes object into a list of integers using ``list(b)``. + +Also, while in previous Python versions, byte strings and Unicode strings +could be exchanged for each other rather freely (barring encoding issues), +strings and bytes are now completely separate concepts. There's no implicit +en-/decoding if you pass an object of the wrong type. A string always +compares unequal to a bytes or bytearray object. + +Lists are constructed with square brackets, separating items with commas: ``[a, +b, c]``. Tuples are constructed by the comma operator (not within square +brackets), with or without enclosing parentheses, but an empty tuple must have +the enclosing parentheses, such as ``a, b, c`` or ``()``. A single item tuple +must have a trailing comma, such as ``(d,)``. + +Objects of type range are created using the :func:`range` function. They don't +support concatenation or repetition, and using :func:`min` or :func:`max` on +them is inefficient. + +Most sequence types support the following operations. The ``in`` and ``not in`` +operations have the same priorities as the comparison operations. The ``+`` and +``*`` operations have the same priority as the corresponding numeric operations. +[3]_ Additional methods are provided for :ref:`typesseq-mutable`. + +This table lists the sequence operations sorted in ascending priority +(operations in the same box have the same priority). In the table, *s* and *t* +are sequences of the same type; *n*, *i*, *j* and *k* are integers. + ++------------------+--------------------------------+----------+ +| Operation | Result | Notes | ++==================+================================+==========+ +| ``x in s`` | ``True`` if an item of *s* is | \(1) | +| | equal to *x*, else ``False`` | | ++------------------+--------------------------------+----------+ +| ``x not in s`` | ``False`` if an item of *s* is | \(1) | +| | equal to *x*, else ``True`` | | ++------------------+--------------------------------+----------+ +| ``s + t`` | the concatenation of *s* and | \(6) | +| | *t* | | ++------------------+--------------------------------+----------+ +| ``s * n, n * s`` | *n* shallow copies of *s* | \(2) | +| | concatenated | | ++------------------+--------------------------------+----------+ +| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | ++------------------+--------------------------------+----------+ +| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | ++------------------+--------------------------------+----------+ +| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) | +| | with step *k* | | ++------------------+--------------------------------+----------+ +| ``len(s)`` | length of *s* | | ++------------------+--------------------------------+----------+ +| ``min(s)`` | smallest item of *s* | | ++------------------+--------------------------------+----------+ +| ``max(s)`` | largest item of *s* | | ++------------------+--------------------------------+----------+ +| ``s.index(i)`` | index of the first occurence | | +| | of *i* in *s* | | ++------------------+--------------------------------+----------+ +| ``s.count(i)`` | total number of occurences of | | +| | *i* in *s* | | ++------------------+--------------------------------+----------+ + +Sequence types also support comparisons. In particular, tuples and lists are +compared lexicographically by comparing corresponding elements. This means that +to compare equal, every element must compare equal and the two sequences must be +of the same type and have the same length. (For full details see +:ref:`comparisons` in the language reference.) .. index:: triple: operations on; sequence; types @@ -839,67 +914,18 @@ pair: slice; operation operator: in operator: not in - single: count() (sequence method) - single: index() (sequence method) - -+--------------------------+--------------------------------+----------+ -| Operation | Result | Notes | -+==========================+================================+==========+ -| ``x in s`` | ``True`` if an item of *s* is | \(1) | -| | equal to *x*, else ``False`` | | -+--------------------------+--------------------------------+----------+ -| ``x not in s`` | ``False`` if an item of *s* is | \(1) | -| | equal to *x*, else ``True`` | | -+--------------------------+--------------------------------+----------+ -| ``s + t`` | the concatenation of *s* and | (6)(7) | -| | *t* | | -+--------------------------+--------------------------------+----------+ -| ``s * n`` or | equivalent to adding *s* to | (2)(7) | -| ``n * s`` | itself *n* times | | -+--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) | -| | with step *k* | | -+--------------------------+--------------------------------+----------+ -| ``len(s)`` | length of *s* | | -+--------------------------+--------------------------------+----------+ -| ``min(s)`` | smallest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``max(s)`` | largest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ - -Sequences of the same type also support comparisons. In particular, tuples -and lists are compared lexicographically by comparing corresponding elements. -This means that to compare equal, every element must compare equal and the -two sequences must be of the same type and have the same length. (For full -details see :ref:`comparisons` in the language reference.) Notes: (1) - While the ``in`` and ``not in`` operations are used only for simple - containment testing in the general case, some specialised sequences - (such as :class:`str`, :class:`bytes` and :class:`bytearray`) also use - them for subsequence testing:: - - >>> "gg" in "eggs" - True + When *s* is a string object, the ``in`` and ``not in`` operations act like a + substring test. (2) Values of *n* less than ``0`` are treated as ``0`` (which yields an empty - sequence of the same type as *s*). Note that items in the sequence *s* - are not copied; they are referenced multiple times. This often haunts - new Python programmers; consider:: + sequence of the same type as *s*). Note also that the copies are shallow; + nested structures are not copied. This often haunts new Python programmers; + consider: >>> lists = [[]] * 3 >>> lists @@ -909,9 +935,9 @@ [[3], [3], [3]] What has happened is that ``[[]]`` is a one-element list containing an empty - list, so all three elements of ``[[]] * 3`` are references to this single empty + list, so all three elements of ``[[]] * 3`` are (pointers to) this single empty list. Modifying any of the elements of ``lists`` modifies this single list. - You can create a list of different lists this way:: + You can create a list of different lists this way: >>> lists = [[] for i in range(3)] >>> lists[0].append(3) @@ -920,9 +946,6 @@ >>> lists [[3], [5], [7]] - Further explanation is available in the FAQ entry - :ref:`faq-multidimensional-list`. - (3) If *i* or *j* is negative, the index is relative to the end of the string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note that ``-0`` is @@ -945,539 +968,33 @@ If *k* is ``None``, it is treated like ``1``. (6) - Concatenating immutable sequences always results in a new object. This - means that building up a sequence by repeated concatenation will have a - quadratic runtime cost in the total sequence length. To get a linear - runtime cost, you must switch to one of the alternatives below: + Concatenating immutable strings always results in a new object. This means + that building up a string by repeated concatenation will have a quadratic + runtime cost in the total string length. To get a linear runtime cost, + you must switch to one of the alternatives below: * if concatenating :class:`str` objects, you can build a list and use - :meth:`str.join` at the end or else write to an :class:`io.StringIO` - instance and retrieve its value when complete + :meth:`str.join` at the end; * if concatenating :class:`bytes` objects, you can similarly use - :meth:`bytes.join` or :class:`io.BytesIO`, or you can do in-place - concatenation with a :class:`bytearray` object. :class:`bytearray` - objects are mutable and have an efficient overallocation mechanism - - * if concatenating :class:`tuple` objects, extend a :class:`list` instead - - * for other types, investigate the relevant class documentation - - -(7) - Some sequence types (such as :class:`range`) only support item sequences - that follow specific patterns, and hence don't support sequence - concatenation or repetition. - -(8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - When supported, the additional arguments to the index method allow - efficient searching of subsections of the sequence. Passing the extra - arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. - - -.. _typesseq-immutable: - -Immutable Sequence Types ------------------------- - -.. index:: - triple: immutable; sequence; types - object: tuple - builtin: hash - -The only operation that immutable sequence types generally implement that is -not also implemented by mutable sequence types is support for the :func:`hash` -built-in. - -This support allows immutable sequences, such as :class:`tuple` instances, to -be used as :class:`dict` keys and stored in :class:`set` and :class:`frozenset` -instances. - -Attempting to hash an immutable sequence that contains unhashable values will -result in :exc:`TypeError`. - - -.. _typesseq-mutable: - -Mutable Sequence Types ----------------------- - -.. index:: - triple: mutable; sequence; types - object: list - object: bytearray - -The operations in the following table are defined on mutable sequence types. -The :class:`collections.abc.MutableSequence` ABC is provided to make it -easier to correctly implement these operations on custom sequence types. - -In the table *s* is an instance of a mutable sequence type, *t* is any -iterable object and *x* is an arbitrary object that meets any type -and value restrictions imposed by *s* (for example, :class:`bytearray` only -accepts integers that meet the value restriction ``0 <= x <= 255``). - - -.. index:: - triple: operations on; sequence; types - triple: operations on; list; type - pair: subscript; assignment - pair: slice; assignment - statement: del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) - -+------------------------------+--------------------------------+---------------------+ -| Operation | Result | Notes | -+==============================+================================+=====================+ -| ``s[i] = x`` | item *i* of *s* is replaced by | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j] = t`` | slice of *s* from *i* to *j* | | -| | is replaced by the contents of | | -| | the iterable *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | -| | are replaced by those of *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j:k]`` | removes the elements of | | -| | ``s[i:j:k]`` from the list | | -+------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from ``s`` | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of ``s``| \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | -| | most part the same as | | -| | ``s[len(s):len(s)] = t``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | -| | repeated *n* times | | -+------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop([i])`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i] == x`` | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - - -Notes: - -(1) - *t* must have the same length as the slice it is replacing. - -(2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. - -(3) - ``remove`` raises :exc:`ValueError` when *x* is not found in *s*. - -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. - -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`) - - .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. - -(6) - The value *n* is an integer, or an object implementing - :meth:`~object.__index__`. Zero and negative values of *n* clear - the sequence. Items in the sequence are not copied; they are referenced - multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. - - -.. _typesseq-list: - -Lists ------ - -.. index:: object: list - -Lists are mutable sequences, typically used to store collections of -homogeneous items (where the precise degree of similarity will vary by -application). - -.. class:: list([iterable]) - - Lists may be constructed in several ways: - - * Using a pair of square brackets to denote the empty list: ``[]`` - * Using square brackets, separating items with commas: ``[a]``, ``[a, b, c]`` - * Using a list comprehension: ``[x for x in iterable]`` - * Using the type constructor: ``list()`` or ``list(iterable)`` - - The constructor builds a list whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a list, a copy is made and returned, similar to ``iterable[:]``. - For example, ``list('abc')`` returns ``['a', 'b', 'c']`` and - ``list( (1, 2, 3) )`` returns ``[1, 2, 3]``. - If no argument is given, the constructor creates a new empty list, ``[]``. - - - Many other operations also produce lists, including the :func:`sorted` - built-in. - - Lists implement all of the :ref:`common ` and - :ref:`mutable ` sequence operations. Lists also provide the - following additional method: - - .. method:: list.sort(*, key=None, reverse=None) - - This method sorts the list in place, using only ``<`` comparisons - between items. Exceptions are not suppressed - if any comparison operations - fail, the entire sort operation will fail (and the list will likely be left - in a partially modified state). - - :meth:`sort` accepts two arguments that can only be passed by keyword - (:ref:`keyword-only arguments `): - - *key* specifies a function of one argument that is used to extract a - comparison key from each list element (for example, ``key=str.lower``). - The key corresponding to each item in the list is calculated once and - then used for the entire sorting process. The default value of ``None`` - means that list items are sorted directly without calculating a separate - key value. - - The :func:`functools.cmp_to_key` utility is available to convert a 2.x - style *cmp* function to a *key* function. - - *reverse* is a boolean value. If set to ``True``, then the list elements - are sorted as if each comparison were reversed. - - This method modifies the sequence in place for economy of space when - sorting a large sequence. To remind users that it operates by side - effect, it does not return the sorted sequence (use :func:`sorted` to - explicitly request a new sorted list instance). - - The :meth:`sort` method is guaranteed to be stable. A sort is stable if it - guarantees not to change the relative order of elements that compare equal - --- this is helpful for sorting in multiple passes (for example, sort by - department, then by salary grade). - - .. impl-detail:: - - While a list is being sorted, the effect of attempting to mutate, or even - inspect, the list is undefined. The C implementation of Python makes the - list appear empty for the duration, and raises :exc:`ValueError` if it can - detect that the list has been mutated during a sort. - - -.. _typesseq-tuple: - -Tuples ------- - -.. index:: object: tuple - -Tuples are immutable sequences, typically used to store collections of -heterogeneous data (such as the 2-tuples produced by the :func:`enumerate` -built-in). Tuples are also used for cases where an immutable sequence of -homogeneous data is needed (such as allowing storage in a :class:`set` or -:class:`dict` instance). - -.. class:: tuple([iterable]) - - Tuples may be constructed in a number of ways: - - * Using a pair of parentheses to denote the empty tuple: ``()`` - * Using a trailing comma for a singleton tuple: ``a,`` or ``(a,)`` - * Separating items with commas: ``a, b, c`` or ``(a, b, c)`` - * Using the :func:`tuple` built-in: ``tuple()`` or ``tuple(iterable)`` - - The constructor builds a tuple whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a tuple, it is returned unchanged. For example, - ``tuple('abc')`` returns ``('a', 'b', 'c')`` and - ``tuple( [1, 2, 3] )`` returns ``(1, 2, 3)``. - If no argument is given, the constructor creates a new empty tuple, ``()``. - - Note that it is actually the comma which makes a tuple, not the parentheses. - The parentheses are optional, except in the empty tuple case, or - when they are needed to avoid syntactic ambiguity. For example, - ``f(a, b, c)`` is a function call with three arguments, while - ``f((a, b, c))`` is a function call with a 3-tuple as the sole argument. - - Tuples implement all of the :ref:`common ` sequence - operations. - -For heterogeneous collections of data where access by name is clearer than -access by index, :func:`collections.namedtuple` may be a more appropriate -choice than a simple tuple object. - - -.. _typesseq-range: - -Ranges ------- - -.. index:: object: range - -The :class:`range` type represents an immutable sequence of numbers and is -commonly used for looping a specific number of times in :keyword:`for` -loops. - -.. class:: range(stop) - range(start, stop[, step]) - - The arguments to the range constructor must be integers (either built-in - :class:`int` or any object that implements the ``__index__`` special - method). If the *step* argument is omitted, it defaults to ``1``. - If the *start* argument is omitted, it defaults to ``0``. - If *step* is zero, :exc:`ValueError` is raised. - - For a positive *step*, the contents of a range ``r`` are determined by the - formula ``r[i] = start + step*i`` where ``i >= 0`` and - ``r[i] < stop``. - - For a negative *step*, the contents of the range are still determined by - the formula ``r[i] = start + step*i``, but the constraints are ``i >= 0`` - and ``r[i] > stop``. - - A range object will be empty if ``r[0]`` does not meet the value - constraint. Ranges do support negative indices, but these are interpreted - as indexing from the end of the sequence determined by the positive - indices. - - Ranges containing absolute values larger than :data:`sys.maxsize` are - permitted but some features (such as :func:`len`) may raise - :exc:`OverflowError`. - - Range examples:: - - >>> list(range(10)) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> list(range(1, 11)) - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - >>> list(range(0, 30, 5)) - [0, 5, 10, 15, 20, 25] - >>> list(range(0, 10, 3)) - [0, 3, 6, 9] - >>> list(range(0, -10, -1)) - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] - >>> list(range(0)) - [] - >>> list(range(1, 0)) - [] - - Ranges implement all of the :ref:`common ` sequence operations - except concatenation and repetition (due to the fact that range objects can - only represent sequences that follow a strict pattern and repetition and - concatenation will usually violate that pattern). - - .. data: start - - The value of the *start* parameter (or ``0`` if the parameter was - not supplied) - - .. data: stop - - The value of the *stop* parameter - - .. data: step - - The value of the *step* parameter (or ``1`` if the parameter was - not supplied) - -The advantage of the :class:`range` type over a regular :class:`list` or -:class:`tuple` is that a :class:`range` object will always take the same -(small) amount of memory, no matter the size of the range it represents (as it -only stores the ``start``, ``stop`` and ``step`` values, calculating individual -items and subranges as needed). - -Range objects implement the :class:`collections.abc.Sequence` ABC, and provide -features such as containment tests, element index lookup, slicing and -support for negative indices (see :ref:`typesseq`): - - >>> r = range(0, 20, 2) - >>> r - range(0, 20, 2) - >>> 11 in r - False - >>> 10 in r - True - >>> r.index(10) - 5 - >>> r[5] - 10 - >>> r[:5] - range(0, 10, 2) - >>> r[-1] - 18 - -Testing range objects for equality with ``==`` and ``!=`` compares -them as sequences. That is, two range objects are considered equal if -they represent the same sequence of values. (Note that two range -objects that compare equal might have different :attr:`~range.start`, -:attr:`~range.stop` and :attr:`~range.step` attributes, for example -``range(0) == range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.) - -.. versionchanged:: 3.2 - Implement the Sequence ABC. - Support slicing and negative indices. - Test :class:`int` objects for membership in constant time instead of - iterating through all items. - -.. versionchanged:: 3.3 - Define '==' and '!=' to compare range objects based on the - sequence of values they define (instead of comparing based on - object identity). - -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` - attributes. - - -.. index:: - single: string; text sequence type - single: str (built-in class); (see also string) - object: string - -.. _textseq: - -Text Sequence Type --- :class:`str` -=================================== - -Textual data in Python is handled with :class:`str` objects, or :dfn:`strings`. -Strings are immutable -:ref:`sequences ` of Unicode code points. String literals are -written in a variety of ways: - -* Single quotes: ``'allows embedded "double" quotes'`` -* Double quotes: ``"allows embedded 'single' quotes"``. -* Triple quoted: ``'''Three single quotes'''``, ``"""Three double quotes"""`` - -Triple quoted strings may span multiple lines - all associated whitespace will -be included in the string literal. - -String literals that are part of a single expression and have only whitespace -between them will be implicitly converted to a single string literal. That -is, ``("spam " "eggs") == "spam eggs"``. - -See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that -disables most escape sequence processing. - -Strings may also be created from other objects using the :class:`str` -constructor. - -Since there is no separate "character" type, indexing a string produces -strings of length 1. That is, for a non-empty string *s*, ``s[0] == s[0:1]``. - -.. index:: - object: io.StringIO - -There is also no mutable string type, but :meth:`str.join` or -:class:`io.StringIO` can be used to efficiently construct strings from -multiple fragments. - -.. versionchanged:: 3.3 - For backwards compatibility with the Python 2 series, the ``u`` prefix is - once again permitted on string literals. It has no effect on the meaning - of string literals and cannot be combined with the ``r`` prefix. - - -.. index:: - single: string; str (built-in class) - -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - - Return a :ref:`string ` version of *object*. If *object* is not - provided, returns the empty string. Otherwise, the behavior of ``str()`` - depends on whether *encoding* or *errors* is given, as follows. - - If neither *encoding* nor *errors* is given, ``str(object)`` returns - :meth:`object.__str__() `, which is the "informal" or nicely - printable string representation of *object*. For string objects, this is - the string itself. If *object* does not have a :meth:`~object.__str__` - method, then :func:`str` falls back to returning - :meth:`repr(object) `. - - .. index:: - single: buffer protocol; str (built-in class) - single: bytes; str (built-in class) - - If at least one of *encoding* or *errors* is given, *object* should be a - :term:`bytes-like object` (e.g. :class:`bytes` or :class:`bytearray`). In - this case, if *object* is a :class:`bytes` (or :class:`bytearray`) object, - then ``str(bytes, encoding, errors)`` is equivalent to - :meth:`bytes.decode(encoding, errors) `. Otherwise, the bytes - object underlying the buffer object is obtained before calling - :meth:`bytes.decode`. See :ref:`binaryseq` and - :ref:`bufferobjects` for information on buffer objects. - - Passing a :class:`bytes` object to :func:`str` without the *encoding* - or *errors* arguments falls under the first case of returning the informal - string representation (see also the :option:`-b` command-line option to - Python). For example:: - - >>> str(b'Zoot!') - "b'Zoot!'" - - For more information on the ``str`` class and its methods, see - :ref:`textseq` and the :ref:`string-methods` section below. To output - formatted strings, see the :ref:`string-formatting` section. In addition, - see the :ref:`stringservices` section. - - -.. index:: - pair: string; methods + :meth:`bytes.join`, or you can do in-place concatenation with a + :class:`bytearray` object. :class:`bytearray` objects are mutable and + have an efficient overallocation mechanism. + .. _string-methods: String Methods -------------- -.. index:: - module: re - -Strings implement all of the :ref:`common ` sequence -operations, along with the additional methods described below. - -Strings also support two styles of string formatting, one providing a large -degree of flexibility and customization (see :meth:`str.format`, -:ref:`formatstrings` and :ref:`string-formatting`) and the other based on C -``printf`` style formatting that handles a narrower range of types and is -slightly harder to use correctly, but is often faster for the cases it can -handle (:ref:`old-string-formatting`). - -The :ref:`textservices` section of the standard library covers a number of -other modules that provide various text related utilities (including regular -expression support in the :mod:`re` module). +.. index:: pair: string; methods + +String objects support the methods listed below. + +In addition, Python's strings support the sequence type methods described in the +:ref:`typesseq` section. To output formatted strings, see the +:ref:`string-formatting` section. Also, see the :mod:`re` module for string +functions based on regular expressions. .. method:: str.capitalize() @@ -1505,9 +1022,7 @@ .. method:: str.center(width[, fillchar]) Return centered in a string of length *width*. Padding is done using the - specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. - + specified *fillchar* (default is a space). .. method:: str.count(sub[, start[, end]]) @@ -1525,7 +1040,7 @@ a :exc:`UnicodeError`. Other possible values are ``'ignore'``, ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a + :func:`codecs.register_error`, see section :ref:`codec-base-classes`. For a list of possible encodings, see section :ref:`standard-encodings`. .. versionchanged:: 3.1 @@ -1540,32 +1055,21 @@ at that position. -.. method:: str.expandtabs(tabsize=8) - - Return a copy of the string where all tab characters are replaced by one or - more spaces, depending on the current column and the given tab size. Tab - positions occur every *tabsize* characters (default is 8, giving tab - positions at columns 0, 8, 16 and so on). To expand the string, the current - column is set to zero and the string is examined character by character. If - the character is a tab (``\t``), one or more space characters are inserted - in the result until the current column is equal to the next tab position. - (The tab character itself is not copied.) If the character is a newline - (``\n``) or return (``\r``), it is copied and the current column is reset to - zero. Any other character is copied unchanged and the current column is - incremented by one regardless of how the character is represented when - printed. - - >>> '01\t012\t0123\t01234'.expandtabs() - '01 012 0123 01234' - >>> '01\t012\t0123\t01234'.expandtabs(4) - '01 012 0123 01234' +.. method:: str.expandtabs([tabsize]) + + Return a copy of the string where all tab characters are replaced by zero or + more spaces, depending on the current column and the given tab size. The + column number is reset to zero after each newline occurring in the string. + If *tabsize* is not given, a tab size of ``8`` characters is assumed. This + doesn't understand other non-printing characters or escape sequences. .. method:: str.find(sub[, start[, end]]) - Return the lowest index in the string where substring *sub* is found within - the slice ``s[start:end]``. Optional arguments *start* and *end* are - interpreted as in slice notation. Return ``-1`` if *sub* is not found. + Return the lowest index in the string where substring *sub* is found, such + that *sub* is contained in the slice ``s[start:end]``. Optional arguments + *start* and *end* are interpreted as in slice notation. Return ``-1`` if + *sub* is not found. .. note:: @@ -1596,7 +1100,7 @@ .. method:: str.format_map(mapping) Similar to ``str.format(**mapping)``, except that ``mapping`` is - used directly and not copied to a :class:`dict`. This is useful + used directly and not copied to a :class:`dict` . This is useful if for example ``mapping`` is a dict subclass: >>> class Default(dict): @@ -1611,8 +1115,7 @@ .. method:: str.index(sub[, start[, end]]) - Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. + Like :meth:`find`, but raise :exc:`ValueError` when the substring is not found. .. method:: str.isalnum() @@ -1655,8 +1158,6 @@ Return true if the string is a valid identifier according to the language definition, section :ref:`identifiers`. - Use :func:`keyword.iskeyword` to test for reserved identifiers such as - :keyword:`def` and :keyword:`class`. .. method:: str.islower() @@ -1715,9 +1216,9 @@ .. method:: str.ljust(width[, fillchar]) - Return the string left justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. + Return the string left justified in a string of length *width*. Padding is done + using the specified *fillchar* (default is a space). The original string is + returned if *width* is less than or equal to ``len(s)``. .. method:: str.lower() @@ -1734,7 +1235,7 @@ Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a prefix; rather, all combinations of its values are stripped:: + argument is not a prefix; rather, all combinations of its values are stripped: >>> ' spacious '.lstrip() 'spacious ' @@ -1787,9 +1288,9 @@ .. method:: str.rjust(width[, fillchar]) - Return the string right justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. + Return the string right justified in a string of length *width*. Padding is done + using the specified *fillchar* (default is a space). The original string is + returned if *width* is less than or equal to ``len(s)``. .. method:: str.rpartition(sep) @@ -1814,7 +1315,7 @@ Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + argument is not a suffix; rather, all combinations of its values are stripped: >>> ' spacious '.rstrip() ' spacious' @@ -1827,8 +1328,8 @@ Return a list of the words in the string, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done (thus, the list will have at most ``maxsplit+1`` elements). If *maxsplit* is not - specified or ``-1``, then there is no limit on the number of splits - (all possible splits are made). + specified, then there is no limit on the number of splits (all possible + splits are made). If *sep* is given, consecutive delimiters are not grouped together and are deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns @@ -1836,15 +1337,6 @@ (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). Splitting an empty string with a specified separator returns ``['']``. - For example:: - - >>> '1,2,3'.split(',') - ['1', '2', '3'] - >>> '1,2,3'.split(',', maxsplit=1) - ['1', '2,3'] - >>> '1,2,,3,'.split(',') - ['1', '2', '', '3', ''] - If *sep* is not specified or is ``None``, a different splitting algorithm is applied: runs of consecutive whitespace are regarded as a single separator, and the result will contain no empty strings at the start or end if the @@ -1852,18 +1344,9 @@ string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: - - >>> '1 2 3'.split() - ['1', '2', '3'] - >>> '1 2 3'.split(maxsplit=1) - ['1', '2 3'] - >>> ' 1 2 3 '.split() - ['1', '2', '3'] - - -.. index:: - single: universal newlines; str.splitlines method + For example, ``' 1 2 3 '.split()`` returns ``['1', '2', '3']``, and + ``' 1 2 3 '.split(None, 1)`` returns ``['1', '2 3 ']``. + .. method:: str.splitlines([keepends]) @@ -1871,62 +1354,6 @@ breaks are not included in the resulting list unless *keepends* is given and true. - This method splits on the following line boundaries. In particular, the - boundaries are a superset of :term:`universal newlines`. - - +-----------------------+-----------------------------+ - | Representation | Description | - +=======================+=============================+ - | ``\n`` | Line Feed | - +-----------------------+-----------------------------+ - | ``\r`` | Carriage Return | - +-----------------------+-----------------------------+ - | ``\r\n`` | Carriage Return + Line Feed | - +-----------------------+-----------------------------+ - | ``\v`` or ``\x0b`` | Line Tabulation | - +-----------------------+-----------------------------+ - | ``\f`` or ``\x0c`` | Form Feed | - +-----------------------+-----------------------------+ - | ``\x1c`` | File Separator | - +-----------------------+-----------------------------+ - | ``\x1d`` | Group Separator | - +-----------------------+-----------------------------+ - | ``\x1e`` | Record Separator | - +-----------------------+-----------------------------+ - | ``\x85`` | Next Line (C1 Control Code) | - +-----------------------+-----------------------------+ - | ``\u2028`` | Line Separator | - +-----------------------+-----------------------------+ - | ``\u2029`` | Paragraph Separator | - +-----------------------+-----------------------------+ - - .. versionchanged:: 3.2 - - ``\v`` and ``\f`` added to list of line boundaries. - - For example:: - - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() - ['ab c', '', 'de fg', 'kl'] - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] - - Unlike :meth:`~str.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> "".splitlines() - [] - >>> "One line\n".splitlines() - ['One line'] - - For comparison, ``split('\n')`` gives:: - - >>> ''.split('\n') - [''] - >>> 'Two lines\n'.split('\n') - ['Two lines', ''] - .. method:: str.startswith(prefix[, start[, end]]) @@ -1942,23 +1369,13 @@ The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + values are stripped: >>> ' spacious '.strip() 'spacious' >>> 'www.example.com'.strip('cmowz.') 'example' - The outermost leading and trailing *chars* argument values are stripped - from the string. Characters are removed from the leading end until - reaching a string character that is not contained in the set of - characters in *chars*. A similar action takes place on the trailing end. - For example:: - - >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' - >>> comment_string.strip('.#! ') - 'Section 3.2.1 Issue #32' - .. method:: str.swapcase() @@ -1972,11 +1389,6 @@ Return a titlecased version of the string where words start with an uppercase character and the remaining characters are lowercase. - For example:: - - >>> 'Hello world'.title() - 'Hello World' - The algorithm uses a simple language-independent definition of a word as groups of consecutive letters. The definition works in many contexts but it means that apostrophes in contractions and possessives form word @@ -1989,31 +1401,30 @@ >>> import re >>> def titlecase(s): - ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0)[0].upper() + - ... mo.group(0)[1:].lower(), - ... s) - ... + return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", + lambda mo: mo.group(0)[0].upper() + + mo.group(0)[1:].lower(), + s) + >>> titlecase("they're bill's friends.") "They're Bill's Friends." -.. method:: str.translate(table) - - Return a copy of the string in which each character has been mapped through - the given translation table. The table must be an object that implements - indexing via :meth:`__getitem__`, typically a :term:`mapping` or - :term:`sequence`. When indexed by a Unicode ordinal (an integer), the - table object can do any of the following: return a Unicode ordinal or a - string, to map the character to one or more other characters; return - ``None``, to delete the character from the return string; or raise a - :exc:`LookupError` exception, to map the character to itself. +.. method:: str.translate(map) + + Return a copy of the *s* where all characters have been mapped through the + *map* which must be a dictionary of Unicode ordinals (integers) to Unicode + ordinals, strings or ``None``. Unmapped characters are left untouched. + Characters mapped to ``None`` are deleted. You can use :meth:`str.maketrans` to create a translation map from character-to-character mappings in different formats. - See also the :mod:`codecs` module for a more flexible approach to custom - character mappings. + .. note:: + + An even more flexible approach is to create a custom character mapping + codec using the :mod:`codecs` module (see :mod:`encodings.cp1251` for an + example). .. method:: str.upper() @@ -2030,25 +1441,16 @@ .. method:: str.zfill(width) - Return a copy of the string left filled with ASCII ``'0'`` digits to - make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) - is handled by inserting the padding *after* the sign character rather - than before. The original string is returned if *width* is less than - or equal to ``len(s)``. - - For example:: - - >>> "42".zfill(5) - '00042' - >>> "-42".zfill(5) - '-0042' + Return the numeric string left filled with zeros in a string of length + *width*. A sign prefix is handled correctly. The original string is + returned if *width* is less than or equal to ``len(s)``. .. _old-string-formatting: -``printf``-style String Formatting ----------------------------------- +Old String Formatting Operations +-------------------------------- .. index:: single: formatting, string (%) @@ -2060,19 +1462,23 @@ single: % formatting single: % interpolation +.. XXX is the note enough? + .. note:: - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :meth:`str.format` interface - helps avoid these errors, and also provides a generally more powerful, - flexible and extensible approach to formatting text. + The formatting operations described here are modelled on C's printf() + syntax. They only support formatting of certain builtin types. The + use of a binary operator means that care may be needed in order to + format tuples and dictionaries correctly. As the new + :ref:`string-formatting` syntax is more flexible and handles tuples and + dictionaries naturally, it is recommended for new code. However, there + are no current plans to deprecate printf-style formatting. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a string), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to the using :c:func:`sprintf` in the C language. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -2230,232 +1636,229 @@ ``%f`` conversions for numbers whose absolute value is over 1e50 are no longer replaced by ``%g`` conversions. - .. index:: - single: buffer protocol; binary sequence types - -.. _binaryseq: - -Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview` -================================================================================= + module: string + module: re + +Additional string operations are defined in standard modules :mod:`string` and +:mod:`re`. + + +.. _typesseq-range: + +Range Type +---------- + +.. index:: object: range + +The :class:`range` type is an immutable sequence which is commonly used for +looping. The advantage of the :class:`range` type is that an :class:`range` +object will always take the same amount of memory, no matter the size of the +range it represents. + +Range objects have relatively little behavior: they support indexing, contains, +iteration, the :func:`len` function, and the following methods: + +.. method:: range.count(x) + + Return the number of *i*'s for which ``s[i] == x``. + + .. versionadded:: 3.2 + +.. method:: range.index(x) + + Return the smallest *i* such that ``s[i] == x``. Raises + :exc:`ValueError` when *x* is not in the range. + + .. versionadded:: 3.2 + + +.. _typesseq-mutable: + +Mutable Sequence Types +---------------------- .. index:: - object: bytes + triple: mutable; sequence; types + object: list object: bytearray - object: memoryview - module: array - -The core built-in types for manipulating binary data are :class:`bytes` and -:class:`bytearray`. They are supported by :class:`memoryview` which uses -the :ref:`buffer protocol ` to access the memory of other -binary objects without needing to make a copy. - -The :mod:`array` module supports efficient storage of basic data types like -32-bit integers and IEEE754 double-precision floating values. - -.. _typebytes: - -Bytes ------ - -.. index:: object: bytes - -Bytes objects are immutable sequences of single bytes. Since many major -binary protocols are based on the ASCII text encoding, bytes objects offer -several methods that are only valid when working with ASCII compatible -data and are closely related to string objects in a variety of other ways. - -Firstly, the syntax for bytes literals is largely the same as that for string -literals, except that a ``b`` prefix is added: - -* Single quotes: ``b'still allows embedded "double" quotes'`` -* Double quotes: ``b"still allows embedded 'single' quotes"``. -* Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` - -Only ASCII characters are permitted in bytes literals (regardless of the -declared source code encoding). Any binary values over 127 must be entered -into bytes literals using the appropriate escape sequence. - -As with string literals, bytes literals may also use a ``r`` prefix to disable -processing of escape sequences. See :ref:`strings` for more about the various -forms of bytes literal, including supported escape sequences. - -While bytes literals and representations are based on ASCII text, bytes -objects actually behave like immutable sequences of integers, with each -value in the sequence restricted such that ``0 <= x < 256`` (attempts to -violate this restriction will trigger :exc:`ValueError`. This is done -deliberately to emphasise that while many binary formats include ASCII based -elements and can be usefully manipulated with some text-oriented algorithms, -this is not generally the case for arbitrary binary data (blindly applying -text processing algorithms to binary data formats that are not ASCII -compatible will usually lead to data corruption). - -In addition to the literal forms, bytes objects can be created in a number of -other ways: - -* A zero-filled bytes object of a specified length: ``bytes(10)`` -* From an iterable of integers: ``bytes(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytes(obj)`` - -Also see the :ref:`bytes ` built-in. - -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytes type has an additional class method to read data in that format: - -.. classmethod:: bytes.fromhex(string) - - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII spaces being ignored. - - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' - -A reverse conversion function exists to transform a bytes object into its -hexadecimal representation. - -.. method:: bytes.hex() - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' - - .. versionadded:: 3.5 - -Since bytes objects are sequences of integers (akin to a tuple), for a bytes -object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes -object of length 1. (This contrasts with text strings, where both indexing -and slicing will produce a string of length 1) - -The representation of bytes objects uses the literal format (``b'...'``) -since it is often more useful than e.g. ``bytes([46, 46, 46])``. You can -always convert a bytes object into a list of integers using ``list(b)``. - -.. note:: - For Python 2.x users: In the Python 2.x series, a variety of implicit - conversions between 8-bit strings (the closest thing 2.x offers to a - built-in binary data type) and Unicode strings were permitted. This was a - backwards compatibility workaround to account for the fact that Python - originally only supported 8-bit text, and Unicode text was a later - addition. In Python 3.x, those implicit conversions are gone - conversions - between 8-bit binary data and Unicode text must be explicit, and bytes and - string objects will always compare unequal. - - -.. _typebytearray: - -Bytearray Objects ------------------ - -.. index:: object: bytearray - -:class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. There is no dedicated literal syntax for bytearray objects, instead -they are always created by calling the constructor: - -* Creating an empty instance: ``bytearray()`` -* Creating a zero-filled instance with a given length: ``bytearray(10)`` -* From an iterable of integers: ``bytearray(range(20))`` -* Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` - -As bytearray objects are mutable, they support the -:ref:`mutable ` sequence operations in addition to the -common bytes and bytearray operations described in :ref:`bytes-methods`. - -Also see the :ref:`bytearray ` built-in. - -Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal -numbers are a commonly used format for describing binary data. Accordingly, -the bytearray type has an additional class method to read data in that format: - -.. classmethod:: bytearray.fromhex(string) - - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII spaces being ignored. - - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') - -A reverse conversion function exists to transform a bytearray object into its -hexadecimal representation. - -.. method:: bytearray.hex() - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' - - .. versionadded:: 3.5 - -Since bytearray objects are sequences of integers (akin to a list), for a -bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be -a bytearray object of length 1. (This contrasts with text strings, where -both indexing and slicing will produce a string of length 1) - -The representation of bytearray objects uses the bytes literal format -(``bytearray(b'...')``) since it is often more useful than e.g. -``bytearray([46, 46, 46])``. You can always convert a bytearray object into -a list of integers using ``list(b)``. + +List and bytearray objects support additional operations that allow in-place +modification of the object. Other mutable sequence types (when added to the +language) should also support these operations. Strings and tuples are +immutable sequence types: such objects cannot be modified once created. The +following operations are defined on mutable sequence types (where *x* is an +arbitrary object). + +Note that while lists allow their items to be of any type, bytearray object +"items" are all integers in the range 0 <= x < 256. + +.. index:: + triple: operations on; sequence; types + triple: operations on; list; type + pair: subscript; assignment + pair: slice; assignment + statement: del + single: append() (sequence method) + single: extend() (sequence method) + single: count() (sequence method) + single: clear() (sequence method) + single: copy() (sequence method) + single: index() (sequence method) + single: insert() (sequence method) + single: pop() (sequence method) + single: remove() (sequence method) + single: reverse() (sequence method) + single: sort() (sequence method) + ++------------------------------+--------------------------------+---------------------+ +| Operation | Result | Notes | ++==============================+================================+=====================+ +| ``s[i] = x`` | item *i* of *s* is replaced by | | +| | *x* | | ++------------------------------+--------------------------------+---------------------+ +| ``s[i:j] = t`` | slice of *s* from *i* to *j* | | +| | is replaced by the contents of | | +| | the iterable *t* | | ++------------------------------+--------------------------------+---------------------+ +| ``del s[i:j]`` | same as ``s[i:j] = []`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | +| | are replaced by those of *t* | | ++------------------------------+--------------------------------+---------------------+ +| ``del s[i:j:k]`` | removes the elements of | | +| | ``s[i:j:k]`` from the list | | ++------------------------------+--------------------------------+---------------------+ +| ``s.append(x)`` | same as ``s[len(s):len(s)] = | | +| | [x]`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s.extend(x)`` | same as ``s[len(s):len(s)] = | \(2) | +| | x`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s.clear()`` | remove all items from ``s`` | | +| | | | ++------------------------------+--------------------------------+---------------------+ +| ``s.copy()`` | return a shallow copy of ``s`` | | +| | | | ++------------------------------+--------------------------------+---------------------+ +| ``s.count(x)`` | return number of *i*'s for | | +| | which ``s[i] == x`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s.index(x[, i[, j]])`` | return smallest *k* such that | \(3) | +| | ``s[k] == x`` and ``i <= k < | | +| | j`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | \(4) | ++------------------------------+--------------------------------+---------------------+ +| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | \(5) | +| | return x`` | | ++------------------------------+--------------------------------+---------------------+ +| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | \(3) | ++------------------------------+--------------------------------+---------------------+ +| ``s.reverse()`` | reverses the items of *s* in | \(6) | +| | place | | ++------------------------------+--------------------------------+---------------------+ +| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) | ++------------------------------+--------------------------------+---------------------+ + + +Notes: + +(1) + *t* must have the same length as the slice it is replacing. + +(2) + *x* can be any iterable object. + +(3) + Raises :exc:`ValueError` when *x* is not found in *s*. When a negative index is + passed as the second or third parameter to the :meth:`index` method, the sequence + length is added, as for slice indices. If it is still negative, it is truncated + to zero, as for slice indices. + +(4) + When a negative index is passed as the first parameter to the :meth:`insert` + method, the sequence length is added, as for slice indices. If it is still + negative, it is truncated to zero, as for slice indices. + +(5) + The optional argument *i* defaults to ``-1``, so that by default the last + item is removed and returned. + +(6) + The :meth:`sort` and :meth:`reverse` methods modify the sequence in place for + economy of space when sorting or reversing a large sequence. To remind you + that they operate by side effect, they don't return the sorted or reversed + sequence. + +(7) + The :meth:`sort` method takes optional arguments for controlling the + comparisons. Each must be specified as a keyword argument. + + *key* specifies a function of one argument that is used to extract a comparison + key from each list element: ``key=str.lower``. The default value is ``None``. + Use :func:`functools.cmp_to_key` to convert an + old-style *cmp* function to a *key* function. + + + *reverse* is a boolean value. If set to ``True``, then the list elements are + sorted as if each comparison were reversed. + + The :meth:`sort` method is guaranteed to be stable. A + sort is stable if it guarantees not to change the relative order of elements + that compare equal --- this is helpful for sorting in multiple passes (for + example, sort by department, then by salary grade). + + .. impl-detail:: + + While a list is being sorted, the effect of attempting to mutate, or even + inspect, the list is undefined. The C implementation of Python makes the + list appear empty for the duration, and raises :exc:`ValueError` if it can + detect that the list has been mutated during a sort. + +(8) + :meth:`sort` is not supported by :class:`bytearray` objects. + + .. versionadded:: 3.3 + :meth:`clear` and :meth:`!copy` methods. .. _bytes-methods: -Bytes and Bytearray Operations ------------------------------- +Bytes and Byte Array Methods +---------------------------- .. index:: pair: bytes; methods pair: bytearray; methods -Both bytes and bytearray objects support the :ref:`common ` -sequence operations. They interoperate not just with operands of the same -type, but with any :term:`bytes-like object`. Due to this flexibility, they can be -freely mixed in operations without causing errors. However, the return type -of the result may depend on the order of operands. +Bytes and bytearray objects, being "strings of bytes", have all methods found on +strings, with the exception of :func:`encode`, :func:`format` and +:func:`isidentifier`, which do not make sense with these types. For converting +the objects to strings, they have a :func:`decode` method. + +Wherever one of these methods needs to interpret the bytes as characters +(e.g. the :func:`is...` methods), the ASCII character set is assumed. + +.. versionadded:: 3.3 + The functions :func:`count`, :func:`find`, :func:`index`, + :func:`rfind` and :func:`rindex` have additional semantics compared to + the corresponding string functions: They also accept an integer in + range 0 to 255 (a byte) as their first argument. .. note:: The methods on bytes and bytearray objects don't accept strings as their arguments, just as the methods on strings don't accept bytes as their - arguments. For example, you have to write:: + arguments. For example, you have to write :: a = "abc" b = a.replace("a", "f") - and:: + and :: a = b"abc" b = a.replace(b"a", b"f") -Some bytes and bytearray operations assume the use of ASCII compatible -binary formats, and hence should be avoided when working with arbitrary -binary data. These restrictions are covered below. - -.. note:: - Using these ASCII based operations to manipulate binary data that is not - stored in an ASCII based format may lead to data corruption. - -The following methods on bytes and bytearray objects can be used with -arbitrary binary data. - -.. method:: bytes.count(sub[, start[, end]]) - bytearray.count(sub[, start[, end]]) - - Return the number of non-overlapping occurrences of subsequence *sub* in - the range [*start*, *end*]. Optional arguments *start* and *end* are - interpreted as in slice notation. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - .. method:: bytes.decode(encoding="utf-8", errors="strict") bytearray.decode(encoding="utf-8", errors="strict") @@ -2465,76 +1868,44 @@ error handling scheme. The default for *errors* is ``'strict'``, meaning that encoding errors raise a :exc:`UnicodeError`. Other possible values are ``'ignore'``, ``'replace'`` and any other name registered via - :func:`codecs.register_error`, see section :ref:`error-handlers`. For a + :func:`codecs.register_error`, see section :ref:`codec-base-classes`. For a list of possible encodings, see section :ref:`standard-encodings`. - .. note:: - - Passing the *encoding* argument to :class:`str` allows decoding any - :term:`bytes-like object` directly, without needing to make a temporary - bytes or bytearray object. - .. versionchanged:: 3.1 Added support for keyword arguments. -.. method:: bytes.endswith(suffix[, start[, end]]) - bytearray.endswith(suffix[, start[, end]]) - - Return ``True`` if the binary data ends with the specified *suffix*, - otherwise return ``False``. *suffix* can also be a tuple of suffixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The suffix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.find(sub[, start[, end]]) - bytearray.find(sub[, start[, end]]) - - Return the lowest index in the data where the subsequence *sub* is found, - such that *sub* is contained in the slice ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` if *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. note:: - - The :meth:`~bytes.find` method should be used only if you need to know the - position of *sub*. To check if *sub* is a substring or not, use the - :keyword:`in` operator:: - - >>> b'Py' in b'Python' - True - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.index(sub[, start[, end]]) - bytearray.index(sub[, start[, end]]) - - Like :meth:`~bytes.find`, but raise :exc:`ValueError` when the - subsequence is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.join(iterable) - bytearray.join(iterable) - - Return a bytes or bytearray object which is the concatenation of the - binary data sequences in the :term:`iterable` *iterable*. A - :exc:`TypeError` will be raised if there are any values in *iterable* - that are not :term:`bytes-like objects `, including - :class:`str` objects. The separator between elements is the contents - of the bytes or bytearray object providing this method. +The bytes and bytearray types have an additional class method: + +.. classmethod:: bytes.fromhex(string) + bytearray.fromhex(string) + + This :class:`bytes` class method returns a bytes or bytearray object, + decoding the given string object. The string must contain two hexadecimal + digits per byte, spaces are ignored. + + >>> bytes.fromhex('f0 f1f2 ') + b'\xf0\xf1\xf2' + + +The maketrans and translate methods differ in semantics from the versions +available on strings: + +.. method:: bytes.translate(table[, delete]) + bytearray.translate(table[, delete]) + + Return a copy of the bytes or bytearray object where all bytes occurring in + the optional argument *delete* are removed, and the remaining bytes have been + mapped through the given translation table, which must be a bytes object of + length 256. + + You can use the :func:`bytes.maketrans` method to create a translation table. + + Set the *table* argument to ``None`` for translations that only delete + characters:: + + >>> b'read this short text'.translate(None, b'aeiou') + b'rd ths shrt txt' .. staticmethod:: bytes.maketrans(from, to) @@ -2542,803 +1913,477 @@ This static method returns a translation table usable for :meth:`bytes.translate` that will map each character in *from* into the - character at the same position in *to*; *from* and *to* must both be - :term:`bytes-like objects ` and have the same length. + character at the same position in *to*; *from* and *to* must be bytes objects + and have the same length. .. versionadded:: 3.1 -.. method:: bytes.partition(sep) - bytearray.partition(sep) - - Split the sequence at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator, and the part - after the separator. If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) - - Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. - - The subsequence to search for and its replacement may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.rfind(sub[, start[, end]]) - bytearray.rfind(sub[, start[, end]]) - - Return the highest index in the sequence where the subsequence *sub* is - found, such that *sub* is contained within ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` on failure. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rindex(sub[, start[, end]]) - bytearray.rindex(sub[, start[, end]]) - - Like :meth:`~bytes.rfind` but raises :exc:`ValueError` when the - subsequence *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) - - Split the sequence at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator, and the part - after the separator. If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.startswith(prefix[, start[, end]]) - bytearray.startswith(prefix[, start[, end]]) - - Return ``True`` if the binary data starts with the specified *prefix*, - otherwise return ``False``. *prefix* can also be a tuple of prefixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The prefix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.translate(table[, delete]) - bytearray.translate(table[, delete]) - - Return a copy of the bytes or bytearray object where all bytes occurring in - the optional argument *delete* are removed, and the remaining bytes have - been mapped through the given translation table, which must be a bytes - object of length 256. - - You can use the :func:`bytes.maketrans` method to create a translation - table. - - Set the *table* argument to ``None`` for translations that only delete - characters:: - - >>> b'read this short text'.translate(None, b'aeiou') - b'rd ths shrt txt' - - -The following methods on bytes and bytearray objects have default behaviours -that assume the use of ASCII compatible binary formats, but can still be used -with arbitrary binary data by passing appropriate arguments. Note that all of -the bytearray methods in this section do *not* operate in place, and instead -produce new objects. - -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) - - Return a copy of the object centered in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) - - Return a copy of the object left justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) - - Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; - rather, all combinations of its values are stripped:: - - >>> b' spacious '.lstrip() - b'spacious ' - >>> b'www.example.com'.lstrip(b'cmowz.') - b'example.com' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) - - Return a copy of the object right justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rsplit(sep=None, maxsplit=-1) - bytearray.rsplit(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits - are done, the *rightmost* ones. If *sep* is not specified or ``None``, - any subsequence consisting solely of ASCII whitespace is a separator. - Except for splitting from the right, :meth:`rsplit` behaves like - :meth:`split` which is described in detail below. - - -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) - - Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, - all combinations of its values are stripped:: - - >>> b' spacious '.rstrip() - b' spacious' - >>> b'mississippi'.rstrip(b'ipz') - b'mississ' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.split(sep=None, maxsplit=-1) - bytearray.split(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given and non-negative, at most - *maxsplit* splits are done (thus, the list will have at most ``maxsplit+1`` - elements). If *maxsplit* is not specified or is ``-1``, then there is no - limit on the number of splits (all possible splits are made). - - If *sep* is given, consecutive delimiters are not grouped together and are - deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` - returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any - :term:`bytes-like object`. - - For example:: - - >>> b'1,2,3'.split(b',') - [b'1', b'2', b'3'] - >>> b'1,2,3'.split(b',', maxsplit=1) - [b'1', b'2,3'] - >>> b'1,2,,3,'.split(b',') - [b'1', b'2', b'', b'3', b''] - - If *sep* is not specified or is ``None``, a different splitting algorithm - is applied: runs of consecutive ASCII whitespace are regarded as a single - separator, and the result will contain no empty strings at the start or - end if the sequence has leading or trailing whitespace. Consequently, - splitting an empty sequence or a sequence consisting solely of ASCII - whitespace without a specified separator returns ``[]``. - - For example:: - - - >>> b'1 2 3'.split() - [b'1', b'2', b'3'] - >>> b'1 2 3'.split(maxsplit=1) - [b'1', b'2 3'] - >>> b' 1 2 3 '.split() - [b'1', b'2', b'3'] - - -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) - - Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is - not a prefix or suffix; rather, all combinations of its values are - stripped:: - - >>> b' spacious '.strip() - b'spacious' - >>> b'www.example.com'.strip(b'cmowz.') - b'example' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -The following methods on bytes and bytearray objects assume the use of ASCII -compatible binary formats and should not be applied to arbitrary binary data. -Note that all of the bytearray methods in this section do *not* operate in -place, and instead produce new objects. - -.. method:: bytes.capitalize() - bytearray.capitalize() - - Return a copy of the sequence with each byte interpreted as an ASCII - character, and the first byte capitalized and the rest lowercased. - Non-ASCII byte values are passed through unchanged. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.expandtabs(tabsize=8) - bytearray.expandtabs(tabsize=8) - - Return a copy of the sequence where all ASCII tab characters are replaced - by one or more ASCII spaces, depending on the current column and the given - tab size. Tab positions occur every *tabsize* bytes (default is 8, - giving tab positions at columns 0, 8, 16 and so on). To expand the - sequence, the current column is set to zero and the sequence is examined - byte by byte. If the byte is an ASCII tab character (``b'\t'``), one or - more space characters are inserted in the result until the current column - is equal to the next tab position. (The tab character itself is not - copied.) If the current byte is an ASCII newline (``b'\n'``) or - carriage return (``b'\r'``), it is copied and the current column is reset - to zero. Any other byte value is copied unchanged and the current column - is incremented by one regardless of how the byte value is represented when - printed:: - - >>> b'01\t012\t0123\t01234'.expandtabs() - b'01 012 0123 01234' - >>> b'01\t012\t0123\t01234'.expandtabs(4) - b'01 012 0123 01234' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.isalnum() - bytearray.isalnum() - - Return true if all bytes in the sequence are alphabetical ASCII characters - or ASCII decimal digits and the sequence is not empty, false otherwise. - Alphabetic ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. ASCII decimal - digits are those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'ABCabc1'.isalnum() - True - >>> b'ABC abc1'.isalnum() - False - - -.. method:: bytes.isalpha() - bytearray.isalpha() - - Return true if all bytes in the sequence are alphabetic ASCII characters - and the sequence is not empty, false otherwise. Alphabetic ASCII - characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - For example:: - - >>> b'ABCabc'.isalpha() - True - >>> b'ABCabc1'.isalpha() - False - - -.. method:: bytes.isdigit() - bytearray.isdigit() - - Return true if all bytes in the sequence are ASCII decimal digits - and the sequence is not empty, false otherwise. ASCII decimal digits are - those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'1234'.isdigit() - True - >>> b'1.23'.isdigit() - False - - -.. method:: bytes.islower() - bytearray.islower() - - Return true if there is at least one lowercase ASCII character - in the sequence and no uppercase ASCII characters, false otherwise. - - For example:: - - >>> b'hello world'.islower() - True - >>> b'Hello world'.islower() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.isspace() - bytearray.isspace() - - Return true if all bytes in the sequence are ASCII whitespace and the - sequence is not empty, false otherwise. ASCII whitespace characters are - those byte values in the sequence ``b' \t\n\r\x0b\f'`` (space, tab, newline, - carriage return, vertical tab, form feed). - - -.. method:: bytes.istitle() - bytearray.istitle() - - Return true if the sequence is ASCII titlecase and the sequence is not - empty, false otherwise. See :meth:`bytes.title` for more details on the - definition of "titlecase". - - For example:: - - >>> b'Hello World'.istitle() - True - >>> b'Hello world'.istitle() - False - - -.. method:: bytes.isupper() - bytearray.isupper() - - Return true if there is at least one uppercase alphabetic ASCII character - in the sequence and no lowercase ASCII characters, false otherwise. - - For example:: - - >>> b'HELLO WORLD'.isupper() - True - >>> b'Hello world'.isupper() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.lower() - bytearray.lower() - - Return a copy of the sequence with all the uppercase ASCII characters - converted to their corresponding lowercase counterpart. - - For example:: - - >>> b'Hello World'.lower() - b'hello world' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - +.. _types-set: + +Set Types --- :class:`set`, :class:`frozenset` +============================================== + +.. index:: object: set + +A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. +Common uses include membership testing, removing duplicates from a sequence, and +computing mathematical operations such as intersection, union, difference, and +symmetric difference. +(For other containers see the built in :class:`dict`, :class:`list`, +and :class:`tuple` classes, and the :mod:`collections` module.) + +Like other collections, sets support ``x in set``, ``len(set)``, and ``for x in +set``. Being an unordered collection, sets do not record element position or +order of insertion. Accordingly, sets do not support indexing, slicing, or +other sequence-like behavior. + +There are currently two built-in set types, :class:`set` and :class:`frozenset`. +The :class:`set` type is mutable --- the contents can be changed using methods +like :meth:`add` and :meth:`remove`. Since it is mutable, it has no hash value +and cannot be used as either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- its contents cannot be +altered after it is created; it can therefore be used as a dictionary key or as +an element of another set. + +Non-empty sets (not frozensets) can be created by placing a comma-separated list +of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the +:class:`set` constructor. + +The constructors for both classes work the same: + +.. class:: set([iterable]) + frozenset([iterable]) + + Return a new set or frozenset object whose elements are taken from + *iterable*. The elements of a set must be hashable. To represent sets of + sets, the inner sets must be :class:`frozenset` objects. If *iterable* is + not specified, a new empty set is returned. + + Instances of :class:`set` and :class:`frozenset` provide the following + operations: + + .. describe:: len(s) + + Return the cardinality of set *s*. + + .. describe:: x in s + + Test *x* for membership in *s*. + + .. describe:: x not in s + + Test *x* for non-membership in *s*. + + .. method:: isdisjoint(other) + + Return True if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. + + .. method:: issubset(other) + set <= other + + Test whether every element in the set is in *other*. + + .. method:: set < other + + Test whether the set is a true subset of *other*, that is, + ``set <= other and set != other``. + + .. method:: issuperset(other) + set >= other + + Test whether every element in *other* is in the set. + + .. method:: set > other + + Test whether the set is a true superset of *other*, that is, ``set >= + other and set != other``. + + .. method:: union(other, ...) + set | other | ... + + Return a new set with elements from the set and all others. + + .. method:: intersection(other, ...) + set & other & ... + + Return a new set with elements common to the set and all others. + + .. method:: difference(other, ...) + set - other - ... + + Return a new set with elements in the set that are not in the others. + + .. method:: symmetric_difference(other) + set ^ other + + Return a new set with elements in either the set or *other* but not both. + + .. method:: copy() + + Return a new set with a shallow copy of *s*. + + + Note, the non-operator versions of :meth:`union`, :meth:`intersection`, + :meth:`difference`, and :meth:`symmetric_difference`, :meth:`issubset`, and + :meth:`issuperset` methods will accept any iterable as an argument. In + contrast, their operator based counterparts require their arguments to be + sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` + in favor of the more readable ``set('abc').intersection('cbs')``. + + Both :class:`set` and :class:`frozenset` support set to set comparisons. Two + sets are equal if and only if every element of each set is contained in the + other (each is a subset of the other). A set is less than another set if and + only if the first set is a proper subset of the second set (is a subset, but + is not equal). A set is greater than another set if and only if the first set + is a proper superset of the second set (is a superset, but is not equal). + + Instances of :class:`set` are compared to instances of :class:`frozenset` + based on their members. For example, ``set('abc') == frozenset('abc')`` + returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. + + The subset and equality comparisons do not generalize to a complete ordering + function. For example, any two disjoint sets are not equal and are not + subsets of each other, so *all* of the following return ``False``: ``ab``. + + Since sets only define partial ordering (subset relationships), the output of + the :meth:`list.sort` method is undefined for lists of sets. + + Set elements, like dictionary keys, must be :term:`hashable`. + + Binary operations that mix :class:`set` instances with :class:`frozenset` + return the type of the first operand. For example: ``frozenset('ab') | + set('bc')`` returns an instance of :class:`frozenset`. + + The following table lists operations available for :class:`set` that do not + apply to immutable instances of :class:`frozenset`: + + .. method:: update(other, ...) + set |= other | ... + + Update the set, adding elements from all others. + + .. method:: intersection_update(other, ...) + set &= other & ... + + Update the set, keeping only elements found in it and all others. + + .. method:: difference_update(other, ...) + set -= other | ... + + Update the set, removing elements found in others. + + .. method:: symmetric_difference_update(other) + set ^= other + + Update the set, keeping only elements found in either set, but not in both. + + .. method:: add(elem) + + Add element *elem* to the set. + + .. method:: remove(elem) + + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. + + .. method:: discard(elem) + + Remove element *elem* from the set if it is present. + + .. method:: pop() + + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. + + .. method:: clear() + + Remove all elements from the set. + + + Note, the non-operator versions of the :meth:`update`, + :meth:`intersection_update`, :meth:`difference_update`, and + :meth:`symmetric_difference_update` methods will accept any iterable as an + argument. + + Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and + :meth:`discard` methods may be a set. To support searching for an equivalent + frozenset, the *elem* set is temporarily mutated during the search and then + restored. During the search, the *elem* set should not be read or mutated + since it does not have a meaningful value. + + +.. _typesmapping: + +Mapping Types --- :class:`dict` +=============================== .. index:: - single: universal newlines; bytes.splitlines method - single: universal newlines; bytearray.splitlines method - -.. method:: bytes.splitlines(keepends=False) - bytearray.splitlines(keepends=False) - - Return a list of the lines in the binary sequence, breaking at ASCII - line boundaries. This method uses the :term:`universal newlines` approach - to splitting lines. Line breaks are not included in the resulting list - unless *keepends* is given and true. - - For example:: - - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines() - [b'ab c', b'', b'de fg', b'kl'] - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - [b'ab c\n', b'\n', b'de fg\r', b'kl\r\n'] - - Unlike :meth:`~bytes.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> b"".split(b'\n'), b"Two lines\n".split(b'\n') - ([b''], [b'Two lines', b'']) - >>> b"".splitlines(), b"One line\n".splitlines() - ([], [b'One line']) - - -.. method:: bytes.swapcase() - bytearray.swapcase() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart and vice-versa. - - For example:: - - >>> b'Hello World'.swapcase() - b'hELLO wORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - Unlike :func:`str.swapcase()`, it is always the case that - ``bin.swapcase().swapcase() == bin`` for the binary versions. Case - conversions are symmetrical in ASCII, even though that is not generally - true for arbitrary Unicode code points. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.title() - bytearray.title() - - Return a titlecased version of the binary sequence where words start with - an uppercase ASCII character and the remaining characters are lowercase. - Uncased byte values are left unmodified. - - For example:: - - >>> b'Hello world'.title() - b'Hello World' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - All other byte values are uncased. - - The algorithm uses a simple language-independent definition of a word as - groups of consecutive letters. The definition works in many contexts but - it means that apostrophes in contractions and possessives form word - boundaries, which may not be the desired result:: - - >>> b"they're bill's friends from the UK".title() - b"They'Re Bill'S Friends From The Uk" - - A workaround for apostrophes can be constructed using regular expressions:: - - >>> import re - >>> def titlecase(s): - ... return re.sub(rb"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0)[0:1].upper() + - ... mo.group(0)[1:].lower(), - ... s) - ... - >>> titlecase(b"they're bill's friends.") - b"They're Bill's Friends." - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.upper() - bytearray.upper() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart. - - For example:: - - >>> b'Hello World'.upper() - b'HELLO WORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.zfill(width) - bytearray.zfill(width) - - Return a copy of the sequence left filled with ASCII ``b'0'`` digits to - make a sequence of length *width*. A leading sign prefix (``b'+'``/ - ``b'-'`` is handled by inserting the padding *after* the sign character - rather than before. For :class:`bytes` objects, the original sequence is - returned if *width* is less than or equal to ``len(seq)``. - - For example:: - - >>> b"42".zfill(5) - b'00042' - >>> b"-42".zfill(5) - b'-0042' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. _bytes-formatting: - -``printf``-style Bytes Formatting ----------------------------------- - -.. index:: - single: formatting, bytes (%) - single: formatting, bytearray (%) - single: interpolation, bytes (%) - single: interpolation, bytearray (%) - single: bytes; formatting - single: bytearray; formatting - single: bytes; interpolation - single: bytearray; interpolation - single: printf-style formatting - single: sprintf-style formatting - single: % formatting - single: % interpolation - -.. note:: - - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). If the value being printed may be a tuple or - dictionary, wrap it in a tuple. - -Bytes objects (``bytes``/``bytearray``) have one unique built-in operation: -the ``%`` operator (modulo). -This is also known as the bytes *formatting* or *interpolation* operator. -Given ``format % values`` (where *format* is a bytes object), ``%`` conversion -specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. - -If *format* requires a single argument, *values* may be a single non-tuple -object. [5]_ Otherwise, *values* must be a tuple with exactly the number of -items specified by the format bytes object, or a single mapping object (for -example, a dictionary). - -A conversion specifier contains two or more characters and has the following -components, which must occur in this order: - -#. The ``'%'`` character, which marks the start of the specifier. - -#. Mapping key (optional), consisting of a parenthesised sequence of characters - (for example, ``(somename)``). - -#. Conversion flags (optional), which affect the result of some conversion - types. - -#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the - actual width is read from the next element of the tuple in *values*, and the - object to convert comes after the minimum field width and optional precision. - -#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If - specified as ``'*'`` (an asterisk), the actual precision is read from the next - element of the tuple in *values*, and the value to convert comes after the - precision. - -#. Length modifier (optional). - -#. Conversion type. - -When the right argument is a dictionary (or other mapping type), then the -formats in the bytes object *must* include a parenthesised mapping key into that -dictionary inserted immediately after the ``'%'`` character. The mapping key -selects the value to be formatted from the mapping. For example: - - >>> print(b'%(language)s has %(number)03d quote types.' % - ... {b'language': b"Python", b"number": 2}) - b'Python has 002 quote types.' - -In this case no ``*`` specifiers may occur in a format (since they require a -sequential parameter list). - -The conversion flag characters are: - -+---------+---------------------------------------------------------------------+ -| Flag | Meaning | -+=========+=====================================================================+ -| ``'#'`` | The value conversion will use the "alternate form" (where defined | -| | below). | -+---------+---------------------------------------------------------------------+ -| ``'0'`` | The conversion will be zero padded for numeric values. | -+---------+---------------------------------------------------------------------+ -| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` | -| | conversion if both are given). | -+---------+---------------------------------------------------------------------+ -| ``' '`` | (a space) A blank should be left before a positive number (or empty | -| | string) produced by a signed conversion. | -+---------+---------------------------------------------------------------------+ -| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion | -| | (overrides a "space" flag). | -+---------+---------------------------------------------------------------------+ - -A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it -is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``. - -The conversion types are: - -+------------+-----------------------------------------------------+-------+ -| Conversion | Meaning | Notes | -+============+=====================================================+=======+ -| ``'d'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'i'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'o'`` | Signed octal value. | \(1) | -+------------+-----------------------------------------------------+-------+ -| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(8) | -+------------+-----------------------------------------------------+-------+ -| ``'x'`` | Signed hexadecimal (lowercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'X'`` | Signed hexadecimal (uppercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'c'`` | Single byte (accepts integer or single | | -| | byte objects). | | -+------------+-----------------------------------------------------+-------+ -| ``'b'`` | Bytes (any object that follows the | \(5) | -| | :ref:`buffer protocol ` or has | | -| | :meth:`__bytes__`). | | -+------------+-----------------------------------------------------+-------+ -| ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'a'`` | Bytes (converts any Python object using | \(5) | -| | ``repr(obj).encode('ascii','backslashreplace)``). | | -+------------+-----------------------------------------------------+-------+ -| ``'r'`` | ``'r'`` is an alias for ``'a'`` and should only | \(7) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'%'`` | No argument is converted, results in a ``'%'`` | | -| | character in the result. | | -+------------+-----------------------------------------------------+-------+ - -Notes: - -(1) - The alternate form causes a leading zero (``'0'``) to be inserted between - left-hand padding and the formatting of the number if the leading character - of the result is not already a zero. - -(2) - The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether - the ``'x'`` or ``'X'`` format was used) to be inserted between left-hand padding - and the formatting of the number if the leading character of the result is not - already a zero. - -(3) - The alternate form causes the result to always contain a decimal point, even if - no digits follow it. - - The precision determines the number of digits after the decimal point and - defaults to 6. - -(4) - The alternate form causes the result to always contain a decimal point, and - trailing zeroes are not removed as they would otherwise be. - - The precision determines the number of significant digits before and after the - decimal point and defaults to 6. - -(5) - If precision is ``N``, the output is truncated to ``N`` characters. - -(6) - ``b'%s'`` is deprecated, but will not be removed during the 3.x series. - -(7) - ``b'%r'`` is deprecated, but will not be removed during the 3.x series. - -(8) - See :pep:`237`. - -.. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - -.. seealso:: :pep:`461`. -.. versionadded:: 3.5 + object: mapping + object: dictionary + triple: operations on; mapping; types + triple: operations on; dictionary; type + statement: del + builtin: len + +A :dfn:`mapping` object maps :term:`hashable` values to arbitrary objects. +Mappings are mutable objects. There is currently only one standard mapping +type, the :dfn:`dictionary`. (For other containers see the built in +:class:`list`, :class:`set`, and :class:`tuple` classes, and the +:mod:`collections` module.) + +A dictionary's keys are *almost* arbitrary values. Values that are not +:term:`hashable`, that is, values containing lists, dictionaries or other +mutable types (that are compared by value rather than by object identity) may +not be used as keys. Numeric types used for keys obey the normal rules for +numeric comparison: if two numbers compare equal (such as ``1`` and ``1.0``) +then they can be used interchangeably to index the same dictionary entry. (Note +however, that since computers store floating-point numbers as approximations it +is usually unwise to use them as dictionary keys.) + +Dictionaries can be created by placing a comma-separated list of ``key: value`` +pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: +'jack', 4127: 'sjoerd'}``, or by the :class:`dict` constructor. + +.. class:: dict([arg]) + + Return a new dictionary initialized from an optional positional argument or + from a set of keyword arguments. If no arguments are given, return a new + empty dictionary. If the positional argument *arg* is a mapping object, + return a dictionary mapping the same keys to the same values as does the + mapping object. Otherwise the positional argument must be a sequence, a + container that supports iteration, or an iterator object. The elements of + the argument must each also be of one of those kinds, and each must in turn + contain exactly two objects. The first is used as a key in the new + dictionary, and the second as the key's value. If a given key is seen more + than once, the last value associated with it is retained in the new + dictionary. + + If keyword arguments are given, the keywords themselves with their associated + values are added as items to the dictionary. If a key is specified both in + the positional argument and as a keyword argument, the value associated with + the keyword is retained in the dictionary. For example, these all return a + dictionary equal to ``{"one": 1, "two": 2}``: + + * ``dict(one=1, two=2)`` + * ``dict({'one': 1, 'two': 2})`` + * ``dict(zip(('one', 'two'), (1, 2)))`` + * ``dict([['two', 2], ['one', 1]])`` + + The first example only works for keys that are valid Python identifiers; the + others work with any valid keys. + + + These are the operations that dictionaries support (and therefore, custom + mapping types should support too): + + .. describe:: len(d) + + Return the number of items in the dictionary *d*. + + .. describe:: d[key] + + Return the item of *d* with key *key*. Raises a :exc:`KeyError` if *key* is + not in the map. + + If a subclass of dict defines a method :meth:`__missing__`, if the key *key* + is not present, the ``d[key]`` operation calls that method with the key *key* + as argument. The ``d[key]`` operation then returns or raises whatever is + returned or raised by the ``__missing__(key)`` call if the key is not + present. No other operations or methods invoke :meth:`__missing__`. If + :meth:`__missing__` is not defined, :exc:`KeyError` is raised. + :meth:`__missing__` must be a method; it cannot be an instance variable:: + + >>> class Counter(dict): + ... def __missing__(self, key): + ... return 0 + >>> c = Counter() + >>> c['red'] + 0 + >>> c['red'] += 1 + >>> c['red'] + 1 + + See :class:`collections.Counter` for a complete implementation including + other methods helpful for accumulating and managing tallies. + + .. versionchanged:: 3.3 + If the dict is modified during the lookup, a :exc:`RuntimeError` + exception is now raised. + + .. describe:: d[key] = value + + Set ``d[key]`` to *value*. + + .. describe:: del d[key] + + Remove ``d[key]`` from *d*. Raises a :exc:`KeyError` if *key* is not in the + map. + + .. describe:: key in d + + Return ``True`` if *d* has a key *key*, else ``False``. + + .. describe:: key not in d + + Equivalent to ``not key in d``. + + .. describe:: iter(d) + + Return an iterator over the keys of the dictionary. This is a shortcut + for ``iter(d.keys())``. + + .. method:: clear() + + Remove all items from the dictionary. + + .. method:: copy() + + Return a shallow copy of the dictionary. + + .. classmethod:: fromkeys(seq[, value]) + + Create a new dictionary with keys from *seq* and values set to *value*. + + :meth:`fromkeys` is a class method that returns a new dictionary. *value* + defaults to ``None``. + + .. method:: get(key[, default]) + + Return the value for *key* if *key* is in the dictionary, else *default*. + If *default* is not given, it defaults to ``None``, so that this method + never raises a :exc:`KeyError`. + + .. method:: items() + + Return a new view of the dictionary's items (``(key, value)`` pairs). See + below for documentation of view objects. + + .. method:: keys() + + Return a new view of the dictionary's keys. See below for documentation of + view objects. + + .. method:: pop(key[, default]) + + If *key* is in the dictionary, remove it and return its value, else return + *default*. If *default* is not given and *key* is not in the dictionary, + a :exc:`KeyError` is raised. + + .. method:: popitem() + + Remove and return an arbitrary ``(key, value)`` pair from the dictionary. + + :meth:`popitem` is useful to destructively iterate over a dictionary, as + often used in set algorithms. If the dictionary is empty, calling + :meth:`popitem` raises a :exc:`KeyError`. + + .. method:: setdefault(key[, default]) + + If *key* is in the dictionary, return its value. If not, insert *key* + with a value of *default* and return *default*. *default* defaults to + ``None``. + + .. method:: update([other]) + + Update the dictionary with the key/value pairs from *other*, overwriting + existing keys. Return ``None``. + + :meth:`update` accepts either another dictionary object or an iterable of + key/value pairs (as tuples or other iterables of length two). If keyword + arguments are specified, the dictionary is then updated with those + key/value pairs: ``d.update(red=1, blue=2)``. + + .. method:: values() + + Return a new view of the dictionary's values. See below for documentation of + view objects. + + +.. _dict-views: + +Dictionary view objects +----------------------- + +The objects returned by :meth:`dict.keys`, :meth:`dict.values` and +:meth:`dict.items` are *view objects*. They provide a dynamic view on the +dictionary's entries, which means that when the dictionary changes, the view +reflects these changes. + +Dictionary views can be iterated over to yield their respective data, and +support membership tests: + +.. describe:: len(dictview) + + Return the number of entries in the dictionary. + +.. describe:: iter(dictview) + + Return an iterator over the keys, values or items (represented as tuples of + ``(key, value)``) in the dictionary. + + Keys and values are iterated over in an arbitrary order which is non-random, + varies across Python implementations, and depends on the dictionary's history + of insertions and deletions. If keys, values and items views are iterated + over with no intervening modifications to the dictionary, the order of items + will directly correspond. This allows the creation of ``(value, key)`` pairs + using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to + create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``. + + Iterating views while adding or deleting entries in the dictionary may raise + a :exc:`RuntimeError` or fail to iterate over all entries. + +.. describe:: x in dictview + + Return ``True`` if *x* is in the underlying dictionary's keys, values or + items (in the latter case, *x* should be a ``(key, value)`` tuple). + + +Keys views are set-like since their entries are unique and hashable. If all +values are hashable, so that ``(key, value)`` pairs are unique and hashable, +then the items view is also set-like. (Values views are not treated as set-like +since the entries are generally not unique.) For set-like views, all of the +operations defined for the abstract base class :class:`collections.Set` are +available (for example, ``==``, ``<``, or ``^``). + +An example of dictionary view usage:: + + >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} + >>> keys = dishes.keys() + >>> values = dishes.values() + + >>> # iteration + >>> n = 0 + >>> for val in values: + ... n += val + >>> print(n) + 504 + + >>> # keys and values are iterated over in the same order + >>> list(keys) + ['eggs', 'bacon', 'sausage', 'spam'] + >>> list(values) + [2, 1, 1, 500] + + >>> # view objects are dynamic and reflect dict changes + >>> del dishes['eggs'] + >>> del dishes['sausage'] + >>> list(keys) + ['spam', 'bacon'] + + >>> # set operations + >>> keys & {'eggs', 'bacon', 'salad'} + {'bacon'} + >>> keys ^ {'sausage', 'juice'} + {'juice', 'sausage', 'bacon', 'spam'} + .. _typememoryview: -Memory Views ------------- +memoryview type +=============== :class:`memoryview` objects allow Python code to access the internal data of an object that supports the :ref:`buffer protocol ` without @@ -3363,8 +2408,10 @@ the view. The :class:`~memoryview.itemsize` attribute will give you the number of bytes in a single element. - A :class:`memoryview` supports slicing and indexing to expose its data. - One-dimensional slicing will result in a subview:: + A :class:`memoryview` supports slicing to expose its data. If + :class:`~memoryview.format` is one of the native format specifiers + from the :mod:`struct` module, indexing will return a single element + with the correct type. Full slicing will result in a subview:: >>> v = memoryview(b'abcefg') >>> v[1] @@ -3376,29 +2423,25 @@ >>> bytes(v[1:4]) b'bce' - If :class:`~memoryview.format` is one of the native format specifiers - from the :mod:`struct` module, indexing with an integer or a tuple of - integers is also supported and returns a single *element* with - the correct type. One-dimensional memoryviews can be indexed - with an integer or a one-integer tuple. Multi-dimensional memoryviews - can be indexed with tuples of exactly *ndim* integers where *ndim* is - the number of dimensions. Zero-dimensional memoryviews can be indexed - with the empty tuple. - - Here is an example with a non-byte format:: + Other native formats:: >>> import array >>> a = array.array('l', [-11111111, 22222222, -33333333, 44444444]) - >>> m = memoryview(a) - >>> m[0] + >>> a[0] -11111111 - >>> m[-1] + >>> a[-1] 44444444 - >>> m[::2].tolist() + >>> a[2:3].tolist() + [-33333333] + >>> a[::2].tolist() [-11111111, -33333333] - - If the underlying object is writable, the memoryview supports - one-dimensional slice assignment. Resizing is not allowed:: + >>> a[::-1].tolist() + [44444444, -33333333, 22222222, -11111111] + + .. versionadded:: 3.3 + + If the underlying object is writable, the memoryview supports slice + assignment. Resizing is not allowed:: >>> data = bytearray(b'abcefg') >>> v = memoryview(data) @@ -3412,15 +2455,14 @@ bytearray(b'z123fg') >>> v[2:3] = b'spam' Traceback (most recent call last): - File "", line 1, in + File "", line 1, in ValueError: memoryview assignment: lvalue and rvalue have different structures >>> v[2:6] = b'spam' >>> data bytearray(b'z1spam') - One-dimensional memoryviews of hashable (read-only) types with formats - 'B', 'b' or 'c' are also hashable. The hash is defined as - ``hash(m) == hash(m.tobytes())``:: + Memoryviews of hashable (read-only) types are also hashable. The hash + is defined as ``hash(m) == hash(m.tobytes())``:: >>> v = memoryview(b'abcefg') >>> hash(v) == hash(b'abcefg') @@ -3430,67 +2472,24 @@ >>> hash(v[::-2]) == hash(b'abcefg'[::-2]) True + Hashing of multi-dimensional objects is supported:: + + >>> buf = bytes(list(range(12))) + >>> x = memoryview(buf) + >>> y = x.cast('B', shape=[2,2,3]) + >>> x.tolist() + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + >>> y.tolist() + [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]] + >>> hash(x) == hash(y) == hash(y.tobytes()) + True + .. versionchanged:: 3.3 - One-dimensional memoryviews can now be sliced. - One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable. - - .. versionchanged:: 3.4 - memoryview is now registered automatically with - :class:`collections.abc.Sequence` - - .. versionchanged:: 3.5 - memoryviews can now be indexed with tuple of integers. + Memoryview objects are now hashable. + :class:`memoryview` has several methods: - .. method:: __eq__(exporter) - - A memoryview and a :pep:`3118` exporter are equal if their shapes are - equivalent and if all corresponding values are equal when the operands' - respective format codes are interpreted using :mod:`struct` syntax. - - For the subset of :mod:`struct` format strings currently supported by - :meth:`tolist`, ``v`` and ``w`` are equal if ``v.tolist() == w.tolist()``:: - - >>> import array - >>> a = array.array('I', [1, 2, 3, 4, 5]) - >>> b = array.array('d', [1.0, 2.0, 3.0, 4.0, 5.0]) - >>> c = array.array('b', [5, 3, 1]) - >>> x = memoryview(a) - >>> y = memoryview(b) - >>> x == a == y == b - True - >>> x.tolist() == a.tolist() == y.tolist() == b.tolist() - True - >>> z = y[::-2] - >>> z == c - True - >>> z.tolist() == c.tolist() - True - - If either format string is not supported by the :mod:`struct` module, - then the objects will always compare as unequal (even if the format - strings and buffer contents are identical):: - - >>> from ctypes import BigEndianStructure, c_long - >>> class BEPoint(BigEndianStructure): - ... _fields_ = [("x", c_long), ("y", c_long)] - ... - >>> point = BEPoint(100, 200) - >>> a = memoryview(point) - >>> b = memoryview(point) - >>> a == point - False - >>> a == b - False - - Note that, as with floating point numbers, ``v is w`` does *not* imply - ``v == w`` for memoryview objects. - - .. versionchanged:: 3.3 - Previous versions compared the raw memory disregarding the item format - and the logical array structure. - .. method:: tobytes() Return the data in the buffer as a bytestring. This is equivalent to @@ -3503,20 +2502,7 @@ b'abc' For non-contiguous arrays the result is equal to the flattened list - representation with all elements converted to bytes. :meth:`tobytes` - supports all format strings, including those that are not in - :mod:`struct` module syntax. - - .. method:: hex() - - Return a string object containing two hexadecimal digits for each - byte in the buffer. :: - - >>> m = memoryview(b"abc") - >>> m.hex() - '616263' - - .. versionadded:: 3.5 + representation with all elements converted to bytes. .. method:: tolist() @@ -3530,11 +2516,6 @@ >>> m.tolist() [1.1, 2.2, 3.3] - .. versionchanged:: 3.3 - :meth:`tolist` now supports all single character native formats in - :mod:`struct` module syntax as well as multi-dimensional - representations. - .. method:: release() Release the underlying buffer exposed by the memoryview object. Many @@ -3573,11 +2554,8 @@ Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view will be one-dimensional. The return value is a new memoryview, but - the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous` - and C-contiguous -> 1D. - - The destination format is restricted to a single element native format in - :mod:`struct` syntax. One of the formats must be a byte format + the buffer itself is not copied. Supported casts are 1D -> C-contiguous + and C-contiguous -> 1D. One of the formats must be a byte format ('B', 'b' or 'c'). The byte length of the result must be the same as the original length. @@ -3643,7 +2621,7 @@ >>> z.nbytes 48 - Cast 1D/unsigned char to 2D/unsigned long:: + Cast 1D/unsigned char to to 2D/unsigned long:: >>> buf = struct.pack("L"*6, *list(range(6))) >>> x = memoryview(buf) @@ -3657,9 +2635,6 @@ .. versionadded:: 3.3 - .. versionchanged:: 3.5 - The source format is no longer restricted when casting to a byte view. - There are also several readonly attributes available: .. attribute:: obj @@ -3718,11 +2693,25 @@ A string containing the format (in :mod:`struct` module style) for each element in the view. A memoryview can be created from exporters with arbitrary format strings, but some methods (e.g. :meth:`tolist`) are - restricted to native single element formats. - - .. versionchanged:: 3.3 - format ``'B'`` is now handled according to the struct module syntax. - This means that ``memoryview(b'abc')[0] == b'abc'[0] == 97``. + restricted to native single element formats. Special care must be taken + when comparing memoryviews. Since comparisons are required to return a + value for ``==`` and ``!=``, two memoryviews referencing the same + exporter can compare as not-equal if the exporter's format is not + understood:: + + >>> from ctypes import BigEndianStructure, c_long + >>> class BEPoint(BigEndianStructure): + ... _fields_ = [("x", c_long), ("y", c_long)] + ... + >>> point = BEPoint(100, 200) + >>> a = memoryview(point) + >>> b = memoryview(point) + >>> a == b + False + >>> a.tolist() + Traceback (most recent call last): + File "", line 1, in + NotImplementedError: memoryview: unsupported format T{>l:x:>l:y:} .. attribute:: itemsize @@ -3745,519 +2734,36 @@ .. attribute:: shape A tuple of integers the length of :attr:`ndim` giving the shape of the - memory as an N-dimensional array. - - .. versionchanged:: 3.3 - An empty tuple instead of None when ndim = 0. + memory as a N-dimensional array. .. attribute:: strides A tuple of integers the length of :attr:`ndim` giving the size in bytes to access each element for each dimension of the array. - .. versionchanged:: 3.3 - An empty tuple instead of None when ndim = 0. - .. attribute:: suboffsets Used internally for PIL-style arrays. The value is informational only. .. attribute:: c_contiguous - A bool indicating whether the memory is C-:term:`contiguous`. + A bool indicating whether the memory is C-contiguous. .. versionadded:: 3.3 .. attribute:: f_contiguous - A bool indicating whether the memory is Fortran :term:`contiguous`. + A bool indicating whether the memory is Fortran contiguous. .. versionadded:: 3.3 .. attribute:: contiguous - A bool indicating whether the memory is :term:`contiguous`. + A bool indicating whether the memory is contiguous. .. versionadded:: 3.3 -.. _types-set: - -Set Types --- :class:`set`, :class:`frozenset` -============================================== - -.. index:: object: set - -A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. -Common uses include membership testing, removing duplicates from a sequence, and -computing mathematical operations such as intersection, union, difference, and -symmetric difference. -(For other containers see the built-in :class:`dict`, :class:`list`, -and :class:`tuple` classes, and the :mod:`collections` module.) - -Like other collections, sets support ``x in set``, ``len(set)``, and ``for x in -set``. Being an unordered collection, sets do not record element position or -order of insertion. Accordingly, sets do not support indexing, slicing, or -other sequence-like behavior. - -There are currently two built-in set types, :class:`set` and :class:`frozenset`. -The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. - -Non-empty sets (not frozensets) can be created by placing a comma-separated list -of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the -:class:`set` constructor. - -The constructors for both classes work the same: - -.. class:: set([iterable]) - frozenset([iterable]) - - Return a new set or frozenset object whose elements are taken from - *iterable*. The elements of a set must be :term:`hashable`. To - represent sets of sets, the inner sets must be :class:`frozenset` - objects. If *iterable* is not specified, a new empty set is - returned. - - Instances of :class:`set` and :class:`frozenset` provide the following - operations: - - .. describe:: len(s) - - Return the cardinality of set *s*. - - .. describe:: x in s - - Test *x* for membership in *s*. - - .. describe:: x not in s - - Test *x* for non-membership in *s*. - - .. method:: isdisjoint(other) - - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. - - .. method:: issubset(other) - set <= other - - Test whether every element in the set is in *other*. - - .. method:: set < other - - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. - - .. method:: issuperset(other) - set >= other - - Test whether every element in *other* is in the set. - - .. method:: set > other - - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. - - .. method:: union(other, ...) - set | other | ... - - Return a new set with elements from the set and all others. - - .. method:: intersection(other, ...) - set & other & ... - - Return a new set with elements common to the set and all others. - - .. method:: difference(other, ...) - set - other - ... - - Return a new set with elements in the set that are not in the others. - - .. method:: symmetric_difference(other) - set ^ other - - Return a new set with elements in either the set or *other* but not both. - - .. method:: copy() - - Return a new set with a shallow copy of *s*. - - - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, and :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. - - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). - - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. - - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. - - Set elements, like dictionary keys, must be :term:`hashable`. - - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. - - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: - - .. method:: update(other, ...) - set |= other | ... - - Update the set, adding elements from all others. - - .. method:: intersection_update(other, ...) - set &= other & ... - - Update the set, keeping only elements found in it and all others. - - .. method:: difference_update(other, ...) - set -= other | ... - - Update the set, removing elements found in others. - - .. method:: symmetric_difference_update(other) - set ^= other - - Update the set, keeping only elements found in either set, but not in both. - - .. method:: add(elem) - - Add element *elem* to the set. - - .. method:: remove(elem) - - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. - - .. method:: discard(elem) - - Remove element *elem* from the set if it is present. - - .. method:: pop() - - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. - - .. method:: clear() - - Remove all elements from the set. - - - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. - - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, the *elem* set is temporarily mutated during the search and then - restored. During the search, the *elem* set should not be read or mutated - since it does not have a meaningful value. - - -.. _typesmapping: - -Mapping Types --- :class:`dict` -=============================== - -.. index:: - object: mapping - object: dictionary - triple: operations on; mapping; types - triple: operations on; dictionary; type - statement: del - builtin: len - -A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in -:class:`list`, :class:`set`, and :class:`tuple` classes, and the -:mod:`collections` module.) - -A dictionary's keys are *almost* arbitrary values. Values that are not -:term:`hashable`, that is, values containing lists, dictionaries or other -mutable types (that are compared by value rather than by object identity) may -not be used as keys. Numeric types used for keys obey the normal rules for -numeric comparison: if two numbers compare equal (such as ``1`` and ``1.0``) -then they can be used interchangeably to index the same dictionary entry. (Note -however, that since computers store floating-point numbers as approximations it -is usually unwise to use them as dictionary keys.) - -Dictionaries can be created by placing a comma-separated list of ``key: value`` -pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: -'jack', 4127: 'sjoerd'}``, or by the :class:`dict` constructor. - -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) - - Return a new dictionary initialized from an optional positional argument - and a possibly empty set of keyword arguments. - - If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. - - If keyword arguments are given, the keyword arguments and their values are - added to the dictionary created from the positional argument. If a key - being added is already present, the value from the keyword argument - replaces the value from the positional argument. - - To illustrate, the following examples all return a dictionary equal to - ``{"one": 1, "two": 2, "three": 3}``:: - - >>> a = dict(one=1, two=2, three=3) - >>> b = {'one': 1, 'two': 2, 'three': 3} - >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) - >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) - >>> e = dict({'three': 3, 'one': 1, 'two': 2}) - >>> a == b == c == d == e - True - - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - - - These are the operations that dictionaries support (and therefore, custom - mapping types should support too): - - .. describe:: len(d) - - Return the number of items in the dictionary *d*. - - .. describe:: d[key] - - Return the item of *d* with key *key*. Raises a :exc:`KeyError` if *key* is - not in the map. - - .. index:: __missing__() - - If a subclass of dict defines a method :meth:`__missing__` and *key* - is not present, the ``d[key]`` operation calls that method with the key *key* - as argument. The ``d[key]`` operation then returns or raises whatever is - returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: - - >>> class Counter(dict): - ... def __missing__(self, key): - ... return 0 - >>> c = Counter() - >>> c['red'] - 0 - >>> c['red'] += 1 - >>> c['red'] - 1 - - The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used - by :class:`collections.defaultdict`. - - .. describe:: d[key] = value - - Set ``d[key]`` to *value*. - - .. describe:: del d[key] - - Remove ``d[key]`` from *d*. Raises a :exc:`KeyError` if *key* is not in the - map. - - .. describe:: key in d - - Return ``True`` if *d* has a key *key*, else ``False``. - - .. describe:: key not in d - - Equivalent to ``not key in d``. - - .. describe:: iter(d) - - Return an iterator over the keys of the dictionary. This is a shortcut - for ``iter(d.keys())``. - - .. method:: clear() - - Remove all items from the dictionary. - - .. method:: copy() - - Return a shallow copy of the dictionary. - - .. classmethod:: fromkeys(seq[, value]) - - Create a new dictionary with keys from *seq* and values set to *value*. - - :meth:`fromkeys` is a class method that returns a new dictionary. *value* - defaults to ``None``. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the dictionary, else *default*. - If *default* is not given, it defaults to ``None``, so that this method - never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the dictionary's items (``(key, value)`` pairs). - See the :ref:`documentation of view objects `. - - .. method:: keys() - - Return a new view of the dictionary's keys. See the :ref:`documentation - of view objects `. - - .. method:: pop(key[, default]) - - If *key* is in the dictionary, remove it and return its value, else return - *default*. If *default* is not given and *key* is not in the dictionary, - a :exc:`KeyError` is raised. - - .. method:: popitem() - - Remove and return an arbitrary ``(key, value)`` pair from the dictionary. - - :meth:`popitem` is useful to destructively iterate over a dictionary, as - often used in set algorithms. If the dictionary is empty, calling - :meth:`popitem` raises a :exc:`KeyError`. - - .. method:: setdefault(key[, default]) - - If *key* is in the dictionary, return its value. If not, insert *key* - with a value of *default* and return *default*. *default* defaults to - ``None``. - - .. method:: update([other]) - - Update the dictionary with the key/value pairs from *other*, overwriting - existing keys. Return ``None``. - - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. - - .. method:: values() - - Return a new view of the dictionary's values. See the - :ref:`documentation of view objects `. - - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs. Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - -.. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. - - -.. _dict-views: - -Dictionary view objects ------------------------ - -The objects returned by :meth:`dict.keys`, :meth:`dict.values` and -:meth:`dict.items` are *view objects*. They provide a dynamic view on the -dictionary's entries, which means that when the dictionary changes, the view -reflects these changes. - -Dictionary views can be iterated over to yield their respective data, and -support membership tests: - -.. describe:: len(dictview) - - Return the number of entries in the dictionary. - -.. describe:: iter(dictview) - - Return an iterator over the keys, values or items (represented as tuples of - ``(key, value)``) in the dictionary. - - Keys and values are iterated over in an arbitrary order which is non-random, - varies across Python implementations, and depends on the dictionary's history - of insertions and deletions. If keys, values and items views are iterated - over with no intervening modifications to the dictionary, the order of items - will directly correspond. This allows the creation of ``(value, key)`` pairs - using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to - create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``. - - Iterating views while adding or deleting entries in the dictionary may raise - a :exc:`RuntimeError` or fail to iterate over all entries. - -.. describe:: x in dictview - - Return ``True`` if *x* is in the underlying dictionary's keys, values or - items (in the latter case, *x* should be a ``(key, value)`` tuple). - - -Keys views are set-like since their entries are unique and hashable. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like -since the entries are generally not unique.) For set-like views, all of the -operations defined for the abstract base class :class:`collections.abc.Set` are -available (for example, ``==``, ``<``, or ``^``). - -An example of dictionary view usage:: - - >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} - >>> keys = dishes.keys() - >>> values = dishes.values() - - >>> # iteration - >>> n = 0 - >>> for val in values: - ... n += val - >>> print(n) - 504 - - >>> # keys and values are iterated over in the same order - >>> list(keys) - ['eggs', 'bacon', 'sausage', 'spam'] - >>> list(values) - [2, 1, 1, 500] - - >>> # view objects are dynamic and reflect dict changes - >>> del dishes['eggs'] - >>> del dishes['sausage'] - >>> list(keys) - ['spam', 'bacon'] - - >>> # set operations - >>> keys & {'eggs', 'bacon', 'salad'} - {'bacon'} - >>> keys ^ {'sausage', 'juice'} - {'juice', 'sausage', 'bacon', 'spam'} - - .. _typecontextmanager: Context Manager Types @@ -4310,8 +2816,8 @@ The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`__exit__` - method has actually failed. + context management code (such as ``contextlib.nested``) to easily detect whether + or not an :meth:`__exit__` method has actually failed. Python defines several context managers to support easy thread synchronisation, prompt closure of files or other objects, and simpler manipulation of the active @@ -4354,12 +2860,12 @@ foo`` does not require a module object named *foo* to exist, rather it requires an (external) *definition* for a module named *foo* somewhere.) -A special attribute of every module is :attr:`~object.__dict__`. This is the -dictionary containing the module's symbol table. Modifying this dictionary will -actually change the module's symbol table, but direct assignment to the -:attr:`__dict__` attribute is not possible (you can write -``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but you can't write -``m.__dict__ = {}``). Modifying :attr:`__dict__` directly is not recommended. +A special attribute of every module is :attr:`__dict__`. This is the dictionary +containing the module's symbol table. Modifying this dictionary will actually +change the module's symbol table, but direct assignment to the :attr:`__dict__` +attribute is not possible (you can write ``m.__dict__['a'] = 1``, which defines +``m.a`` to be ``1``, but you can't write ``m.__dict__ = {}``). Modifying +:attr:`__dict__` directly is not recommended. Modules built into the interpreter are written like this: ````. If loaded from a file, they are written as ``>> class C: - ... def method(self): - ... pass - ... - >>> c = C() - >>> c.method.whoami = 'my name is method' # can't set on the method - Traceback (most recent call last): - File "", line 1, in - AttributeError: 'method' object has no attribute 'whoami' - >>> c.method.__func__.whoami = 'my name is method' - >>> c.method.whoami - 'my name is method' +bound methods is disallowed. Attempting to set a method attribute results in a +:exc:`TypeError` being raised. In order to set a method attribute, you need to +explicitly set it on the underlying function object:: + + class C: + def method(self): + pass + + c = C() + c.method.__func__.whoami = 'my name is c' See :ref:`types` for more information. @@ -4495,10 +2995,11 @@ The Ellipsis Object ------------------- -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named -:const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the -:const:`Ellipsis` singleton. +This object is commonly used by slicing (see :ref:`slicings`), but may also +be used in other situations where a sentinel value other than :const:`None` +is needed. It supports no special operations. There is exactly one ellipsis +object, named :const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` +produces the :const:`Ellipsis` singleton. It is written as ``Ellipsis`` or ``...``. @@ -4594,7 +3095,7 @@ This method can be overridden by a metaclass to customize the method resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. + its result is stored in :attr:`__mro__`. .. method:: class.__subclasses__ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/string.rst --- a/Doc/library/string.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/string.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,7 @@ .. seealso:: - :ref:`textseq` + :ref:`typesseq` :ref:`string-methods` @@ -91,22 +91,18 @@ .. method:: format(format_string, *args, **kwargs) - :meth:`format` is the primary API method. It takes a format string and - an arbitrary set of positional and keyword arguments. + :meth:`format` is the primary API method. It takes a format template + string, and an arbitrary set of positional and keyword argument. :meth:`format` is just a wrapper that calls :meth:`vformat`. - .. deprecated:: 3.5 - Passing a format string as keyword argument *format_string* has been - deprecated. - .. method:: vformat(format_string, args, kwargs) This function does the actual work of formatting. It is exposed as a separate function for cases where you want to pass in a predefined dictionary of arguments, rather than unpacking and repacking the - dictionary as individual arguments using the ``*args`` and ``**kwargs`` - syntax. :meth:`vformat` does the work of breaking up the format string - into character data and replacement fields. It calls the various + dictionary as individual arguments using the ``*args`` and ``**kwds`` + syntax. :meth:`vformat` does the work of breaking up the format template + string into character data and replacement fields. It calls the various methods described below. In addition, the :class:`Formatter` defines a number of methods that are @@ -177,8 +173,7 @@ Converts the value (returned by :meth:`get_field`) given a conversion type (as in the tuple returned by the :meth:`parse` method). The default - version understands 's' (str), 'r' (repr) and 'a' (ascii) conversion - types. + version understands 'r' (repr) and 's' (str) conversion types. .. _formatstrings: @@ -297,18 +292,18 @@ .. productionlist:: sf format_spec: [[`fill`]`align`][`sign`][#][0][`width`][,][.`precision`][`type`] - fill: + fill: align: "<" | ">" | "=" | "^" sign: "+" | "-" | " " width: `integer` precision: `integer` type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" -If a valid *align* value is specified, it can be preceded by a *fill* -character that can be any character and defaults to a space if omitted. -Note that it is not possible to use ``{`` and ``}`` as *fill* char while -using the :meth:`str.format` method; this limitation however doesn't -affect the :func:`format` function. +The *fill* character can be any character other than '{' or '}'. The presence +of a fill character is signaled by the character following it, which must be +one of the alignment options. If the second character of *format_spec* is not +a valid alignment option, then it is assumed that both the fill character and +the alignment option are absent. The meaning of the various alignment options is as follows: @@ -373,9 +368,9 @@ *width* is a decimal integer defining the minimum field width. If not specified, then the field width will be determined by the content. -Preceding the *width* field by a zero (``'0'``) character enables -sign-aware zero-padding for numeric types. This is equivalent to a *fill* -character of ``'0'`` with an *alignment* type of ``'='``. +If the *width* field is preceded by a zero (``'0'``) character, this enables +zero-padding. This is equivalent to an *alignment* type of ``'='`` and a *fill* +character of ``'0'``. The *precision* is a decimal number indicating how many digits should be displayed after the decimal point for a floating point value formatted with @@ -436,13 +431,12 @@ +=========+==========================================================+ | ``'e'`` | Exponent notation. Prints the number in scientific | | | notation using the letter 'e' to indicate the exponent. | - | | The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | | | upper case 'E' as the separator character. | +---------+----------------------------------------------------------+ | ``'f'`` | Fixed point. Displays the number as a fixed-point | - | | number. The default precision is ``6``. | + | | number. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed point. Same as ``'f'``, but converts ``nan`` to | | | ``NAN`` and ``inf`` to ``INF``. | @@ -469,7 +463,7 @@ | | the precision. | | | | | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. The default precision is ``6``. | + | | precision of ``1``. | +---------+----------------------------------------------------------+ | ``'G'`` | General format. Same as ``'g'`` except switches to | | | ``'E'`` if the number gets too large. The | @@ -482,12 +476,10 @@ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | Similar to ``'g'``, except that fixed-point notation, | - | | when used, has at least one digit past the decimal point.| - | | The default precision is as high as needed to represent | - | | the particular value. The overall effect is to match the | - | | output of :func:`str` as altered by the other format | - | | modifiers. | + | None | Similar to ``'g'``, except with at least one digit past | + | | the decimal point and a default precision of 12. This is | + | | intended to match :func:`str`, except you can add the | + | | other format modifiers. | +---------+----------------------------------------------------------+ @@ -617,7 +609,7 @@ 3232235521 >>> >>> width = 5 - >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE + >>> for num in range(5,12): ... for base in 'dXob': ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') ... print() @@ -644,14 +636,12 @@ * ``$$`` is an escape; it is replaced with a single ``$``. * ``$identifier`` names a substitution placeholder matching a mapping key of - ``"identifier"``. By default, ``"identifier"`` is restricted to any - case-insensitive ASCII alphanumeric string (including underscores) that - starts with an underscore or ASCII letter. The first non-identifier - character after the ``$`` character terminates this placeholder - specification. + ``"identifier"``. By default, ``"identifier"`` must spell a Python + identifier. The first non-identifier character after the ``$`` character + terminates this placeholder specification. -* ``${identifier}`` is equivalent to ``$identifier``. It is required when - valid identifier characters follow the placeholder but are not part of the +* ``${identifier}`` is equivalent to ``$identifier``. It is required when valid + identifier characters follow the placeholder but are not part of the placeholder, such as ``"${noun}ification"``. Any other appearance of ``$`` in the string will result in a :exc:`ValueError` @@ -697,7 +687,7 @@ This is the object passed to the constructor's *template* argument. In general, you shouldn't change it, but read-only access is not enforced. -Here is an example of how to use a Template:: +Here is an example of how to use a Template: >>> from string import Template >>> s = Template('$who likes $what') @@ -706,11 +696,11 @@ >>> d = dict(who='tim') >>> Template('Give $who $100').substitute(d) Traceback (most recent call last): - ... - ValueError: Invalid placeholder in string: line 1, col 11 + [...] + ValueError: Invalid placeholder in string: line 1, col 10 >>> Template('$who likes $what').substitute(d) Traceback (most recent call last): - ... + [...] KeyError: 'what' >>> Template('$who likes $what').safe_substitute(d) 'tim likes $what' diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/stringprep.rst --- a/Doc/library/stringprep.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/stringprep.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,6 +3,7 @@ .. module:: stringprep :synopsis: String preparation, as per RFC 3453 + :deprecated: .. moduleauthor:: Martin v. Löwis .. sectionauthor:: Martin v. Löwis diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/strings.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/strings.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,27 @@ +.. _stringservices: + +*************** +String Services +*************** + +The modules described in this chapter provide a wide range of string +manipulation operations. + +In addition, Python's built-in string classes support the sequence type methods +described in the :ref:`typesseq` section, and also the string-specific methods +described in the :ref:`string-methods` section. To output formatted strings, +see the :ref:`string-formatting` section. Also, see the :mod:`re` module for +string functions based on regular expressions. + + +.. toctree:: + + string.rst + re.rst + struct.rst + difflib.rst + textwrap.rst + codecs.rst + unicodedata.rst + stringprep.rst + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/struct.rst --- a/Doc/library/struct.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/struct.rst Mon Jan 25 17:05:13 2016 +0100 @@ -24,14 +24,6 @@ or omit implicit pad bytes, use ``standard`` size and alignment instead of ``native`` size and alignment: see :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* -argument. This refers to objects that implement the :ref:`bufferobjects` and -provide either a readable or read-writable buffer. The most common types used -for that purpose are :class:`bytes` and :class:`bytearray`, but many other types -that can be viewed as an array of bytes implement the buffer protocol, so that -they can be read/filled without additional copying from a :class:`bytes` object. - - Functions and Exceptions ------------------------ @@ -55,7 +47,7 @@ Pack the values *v1*, *v2*, ... according to the format string *fmt* and write the packed bytes into the writable buffer *buffer* starting at - position *offset*. Note that *offset* is a required argument. + position *offset*. Note that *offset* is a required argument. .. function:: unpack(fmt, buffer) @@ -74,19 +66,6 @@ format (``len(buffer[offset:])`` must be at least ``calcsize(fmt)``). -.. function:: iter_unpack(fmt, buffer) - - Iteratively unpack from the buffer *buffer* according to the format - string *fmt*. This function returns an iterator which will read - equally-sized chunks from the buffer until all its contents have been - consumed. The buffer's size in bytes must be a multiple of the amount - of data required by the format, as reflected by :func:`calcsize`. - - Each iteration yields a tuple as specified by the format string. - - .. versionadded:: 3.4 - - .. function:: calcsize(fmt) Return the size of the struct (and hence of the bytes object produced by @@ -303,7 +282,7 @@ For the ``'?'`` format character, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and -any non-zero value will be ``True`` when unpacking. +any non-zero value will be True when unpacking. @@ -409,13 +388,6 @@ (``len(buffer[offset:])`` must be at least :attr:`self.size`). - .. method:: iter_unpack(buffer) - - Identical to the :func:`iter_unpack` function, using the compiled format. - (``len(buffer)`` must be a multiple of :attr:`self.size`). - - .. versionadded:: 3.4 - .. attribute:: format The format string used to construct this Struct object. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/subprocess.rst --- a/Doc/library/subprocess.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/subprocess.rst Mon Jan 25 17:05:13 2016 +0100 @@ -9,7 +9,7 @@ The :mod:`subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to -replace several older modules and functions:: +replace several other, older modules and functions, such as:: os.system os.spawn* @@ -22,102 +22,168 @@ :pep:`324` -- PEP proposing the subprocess module -Using the :mod:`subprocess` Module ----------------------------------- +Using the subprocess Module +--------------------------- -The recommended approach to invoking subprocesses is to use the :func:`run` -function for all use cases it can handle. For more advanced use cases, the -underlying :class:`Popen` interface can be used directly. +The recommended approach to invoking subprocesses is to use the following +convenience functions for all use cases they can handle. For more advanced +use cases, the underlying :class:`Popen` interface can be used directly. -The :func:`run` function was added in Python 3.5; if you need to retain -compatibility with older versions, see the :ref:`call-function-trio` section. - -.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - shell=False, timeout=None, check=False) +.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) Run the command described by *args*. Wait for command to complete, then - return a :class:`CompletedProcess` instance. + return the :attr:`returncode` attribute. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the use of keyword-only notation in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - apart from *timeout*, - *input* and *check*, all the arguments to this function are passed through to - that interface. + same as that of the :class:`Popen` constructor - this function passes all + supplied arguments other than *timeout* directly through to that interface. - This does not capture stdout or stderr by default. To do so, pass - :data:`PIPE` for the *stdout* and/or *stderr* arguments. - - The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout - expires, the child process will be killed and waited for. The + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The :exc:`TimeoutExpired` exception will be re-raised after the child process has terminated. - The *input* argument is passed to :meth:`Popen.communicate` and thus to the - subprocess's stdin. If used it must be a byte sequence, or a string if - ``universal_newlines=True``. When used, the internal :class:`Popen` object - is automatically created with ``stdin=PIPE``, and the *stdin* argument may - not be used as well. + Examples:: - If *check* is True, and the process exits with a non-zero exit code, a - :exc:`CalledProcessError` exception will be raised. Attributes of that - exception hold the arguments, the exit code, and stdout and stderr if they - were captured. + >>> subprocess.call(["ls", "-l"]) + 0 + + >>> subprocess.call("exit 1", shell=True) + 1 + + .. warning:: + + Invoking the system shell with ``shell=True`` can be a security hazard + if combined with untrusted input. See the warning under + :ref:`frequently-used-arguments` for details. + + .. note:: + + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this function. As + the pipes are not being read in the current process, the child + process may block if it generates enough output to a pipe to fill up + the OS pipe buffer. + + .. versionchanged:: 3.3 + *timeout* was added. + + +.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) + + Run command with arguments. Wait for command to complete. If the return + code was zero then return, otherwise raise :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`returncode` attribute. + + The arguments shown above are merely the most common ones, described below + in :ref:`frequently-used-arguments` (hence the use of keyword-only notation + in the abbreviated signature). The full function signature is largely the + same as that of the :class:`Popen` constructor - this function passes all + supplied arguments other than *timeout* directly through to that interface. + + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The + :exc:`TimeoutExpired` exception will be re-raised after the child process + has terminated. Examples:: - >>> subprocess.run(["ls", "-l"]) # doesn't capture output - CompletedProcess(args=['ls', '-l'], returncode=0) + >>> subprocess.check_call(["ls", "-l"]) + 0 - >>> subprocess.run("exit 1", shell=True, check=True) + >>> subprocess.check_call("exit 1", shell=True) Traceback (most recent call last): - ... + ... subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) - CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, - stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') + .. warning:: - .. versionadded:: 3.5 + Invoking the system shell with ``shell=True`` can be a security hazard + if combined with untrusted input. See the warning under + :ref:`frequently-used-arguments` for details. -.. class:: CompletedProcess + .. note:: - The return value from :func:`run`, representing a process that has finished. + Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this function. As + the pipes are not being read in the current process, the child + process may block if it generates enough output to a pipe to fill up + the OS pipe buffer. - .. attribute:: args + .. versionchanged:: 3.3 + *timeout* was added. - The arguments used to launch the process. This may be a list or a string. - .. attribute:: returncode +.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) - Exit status of the child process. Typically, an exit status of 0 indicates - that it ran successfully. + Run command with arguments and return its output as a byte string. - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). + If the return code was non-zero it raises a :exc:`CalledProcessError`. The + :exc:`CalledProcessError` object will have the return code in the + :attr:`returncode` attribute and any output in the :attr:`output` + attribute. - .. attribute:: stdout + The arguments shown above are merely the most common ones, described below + in :ref:`frequently-used-arguments` (hence the use of keyword-only notation + in the abbreviated signature). The full function signature is largely the + same as that of the :class:`Popen` constructor - this functions passes all + supplied arguments other than *timeout* directly through to that interface. + In addition, *stdout* is not permitted as an argument, as it is used + internally to collect the output from the subprocess. - Captured stdout from the child process. A bytes sequence, or a string if - :func:`run` was called with ``universal_newlines=True``. None if stdout - was not captured. + The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout + expires, the child process will be killed and then waited for again. The + :exc:`TimeoutExpired` exception will be re-raised after the child process + has terminated. - If you ran the process with ``stderr=subprocess.STDOUT``, stdout and - stderr will be combined in this attribute, and :attr:`stderr` will be - None. + Examples:: - .. attribute:: stderr + >>> subprocess.check_output(["echo", "Hello World!"]) + b'Hello World!\n' - Captured stderr from the child process. A bytes sequence, or a string if - :func:`run` was called with ``universal_newlines=True``. None if stderr - was not captured. + >>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True) + 'Hello World!\n' - .. method:: check_returncode() + >>> subprocess.check_output("exit 1", shell=True) + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`. + By default, this function will return the data as encoded bytes. The actual + encoding of the output data may depend on the command being invoked, so the + decoding to text will often need to be handled at the application level. - .. versionadded:: 3.5 + This behaviour may be overridden by setting *universal_newlines* to + :const:`True` as described below in :ref:`frequently-used-arguments`. + + To also capture standard error in the result, use + ``stderr=subprocess.STDOUT``:: + + >>> subprocess.check_output( + ... "ls non_existent_file; exit 0", + ... stderr=subprocess.STDOUT, + ... shell=True) + 'ls: non_existent_file: No such file or directory\n' + + .. versionadded:: 3.1 + + .. warning:: + + Invoking the system shell with ``shell=True`` can be a security hazard + if combined with untrusted input. See the warning under + :ref:`frequently-used-arguments` for details. + + .. note:: + + Do not use ``stderr=PIPE`` with this function. As the pipe is not being + read in the current process, the child process may block if it + generates enough output to the pipe to fill up the OS pipe buffer. + + .. versionchanged:: 3.3 + *timeout* was added. + .. data:: DEVNULL @@ -132,7 +198,7 @@ Special value that can be used as the *stdin*, *stdout* or *stderr* argument to :class:`Popen` and indicates that a pipe to the standard stream should be - opened. Most useful with :meth:`Popen.communicate`. + opened. .. data:: STDOUT @@ -142,76 +208,6 @@ output. -.. exception:: SubprocessError - - Base class for all other exceptions from this module. - - .. versionadded:: 3.3 - - -.. exception:: TimeoutExpired - - Subclass of :exc:`SubprocessError`, raised when a timeout expires - while waiting for a child process. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: timeout - - Timeout in seconds. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - -.. exception:: CalledProcessError - - Subclass of :exc:`SubprocessError`, raised when a process run by - :func:`check_call` or :func:`check_output` returns a non-zero exit status. - - .. attribute:: returncode - - Exit status of the child process. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - - .. _frequently-used-arguments: Frequently Used Arguments @@ -241,48 +237,36 @@ :data:`STDOUT`, which indicates that the stderr data from the child process should be captured into the same file handle as for *stdout*. - .. index:: - single: universal newlines; subprocess module + When *stdout* or *stderr* are pipes and *universal_newlines* is + :const:`True` then the output data is assumed to be encoded as UTF-8 and + will automatically be decoded to text. All line endings will be converted + to ``'\n'`` as described for the universal newlines ``'U'`` mode argument + to :func:`open`. - If *universal_newlines* is ``False`` the file objects *stdin*, *stdout* and - *stderr* will be opened as binary streams, and no line ending conversion is - done. + If *shell* is :const:`True`, the specified command will be executed through + the shell. This can be useful if you are using Python primarily for the + enhanced control flow it offers over most system shells and still want + access to other shell features such as filename wildcards, shell pipes and + environment variable expansion. - If *universal_newlines* is ``True``, these file objects - will be opened as text streams in :term:`universal newlines` mode - using the encoding returned by :func:`locale.getpreferredencoding(False) - `. For *stdin*, line ending characters - ``'\n'`` in the input will be converted to the default line separator - :data:`os.linesep`. For *stdout* and *stderr*, all line endings in the - output will be converted to ``'\n'``. For more information see the - documentation of the :class:`io.TextIOWrapper` class when the *newline* - argument to its constructor is ``None``. + .. warning:: - .. note:: + Executing shell commands that incorporate unsanitized input from an + untrusted source makes a program vulnerable to `shell injection + `_, + a serious security flaw which can result in arbitrary command execution. + For this reason, the use of *shell=True* is **strongly discouraged** in cases + where the command string is constructed from external input:: - The newlines attribute of the file objects :attr:`Popen.stdin`, - :attr:`Popen.stdout` and :attr:`Popen.stderr` are not updated by - the :meth:`Popen.communicate` method. + >>> from subprocess import call + >>> filename = input("What file would you like to display?\n") + What file would you like to display? + non_existent; rm -rf / # + >>> call("cat " + filename, shell=True) # Uh-oh. This will end badly... - If *shell* is ``True``, the specified command will be executed through - the shell. This can be useful if you are using Python primarily for the - enhanced control flow it offers over most system shells and still want - convenient access to other shell features such as shell pipes, filename - wildcards, environment variable expansion, and expansion of ``~`` to a - user's home directory. However, note that Python itself offers - implementations of many shell-like features (in particular, :mod:`glob`, - :mod:`fnmatch`, :func:`os.walk`, :func:`os.path.expandvars`, - :func:`os.path.expanduser`, and :mod:`shutil`). - - .. versionchanged:: 3.3 - When *universal_newlines* is ``True``, the class uses the encoding - :func:`locale.getpreferredencoding(False) ` - instead of ``locale.getpreferredencoding()``. See the - :class:`io.TextIOWrapper` class for more information on this change. - - .. note:: - - Read the `Security Considerations`_ section before using ``shell=True``. + ``shell=False`` disables all shell based features, but does not suffer + from this vulnerability; see the Note in the :class:`Popen` constructor + documentation for helpful hints in getting ``shell=False`` to work. These options, along with all of the other options, are described in more detail in the :class:`Popen` constructor documentation. @@ -297,27 +281,24 @@ functions. -.. class:: Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, \ - stderr=None, preexec_fn=None, close_fds=True, shell=False, \ - cwd=None, env=None, universal_newlines=False, \ - startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=()) +.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, pass_fds=()) - Execute a child program in a new process. On POSIX, the class uses - :meth:`os.execvp`-like behavior to execute the child program. On Windows, - the class uses the Windows ``CreateProcess()`` function. The arguments to - :class:`Popen` are as follows. + Arguments are: - *args* should be a sequence of program arguments or else a single string. - By default, the program to execute is the first item in *args* if *args* is - a sequence. If *args* is a string, the interpretation is - platform-dependent and described below. See the *shell* and *executable* - arguments for additional differences from the default behavior. Unless - otherwise stated, it is recommended to pass *args* as a sequence. + *args* should be a string, or a sequence of program arguments. The program + to execute is normally the first item in the args sequence or the string if + a string is given, but can be explicitly set by using the *executable* + argument. When *executable* is given, the first item in the args sequence + is still treated by most programs as the command name, which can then be + different from the actual executable name. On Unix, it becomes the display + name for the executing program in utilities such as :program:`ps`. - On POSIX, if *args* is a string, the string is interpreted as the name or - path of the program to execute. However, this can only be done if not - passing arguments to the program. + On Unix, with *shell=False* (default): In this case, the Popen class uses + :meth:`os.execvp` like behavior to execute the child program. + *args* should normally be a + sequence. If a string is specified for *args*, it will be used as the name + or path of the program to execute; this will only work if the program is + being given no arguments. .. note:: @@ -338,64 +319,48 @@ used in the shell (such as filenames containing spaces or the *echo* command shown above) are single list elements. - On Windows, if *args* is a sequence, it will be converted to a string in a - manner described in :ref:`converting-argument-sequence`. This is because - the underlying ``CreateProcess()`` operates on strings. - - The *shell* argument (which defaults to *False*) specifies whether to use - the shell as the program to execute. If *shell* is *True*, it is - recommended to pass *args* as a string rather than as a sequence. - - On POSIX with ``shell=True``, the shell defaults to :file:`/bin/sh`. If - *args* is a string, the string specifies the command - to execute through the shell. This means that the string must be + On Unix, with *shell=True*: If args is a string, it specifies the command + string to execute through the shell. This means that the string must be formatted exactly as it would be when typed at the shell prompt. This includes, for example, quoting or backslash escaping filenames with spaces in them. If *args* is a sequence, the first item specifies the command string, and any additional items will be treated as additional arguments to the shell - itself. That is to say, :class:`Popen` does the equivalent of:: + itself. That is to say, *Popen* does the equivalent of:: Popen(['/bin/sh', '-c', args[0], args[1], ...]) - On Windows with ``shell=True``, the :envvar:`COMSPEC` environment variable - specifies the default shell. The only time you need to specify - ``shell=True`` on Windows is when the command you wish to execute is built - into the shell (e.g. :command:`dir` or :command:`copy`). You do not need - ``shell=True`` to run a batch file or console-based executable. + .. warning:: + + Enabling this option can be a security hazard if combined with untrusted + input. See the warning under :ref:`frequently-used-arguments` + for details. + + On Windows: the :class:`Popen` class uses CreateProcess() to execute the + child program, which operates on strings. If *args* is a sequence, it will + be converted to a string in a manner described in + :ref:`converting-argument-sequence`. + + *bufsize*, if given, has the same meaning as the corresponding argument to the + built-in open() function: :const:`0` means unbuffered, :const:`1` means line + buffered, any other positive value means use a buffer of (approximately) that + size. A negative *bufsize* means to use the system default, which usually means + fully buffered. The default value for *bufsize* is :const:`0` (unbuffered). .. note:: - Read the `Security Considerations`_ section before using ``shell=True``. + If you experience performance issues, it is recommended that you try to + enable buffering by setting *bufsize* to either -1 or a large enough + positive value (such as 4096). - *bufsize* will be supplied as the corresponding argument to the - :func:`open` function when creating the stdin/stdout/stderr pipe - file objects: - - - :const:`0` means unbuffered (read and write are one - system call and can return short) - - :const:`1` means line buffered - (only usable if ``universal_newlines=True`` i.e., in a text mode) - - any other positive value means use a buffer of approximately that - size - - negative bufsize (the default) means the system default of - io.DEFAULT_BUFFER_SIZE will be used. - - .. versionchanged:: 3.3.1 - *bufsize* now defaults to -1 to enable buffering by default to match the - behavior that most code expects. In versions prior to Python 3.2.4 and - 3.3.1 it incorrectly defaulted to :const:`0` which was unbuffered - and allowed short reads. This was unintentional and did not match the - behavior of Python 2 as most code expected. - - The *executable* argument specifies a replacement program to execute. It - is very seldom needed. When ``shell=False``, *executable* replaces the - program to execute specified by *args*. However, the original *args* is - still passed to the program. Most programs treat the program specified - by *args* as the command name, which can then be different from the program - actually executed. On POSIX, the *args* name - becomes the display name for the executable in utilities such as - :program:`ps`. If ``shell=True``, on POSIX the *executable* argument - specifies a replacement shell for the default :file:`/bin/sh`. + The *executable* argument specifies the program to execute. It is very seldom + needed: Usually, the program to execute is defined by the *args* argument. If + ``shell=True``, the *executable* argument specifies which shell to use. On Unix, + the default shell is :file:`/bin/sh`. On Windows, the default shell is + specified by the :envvar:`COMSPEC` environment variable. The only reason you + would need to specify ``shell=True`` on Windows is where the command you + wish to execute is actually built in to the shell, eg ``dir``, ``copy``. + You don't need ``shell=True`` to run a batch file, nor to run a console-based + executable. *stdin*, *stdout* and *stderr* specify the executed program's standard input, standard output and standard error file handles, respectively. Valid values @@ -410,7 +375,7 @@ If *preexec_fn* is set to a callable object, this object will be called in the child process just before the child is executed. - (POSIX only) + (Unix only) .. warning:: @@ -428,8 +393,8 @@ common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. (POSIX only). - The default varies by platform: Always true on POSIX. On Windows it is + :const:`2` will be closed before the child process is executed. (Unix only). + The default varies by platform: Always true on Unix. On Windows it is true when *stdin*/*stdout*/*stderr* are :const:`None`, false otherwise. On Windows, if *close_fds* is true then no handles will be inherited by the child process. Note that on Windows, you cannot set *close_fds* to true and @@ -441,26 +406,26 @@ *pass_fds* is an optional sequence of file descriptors to keep open between the parent and child. Providing any *pass_fds* forces - *close_fds* to be :const:`True`. (POSIX only) + *close_fds* to be :const:`True`. (Unix only) .. versionadded:: 3.2 The *pass_fds* parameter was added. - If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. In particular, the function looks for - *executable* (or for the first item in *args*) relative to *cwd* if the - executable path is a relative path. + If *cwd* is not ``None``, the child's current directory will be changed to *cwd* + before it is executed. Note that this directory is not considered when + searching the executable, so you can't specify the program's path relative to + *cwd*. - If *restore_signals* is true (the default) all signals that Python has set to + If *restore_signals* is True (the default) all signals that Python has set to SIG_IGN are restored to SIG_DFL in the child process before the exec. Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. - (POSIX only) + (Unix only) .. versionchanged:: 3.2 *restore_signals* was added. - If *start_new_session* is true the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) + If *start_new_session* is True the setsid() system call will be made in the + child process prior to the execution of the subprocess. (Unix only) .. versionchanged:: 3.2 *start_new_session* was added. @@ -477,10 +442,18 @@ .. _side-by-side assembly: http://en.wikipedia.org/wiki/Side-by-Side_Assembly - If *universal_newlines* is ``True``, the file objects *stdin*, *stdout* - and *stderr* are opened as text streams in universal newlines mode, as - described above in :ref:`frequently-used-arguments`, otherwise they are - opened as binary streams. + If *universal_newlines* is :const:`True`, the file objects stdout and stderr are + opened as text files, but lines may be terminated by any of ``'\n'``, the Unix + end-of-line convention, ``'\r'``, the old Macintosh convention or ``'\r\n'``, the + Windows convention. All of these external representations are seen as ``'\n'`` + by the Python program. + + .. note:: + + This feature is only available if Python is built with universal newline + support (the default). Also, the newlines attribute of the file objects + :attr:`stdout`, :attr:`stdin` and :attr:`stderr` are not updated by the + :meth:`communicate` method. If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. @@ -527,21 +500,14 @@ The :exc:`SubprocessError` base class was added. -Security Considerations ------------------------ +Security +^^^^^^^^ -Unlike some other popen functions, this implementation will never -implicitly call a system shell. This means that all characters, -including shell metacharacters, can safely be passed to child processes. -If the shell is invoked explicitly, via ``shell=True``, it is the application's -responsibility to ensure that all whitespace and metacharacters are -quoted appropriately to avoid -`shell injection `_ -vulnerabilities. - -When using ``shell=True``, the :func:`shlex.quote` function can be -used to properly escape whitespace and shell metacharacters in strings -that are going to be used to construct shell commands. +Unlike some other popen functions, this implementation will never call a +system shell implicitly. This means that all characters, including shell +metacharacters, can safely be passed to child processes. Obviously, if the +shell is invoked explicitly, then it is the application's responsibility to +ensure that all whitespace and metacharacters are quoted appropriately. Popen Objects @@ -552,40 +518,29 @@ .. method:: Popen.poll() - Check if child process has terminated. Set and return - :attr:`~Popen.returncode` attribute. + Check if child process has terminated. Set and return :attr:`returncode` + attribute. .. method:: Popen.wait(timeout=None) - Wait for child process to terminate. Set and return - :attr:`~Popen.returncode` attribute. + Wait for child process to terminate. Set and return :attr:`returncode` + attribute. If the process does not terminate after *timeout* seconds, raise a :exc:`TimeoutExpired` exception. It is safe to catch this exception and retry the wait. - .. note:: + .. warning:: - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` - and the child process generates enough output to a pipe such that - it blocks waiting for the OS pipe buffer to accept more data. - Use :meth:`Popen.communicate` when using pipes to avoid that. - - .. note:: - - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. + This will deadlock when using ``stdout=PIPE`` and/or + ``stderr=PIPE`` and the child process generates enough output to + a pipe such that it blocks waiting for the OS pipe buffer to + accept more data. Use :meth:`communicate` to avoid that. .. versionchanged:: 3.3 *timeout* was added. - .. deprecated:: 3.4 - - Do not use the *endtime* parameter. It is was unintentionally - exposed in 3.3 but was left undocumented as it was intended to be - private for internal use. Use *timeout* instead. .. method:: Popen.communicate(input=None, timeout=None) @@ -595,8 +550,7 @@ ``None``, if no data should be sent to the child. The type of *input* must be bytes or, if *universal_newlines* was ``True``, a string. - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - The data will be bytes or, if *universal_newlines* was ``True``, strings. + :meth:`communicate` returns a tuple ``(stdoutdata, stderrdata)``. Note that if you want to send data to the process's stdin, you need to create the Popen object with ``stdin=PIPE``. Similarly, to get anything other than @@ -653,45 +607,31 @@ The following attributes are also available: -.. attribute:: Popen.args +.. warning:: - The *args* argument as it was passed to :class:`Popen` -- a - sequence of program arguments or else a single string. + Use :meth:`communicate` rather than :attr:`.stdin.write `, + :attr:`.stdout.read ` or :attr:`.stderr.read ` to avoid + deadlocks due to any of the other OS pipe buffers filling up and blocking the + child process. - .. versionadded:: 3.3 .. attribute:: Popen.stdin - If the *stdin* argument was :data:`PIPE`, this attribute is a writeable - stream object as returned by :func:`open`. If the *universal_newlines* - argument was ``True``, the stream is a text stream, otherwise it is a byte - stream. If the *stdin* argument was not :data:`PIPE`, this attribute is - ``None``. + If the *stdin* argument was :data:`PIPE`, this attribute is a :term:`file + object` that provides input to the child process. Otherwise, it is ``None``. .. attribute:: Popen.stdout - If the *stdout* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - output from the child process. If the *universal_newlines* argument was - ``True``, the stream is a text stream, otherwise it is a byte stream. If the - *stdout* argument was not :data:`PIPE`, this attribute is ``None``. + If the *stdout* argument was :data:`PIPE`, this attribute is a :term:`file + object` that provides output from the child process. Otherwise, it is ``None``. .. attribute:: Popen.stderr - If the *stderr* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - error output from the child process. If the *universal_newlines* argument was - ``True``, the stream is a text stream, otherwise it is a byte stream. If the - *stderr* argument was not :data:`PIPE`, this attribute is ``None``. - -.. warning:: - - Use :meth:`~Popen.communicate` rather than :attr:`.stdin.write `, - :attr:`.stdout.read ` or :attr:`.stderr.read ` to avoid - deadlocks due to any of the other OS pipe buffers filling up and blocking the - child process. + If the *stderr* argument was :data:`PIPE`, this attribute is a :term:`file + object` that provides error output from the child process. Otherwise, it is + ``None``. .. attribute:: Popen.pid @@ -709,7 +649,7 @@ hasn't terminated yet. A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). + ``N`` (Unix only). Windows Popen Helpers @@ -805,6 +745,8 @@ The new process has a new console, instead of inheriting its parent's console (the default). + This flag is always set when :class:`Popen` is created with ``shell=True``. + .. data:: CREATE_NEW_PROCESS_GROUP A :class:`Popen` ``creationflags`` parameter to specify that a new process @@ -813,117 +755,11 @@ This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified. -.. _call-function-trio: - -Older high-level API --------------------- - -Prior to Python 3.5, these three functions comprised the high level API to -subprocess. You can now use :func:`run` in many cases, but lots of existing code -calls these functions. - -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) - - Run the command described by *args*. Wait for command to complete, then - return the :attr:`~Popen.returncode` attribute. - - This is equivalent to:: - - run(...).returncode - - (except that the *input* and *check* parameters are not supported) - - The arguments shown above are merely the most - common ones. The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) - - Run command with arguments. Wait for command to complete. If the return - code was zero then return, otherwise raise :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute. - - This is equivalent to:: - - run(..., check=True) - - (except that the *input* parameter is not supported) - - The arguments shown above are merely the most - common ones. The full function signature is largely the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) - - Run command with arguments and return its output. - - If the return code was non-zero it raises a :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute and any output in the - :attr:`~CalledProcessError.output` attribute. - - This is equivalent to:: - - run(..., check=True, stdout=PIPE).stdout - - The arguments shown above are merely the most common ones. - The full function signature is largely the same as that of :func:`run` - - most arguments are passed directly through to that interface. - However, explicitly passing ``input=None`` to inherit the parent's - standard input file handle is not supported. - - By default, this function will return the data as encoded bytes. The actual - encoding of the output data may depend on the command being invoked, so the - decoding to text will often need to be handled at the application level. - - This behaviour may be overridden by setting *universal_newlines* to - ``True`` as described above in :ref:`frequently-used-arguments`. - - To also capture standard error in the result, use - ``stderr=subprocess.STDOUT``:: - - >>> subprocess.check_output( - ... "ls non_existent_file; exit 0", - ... stderr=subprocess.STDOUT, - ... shell=True) - 'ls: non_existent_file: No such file or directory\n' - - .. versionadded:: 3.1 - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.4 - Support for the *input* keyword argument was added. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the subprocess Module +---------------------------------------------------- In this section, "a becomes b" means that b can be used as a replacement for a. @@ -935,11 +771,11 @@ In addition, the replacements using :func:`check_output` will fail with a :exc:`CalledProcessError` if the requested operation produces a non-zero - return code. The output is still available as the - :attr:`~CalledProcessError.output` attribute of the raised exception. + return code. The output is still available as the ``output`` attribute of + the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the subprocess module. Replacing /bin/sh shell backquote @@ -968,7 +804,7 @@ to receive a SIGPIPE if p2 exits before p1. Alternatively, for trusted input, the shell's own pipeline support may still -be used directly:: +be used directly: output=`dmesg | grep hda` # becomes @@ -1068,7 +904,7 @@ if rc is not None and rc >> 8: print("There were some errors") ==> - process = Popen(cmd, stdin=PIPE) + process = Popen(cmd, 'w', stdin=PIPE) ... process.stdin.close() if process.wait() != 0: @@ -1087,7 +923,7 @@ (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) ==> - p = Popen("somestring", shell=True, bufsize=bufsize, + p = Popen(["somestring"], shell=True, bufsize=bufsize, stdin=PIPE, stdout=PIPE, close_fds=True) (child_stdout, child_stdin) = (p.stdout, p.stdin) @@ -1125,12 +961,10 @@ Return ``(status, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and - return a 2-tuple ``(status, output)``. Universal newlines mode is used; - see the notes on :ref:`frequently-used-arguments` for more details. - - A trailing newline is stripped from the output. - The exit status for the command can be interpreted + Execute the string *cmd* in a shell with :func:`os.popen` and return a 2-tuple + ``(status, output)``. *cmd* is actually run as ``{ cmd ; } 2>&1``, so that the + returned output will contain output or error messages. A trailing newline is + stripped from the output. The exit status for the command can be interpreted according to the rules for the C function :c:func:`wait`. Example:: >>> subprocess.getstatusoutput('ls /bin/ls') @@ -1140,10 +974,7 @@ >>> subprocess.getstatusoutput('/bin/junk') (256, 'sh: /bin/junk: not found') - Availability: POSIX & Windows - - .. versionchanged:: 3.3.4 - Windows support added + Availability: UNIX. .. function:: getoutput(cmd) @@ -1156,10 +987,7 @@ >>> subprocess.getoutput('ls /bin/ls') '/bin/ls' - Availability: POSIX & Windows - - .. versionchanged:: 3.3.4 - Windows support added + Availability: UNIX. Notes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sunau.rst --- a/Doc/library/sunau.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sunau.rst Mon Jan 25 17:05:13 2016 +0100 @@ -54,8 +54,8 @@ Note that it does not allow read/write files. - A *mode* of ``'r'`` returns an :class:`AU_read` object, while a *mode* of ``'w'`` - or ``'wb'`` returns an :class:`AU_write` object. + A *mode* of ``'r'`` returns a :class:`AU_read` object, while a *mode* of ``'w'`` + or ``'wb'`` returns a :class:`AU_write` object. .. function:: openfp(file, mode) @@ -150,9 +150,8 @@ .. method:: AU_read.getparams() - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. + Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype, + compname)``, equivalent to output of the :meth:`get\*` methods. .. method:: AU_read.readframes(n) @@ -212,9 +211,6 @@ Set the sample width (in bytes.) - .. versionchanged:: 3.4 - Added support for 24-bit samples. - .. method:: AU_write.setframerate(n) @@ -250,17 +246,11 @@ Write audio frames, without correcting *nframes*. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - .. method:: AU_write.writeframes(data) Write audio frames and make sure *nframes* is correct. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - .. method:: AU_write.close() diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/superseded.rst --- a/Doc/library/superseded.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -.. _superseded: - -****************** -Superseded Modules -****************** - -The modules described in this chapter are deprecated and only kept for -backwards compatibility. They have been superseded by other modules. - - -.. toctree:: - - optparse.rst - imp.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/symtable.rst --- a/Doc/library/symtable.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/symtable.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,6 @@ .. module:: symtable :synopsis: Interface to the compiler's internal symbol tables. -**Source code:** :source:`Lib/symtable.py` - --------------- - .. moduleauthor:: Jeremy Hylton .. sectionauthor:: Benjamin Peterson @@ -71,6 +67,10 @@ Return ``True`` if the block uses ``exec``. + .. method:: has_import_star() + + Return ``True`` if the block uses a starred from-import. + .. method:: get_identifiers() Return a list of names of symbols in this table. @@ -185,4 +185,4 @@ .. method:: get_namespace() Return the namespace bound to this name. If more than one namespace is - bound, :exc:`ValueError` is raised. + bound, a :exc:`ValueError` is raised. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sys.rst --- a/Doc/library/sys.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sys.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,12 +12,11 @@ .. data:: abiflags - On POSIX systems where Python was built with the standard ``configure`` + On POSIX systems where Python is build with the standard ``configure`` script, this contains the ABI flags as specified by :pep:`3149`. .. versionadded:: 3.2 - .. data:: argv The list of command line arguments passed to a Python script. ``argv[0]`` is the @@ -30,33 +29,6 @@ command line, see the :mod:`fileinput` module. -.. data:: base_exec_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - -.. data:: base_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - .. data:: byteorder An indicator of the native byte order. This will have the value ``'big'`` on @@ -108,22 +80,6 @@ This function should be used for internal and specialized purposes only. -.. function:: _debugmallocstats() - - Print low-level information to stderr about the state of CPython's memory - allocator. - - If Python is configured --with-pydebug, it also performs some expensive - internal consistency checks. - - .. versionadded:: 3.3 - - .. impl-detail:: - - This function is specific to CPython. The exact output format is not - defined here, and may change. - - .. data:: dllhandle Integer specifying the handle of the Python DLL. Availability: Windows. @@ -167,7 +123,7 @@ .. data:: dont_write_bytecode - If this is true, Python won't try to write ``.pyc`` files on the + If this is true, Python won't try to write ``.pyc`` or ``.pyo`` files on the import of source modules. This value is initially set to ``True`` or ``False`` depending on the :option:`-B` command line option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable, but you can set it @@ -216,6 +172,21 @@ a traceback object (see the Reference Manual) which encapsulates the call stack at the point where the exception originally occurred. + .. warning:: + + Assigning the *traceback* return value to a local variable in a function + that is handling an exception will cause a circular reference. Since most + functions don't need access to the traceback, the best solution is to use + something like ``exctype, value = sys.exc_info()[:2]`` to extract only the + exception type and value. If you do need the traceback, make sure to + delete it after use (best done with a :keyword:`try` + ... :keyword:`finally` statement) or to call :func:`exc_info` in a + function that does not itself handle an exception. + + Such cycles are normally automatically reclaimed when garbage collection + is enabled and they become unreachable, but it remains more efficient to + avoid creating cycles. + .. data:: exec_prefix @@ -228,13 +199,6 @@ installed in :file:`{exec_prefix}/lib/python{X.Y}/lib-dynload`, where *X.Y* is the version number of Python, for example ``3.2``. - .. note:: - - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. - .. data:: executable @@ -255,7 +219,7 @@ (defaulting to zero), or another type of object. If it is an integer, zero is considered "successful termination" and any nonzero value is considered "abnormal termination" by shells and the like. Most systems require it to be - in the range 0--127, and produce undefined results otherwise. Some systems + in the range 0-127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of @@ -268,11 +232,6 @@ the process when called from the main thread, and the exception is not intercepted. - .. versionchanged:: 3.6 - If an error occurs in the cleanup after the Python interpreter - has caught :exc:`SystemExit` (such as an error flushing buffered data - in the standard streams), the exit status is changed to 120. - .. data:: flags @@ -315,8 +274,6 @@ programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - +---------------------+----------------+--------------------------------------------------+ | attribute | float.h macro | explanation | +=====================+================+==================================================+ @@ -388,21 +345,6 @@ .. versionadded:: 3.1 -.. function:: getallocatedblocks() - - Return the number of memory blocks currently allocated by the interpreter, - regardless of their size. This function is mainly useful for tracking - and debugging memory leaks. Because of the interpreter's internal - caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more - predictable results. - - If a Python build or implementation cannot reasonably compute this - information, :func:`getallocatedblocks()` is allowed to return 0 instead. - - .. versionadded:: 3.4 - - .. function:: getcheckinterval() Return the interpreter's "check interval"; see :func:`setcheckinterval`. @@ -419,10 +361,9 @@ .. function:: getdlopenflags() - Return the current value of the flags that are used for - :c:func:`dlopen` calls. Symbolic names for the flag values can be - found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. - :data:`os.RTLD_LAZY`). Availability: Unix. + Return the current value of the flags that are used for :c:func:`dlopen` calls. + The flag constants are defined in the :mod:`ctypes` and :mod:`DLFCN` modules. + Availability: Unix. .. function:: getfilesystemencoding() @@ -433,7 +374,7 @@ * On Mac OS X, the encoding is ``'utf-8'``. * On Unix, the encoding is the user's preference according to the result of - nl_langinfo(CODESET). + nl_langinfo(CODESET), or ``'utf-8'`` if ``nl_langinfo(CODESET)`` failed. * On Windows NT+, file names are Unicode natively, so no conversion is performed. :func:`getfilesystemencoding` still returns ``'mbcs'``, as @@ -444,7 +385,8 @@ * On Windows 9x, the encoding is ``'mbcs'``. .. versionchanged:: 3.2 - :func:`getfilesystemencoding` result cannot be ``None`` anymore. + On Unix, use ``'utf-8'`` instead of ``None`` if ``nl_langinfo(CODESET)`` + failed. :func:`getfilesystemencoding` result cannot be ``None``. .. function:: getrefcount(object) @@ -469,9 +411,6 @@ does not have to hold true for third-party extensions as it is implementation specific. - Only the memory consumption directly attributed to the object is - accounted for, not the memory consumption of objects it refers to. - If given, *default* will be returned if the object does not provide means to retrieve the size. Otherwise a :exc:`TypeError` will be raised. @@ -581,18 +520,6 @@ *service_pack_major*, *suite_mask*, and *product_type*. -.. function:: get_coroutine_wrapper() - - Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. data:: hash_info A :term:`struct sequence` giving parameters of the numeric hash @@ -613,20 +540,9 @@ | :const:`imag` | multiplier used for the imaginary part of a | | | complex number | +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ - .. versionadded:: 3.2 - .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* - .. data:: hexversion @@ -646,56 +562,35 @@ :term:`struct sequence` :data:`sys.version_info` may be used for a more human-friendly encoding of the same information. - More details of ``hexversion`` can be found at :ref:`apiabiversion`. + The ``hexversion`` is a 32-bit number with the following layout: + +-------------------------+------------------------------------------------+ + | Bits (big endian order) | Meaning | + +=========================+================================================+ + | :const:`1-8` | ``PY_MAJOR_VERSION`` (the ``2`` in | + | | ``2.1.0a3``) | + +-------------------------+------------------------------------------------+ + | :const:`9-16` | ``PY_MINOR_VERSION`` (the ``1`` in | + | | ``2.1.0a3``) | + +-------------------------+------------------------------------------------+ + | :const:`17-24` | ``PY_MICRO_VERSION`` (the ``0`` in | + | | ``2.1.0a3``) | + +-------------------------+------------------------------------------------+ + | :const:`25-28` | ``PY_RELEASE_LEVEL`` (``0xA`` for alpha, | + | | ``0xB`` for beta, ``0xC`` for release | + | | candidate and ``0xF`` for final) | + +-------------------------+------------------------------------------------+ + | :const:`29-32` | ``PY_RELEASE_SERIAL`` (the ``3`` in | + | | ``2.1.0a3``, zero for final releases) | + +-------------------------+------------------------------------------------+ -.. data:: implementation - - An object containing information about the implementation of the - currently running Python interpreter. The following attributes are - required to exist in all Python implementations. - - *name* is the implementation's identifier, e.g. ``'cpython'``. The actual - string is defined by the Python implementation, but it is guaranteed to be - lower case. - - *version* is a named tuple, in the same format as - :data:`sys.version_info`. It represents the version of the Python - *implementation*. This has a distinct meaning from the specific - version of the Python *language* to which the currently running - interpreter conforms, which ``sys.version_info`` represents. For - example, for PyPy 1.8 ``sys.implementation.version`` might be - ``sys.version_info(1, 8, 0, 'final', 0)``, whereas ``sys.version_info`` - would be ``sys.version_info(2, 7, 2, 'final', 0)``. For CPython they - are the same value, since it is the reference implementation. - - *hexversion* is the implementation version in hexadecimal format, like - :data:`sys.hexversion`. - - *cache_tag* is the tag used by the import machinery in the filenames of - cached modules. By convention, it would be a composite of the - implementation's name and version, like ``'cpython-33'``. However, a - Python implementation may use some other value if appropriate. If - ``cache_tag`` is set to ``None``, it indicates that module caching should - be disabled. - - :data:`sys.implementation` may contain additional attributes specific to - the Python implementation. These non-standard attributes must start with - an underscore, and are not described here. Regardless of its contents, - :data:`sys.implementation` will not change during a run of the interpreter, - nor between implementation versions. (It may change between Python - language versions, however.) See :pep:`421` for more information. - - .. versionadded:: 3.3 - + Thus ``2.1.0a3`` is hexversion ``0x020100a3``. .. data:: int_info A :term:`struct sequence` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - +-------------------------+----------------------------------------------+ | Attribute | Explanation | +=========================+==============================================+ @@ -710,17 +605,6 @@ .. versionadded:: 3.1 -.. data:: __interactivehook__ - - When this attribute exists, its value is automatically called (with no - arguments) when the interpreter is launched in :ref:`interactive mode - `. This is done after the :envvar:`PYTHONSTARTUP` file is - read, so that you can set this hook there. The :mod:`site` module - :ref:`sets this `. - - .. versionadded:: 3.4 - - .. function:: intern(string) Enter *string* in the table of "interned" strings and return the interned string @@ -735,14 +619,6 @@ value of :func:`intern` around to benefit from it. -.. function:: is_finalizing() - - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. - - .. versionadded:: 3.5 - - .. data:: last_type last_value last_traceback @@ -779,39 +655,24 @@ .. data:: meta_path - A list of :term:`meta path finder` objects that have their - :meth:`~importlib.abc.MetaPathFinder.find_spec` methods called to see if one - of the objects can find the module to be imported. The - :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at - least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` - attribute is passed in as a second argument. The method returns a - :term:`module spec`, or ``None`` if the module cannot be found. + A list of :term:`finder` objects that have their :meth:`find_module` + methods called to see if one of the objects can find the module to be + imported. The :meth:`find_module` method is called at least with the + absolute name of the module being imported. If the module to be imported is + contained in package then the parent package's :attr:`__path__` attribute + is passed in as a second argument. The method returns ``None`` if + the module cannot be found, else returns a :term:`loader`. - .. seealso:: + :data:`sys.meta_path` is searched before any implicit default finders or + :data:`sys.path`. - :class:`importlib.abc.MetaPathFinder` - The abstract base class defining the interface of finder objects on - :data:`meta_path`. - :class:`importlib.machinery.ModuleSpec` - The concrete class which - :meth:`~importlib.abc.MetaPathFinder.find_spec` should return - instances of. + See :pep:`302` for the original specification. - .. versionchanged:: 3.4 - - :term:`Module specs ` were introduced in Python 3.4, by - :pep:`451`. Earlier versions of Python looked for a method called - :meth:`~importlib.abc.MetaPathFinder.find_module`. - This is still called as a fallback if a :data:`meta_path` entry doesn't - have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. .. data:: modules This is a dictionary that maps module names to modules which have already been loaded. This can be manipulated to force reloading of modules and other tricks. - However, replacing the dictionary will not necessarily work as expected and - deleting essential items from the dictionary may cause Python to fail. .. data:: path @@ -830,9 +691,7 @@ current directory first. Notice that the script directory is inserted *before* the entries inserted as a result of :envvar:`PYTHONPATH`. - A program is free to modify this list for its own purposes. Only strings - and bytes should be added to :data:`sys.path`; all other data types are - ignored during import. + A program is free to modify this list for its own purposes. .. seealso:: @@ -854,15 +713,12 @@ A dictionary acting as a cache for :term:`finder` objects. The keys are paths that have been passed to :data:`sys.path_hooks` and the values are the finders that are found. If a path is a valid file system path but no - finder is found on :data:`sys.path_hooks` then ``None`` is - stored. + explicit finder is found on :data:`sys.path_hooks` then ``None`` is + stored to represent the implicit default finder should be used. If the path + is not an existing path then :class:`imp.NullImporter` is set. Originally specified in :pep:`302`. - .. versionchanged:: 3.3 - ``None`` is stored instead of :class:`imp.NullImporter` when no finder - is found. - .. data:: platform @@ -883,12 +739,14 @@ For other systems, the values are: ================ =========================== - System ``platform`` value + System :data:`platform` value ================ =========================== Linux ``'linux'`` Windows ``'win32'`` Windows/Cygwin ``'cygwin'`` Mac OS X ``'darwin'`` + OS/2 ``'os2'`` + OS/2 EMX ``'os2emx'`` ================ =========================== .. versionchanged:: 3.3 @@ -917,11 +775,6 @@ stored in :file:`{prefix}/include/python{X.Y}`, where *X.Y* is the version number of Python, for example ``3.2``. - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. - .. data:: ps1 ps2 @@ -959,7 +812,7 @@ the interpreter loads extension modules. Among other things, this will enable a lazy resolving of symbols when importing a module, if called as ``sys.setdlopenflags(0)``. To share symbols across extension modules, call as - ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values + ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag modules can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g. :data:`os.RTLD_LAZY`). @@ -993,13 +846,6 @@ that supports a higher limit. This should be done with care, because a too-high limit can lead to a crash. - If the new limit is too low at the current recursion depth, a - :exc:`RecursionError` exception is raised. - - .. versionchanged:: 3.5.1 - A :exc:`RecursionError` exception is now raised if the new limit is too - low at the current recursion depth. - .. function:: setswitchinterval(interval) @@ -1098,46 +944,6 @@ thus not likely to be implemented elsewhere. -.. function:: set_coroutine_wrapper(wrapper) - - Allows intercepting creation of :term:`coroutine` objects (only ones that - are created by an :keyword:`async def` function; generators decorated with - :func:`types.coroutine` or :func:`asyncio.coroutine` will not be - intercepted). - - The *wrapper* argument must be either: - - * a callable that accepts one argument (a coroutine object); - * ``None``, to reset the wrapper. - - If called twice, the new wrapper replaces the previous one. The function - is thread-specific. - - The *wrapper* callable cannot define new coroutines directly or indirectly:: - - def wrapper(coro): - async def wrap(coro): - return await coro - return wrap(coro) - sys.set_coroutine_wrapper(wrapper) - - async def foo(): - pass - - # The following line will fail with a RuntimeError, because - # ``wrapper`` creates a ``wrap(coro)`` coroutine: - foo() - - See also :func:`get_coroutine_wrapper`. - - .. versionadded:: 3.5 - See :pep:`492` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - .. data:: stdin stdout stderr @@ -1151,32 +957,35 @@ statements and for the prompts of :func:`input`; * The interpreter's own prompts and its error messages go to ``stderr``. - These streams are regular :term:`text files ` like those - returned by the :func:`open` function. Their parameters are chosen as - follows: + By default, these streams are regular text streams as returned by the + :func:`open` function. Their parameters are chosen as follows: * The character encoding is platform-dependent. Under Windows, if the stream - is interactive (that is, if its :meth:`isatty` method returns ``True``), the + is interactive (that is, if its :meth:`isatty` method returns True), the console codepage is used, otherwise the ANSI code page. Under other platforms, the locale encoding is used (see :meth:`locale.getpreferredencoding`). Under all platforms though, you can override this value by setting the - :envvar:`PYTHONIOENCODING` environment variable before starting Python. + :envvar:`PYTHONIOENCODING` environment variable. * When interactive, standard streams are line-buffered. Otherwise, they are block-buffered like regular text files. You can override this value with the :option:`-u` command-line option. - .. note:: + To write or read binary data from/to the standard streams, use the + underlying binary :data:`~io.TextIOBase.buffer`. For example, to write + bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. Using + :meth:`io.TextIOBase.detach`, streams can be made binary by default. This + function sets :data:`stdin` and :data:`stdout` to binary:: - To write or read binary data from/to the standard streams, use the - underlying binary :data:`~io.TextIOBase.buffer` object. For example, to - write bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. + def make_streams_binary(): + sys.stdin = sys.stdin.detach() + sys.stdout = sys.stdout.detach() - However, if you are writing a library (and do not control in which - context its code will be executed), be aware that the standard streams - may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + Note that the streams may be replaced with objects (like :class:`io.StringIO`) + that do not support the :attr:`~io.BufferedIOBase.buffer` attribute or the + :meth:`~io.BufferedIOBase.detach` method and can raise :exc:`AttributeError` + or :exc:`io.UnsupportedOperation`. .. data:: __stdin__ @@ -1205,14 +1014,13 @@ A :term:`struct sequence` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - +------------------+---------------------------------------------------------+ | Attribute | Explanation | +==================+=========================================================+ | :const:`name` | Name of the thread implementation: | | | | | | * ``'nt'``: Windows threads | + | | * ``'os2'``: OS/2 threads | | | * ``'pthread'``: POSIX threads | | | * ``'solaris'``: Solaris threads | +------------------+---------------------------------------------------------+ @@ -1307,4 +1115,5 @@ .. rubric:: Citations -.. [C99] ISO/IEC 9899:1999. "Programming languages -- C." A public draft of this standard is available at http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf\ . +.. [C99] ISO/IEC 9899:1999. "Programming languages -- C." A public draft of this standard is available at http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf . + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/sysconfig.rst --- a/Doc/library/sysconfig.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/sysconfig.rst Mon Jan 25 17:05:13 2016 +0100 @@ -83,6 +83,8 @@ located under the user home directory. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. +- *os2*: scheme for OS/2 platforms. +- *os2_home*: scheme for OS/2 patforms, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/syslog.rst --- a/Doc/library/syslog.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/syslog.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,8 +17,7 @@ The module defines the following functions: -.. function:: syslog(message) - syslog(priority, message) +.. function:: syslog([priority,] message) Send the string *message* to the system logger. A trailing newline is added if necessary. Each message is tagged with a priority composed of a @@ -79,14 +78,11 @@ Facilities: :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ````, - :const:`LOG_AUTHPRIV`. + :const:`LOG_CRON` and :const:`LOG_LOCAL0` to :const:`LOG_LOCAL7`. Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ````, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. + :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, :const:`LOG_NOWAIT` + and :const:`LOG_PERROR` if defined in ````. Examples diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tarfile.rst --- a/Doc/library/tarfile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tarfile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,8 +19,7 @@ Some facts and figures: -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. +* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives. * read/write support for the POSIX.1-1988 (ustar) format. @@ -62,23 +61,6 @@ +------------------+---------------------------------------------+ | ``'r:xz'`` | Open for reading with lzma compression. | +------------------+---------------------------------------------+ - | ``'x'`` or | Create a tarfile exclusively without | - | ``'x:'`` | compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it is already exists. | - +------------------+---------------------------------------------+ - | ``'x:gz'`` | Create a tarfile with gzip compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it is already exists. | - +------------------+---------------------------------------------+ - | ``'x:bz2'`` | Create a tarfile with bzip2 compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it is already exists. | - +------------------+---------------------------------------------+ - | ``'x:xz'`` | Create a tarfile with lzma compression. | - | | Raise an :exc:`FileExistsError` exception | - | | if it is already exists. | - +------------------+---------------------------------------------+ | ``'a' or 'a:'`` | Open for appending with no compression. The | | | file is created if it does not exist. | +------------------+---------------------------------------------+ @@ -99,10 +81,6 @@ If *fileobj* is specified, it is used as an alternative to a :term:`file object` opened in binary mode for *name*. It is supposed to be at position 0. - For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, ``'x:gz'``, - ``'x:bz2'``, :func:`tarfile.open` accepts the keyword argument - *compresslevel* to specify the compression level of the file. - For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -144,8 +122,6 @@ | | writing. | +-------------+--------------------------------------------+ - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. .. class:: TarFile @@ -196,13 +172,6 @@ Is raised by :meth:`TarInfo.frombuf` if the buffer it gets is invalid. -The following constants are available at the module level: - -.. data:: ENCODING - - The default character encoding: ``'utf-8'`` on Windows, the value returned by - :func:`sys.getfilesystemencoding` otherwise. - Each of the following constants defines a tar archive format that the :mod:`tarfile` module is able to create. See section :ref:`tar-formats` for @@ -229,15 +198,20 @@ The default format for creating archives. This is currently :const:`GNU_FORMAT`. +The following variables are available on module level: + + +.. data:: ENCODING + + The default character encoding: ``'utf-8'`` on Windows, + :func:`sys.getfilesystemencoding` otherwise. + + .. seealso:: Module :mod:`zipfile` Documentation of the :mod:`zipfile` standard module. - :ref:`archiving-operations` - Documentation of the higher-level archiving facilities provided by the - standard :mod:`shutil` module. - `GNU tar manual, Basic Tar Format `_ Documentation for tar archive files, including GNU tar extensions. @@ -260,7 +234,7 @@ :ref:`tar-examples` section for a use case. .. versionadded:: 3.2 - Added support for the context management protocol. + Added support for the context manager protocol. .. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=0) @@ -271,8 +245,8 @@ In this case, the file object's :attr:`name` attribute is used if it exists. *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append - data to an existing file, ``'w'`` to create a new file overwriting an existing - one or ``'x'`` to create a new file only if it's not exists. + data to an existing file or ``'w'`` to create a new file overwriting an existing + one. If *fileobj* is given, it is used for reading or writing data. If it can be determined, *mode* is overridden by *fileobj*'s mode. *fileobj* will be used @@ -311,16 +285,14 @@ to be handled. The default settings will work for most users. See section :ref:`tar-unicode` for in-depth information. + .. versionchanged:: 3.2 + Use ``'surrogateescape'`` as the default for the *errors* argument. + The *pax_headers* argument is an optional dictionary of strings which will be added as a pax global header if *format* is :const:`PAX_FORMAT`. - .. versionchanged:: 3.2 - Use ``'surrogateescape'`` as the default for the *errors* argument. - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. - -.. classmethod:: TarFile.open(...) +.. method:: TarFile.open(...) Alternative constructor. The :func:`tarfile.open` function is actually a shortcut to this classmethod. @@ -349,15 +321,11 @@ returned by :meth:`getmembers`. -.. method:: TarFile.list(verbose=True, *, members=None) +.. method:: TarFile.list(verbose=True) Print a table of contents to ``sys.stdout``. If *verbose* is :const:`False`, only the names of the members are printed. If it is :const:`True`, output - similar to that of :program:`ls -l` is produced. If optional *members* is - given, it must be a subset of the list returned by :meth:`getmembers`. - - .. versionchanged:: 3.5 - Added the *members* parameter. + similar to that of :program:`ls -l` is produced. .. method:: TarFile.next() @@ -367,7 +335,7 @@ available. -.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False) +.. method:: TarFile.extractall(path=".", members=None) Extract all members from the archive to the current working directory or directory *path*. If optional *members* is given, it must be a subset of the @@ -377,10 +345,6 @@ reset each time a file is created in it. And, if a directory's permissions do not allow writing, extracting files to it will fail. - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. - .. warning:: Never extract archives from untrusted sources without prior inspection. @@ -388,21 +352,14 @@ that have absolute filenames starting with ``"/"`` or filenames with two dots ``".."``. - .. versionchanged:: 3.5 - Added the *numeric_only* parameter. - -.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) +.. method:: TarFile.extract(member, path="", set_attrs=True) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* may be a filename or a :class:`TarInfo` object. You can specify a different directory using *path*. File attributes (owner, mtime, mode) are set unless - *set_attrs* is false. - - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. + *set_attrs* is False. .. note:: @@ -416,18 +373,18 @@ .. versionchanged:: 3.2 Added the *set_attrs* parameter. - .. versionchanged:: 3.5 - Added the *numeric_only* parameter. - .. method:: TarFile.extractfile(member) Extract a member from the archive as a file object. *member* may be a filename - or a :class:`TarInfo` object. If *member* is a regular file or a link, an - :class:`io.BufferedReader` object is returned. Otherwise, :const:`None` is - returned. + or a :class:`TarInfo` object. If *member* is a regular file, a :term:`file-like + object` is returned. If *member* is a link, a file-like object is constructed from + the link's target. If *member* is none of the above, :const:`None` is returned. - .. versionchanged:: 3.3 - Return an :class:`io.BufferedReader` object. + .. note:: + + The file-like object is read-only. It provides the methods + :meth:`read`, :meth:`readline`, :meth:`readlines`, :meth:`seek`, :meth:`tell`, + and :meth:`close`, and also supports iteration over its lines. .. method:: TarFile.add(name, arcname=None, recursive=True, exclude=None, *, filter=None) @@ -504,14 +461,14 @@ Create a :class:`TarInfo` object. -.. classmethod:: TarInfo.frombuf(buf, encoding, errors) +.. method:: TarInfo.frombuf(buf) Create and return a :class:`TarInfo` object from string buffer *buf*. - Raises :exc:`HeaderError` if the buffer is invalid. + Raises :exc:`HeaderError` if the buffer is invalid.. -.. classmethod:: TarInfo.fromtarfile(tarfile) +.. method:: TarInfo.fromtarfile(tarfile) Read the next member from the :class:`TarFile` object *tarfile* and return it as a :class:`TarInfo` object. @@ -555,7 +512,7 @@ :const:`AREGTYPE`, :const:`LNKTYPE`, :const:`SYMTYPE`, :const:`DIRTYPE`, :const:`FIFOTYPE`, :const:`CONTTYPE`, :const:`CHRTYPE`, :const:`BLKTYPE`, :const:`GNUTYPE_SPARSE`. To determine the type of a :class:`TarInfo` object - more conveniently, use the ``is*()`` methods below. + more conveniently, use the ``is_*()`` methods below. .. attribute:: TarInfo.linkname @@ -637,67 +594,6 @@ Return :const:`True` if it is one of character device, block device or FIFO. -.. _tarfile-commandline: - -Command Line Interface ----------------------- - -.. versionadded:: 3.4 - -The :mod:`tarfile` module provides a simple command line interface to interact -with tar archives. - -If you want to create a new tar archive, specify its name after the :option:`-c` -option and then list the filename(s) that should be included:: - - $ python -m tarfile -c monty.tar spam.txt eggs.txt - -Passing a directory is also acceptable:: - - $ python -m tarfile -c monty.tar life-of-brian_1979/ - -If you want to extract a tar archive into the current directory, use -the :option:`-e` option:: - - $ python -m tarfile -e monty.tar - -You can also extract a tar archive into a different directory by passing the -directory's name:: - - $ python -m tarfile -e monty.tar other-dir/ - -For a list of the files in a tar archive, use the :option:`-l` option:: - - $ python -m tarfile -l monty.tar - - -Command line options -~~~~~~~~~~~~~~~~~~~~ - -.. cmdoption:: -l - --list - - List files in a tarfile. - -.. cmdoption:: -c - --create - - Create tarfile from source files. - -.. cmdoption:: -e [] - --extract [] - - Extract tarfile into the current directory if *output_dir* is not specified. - -.. cmdoption:: -t - --test - - Test whether the tarfile is valid or not. - -.. cmdoption:: -v, --verbose - - Verbose output - .. _tar-examples: Examples @@ -776,11 +672,11 @@ * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. The - maximum file size is 8 GiB. This is an old and limited but widely + maximum file size is 8 gigabytes. This is an old and limited but widely supported format. * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and - linknames, files bigger than 8 GiB and sparse files. It is the de facto + linknames, files bigger than 8 gigabytes and sparse files. It is the de facto standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. @@ -833,7 +729,7 @@ appropriately, this conversion may fail. The *errors* argument defines how characters are treated that cannot be -converted. Possible values are listed in section :ref:`error-handlers`. +converted. Possible values are listed in section :ref:`codec-base-classes`. The default scheme is ``'surrogateescape'`` which Python also uses for its file system calls, see :ref:`os-filenames`. @@ -841,3 +737,4 @@ because all the metadata is stored using *UTF-8*. *encoding* is only used in the rare cases when binary pax headers are decoded or when strings with surrogate characters are stored. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/telnetlib.rst --- a/Doc/library/telnetlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/telnetlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -31,7 +31,7 @@ :class:`Telnet` represents a connection to a Telnet server. The instance is initially not connected by default; the :meth:`open` method must be used to establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to + number can be passed to the constructor, to, in which case the connection to the server will be established before the constructor returns. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout @@ -43,17 +43,6 @@ :exc:`EOFError` when the end of the connection is read, because they can return an empty string for other reasons. See the individual descriptions below. - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - .. seealso:: @@ -196,7 +185,7 @@ Read until one from a list of a regular expressions matches. The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The + (:class:`re.RegexObject` instances) or uncompiled (byte strings). The optional second argument is a timeout, in seconds; the default is to block indefinitely. @@ -216,7 +205,7 @@ .. method:: Telnet.set_option_negotiation_callback(callback) Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command + called with the following parameters : callback(telnet socket, command (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tempfile.rst --- a/Doc/library/tempfile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tempfile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,60 +16,46 @@ -------------- -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. +This module generates temporary files and directories. It works on all +supported platforms. It provides three new functions, +:func:`NamedTemporaryFile`, :func:`mkstemp`, and :func:`mkdtemp`, which should +eliminate all remaining need to use the insecure :func:`mktemp` function. +Temporary file names created by this module no longer contain the process ID; +instead a string of six random characters is used. -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. +Also, all the user-callable functions now take additional arguments which +allow direct control over the location and name of temporary files. It is +no longer necessary to use the global *tempdir* variable. To maintain backward compatibility, the argument order is somewhat odd; it is recommended to use keyword arguments for clarity. The module defines the following user-callable items: -.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None) +.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix='', prefix='tmp', dir=None) Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon + The file is created using :func:`mkstemp`. It will be destroyed as soon as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed + collected). Under Unix, the directory entry for the file is removed immediately after the file is created. Other platforms do not support this; your code should not rely on a temporary file created using this function having or not having a visible name in the file system. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - The *mode* parameter defaults to ``'w+b'`` so that the file created can be read and written without being closed. Binary mode is used so that it behaves consistently on all platforms without regard for the data that is stored. *buffering*, *encoding* and *newline* are interpreted as for :func:`open`. - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. + The *dir*, *prefix* and *suffix* parameters are passed to :func:`mkstemp`. The returned object is a true file object on POSIX platforms. On other platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. + underlying true file object. This file-like object can be used in a + :keyword:`with` statement, just like a normal file. - The :py:data:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - .. versionchanged:: 3.5 - - The :py:data:`os.O_TMPFILE` flag is now used if available. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True) +.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix='', prefix='tmp', dir=None, delete=True) This function operates exactly as :func:`TemporaryFile` does, except that the file is guaranteed to have a visible name in the file system (on @@ -84,7 +70,7 @@ be used in a :keyword:`with` statement, just like a normal file. -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None) +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=None, encoding=None, newline=None, suffix='', prefix='tmp', dir=None) This function operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or @@ -96,28 +82,22 @@ causes the file to roll over to an on-disk file regardless of its size. The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.StringIO` object (depending on - whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. + is either a :class:`StringIO` object or a true file object, depending on + whether :func:`rollover` has been called. This file-like object can be + used in a :keyword:`with` statement, just like a normal file. - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. +.. function:: TemporaryDirectory(suffix='', prefix='tmp', dir=None) -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) - - This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. + This function creates a temporary directory using :func:`mkdtemp` + (the supplied arguments are passed directly to the underlying function). The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory + :ref:`context-managers`). On completion of the context (or destruction + of the temporary directory object), the newly created temporary directory and all its contents are removed from the filesystem. - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`as` clause in - the :keyword:`with` statement, if there is one. + The directory name can be retrieved from the :attr:`name` attribute + of the returned object. The directory can be explicitly cleaned up by calling the :func:`cleanup` method. @@ -125,7 +105,7 @@ .. versionadded:: 3.2 -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) +.. function:: mkstemp(suffix='', prefix='tmp', dir=None, text=False) Creates a temporary file in the most secure manner possible. There are no race conditions in the file's creation, assuming that the platform @@ -138,16 +118,15 @@ Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible for deleting the temporary file when done with it. - If *suffix* is not ``None``, the file name will end with that suffix, + If *suffix* is specified, the file name will end with that suffix, otherwise there will be no suffix. :func:`mkstemp` does not put a dot between the file name and the suffix; if you need one, put it at the beginning of *suffix*. - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. + If *prefix* is specified, the file name will begin with that prefix; + otherwise, a default prefix is used. - If *dir* is not ``None``, the file will be created in that directory; + If *dir* is specified, the file will be created in that directory; otherwise, a default directory is used. The default directory is chosen from a platform-dependent list, but the user of the application can control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* @@ -155,12 +134,6 @@ filename will have any nice properties, such as not requiring quoting when passed to external commands via ``os.popen()``. - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - If *text* is specified, it indicates whether to open the file in binary mode (the default) or text mode. On some platforms, this makes no difference. @@ -169,14 +142,8 @@ file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) +.. function:: mkdtemp(suffix='', prefix='tmp', dir=None) Creates a temporary directory in the most secure manner possible. There are no race conditions in the directory's creation. The directory is @@ -190,21 +157,51 @@ :func:`mkdtemp` returns the absolute pathname of the new directory. - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. +.. function:: mktemp(suffix='', prefix='tmp', dir=None) -.. function:: gettempdir() + .. deprecated:: 2.3 + Use :func:`mkstemp` instead. - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. + Return an absolute pathname of a file that did not exist at the time the + call is made. The *prefix*, *suffix*, and *dir* arguments are the same + as for :func:`mkstemp`. - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: + .. warning:: + + Use of this function may introduce a security hole in your program. By + the time you get around to doing anything with the file name it returns, + someone else may have beaten you to the punch. :func:`mktemp` usage can + be replaced easily with :func:`NamedTemporaryFile`, passing it the + ``delete=False`` parameter:: + + >>> f = NamedTemporaryFile(delete=False) + >>> f + ', mode 'w+b' at 0x384698> + >>> f.name + '/var/folders/5q/5qTPn6xq2RaWqk+1Ytw3-U+++TI/-Tmp-/tmpG7V1Y0' + >>> f.write("Hello World!\n") + >>> f.close() + >>> os.unlink(f.name) + >>> os.path.exists(f.name) + False + +The module uses two global variables that tell it how to construct a +temporary name. They are initialized at the first call to any of the +functions above. The caller may change them, but this is discouraged; use +the appropriate function arguments, instead. + + +.. data:: tempdir + + When set to a value other than ``None``, this variable defines the + default value for the *dir* argument to all the functions defined in this + module. + + If ``tempdir`` is unset or ``None`` at any call to any of the above + functions, Python searches a standard list of directories and sets + *tempdir* to the first one which the calling user can create files in. + The list is: #. The directory named by the :envvar:`TMPDIR` environment variable. @@ -222,43 +219,19 @@ #. As a last resort, the current working directory. - The result of this search is cached, see the description of - :data:`tempdir` below. -.. function:: gettempdirb() +.. function:: gettempdir() - Same as :func:`gettempdir` but the return value is in bytes. + Return the directory currently selected to create temporary files in. If + :data:`tempdir` is not ``None``, this simply returns its contents; otherwise, + the search described above is performed, and the result returned. - .. versionadded:: 3.5 .. function:: gettempprefix() Return the filename prefix used to create temporary files. This does not contain the directory component. -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommend approach. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to all the functions defined in this - module. - - If ``tempdir`` is unset or ``None`` at any call to any of the above - functions except :func:`gettempprefix` it is initalized following the - algorithm described in :func:`gettempdir`. - -.. _tempfile-examples: Examples -------- @@ -292,43 +265,3 @@ >>> # directory and contents have been removed - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/test.rst --- a/Doc/library/test.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/test.rst Mon Jan 25 17:05:13 2016 +0100 @@ -80,12 +80,17 @@ ... more test classes ... + def test_main(): + support.run_unittest(MyTestCase1, + MyTestCase2, + ... list other tests ... + ) + if __name__ == '__main__': - unittest.main() + test_main() -This code pattern allows the testing suite to be run by :mod:`test.regrtest`, -on its own as a script that supports the :mod:`unittest` CLI, or via the -``python -m unittest`` CLI. +This boilerplate code allows the testing suite to be run by :mod:`test.regrtest` +as well as on its own as a script. The goal for regression testing is to try to break code. This leads to a few guidelines to be followed: @@ -124,27 +129,22 @@ as what type of input is used. Minimize code duplication by subclassing a basic test class with a class that specifies the input:: - class TestFuncAcceptsSequencesMixin: + class TestFuncAcceptsSequences(unittest.TestCase): func = mySuperWhammyFunction def test_func(self): self.func(self.arg) - class AcceptLists(TestFuncAcceptsSequencesMixin, unittest.TestCase): + class AcceptLists(TestFuncAcceptsSequences): arg = [1, 2, 3] - class AcceptStrings(TestFuncAcceptsSequencesMixin, unittest.TestCase): + class AcceptStrings(TestFuncAcceptsSequences): arg = 'abc' - class AcceptTuples(TestFuncAcceptsSequencesMixin, unittest.TestCase): + class AcceptTuples(TestFuncAcceptsSequences): arg = (1, 2, 3) - When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above - does not have any data and so can't be run by itself, thus it does not - inherit from :class:`unittest.TestCase`. - .. seealso:: @@ -160,21 +160,20 @@ The :mod:`test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under the hood, it uses :mod:`test.regrtest`; the call :program:`python -m -test.regrtest` used in previous Python versions still works. Running the -script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose -name starts with ``test_``, importing them, and executing the function -:func:`test_main` if present or loading the tests via -unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The -names of tests to execute may also be passed to the script. Specifying a single -regression test (:program:`python -m test test_spam`) will minimize output and -only print whether the test passed or failed. +test.regrtest` used in previous Python versions still works). +Running the script by itself automatically starts running all regression +tests in the :mod:`test` package. It does this by finding all modules in the +package whose name starts with ``test_``, importing them, and executing the +function :func:`test_main` if present. The names of tests to execute may also +be passed to the script. Specifying a single regression test (:program:`python +-m test test_spam`) will minimize output and only print +whether the test passed or failed and thus minimize output. Running :mod:`test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line -option. Specifying ``all`` as the value for the ``-u`` option enables all -possible resources: :program:`python -m test -uall`. -If all but one resource is desired (a more common case), a +option. Run :program:`python -m test -uall` to turn on all +resources; specifying ``all`` as an option for ``-u`` enables all +possible resources. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` will run :mod:`test` with all resources except the ``audio`` and @@ -199,7 +198,6 @@ test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -264,15 +262,12 @@ Used when tests are executed by :mod:`test.regrtest`. -.. function:: findfile(filename, subdir=None) +.. function:: findfile(filename) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. - Setting *subdir* indicates a relative path to use to find the file - rather than looking directly in the path directories. - .. function:: run_unittest(\*classes) @@ -366,65 +361,31 @@ New optional arguments *filters* and *quiet*. -.. function:: captured_stdin() - captured_stdout() - captured_stderr() +.. function:: captured_stdout() - A context managers that temporarily replaces the named stream with - :class:`io.StringIO` object. + A context manager that runs the :keyword:`with` statement body using + a :class:`StringIO.StringIO` object as sys.stdout. That object can be + retrieved using the ``as`` clause of the :keyword:`with` statement. - Example use with output streams:: + Example use:: - with captured_stdout() as stdout, captured_stderr() as stderr: + with captured_stdout() as s: print("hello") - print("error", file=sys.stderr) - assert stdout.getvalue() == "hello\n" - assert stderr.getvalue() == "error\n" + assert s.getvalue() == "hello" - Example use with input stream:: - with captured_stdin() as stdin: - stdin.write('hello\n') - stdin.seek(0) - # call test code that consumes from sys.stdin - captured = input() - self.assertEqual(captured, "hello") - - -.. function:: temp_dir(path=None, quiet=False) - - A context manager that creates a temporary directory at *path* and - yields the directory. - - If *path* is None, the temporary directory is created using - :func:`tempfile.mkdtemp`. If *quiet* is ``False``, the context manager - raises an exception on error. Otherwise, if *path* is specified and - cannot be created, only a warning is issued. - - -.. function:: change_cwd(path, quiet=False) +.. function:: temp_cwd(name='tempcwd', quiet=False, path=None) A context manager that temporarily changes the current working - directory to *path* and yields the directory. + directory (CWD). - If *quiet* is ``False``, the context manager raises an exception - on error. Otherwise, it issues only a warning and keeps the current - working directory the same. + An existing path may be provided as *path*, in which case this function + makes no changes to the file system. - -.. function:: temp_cwd(name='tempcwd', quiet=False) - - A context manager that temporarily creates a new directory and - changes the current working directory (CWD). - - The context manager creates a temporary directory in the current - directory with name *name* before temporarily changing the current - working directory. If *name* is None, the temporary directory is - created using :func:`tempfile.mkdtemp`. - - If *quiet* is ``False`` and it is not possible to create or change - the CWD, an error is raised. Otherwise, only a warning is raised - and the original CWD is used. + Otherwise, the new CWD is created in the current directory and it's named + *name*. If *quiet* is ``False`` and it's not possible to create or + change the CWD, an error is raised. If it's ``True``, only a warning + is raised and the original CWD is used. .. function:: temp_umask(umask) @@ -461,7 +422,7 @@ .. function:: make_bad_fd() Create an invalid file descriptor by opening and closing a temporary file, - and returning its descriptor. + and returning its descripor. .. function:: import_module(name, deprecated=False) @@ -486,7 +447,7 @@ *fresh* is an iterable of additional module names that are also removed from the ``sys.modules`` cache before doing the import. - *blocked* is an iterable of module names that are replaced with ``None`` + *blocked* is an iterable of module names that are replaced with :const:`0` in the module cache during the import to ensure that attempts to import them raise :exc:`ImportError`. @@ -497,15 +458,15 @@ Module and package deprecation messages are suppressed during this import if *deprecated* is ``True``. - This function will raise :exc:`ImportError` if the named module cannot be - imported. + This function will raise :exc:`unittest.SkipTest` if the named module + cannot be imported. Example use:: - # Get copies of the warnings module for testing without affecting the - # version being used by the rest of the test suite. One copy uses the - # C implementation, the other is forced to use the pure Python fallback - # implementation + # Get copies of the warnings module for testing without + # affecting the version being used by the rest of the test suite + # One copy uses the C implementation, the other is forced to use + # the pure Python fallback implementation py_warnings = import_fresh_module('warnings', blocked=['_warnings']) c_warnings = import_fresh_module('warnings', fresh=['_warnings']) @@ -554,74 +515,6 @@ run simultaneously, which is a problem for buildbots. -.. function:: load_package_tests(pkg_dir, loader, standard_tests, pattern) - - Generic implementation of the :mod:`unittest` ``load_tests`` protocol for - use in test packages. *pkg_dir* is the root directory of the package; - *loader*, *standard_tests*, and *pattern* are the arguments expected by - ``load_tests``. In simple cases, the test package's ``__init__.py`` - can be the following:: - - import os - from test.support import load_package_tests - - def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) - -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()): - - Returns the set of attributes, functions or methods of *ref_api* not - found on *other_api*, except for a defined list of items to be - ignored in this check specified in *ignore*. - - By default this skips private attributes beginning with '_' but - includes all magic methods, i.e. those starting and ending in '__'. - - .. versionadded:: 3.5 - - -.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) - - Assert that the ``__all__`` variable of *module* contains all public names. - - The module's public names (its API) are detected automatically - based on whether they match the public name convention and were defined in - *module*. - - The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in in order to be detected as a public - API. One case for this is when *module* imports part of its public API from - other modules, possibly a C backend (like ``csv`` and its ``_csv``). - - The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` - attribute. If provided, it will be added to the automatically detected ones. - - The *blacklist* argument can be a set of names that must not be treated as part of - the public API even though their names indicate otherwise. - - Example use:: - - import bar - import foo - import unittest - from test import support - - class MiscTestCase(unittest.TestCase): - def test__all__(self): - support.check__all__(self, foo) - - class OtherTestCase(unittest.TestCase): - def test__all__(self): - extra = {'BAR_CONST', 'FOO_CONST'} - blacklist = {'baz'} # Undocumented name. - # bar imports part of its API from _bar. - support.check__all__(self, bar, ('bar', '_bar'), - extra=extra, blacklist=blacklist) - - .. versionadded:: 3.6 - - The :mod:`test.support` module defines the following classes: .. class:: TransientResource(exc, **kwargs) @@ -655,21 +548,6 @@ Temporarily unset the environment variable ``envvar``. -.. class:: SuppressCrashReport() - - A context manager used to try to prevent crash dialog popups on tests that - are expected to crash a subprocess. - - On Windows, it disables Windows Error Reporting dialogs using - `SetErrorMode `_. - - On UNIX, :func:`resource.setrlimit` is used to set - :attr:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file - creation. - - On both platforms, the old value is restored by :meth:`__exit__`. - - .. class:: WarningsRecorder() Class used to record warnings for unit tests. See documentation of diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/text.rst --- a/Doc/library/text.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,26 +0,0 @@ -.. _stringservices: -.. _textservices: - -************************ -Text Processing Services -************************ - -The modules described in this chapter provide a wide range of string -manipulation operations and other text processing services. - -The :mod:`codecs` module described under :ref:`binaryservices` is also -highly relevant to text processing. In addition, see the documentation for -Python's built-in string type in :ref:`textseq`. - - -.. toctree:: - - string.rst - re.rst - difflib.rst - textwrap.rst - unicodedata.rst - stringprep.rst - readline.rst - rlcompleter.rst - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/textwrap.rst --- a/Doc/library/textwrap.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/textwrap.rst Mon Jan 25 17:05:13 2016 +0100 @@ -10,11 +10,11 @@ -------------- -The :mod:`textwrap` module provides some convenience functions, -as well as :class:`TextWrapper`, the class that does all the work. -If you're just wrapping or filling one or two text strings, the convenience -functions should be good enough; otherwise, you should use an instance of -:class:`TextWrapper` for efficiency. +The :mod:`textwrap` module provides two convenience functions, :func:`wrap` and +:func:`fill`, as well as :class:`TextWrapper`, the class that does all the work, +and a utility function :func:`dedent`. If you're just wrapping or filling one +or two text strings, the convenience functions should be good enough; +otherwise, you should use an instance of :class:`TextWrapper` for efficiency. .. function:: wrap(text, width=70, **kwargs) @@ -25,9 +25,6 @@ Optional keyword arguments correspond to the instance attributes of :class:`TextWrapper`, documented below. *width* defaults to ``70``. - See the :meth:`TextWrapper.wrap` method for additional details on how - :func:`wrap` behaves. - .. function:: fill(text, width=70, **kwargs) @@ -39,30 +36,17 @@ In particular, :func:`fill` accepts exactly the same keyword arguments as :func:`wrap`. +Both :func:`wrap` and :func:`fill` work by creating a :class:`TextWrapper` +instance and calling a single method on it. That instance is not reused, so for +applications that wrap/fill many text strings, it will be more efficient for you +to create your own :class:`TextWrapper` object. -.. function:: shorten(text, width, **kwargs) +Text is preferably wrapped on whitespaces and right after the hyphens in +hyphenated words; only then will long words be broken if necessary, unless +:attr:`TextWrapper.break_long_words` is set to false. - Collapse and truncate the given *text* to fit in the given *width*. - - First the whitespace in *text* is collapsed (all whitespace is replaced by - single spaces). If the result fits in the *width*, it is returned. - Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`placeholder` fit within :attr:`width`:: - - >>> textwrap.shorten("Hello world!", width=12) - 'Hello world!' - >>> textwrap.shorten("Hello world!", width=11) - 'Hello [...]' - >>> textwrap.shorten("Hello world", width=10, placeholder="...") - 'Hello...' - - Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. Note that the whitespace is - collapsed before the text is passed to the :class:`TextWrapper` :meth:`fill` - function, so changing the value of :attr:`.tabsize`, :attr:`.expand_tabs`, - :attr:`.drop_whitespace`, and :attr:`.replace_whitespace` will have no effect. - - .. versionadded:: 3.4 +An additional utility function, :func:`dedent`, is provided to remove +indentation from strings that have unwanted whitespace to the left of the text. .. function:: dedent(text) @@ -88,44 +72,6 @@ print(repr(dedent(s))) # prints 'hello\n world\n' -.. function:: indent(text, prefix, predicate=None) - - Add *prefix* to the beginning of selected lines in *text*. - - Lines are separated by calling ``text.splitlines(True)``. - - By default, *prefix* is added to all lines that do not consist - solely of whitespace (including any line endings). - - For example:: - - >>> s = 'hello\n\n \nworld' - >>> indent(s, ' ') - ' hello\n\n \n world' - - The optional *predicate* argument can be used to control which lines - are indented. For example, it is easy to add *prefix* to even empty - and whitespace-only lines:: - - >>> print(indent(s, '+ ', lambda line: True)) - + hello - + - + - + world - - .. versionadded:: 3.3 - - -:func:`wrap`, :func:`fill` and :func:`shorten` work by creating a -:class:`TextWrapper` instance and calling a single method on it. That -instance is not reused, so for applications that process many text -strings using :func:`wrap` and/or :func:`fill`, it may be more efficient to -create your own :class:`TextWrapper` object. - -Text is preferably wrapped on whitespaces and right after the hyphens in -hyphenated words; only then will long words be broken if necessary, unless -:attr:`TextWrapper.break_long_words` is set to false. - .. class:: TextWrapper(**kwargs) The :class:`TextWrapper` constructor accepts a number of optional keyword @@ -161,22 +107,11 @@ expanded to spaces using the :meth:`expandtabs` method of *text*. - .. attribute:: tabsize - - (default: ``8``) If :attr:`expand_tabs` is true, then all tab characters - in *text* will be expanded to zero or more spaces, depending on the - current column and the given tab size. - - .. versionadded:: 3.3 - - .. attribute:: replace_whitespace - (default: ``True``) If true, after tab expansion but before wrapping, - the :meth:`wrap` method will replace each whitespace character - with a single space. The whitespace characters replaced are - as follows: tab, newline, vertical tab, formfeed, and carriage - return (``'\t\n\v\f\r'``). + (default: ``True``) If true, each whitespace character (as defined by + ``string.whitespace``) remaining after tab expansion will be replaced by a + single space. .. note:: @@ -194,18 +129,15 @@ .. attribute:: drop_whitespace - (default: ``True``) If true, whitespace at the beginning and ending of - every line (after wrapping but before indenting) is dropped. - Whitespace at the beginning of the paragraph, however, is not dropped - if non-whitespace follows it. If whitespace being dropped takes up an - entire line, the whole line is dropped. + (default: ``True``) If true, whitespace that, after wrapping, happens to + end up at the beginning or end of a line is dropped (leading whitespace in + the first line is always preserved, though). .. attribute:: initial_indent (default: ``''``) String that will be prepended to the first line of - wrapped output. Counts towards the length of the first line. The empty - string is not indented. + wrapped output. Counts towards the length of the first line. .. attribute:: subsequent_indent @@ -259,35 +191,19 @@ was to always allow breaking hyphenated words. - .. attribute:: max_lines - - (default: ``None``) If not ``None``, then the output will contain at most - *max_lines* lines, with *placeholder* appearing at the end of the output. - - .. versionadded:: 3.4 - - - .. attribute:: placeholder - - (default: ``' [...]'``) String that will appear at the end of the output - text if it has been truncated. - - .. versionadded:: 3.4 - - - :class:`TextWrapper` also provides some public methods, analogous to the + :class:`TextWrapper` also provides two public methods, analogous to the module-level convenience functions: .. method:: wrap(text) Wraps the single paragraph in *text* (a string) so every line is at most :attr:`width` characters long. All wrapping options are taken from - instance attributes of the :class:`TextWrapper` instance. Returns a list - of output lines, without final newlines. If the wrapped output has no - content, the returned list is empty. + instance attributes of the :class:`TextWrapper` instance. Returns a list + of output lines, without final newlines. .. method:: fill(text) Wraps the single paragraph in *text*, and returns a single string containing the wrapped paragraph. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/threading.rst --- a/Doc/library/threading.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/threading.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,7 +21,7 @@ supported by this module. -This module defines the following functions: +This module defines the following functions and objects: .. function:: active_count() @@ -30,6 +30,16 @@ count is equal to the length of the list returned by :func:`.enumerate`. +.. function:: Condition() + :noindex: + + A factory function that returns a new condition variable object. A condition + variable allows one or more threads to wait until they are notified by another + thread. + + See :ref:`condition-objects`. + + .. function:: current_thread() Return the current :class:`Thread` object, corresponding to the caller's thread @@ -57,13 +67,87 @@ and threads that have not yet been started. -.. function:: main_thread() +.. function:: Event() + :noindex: - Return the main :class:`Thread` object. In normal conditions, the - main thread is the thread from which the Python interpreter was - started. + A factory function that returns a new event object. An event manages a flag + that can be set to true with the :meth:`~Event.set` method and reset to false + with the :meth:`clear` method. The :meth:`wait` method blocks until the flag + is true. - .. versionadded:: 3.4 + See :ref:`event-objects`. + + +.. class:: local + + A class that represents thread-local data. Thread-local data are data whose + values are thread specific. To manage thread-local data, just create an + instance of :class:`local` (or a subclass) and store attributes on it:: + + mydata = threading.local() + mydata.x = 1 + + The instance's values will be different for separate threads. + + For more details and extensive examples, see the documentation string of the + :mod:`_threading_local` module. + + +.. function:: Lock() + + A factory function that returns a new primitive lock object. Once a thread has + acquired it, subsequent attempts to acquire it block, until it is released; any + thread may release it. + + See :ref:`lock-objects`. + + +.. function:: RLock() + + A factory function that returns a new reentrant lock object. A reentrant lock + must be released by the thread that acquired it. Once a thread has acquired a + reentrant lock, the same thread may acquire it again without blocking; the + thread must release it once for each time it has acquired it. + + See :ref:`rlock-objects`. + + +.. function:: Semaphore(value=1) + :noindex: + + A factory function that returns a new semaphore object. A semaphore manages a + counter representing the number of :meth:`release` calls minus the number of + :meth:`acquire` calls, plus an initial value. The :meth:`acquire` method blocks + if necessary until it can return without making the counter negative. If not + given, *value* defaults to 1. + + See :ref:`semaphore-objects`. + + +.. function:: BoundedSemaphore(value=1) + + A factory function that returns a new bounded semaphore object. A bounded + semaphore checks to make sure its current value doesn't exceed its initial + value. If it does, :exc:`ValueError` is raised. In most situations semaphores + are used to guard resources with limited capacity. If the semaphore is released + too many times it's a sign of a bug. If not given, *value* defaults to 1. + + +.. class:: Thread + :noindex: + + A class that represents a thread of control. This class can be safely + subclassed in a limited fashion. + + See :ref:`thread-objects`. + + +.. class:: Timer + :noindex: + + A thread that executes a function after a specified interval has passed. + + See :ref:`timer-objects`. .. function:: settrace(func) @@ -72,7 +156,7 @@ Set a trace function for all threads started from the :mod:`threading` module. The *func* will be passed to :func:`sys.settrace` for each thread, before its - :meth:`~Thread.run` method is called. + :meth:`run` method is called. .. function:: setprofile(func) @@ -81,7 +165,7 @@ Set a profile function for all threads started from the :mod:`threading` module. The *func* will be passed to :func:`sys.setprofile` for each thread, before its - :meth:`~Thread.run` method is called. + :meth:`run` method is called. .. function:: stack_size([size]) @@ -89,16 +173,15 @@ Return the thread stack size used when creating new threads. The optional *size* argument specifies the stack size to be used for subsequently created threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB + integer value of at least 32,768 (32kB). If changing the thread stack size is + unsupported, a :exc:`ThreadError` is raised. If the specified stack size is + invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32kB is currently the minimum supported stack size value to guarantee sufficient stack space for the interpreter itself. Note that some platforms may have particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system + minimum stack size > 32kB or requiring allocation in multiples of the system memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is + information (4kB pages are common; using multiples of 4096 for the stack size is the suggested approach in the absence of more specific information). Availability: Windows, systems with POSIX threads. @@ -115,8 +198,7 @@ .. versionadded:: 3.2 -This module defines a number of classes, which are detailed in the sections -below. +Detailed interfaces for the objects are documented below. The design of this module is loosely based on Java's threading model. However, where Java makes locks and condition variables basic behavior of every object, @@ -129,66 +211,37 @@ All of the methods described below are executed atomically. -Thread-Local Data ------------------ - -Thread-local data is data whose values are thread specific. To manage -thread-local data, just create an instance of :class:`local` (or a -subclass) and store attributes on it:: - - mydata = threading.local() - mydata.x = 1 - -The instance's values will be different for separate threads. - - -.. class:: local() - - A class that represents thread-local data. - - For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. - - .. _thread-objects: Thread Objects -------------- -The :class:`Thread` class represents an activity that is run in a separate -thread of control. There are two ways to specify the activity: by passing a -callable object to the constructor, or by overriding the :meth:`~Thread.run` -method in a subclass. No other methods (except for the constructor) should be -overridden in a subclass. In other words, *only* override the -:meth:`~Thread.__init__` and :meth:`~Thread.run` methods of this class. +This class represents an activity that is run in a separate thread of control. +There are two ways to specify the activity: by passing a callable object to the +constructor, or by overriding the :meth:`run` method in a subclass. No other +methods (except for the constructor) should be overridden in a subclass. In +other words, *only* override the :meth:`__init__` and :meth:`run` methods of +this class. Once a thread object is created, its activity must be started by calling the -thread's :meth:`~Thread.start` method. This invokes the :meth:`~Thread.run` -method in a separate thread of control. +thread's :meth:`start` method. This invokes the :meth:`run` method in a +separate thread of control. Once the thread's activity is started, the thread is considered 'alive'. It -stops being alive when its :meth:`~Thread.run` method terminates -- either -normally, or by raising an unhandled exception. The :meth:`~Thread.is_alive` -method tests whether the thread is alive. +stops being alive when its :meth:`run` method terminates -- either normally, or +by raising an unhandled exception. The :meth:`is_alive` method tests whether the +thread is alive. -Other threads can call a thread's :meth:`~Thread.join` method. This blocks -the calling thread until the thread whose :meth:`~Thread.join` method is -called is terminated. +Other threads can call a thread's :meth:`join` method. This blocks the calling +thread until the thread whose :meth:`join` method is called is terminated. A thread has a name. The name can be passed to the constructor, and read or -changed through the :attr:`~Thread.name` attribute. +changed through the :attr:`name` attribute. A thread can be flagged as a "daemon thread". The significance of this flag is that the entire Python program exits when only daemon threads are left. The initial value is inherited from the creating thread. The flag can be set -through the :attr:`~Thread.daemon` property or the *daemon* constructor -argument. - -.. note:: - Daemon threads are abruptly stopped at shutdown. Their resources (such - as open files, database transactions, etc.) may not be released properly. - If you want your threads to stop gracefully, make them non-daemonic and - use a suitable signalling mechanism such as an :class:`Event`. +through the :attr:`daemon` property or the *daemon* constructor argument. There is a "main thread" object; this corresponds to the initial thread of control in the Python program. It is not a daemon thread. @@ -197,12 +250,12 @@ thread objects corresponding to "alien threads", which are threads of control started outside the threading module, such as directly from C code. Dummy thread objects have limited functionality; they are always considered alive and -daemonic, and cannot be :meth:`~Thread.join`\ ed. They are never deleted, -since it is impossible to detect the termination of alien threads. +daemonic, and cannot be :meth:`join`\ ed. They are never deleted, since it is +impossible to detect the termination of alien threads. -.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={}, *, \ - daemon=None) +.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={}, + verbose=None, *, daemon=None) This constructor should always be called with keyword arguments. Arguments are: @@ -221,6 +274,8 @@ *kwargs* is a dictionary of keyword arguments for the target invocation. Defaults to ``{}``. + *verbose* is a flag used for debugging messages. + If not ``None``, *daemon* explicitly sets whether the thread is daemonic. If ``None`` (the default), the daemonic property is inherited from the current thread. @@ -237,8 +292,7 @@ Start the thread's activity. It must be called at most once per thread object. It arranges for the - object's :meth:`~Thread.run` method to be invoked in a separate thread - of control. + object's :meth:`run` method to be invoked in a separate thread of control. This method will raise a :exc:`RuntimeError` if called more than once on the same thread object. @@ -254,27 +308,25 @@ .. method:: join(timeout=None) - Wait until the thread terminates. This blocks the calling thread until - the thread whose :meth:`~Thread.join` method is called terminates -- either - normally or through an unhandled exception --, or until the optional - timeout occurs. + Wait until the thread terminates. This blocks the calling thread until the + thread whose :meth:`join` method is called terminates -- either normally + or through an unhandled exception -- or until the optional timeout occurs. When the *timeout* argument is present and not ``None``, it should be a floating point number specifying a timeout for the operation in seconds - (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, - you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to - decide whether a timeout happened -- if the thread is still alive, the - :meth:`~Thread.join` call timed out. + (or fractions thereof). As :meth:`join` always returns ``None``, you must + call :meth:`is_alive` after :meth:`join` to decide whether a timeout + happened -- if the thread is still alive, the :meth:`join` call timed out. When the *timeout* argument is not present or ``None``, the operation will block until the thread terminates. - A thread can be :meth:`~Thread.join`\ ed many times. + A thread can be :meth:`join`\ ed many times. - :meth:`~Thread.join` raises a :exc:`RuntimeError` if an attempt is made - to join the current thread as that would cause a deadlock. It is also - an error to :meth:`~Thread.join` a thread before it has been started - and attempts to do so raise the same exception. + :meth:`join` raises a :exc:`RuntimeError` if an attempt is made to join + the current thread as that would cause a deadlock. It is also an error to + :meth:`join` a thread before it has been started and attempts to do so + raises the same exception. .. attribute:: name @@ -291,27 +343,27 @@ .. attribute:: ident The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the - :func:`_thread.get_ident()` function. Thread identifiers may be recycled - when a thread exits and another thread is created. The identifier is - available even after the thread has exited. + been started. This is a nonzero integer. See the :func:`get_ident()` + function. Thread identifiers may be recycled when a thread exits and + another thread is created. The identifier is available even after the + thread has exited. .. method:: is_alive() Return whether the thread is alive. - This method returns ``True`` just before the :meth:`~Thread.run` method - starts until just after the :meth:`~Thread.run` method terminates. The - module function :func:`.enumerate` returns a list of all alive threads. + This method returns ``True`` just before the :meth:`run` method starts + until just after the :meth:`run` method terminates. The module function + :func:`.enumerate` returns a list of all alive threads. .. attribute:: daemon A boolean value indicating whether this thread is a daemon thread (True) - or not (False). This must be set before :meth:`~Thread.start` is called, + or not (False). This must be set before :meth:`start` is called, otherwise :exc:`RuntimeError` is raised. Its initial value is inherited from the creating thread; the main thread is not a daemon thread and - therefore all threads created in the main thread default to - :attr:`~Thread.daemon` = ``False``. + therefore all threads created in the main thread default to :attr:`daemon` + = ``False``. The entire Python program exits when no alive non-daemon threads are left. @@ -324,10 +376,10 @@ .. impl-detail:: - In CPython, due to the :term:`Global Interpreter Lock`, only one thread + Due to the :term:`Global Interpreter Lock`, in CPython only one thread can execute Python code at once (even though certain performance-oriented libraries might overcome this limitation). - If you want your application to make better use of the computational + If you want your application to make better of use of the computational resources of multi-core machines, you are advised to use :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. However, threading is still an appropriate model if you want to run @@ -345,75 +397,64 @@ extension module. A primitive lock is in one of two states, "locked" or "unlocked". It is created -in the unlocked state. It has two basic methods, :meth:`~Lock.acquire` and -:meth:`~Lock.release`. When the state is unlocked, :meth:`~Lock.acquire` -changes the state to locked and returns immediately. When the state is locked, -:meth:`~Lock.acquire` blocks until a call to :meth:`~Lock.release` in another -thread changes it to unlocked, then the :meth:`~Lock.acquire` call resets it -to locked and returns. The :meth:`~Lock.release` method should only be -called in the locked state; it changes the state to unlocked and returns -immediately. If an attempt is made to release an unlocked lock, a -:exc:`RuntimeError` will be raised. +in the unlocked state. It has two basic methods, :meth:`acquire` and +:meth:`release`. When the state is unlocked, :meth:`acquire` changes the state +to locked and returns immediately. When the state is locked, :meth:`acquire` +blocks until a call to :meth:`release` in another thread changes it to unlocked, +then the :meth:`acquire` call resets it to locked and returns. The +:meth:`release` method should only be called in the locked state; it changes the +state to unlocked and returns immediately. If an attempt is made to release an +unlocked lock, a :exc:`RuntimeError` will be raised. -Locks also support the :ref:`context management protocol `. - -When more than one thread is blocked in :meth:`~Lock.acquire` waiting for the -state to turn to unlocked, only one thread proceeds when a :meth:`~Lock.release` -call resets the state to unlocked; which one of the waiting threads proceeds -is not defined, and may vary across implementations. +When more than one thread is blocked in :meth:`acquire` waiting for the state to +turn to unlocked, only one thread proceeds when a :meth:`release` call resets +the state to unlocked; which one of the waiting threads proceeds is not defined, +and may vary across implementations. All methods are executed atomically. -.. class:: Lock() +.. method:: Lock.acquire(blocking=True, timeout=-1) - The class implementing primitive lock objects. Once a thread has acquired a - lock, subsequent attempts to acquire it block, until it is released; any - thread may release it. + Acquire a lock, blocking or non-blocking. - .. versionchanged:: 3.3 - Changed from a factory function to a class. + When invoked without arguments, block until the lock is unlocked, then set it to + locked, and return true. + When invoked with the *blocking* argument set to true, do the same thing as when + called without arguments, and return true. - .. method:: acquire(blocking=True, timeout=-1) + When invoked with the *blocking* argument set to false, do not block. If a call + without an argument would block, return false immediately; otherwise, do the + same thing as when called without arguments, and return true. - Acquire a lock, blocking or non-blocking. + When invoked with the floating-point *timeout* argument set to a positive + value, block for at most the number of seconds specified by *timeout* + and as long as the lock cannot be acquired. A negative *timeout* argument + specifies an unbounded wait. It is forbidden to specify a *timeout* + when *blocking* is false. - When invoked with the *blocking* argument set to ``True`` (the default), - block until the lock is unlocked, then set it to locked and return ``True``. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not (for example if the *timeout* expired). - When invoked with the *blocking* argument set to ``False``, do not block. - If a call with *blocking* set to ``True`` would block, return ``False`` - immediately; otherwise, set the lock to locked and return ``True``. + .. versionchanged:: 3.2 + The *timeout* parameter is new. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` - specifies an unbounded wait. It is forbidden to specify a *timeout* - when *blocking* is false. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not (for example if the *timeout* expired). - .. versionchanged:: 3.2 - The *timeout* parameter is new. +.. method:: Lock.release() - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + Release a lock. + When the lock is locked, reset it to unlocked, and return. If any other threads + are blocked waiting for the lock to become unlocked, allow exactly one of them + to proceed. - .. method:: release() + Do not call this method when the lock is unlocked. - Release a lock. This can be called from any thread, not only the thread - which has acquired the lock. - - When the lock is locked, reset it to unlocked, and return. If any other threads - are blocked waiting for the lock to become unlocked, allow exactly one of them - to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. + There is no return value. .. _rlock-objects: @@ -427,69 +468,55 @@ locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. +To lock the lock, a thread calls its :meth:`acquire` method; this returns once +the thread owns the lock. To unlock the lock, a thread calls its +:meth:`release` method. :meth:`acquire`/:meth:`release` call pairs may be +nested; only the final :meth:`release` (the :meth:`release` of the outermost +pair) resets the lock to unlocked and allows another thread blocked in +:meth:`acquire` to proceed. -Reentrant locks also support the :ref:`context management protocol `. +.. method:: RLock.acquire(blocking=True, timeout=-1) -.. class:: RLock() + Acquire a lock, blocking or non-blocking. - This class implements reentrant lock objects. A reentrant lock must be - released by the thread that acquired it. Once a thread has acquired a - reentrant lock, the same thread may acquire it again without blocking; the - thread must release it once for each time it has acquired it. + When invoked without arguments: if this thread already owns the lock, increment + the recursion level by one, and return immediately. Otherwise, if another + thread owns the lock, block until the lock is unlocked. Once the lock is + unlocked (not owned by any thread), then grab ownership, set the recursion level + to one, and return. If more than one thread is blocked waiting until the lock + is unlocked, only one at a time will be able to grab ownership of the lock. + There is no return value in this case. - Note that ``RLock`` is actually a factory function which returns an instance - of the most efficient version of the concrete RLock class that is supported - by the platform. + When invoked with the *blocking* argument set to true, do the same thing as when + called without arguments, and return true. + When invoked with the *blocking* argument set to false, do not block. If a call + without an argument would block, return false immediately; otherwise, do the + same thing as when called without arguments, and return true. - .. method:: acquire(blocking=True, timeout=-1) + When invoked with the floating-point *timeout* argument set to a positive + value, block for at most the number of seconds specified by *timeout* + and as long as the lock cannot be acquired. Return true if the lock has + been acquired, false if the timeout has elapsed. - Acquire a lock, blocking or non-blocking. + .. versionchanged:: 3.2 + The *timeout* parameter is new. - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. - When invoked with the *blocking* argument set to true, do the same thing as when - called without arguments, and return true. +.. method:: RLock.release() - When invoked with the *blocking* argument set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, do the - same thing as when called without arguments, and return true. + Release a lock, decrementing the recursion level. If after the decrement it is + zero, reset the lock to unlocked (not owned by any thread), and if any other + threads are blocked waiting for the lock to become unlocked, allow exactly one + of them to proceed. If after the decrement the recursion level is still + nonzero, the lock remains locked and owned by the calling thread. - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return true if the lock has - been acquired, false if the timeout has elapsed. + Only call this method when the calling thread owns the lock. A + :exc:`RuntimeError` is raised if this method is called when the lock is + unlocked. - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the decrement it is - zero, reset the lock to unlocked (not owned by any thread), and if any other - threads are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling thread. - - Only call this method when the calling thread owns the lock. A - :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. - - There is no return value. + There is no return value. .. _condition-objects: @@ -498,82 +525,69 @@ ----------------- A condition variable is always associated with some kind of lock; this can be -passed in or one will be created by default. Passing one in is useful when -several condition variables must share the same lock. The lock is part of -the condition object: you don't have to track it separately. +passed in or one will be created by default. (Passing one in is useful when +several condition variables must share the same lock.) -A condition variable obeys the :ref:`context management protocol `: -using the ``with`` statement acquires the associated lock for the duration of -the enclosed block. The :meth:`~Condition.acquire` and -:meth:`~Condition.release` methods also call the corresponding methods of -the associated lock. +A condition variable has :meth:`acquire` and :meth:`release` methods that call +the corresponding methods of the associated lock. It also has a :meth:`wait` +method, and :meth:`notify` and :meth:`notify_all` methods. These three must only +be called when the calling thread has acquired the lock, otherwise a +:exc:`RuntimeError` is raised. -Other methods must be called with the associated lock held. The -:meth:`~Condition.wait` method releases the lock, and then blocks until -another thread awakens it by calling :meth:`~Condition.notify` or -:meth:`~Condition.notify_all`. Once awakened, :meth:`~Condition.wait` -re-acquires the lock and returns. It is also possible to specify a timeout. +The :meth:`wait` method releases the lock, and then blocks until it is awakened +by a :meth:`notify` or :meth:`notify_all` call for the same condition variable in +another thread. Once awakened, it re-acquires the lock and returns. It is also +possible to specify a timeout. -The :meth:`~Condition.notify` method wakes up one of the threads waiting for -the condition variable, if any are waiting. The :meth:`~Condition.notify_all` -method wakes up all threads waiting for the condition variable. +The :meth:`notify` method wakes up one of the threads waiting for the condition +variable, if any are waiting. The :meth:`notify_all` method wakes up all threads +waiting for the condition variable. -Note: the :meth:`~Condition.notify` and :meth:`~Condition.notify_all` methods -don't release the lock; this means that the thread or threads awakened will -not return from their :meth:`~Condition.wait` call immediately, but only when -the thread that called :meth:`~Condition.notify` or :meth:`~Condition.notify_all` -finally relinquishes ownership of the lock. +Note: the :meth:`notify` and :meth:`notify_all` methods don't release the lock; +this means that the thread or threads awakened will not return from their +:meth:`wait` call immediately, but only when the thread that called +:meth:`notify` or :meth:`notify_all` finally relinquishes ownership of the lock. -The typical programming style using condition variables uses the lock to +Tip: the typical programming style using condition variables uses the lock to synchronize access to some shared state; threads that are interested in a -particular change of state call :meth:`~Condition.wait` repeatedly until they -see the desired state, while threads that modify the state call -:meth:`~Condition.notify` or :meth:`~Condition.notify_all` when they change -the state in such a way that it could possibly be a desired state for one -of the waiters. For example, the following code is a generic -producer-consumer situation with unlimited buffer capacity:: +particular change of state call :meth:`wait` repeatedly until they see the +desired state, while threads that modify the state call :meth:`notify` or +:meth:`notify_all` when they change the state in such a way that it could +possibly be a desired state for one of the waiters. For example, the following +code is a generic producer-consumer situation with unlimited buffer capacity:: # Consume one item - with cv: - while not an_item_is_available(): - cv.wait() - get_an_available_item() + cv.acquire() + while not an_item_is_available(): + cv.wait() + get_an_available_item() + cv.release() # Produce one item - with cv: - make_an_item_available() - cv.notify() + cv.acquire() + make_an_item_available() + cv.notify() + cv.release() -The ``while`` loop checking for the application's condition is necessary -because :meth:`~Condition.wait` can return after an arbitrary long time, -and the condition which prompted the :meth:`~Condition.notify` call may -no longer hold true. This is inherent to multi-threaded programming. The -:meth:`~Condition.wait_for` method can be used to automate the condition -checking, and eases the computation of timeouts:: +To choose between :meth:`notify` and :meth:`notify_all`, consider whether one +state change can be interesting for only one or several waiting threads. E.g. +in a typical producer-consumer situation, adding one item to the buffer only +needs to wake up one consumer thread. - # Consume an item - with cv: - cv.wait_for(an_item_is_available) - get_an_available_item() - -To choose between :meth:`~Condition.notify` and :meth:`~Condition.notify_all`, -consider whether one state change can be interesting for only one or several -waiting threads. E.g. in a typical producer-consumer situation, adding one -item to the buffer only needs to wake up one consumer thread. +Note: Condition variables can be, depending on the implementation, subject +to both spurious wakeups (when :meth:`wait` returns without a :meth:`notify` +call) and stolen wakeups (when another thread acquires the lock before the +awoken thread.) For this reason, it is always necessary to verify the state +the thread is waiting for when :meth:`wait` returns and optionally repeat +the call as often as necessary. .. class:: Condition(lock=None) - This class implements condition variable objects. A condition variable - allows one or more threads to wait until they are notified by another thread. - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` or :class:`RLock` object, and it is used as the underlying lock. Otherwise, a new :class:`RLock` object is created and used as the underlying lock. - .. versionchanged:: 3.3 - changed from a factory function to a class. - .. method:: acquire(*args) Acquire the underlying lock. This method calls the corresponding method on @@ -631,9 +645,15 @@ cv.wait() Therefore, the same rules apply as with :meth:`wait`: The lock must be - held when called and is re-acquired on return. The predicate is evaluated + held when called and is re-aquired on return. The predicate is evaluated with the lock held. + Using this method, the consumer example above can be written thus:: + + with cv: + cv.wait_for(an_item_is_available) + get_an_available_item() + .. versionadded:: 3.2 .. method:: notify(n=1) @@ -669,33 +689,20 @@ This is one of the oldest synchronization primitives in the history of computer science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he -used the names ``P()`` and ``V()`` instead of :meth:`~Semaphore.acquire` and -:meth:`~Semaphore.release`). +used :meth:`P` and :meth:`V` instead of :meth:`acquire` and :meth:`release`). A semaphore manages an internal counter which is decremented by each -:meth:`~Semaphore.acquire` call and incremented by each :meth:`~Semaphore.release` -call. The counter can never go below zero; when :meth:`~Semaphore.acquire` -finds that it is zero, it blocks, waiting until some other thread calls -:meth:`~Semaphore.release`. - -Semaphores also support the :ref:`context management protocol `. +:meth:`acquire` call and incremented by each :meth:`release` call. The counter +can never go below zero; when :meth:`acquire` finds that it is zero, it blocks, +waiting until some other thread calls :meth:`release`. .. class:: Semaphore(value=1) - This class implements semaphore objects. A semaphore manages a counter - representing the number of :meth:`release` calls minus the number of - :meth:`acquire` calls, plus an initial value. The :meth:`acquire` method - blocks if necessary until it can return without making the counter negative. - If not given, *value* defaults to 1. - The optional argument gives the initial *value* for the internal counter; it defaults to ``1``. If the *value* given is less than 0, :exc:`ValueError` is raised. - .. versionchanged:: 3.3 - changed from a factory function to a class. - .. method:: acquire(blocking=True, timeout=None) Acquire a semaphore. @@ -703,12 +710,11 @@ When invoked without arguments: if the internal counter is larger than zero on entry, decrement it by one and return immediately. If it is zero on entry, block, waiting until some other thread has called - :meth:`~Semaphore.release` to make it larger than zero. This is done - with proper interlocking so that if multiple :meth:`acquire` calls are - blocked, :meth:`~Semaphore.release` will wake exactly one of them up. - The implementation may pick one at random, so the order in which - blocked threads are awakened should not be relied on. Returns - true (or blocks indefinitely). + :meth:`release` to make it larger than zero. This is done with proper + interlocking so that if multiple :meth:`acquire` calls are blocked, + :meth:`release` will wake exactly one of them up. The implementation may + pick one at random, so the order in which blocked threads are awakened + should not be relied on. Returns true (or blocks indefinitely). When invoked with *blocking* set to false, do not block. If a call without an argument would block, return false immediately; otherwise, @@ -728,18 +734,6 @@ than zero again, wake up that thread. -.. class:: BoundedSemaphore(value=1) - - Class implementing bounded semaphore objects. A bounded semaphore checks to - make sure its current value doesn't exceed its initial value. If it does, - :exc:`ValueError` is raised. In most situations semaphores are used to guard - resources with limited capacity. If the semaphore is released too many times - it's a sign of a bug. If not given, *value* defaults to 1. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. _semaphore-examples: :class:`Semaphore` Example @@ -751,18 +745,17 @@ main thread would initialize the semaphore:: maxconnections = 5 - # ... + ... pool_sema = BoundedSemaphore(value=maxconnections) Once spawned, worker threads call the semaphore's acquire and release methods when they need to connect to the server:: - with pool_sema: - conn = connectdb() - try: - # ... use connection ... - finally: - conn.close() + pool_sema.acquire() + conn = connectdb() + ... use connection ... + conn.close() + pool_sema.release() The use of a bounded semaphore reduces the chance that a programming error which causes the semaphore to be released more than it's acquired will go undetected. @@ -777,19 +770,13 @@ thread signals an event and other threads wait for it. An event object manages an internal flag that can be set to true with the -:meth:`~Event.set` method and reset to false with the :meth:`~Event.clear` -method. The :meth:`~Event.wait` method blocks until the flag is true. +:meth:`~Event.set` method and reset to false with the :meth:`clear` method. The +:meth:`wait` method blocks until the flag is true. .. class:: Event() - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`~Event.set` method and reset to false with the - :meth:`clear` method. The :meth:`wait` method blocks until the flag is true. - The flag is initially false. - - .. versionchanged:: 3.3 - changed from a factory function to a class. + The internal flag is initially false. .. method:: is_set() @@ -811,7 +798,7 @@ Block until the internal flag is true. If the internal flag is true on entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. + :meth:`set` to set the flag to true, or until the optional timeout occurs. When the timeout argument is present and not ``None``, it should be a floating point number specifying a timeout for the operation in seconds @@ -835,11 +822,10 @@ of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` and as such also functions as an example of creating custom threads. -Timers are started, as with threads, by calling their :meth:`~Timer.start` -method. The timer can be stopped (before its action has begun) by calling the -:meth:`~Timer.cancel` method. The interval the timer will wait before -executing its action may not be exactly the same as the interval specified by -the user. +Timers are started, as with threads, by calling their :meth:`start` method. The +timer can be stopped (before its action has begun) by calling the :meth:`cancel` +method. The interval the timer will wait before executing its action may not be +exactly the same as the interval specified by the user. For example:: @@ -850,15 +836,10 @@ t.start() # after 30 seconds, "hello, world" will be printed -.. class:: Timer(interval, function, args=None, kwargs=None) +.. class:: Timer(interval, function, args=[], kwargs={}) Create a timer that will run *function* with arguments *args* and keyword arguments *kwargs*, after *interval* seconds have passed. - If *args* is None (the default) then an empty list will be used. - If *kwargs* is None (the default) then an empty dict will be used. - - .. versionchanged:: 3.3 - changed from a factory function to a class. .. method:: cancel() @@ -873,9 +854,9 @@ This class provides a simple synchronization primitive for use by a fixed number of threads that need to wait for each other. Each of the threads tries to pass -the barrier by calling the :meth:`~Barrier.wait` method and will block until -all of the threads have made the call. At this points, the threads are released -simultaneously. +the barrier by calling the :meth:`wait` method and will block until all of the +threads have made the call. At this points, the threads are released +simultanously. The barrier can be reused any number of times for the same number of threads. @@ -975,21 +956,40 @@ All of the objects provided by this module that have :meth:`acquire` and :meth:`release` methods can be used as context managers for a :keyword:`with` -statement. The :meth:`acquire` method will be called when the block is -entered, and :meth:`release` will be called when the block is exited. Hence, -the following snippet:: - - with some_lock: - # do something... - -is equivalent to:: - - some_lock.acquire() - try: - # do something... - finally: - some_lock.release() +statement. The :meth:`acquire` method will be called when the block is entered, +and :meth:`release` will be called when the block is exited. Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as -:keyword:`with` statement context managers. +:keyword:`with` statement context managers. For example:: + + import threading + + some_rlock = threading.RLock() + + with some_rlock: + print("some_rlock is locked while this executes") + + +.. _threaded-imports: + +Importing in threaded code +-------------------------- + +While the import machinery is thread-safe, there are two key restrictions on +threaded imports due to inherent limitations in the way that thread-safety is +provided: + +* Firstly, other than in the main module, an import should not have the + side effect of spawning a new thread and then waiting for that thread in + any way. Failing to abide by this restriction can lead to a deadlock if + the spawned thread directly or indirectly attempts to import a module. +* Secondly, all import attempts must be completed before the interpreter + starts shutting itself down. This can be most easily achieved by only + performing imports from non-daemon threads created through the threading + module. Daemon threads and threads created directly with the thread + module will require some other form of synchronization to ensure they do + not attempt imports after system shutdown has commenced. Failure to + abide by this restriction will lead to intermittent exceptions and + crashes during interpreter shutdown (as the late imports attempt to + access machinery which is no longer in a valid state). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/time.rst --- a/Doc/library/time.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/time.rst Mon Jan 25 17:05:13 2016 +0100 @@ -62,9 +62,9 @@ the units in which their value or argument is expressed. E.g. on most Unix systems, the clock "ticks" only 50 or 100 times a second. -* On the other hand, the precision of :func:`.time` and :func:`sleep` is better +* On the other hand, the precision of :func:`time` and :func:`sleep` is better than their Unix equivalents: times are expressed as floating point numbers, - :func:`.time` returns the most accurate time available (using Unix + :func:`time` returns the most accurate time available (using Unix :c:func:`gettimeofday` where available), and :func:`sleep` will accept a time with a nonzero fraction (Unix :c:func:`select` is used to implement this, where available). @@ -77,11 +77,6 @@ See :class:`struct_time` for a description of these objects. - .. versionchanged:: 3.3 - The :class:`struct_time` type was extended to provide the :attr:`tm_gmtoff` - and :attr:`tm_zone` attributes when platform supports corresponding - ``struct tm`` members. - * Use the following functions to convert between time representations: +-------------------------+-------------------------+-------------------------+ @@ -133,67 +128,41 @@ On Unix, return the current processor time as a floating point number expressed in seconds. The precision, and in fact the very definition of the meaning of - "processor time", depends on that of the C function of the same name. + "processor time", depends on that of the C function of the same name, but in any + case, this is the function to use for benchmarking Python or timing algorithms. On Windows, this function returns wall-clock seconds elapsed since the first call to this function, as a floating point number, based on the Win32 function :c:func:`QueryPerformanceCounter`. The resolution is typically better than one microsecond. - .. deprecated:: 3.3 - The behaviour of this function depends on the platform: use - :func:`perf_counter` or :func:`process_time` instead, depending on your - requirements, to have a well defined behaviour. - .. function:: clock_getres(clk_id) Return the resolution (precision) of the specified clock *clk_id*. - Availability: Unix. - .. versionadded:: 3.3 - .. function:: clock_gettime(clk_id) Return the time of the specified clock *clk_id*. - Availability: Unix. + .. versionadded:: 3.3 + +.. data:: CLOCK_REALTIME + + System-wide real-time clock. Setting this clock requires appropriate + privileges. .. versionadded:: 3.3 +.. data:: CLOCK_MONOTONIC -.. function:: clock_settime(clk_id, time) - - Set the time of the specified clock *clk_id*. - - Availability: Unix. + Clock that cannot be set and represents monotonic time since some + unspecified starting point. .. versionadded:: 3.3 - -.. data:: CLOCK_HIGHRES - - The Solaris OS has a CLOCK_HIGHRES timer that attempts to use an optimal - hardware source, and may give close to nanosecond resolution. CLOCK_HIGHRES - is the nonadjustable, high-resolution clock. - - Availability: Solaris. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC - - Clock that cannot be set and represents monotonic time since some unspecified - starting point. - - Availability: Unix. - - .. versionadded:: 3.3 - - .. data:: CLOCK_MONOTONIC_RAW Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw @@ -203,40 +172,23 @@ .. versionadded:: 3.3 - .. data:: CLOCK_PROCESS_CPUTIME_ID High-resolution per-process timer from the CPU. - Availability: Unix. - .. versionadded:: 3.3 - -.. data:: CLOCK_REALTIME - - System-wide real-time clock. Setting this clock requires appropriate - privileges. - - Availability: Unix. - - .. versionadded:: 3.3 - - .. data:: CLOCK_THREAD_CPUTIME_ID Thread-specific CPU-time clock. - Availability: Unix. - .. versionadded:: 3.3 - .. function:: ctime([secs]) Convert a time expressed in seconds since the epoch to a string representing local time. If *secs* is not provided or :const:`None`, the current time as - returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to + returned by :func:`time` is used. ``ctime(secs)`` is equivalent to ``asctime(localtime(secs))``. Locale information is not used by :func:`ctime`. @@ -245,36 +197,11 @@ Nonzero if a DST timezone is defined. -.. function:: get_clock_info(name) - - Get information on the specified clock as a namespace object. - Supported clock names and the corresponding functions to read their value - are: - - * ``'clock'``: :func:`time.clock` - * ``'monotonic'``: :func:`time.monotonic` - * ``'perf_counter'``: :func:`time.perf_counter` - * ``'process_time'``: :func:`time.process_time` - * ``'time'``: :func:`time.time` - - The result has the following attributes: - - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise - - *implementation*: The name of the underlying C function used to get - the clock value - - *monotonic*: ``True`` if the clock cannot go backward, - ``False`` otherwise - - *resolution*: The resolution of the clock in seconds (:class:`float`) - - .. versionadded:: 3.3 - - .. function:: gmtime([secs]) Convert a time expressed in seconds since the epoch to a :class:`struct_time` in UTC in which the dst flag is always zero. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. Fractions + :const:`None`, the current time as returned by :func:`time` is used. Fractions of a second are ignored. See above for a description of the :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this function. @@ -283,7 +210,7 @@ .. function:: localtime([secs]) Like :func:`gmtime` but converts to local time. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. The dst + :const:`None`, the current time as returned by :func:`time` is used. The dst flag is set to ``1`` when DST applies to the given time. @@ -292,68 +219,39 @@ This is the inverse function of :func:`localtime`. Its argument is the :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. + UTC. It returns a floating point number, for compatibility with :func:`time`. If the input value cannot be represented as a valid time, either :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on whether the invalid value is caught by Python or the underlying C libraries). The earliest date for which it can generate a time is platform-dependent. -.. function:: monotonic() +.. function:: steady(strict=False) - Return the value (in fractional seconds) of a monotonic clock, i.e. a clock - that cannot go backwards. The clock is not affected by system clock updates. - The reference point of the returned value is undefined, so that only the - difference between the results of consecutive calls is valid. + .. index:: + single: benchmarking - On Windows versions older than Vista, :func:`monotonic` detects - :c:func:`GetTickCount` integer overflow (32 bits, roll-over after 49.7 days). - It increases an internal epoch (reference time) by 2\ :sup:`32` each time - that an overflow is detected. The epoch is stored in the process-local state - and so the value of :func:`monotonic` may be different in two Python - processes running for more than 49 days. On more recent versions of Windows - and on other operating systems, :func:`monotonic` is system-wide. + Return the current time as a floating point number expressed in seconds. + This clock advances at a steady rate relative to real time and it may not be + adjusted. The reference point of the returned value is undefined so only the + difference of consecutive calls is valid. - .. versionadded:: 3.3 - .. versionchanged:: 3.5 - The function is now always available. - - -.. function:: perf_counter() - - Return the value (in fractional seconds) of a performance counter, i.e. a - clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference - point of the returned value is undefined, so that only the difference between - the results of consecutive calls is valid. + If available, a monotonic clock is used. By default, + the function falls back to another clock if the monotonic clock failed or is + not available. If *strict* is True, raise an :exc:`OSError` on error or + :exc:`NotImplementedError` if no monotonic clock is available. .. versionadded:: 3.3 -.. function:: process_time() - - Return the value (in fractional seconds) of the sum of the system and user - CPU time of the current process. It does not include time elapsed during - sleep. It is process-wide by definition. The reference point of the - returned value is undefined, so that only the difference between the results - of consecutive calls is valid. - - .. versionadded:: 3.3 - .. function:: sleep(secs) - Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep - time. The actual suspension time may be less than that requested because any - caught signal will terminate the :func:`sleep` following execution of that - signal's catching routine. Also, the suspension time may be longer than - requested by an arbitrary amount because of the scheduling of other activity - in the system. - - .. versionchanged:: 3.5 - The function now sleeps at least *secs* even if the sleep is interrupted - by a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale). + Suspend execution for the given number of seconds. The argument may be a + floating point number to indicate a more precise sleep time. The actual + suspension time may be less than that requested because any caught signal will + terminate the :func:`sleep` following execution of that signal's catching + routine. Also, the suspension time may be longer than requested by an arbitrary + amount because of the scheduling of other activity in the system. .. function:: strftime(format[, t]) @@ -442,12 +340,6 @@ | ``%Y`` | Year with century as a decimal number. | | | | | | +-----------+------------------------------------------------+-------+ - | ``%z`` | Time zone offset indicating a positive or | | - | | negative time difference from UTC/GMT of the | | - | | form +HHMM or -HHMM, where H represents decimal| | - | | hour digits and M represents decimal minute | | - | | digits [-23:59, +23:59]. | | - +-----------+------------------------------------------------+-------+ | ``%Z`` | Time zone name (no characters if no time zone | | | | exists). | | +-----------+------------------------------------------------+-------+ @@ -476,10 +368,8 @@ >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) 'Thu, 28 Jun 2001 14:17:15 +0000' - Additional directives may be supported on certain platforms, but only the - ones listed here have a meaning standardized by ANSI C. To see the full set - of format codes supported on your platform, consult the :manpage:`strftime(3)` - documentation. + Additional directives may be supported on certain platforms, but only the ones + listed here have a meaning standardized by ANSI C. On some platforms, an optional field width and precision specification can immediately follow the initial ``'%'`` of a directive in the following order; @@ -549,25 +439,16 @@ +-------+-------------------+---------------------------------+ | 8 | :attr:`tm_isdst` | 0, 1 or -1; see below | +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_zone` | abbreviation of timezone name | - +-------+-------------------+---------------------------------+ - | N/A | :attr:`tm_gmtoff` | offset east of UTC in seconds | - +-------+-------------------+---------------------------------+ Note that unlike the C structure, the month value is a range of [1, 12], not - [0, 11]. - - In calls to :func:`mktime`, :attr:`tm_isdst` may be set to 1 when daylight - savings time is in effect, and 0 when it is not. A value of -1 indicates that - this is not known, and will usually result in the correct state being filled in. + [0, 11]. A ``-1`` argument as the daylight + savings flag, passed to :func:`mktime` will usually result in the correct + daylight savings state to be filled in. When a tuple with an incorrect length is passed to a function expecting a :class:`struct_time`, or having elements of the wrong type, a :exc:`TypeError` is raised. - .. versionchanged:: 3.3 - :attr:`tm_gmtoff` and :attr:`tm_zone` attributes are available on platforms - with C library supporting the corresponding fields in ``struct tm``. .. function:: time() @@ -578,6 +459,7 @@ lower value than a previous call if the system clock has been set back between the two calls. + .. data:: timezone The offset of the local (non-DST) timezone, in seconds west of UTC (negative in @@ -636,11 +518,11 @@ it is possible to refer to February 29. :samp:`M{m}.{n}.{d}` - The *d*'th day (0 <= *d* <= 6) of week *n* of month *m* of the year (1 + The *d*'th day (0 <= *d* <= 6) or week *n* of month *m* of the year (1 <= *n* <= 5, 1 <= *m* <= 12, where week 5 means "the last *d* day in month *m*" which may occur in either the fourth or the fifth week). Week 1 is the first week in which the *d*'th day occurs. Day - zero is a Sunday. + zero is Sunday. ``time`` has the same format as ``offset`` except that no leading sign ('-' or '+') is allowed. The default, if time is not given, is 02:00:00. @@ -680,12 +562,12 @@ More object-oriented interface to dates and times. Module :mod:`locale` - Internationalization services. The locale setting affects the interpretation - of many format specifiers in :func:`strftime` and :func:`strptime`. + Internationalization services. The locale settings can affect the return values + for some of the functions in the :mod:`time` module. Module :mod:`calendar` - General calendar-related functions. :func:`~calendar.timegm` is the - inverse of :func:`gmtime` from this module. + General calendar-related functions. :func:`timegm` is the inverse of + :func:`gmtime` from this module. .. rubric:: Footnotes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/timeit.rst --- a/Doc/library/timeit.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/timeit.rst Mon Jan 25 17:05:13 2016 +0100 @@ -14,174 +14,112 @@ -------------- This module provides a simple way to time small bits of Python code. It has both -a :ref:`command-line-interface` as well as a :ref:`callable ` -one. It avoids a number of common traps for measuring execution times. -See also Tim Peters' introduction to the "Algorithms" chapter in the *Python -Cookbook*, published by O'Reilly. +command line as well as callable interfaces. It avoids a number of common traps +for measuring execution times. See also Tim Peters' introduction to the +"Algorithms" chapter in the Python Cookbook, published by O'Reilly. +The module defines the following public class: -Basic Examples --------------- -The following example shows how the :ref:`command-line-interface` -can be used to compare three different expressions: - -.. code-block:: sh - - $ python3 -m timeit '"-".join(str(n) for n in range(100))' - 10000 loops, best of 3: 30.2 usec per loop - $ python3 -m timeit '"-".join([str(n) for n in range(100)])' - 10000 loops, best of 3: 27.5 usec per loop - $ python3 -m timeit '"-".join(map(str, range(100)))' - 10000 loops, best of 3: 23.2 usec per loop - -This can be achieved from the :ref:`python-interface` with:: - - >>> import timeit - >>> timeit.timeit('"-".join(str(n) for n in range(100))', number=10000) - 0.3018611848820001 - >>> timeit.timeit('"-".join([str(n) for n in range(100)])', number=10000) - 0.2727368790656328 - >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) - 0.23702679807320237 - - -Note however that :mod:`timeit` will automatically determine the number of -repetitions only when the command-line interface is used. In the -:ref:`timeit-examples` section you can find more advanced examples. - - -.. _python-interface: - -Python Interface ----------------- - -The module defines three convenience functions and a public class: - - -.. function:: timeit(stmt='pass', setup='pass', timer=, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.timeit` method with *number* executions. - The optional *globals* argument specifies a namespace in which to execute the - code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - -.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=3, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.repeat` method with the given *repeat* - count and *number* executions. The optional *globals* argument specifies a - namespace in which to execute the code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - -.. function:: default_timer() - - The default timer, which is always :func:`time.perf_counter`. - - .. versionchanged:: 3.3 - :func:`time.perf_counter` is now the default timer. - - -.. class:: Timer(stmt='pass', setup='pass', timer=, globals=None) +.. class:: Timer(stmt='pass', setup='pass', timer=) Class for timing execution speed of small code snippets. - The constructor takes a statement to be timed, an additional statement used - for setup, and a timer function. Both statements default to ``'pass'``; - the timer function is platform-dependent (see the module doc string). - *stmt* and *setup* may also contain multiple statements separated by ``;`` - or newlines, as long as they don't contain multi-line string literals. The - statement will by default be executed within timeit's namespace; this behavior - can be controlled by passing a namespace to *globals*. + The constructor takes a statement to be timed, an additional statement used for + setup, and a timer function. Both statements default to ``'pass'``; the timer + function is platform-dependent (see the module doc string). *stmt* and *setup* + may also contain multiple statements separated by ``;`` or newlines, as long as + they don't contain multi-line string literals. - To measure the execution time of the first statement, use the :meth:`.timeit` - method. The :meth:`.repeat` method is a convenience to call :meth:`.timeit` + To measure the execution time of the first statement, use the :meth:`timeit` + method. The :meth:`repeat` method is a convenience to call :meth:`timeit` multiple times and return a list of results. - The execution time of *setup* is excluded from the overall timed execution run. - The *stmt* and *setup* parameters can also take objects that are callable - without arguments. This will embed calls to them in a timer function that - will then be executed by :meth:`.timeit`. Note that the timing overhead is a + without arguments. This will embed calls to them in a timer function that + will then be executed by :meth:`timeit`. Note that the timing overhead is a little larger in this case because of the extra function calls. - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - .. method:: Timer.timeit(number=1000000) +.. method:: Timer.print_exc(file=None) - Time *number* executions of the main statement. This executes the setup - statement once, and then returns the time it takes to execute the main - statement a number of times, measured in seconds as a float. - The argument is the number of times through the loop, defaulting to one - million. The main statement, the setup statement and the timer function - to be used are passed to the constructor. + Helper to print a traceback from the timed code. - .. note:: + Typical use:: - By default, :meth:`.timeit` temporarily turns off :term:`garbage - collection` during the timing. The advantage of this approach is that - it makes independent timings more comparable. This disadvantage is - that GC may be an important component of the performance of the - function being measured. If so, GC can be re-enabled as the first - statement in the *setup* string. For example:: + t = Timer(...) # outside the try/except + try: + t.timeit(...) # or t.repeat(...) + except: + t.print_exc() - timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() + The advantage over the standard traceback is that source lines in the compiled + template will be displayed. The optional *file* argument directs where the + traceback is sent; it defaults to ``sys.stderr``. - .. method:: Timer.repeat(repeat=3, number=1000000) +.. method:: Timer.repeat(repeat=3, number=1000000) - Call :meth:`.timeit` a few times. + Call :meth:`timeit` a few times. - This is a convenience function that calls the :meth:`.timeit` repeatedly, - returning a list of results. The first argument specifies how many times - to call :meth:`.timeit`. The second argument specifies the *number* - argument for :meth:`.timeit`. + This is a convenience function that calls the :meth:`timeit` repeatedly, + returning a list of results. The first argument specifies how many times to + call :meth:`timeit`. The second argument specifies the *number* argument for + :func:`timeit`. - .. note:: + .. note:: - It's tempting to calculate mean and standard deviation from the result - vector and report these. However, this is not very useful. - In a typical case, the lowest value gives a lower bound for how fast - your machine can run the given code snippet; higher values in the - result vector are typically not caused by variability in Python's - speed, but by other processes interfering with your timing accuracy. - So the :func:`min` of the result is probably the only number you - should be interested in. After that, you should look at the entire - vector and apply common sense rather than statistics. + It's tempting to calculate mean and standard deviation from the result vector + and report these. However, this is not very useful. In a typical case, the + lowest value gives a lower bound for how fast your machine can run the given + code snippet; higher values in the result vector are typically not caused by + variability in Python's speed, but by other processes interfering with your + timing accuracy. So the :func:`min` of the result is probably the only number + you should be interested in. After that, you should look at the entire vector + and apply common sense rather than statistics. - .. method:: Timer.print_exc(file=None) +.. method:: Timer.timeit(number=1000000) - Helper to print a traceback from the timed code. + Time *number* executions of the main statement. This executes the setup + statement once, and then returns the time it takes to execute the main statement + a number of times, measured in seconds as a float. The argument is the number + of times through the loop, defaulting to one million. The main statement, the + setup statement and the timer function to be used are passed to the constructor. - Typical use:: + .. note:: - t = Timer(...) # outside the try/except - try: - t.timeit(...) # or t.repeat(...) - except Exception: - t.print_exc() + By default, :meth:`timeit` temporarily turns off :term:`garbage collection` + during the timing. The advantage of this approach is that it makes + independent timings more comparable. This disadvantage is that GC may be + an important component of the performance of the function being measured. + If so, GC can be re-enabled as the first statement in the *setup* string. + For example:: - The advantage over the standard traceback is that source lines in the - compiled template will be displayed. The optional *file* argument directs - where the traceback is sent; it defaults to :data:`sys.stderr`. + timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() -.. _command-line-interface: +The module also defines two convenience functions: -Command-Line Interface +.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=3, number=1000000) + + Create a :class:`Timer` instance with the given statement, setup code and timer + function and run its :meth:`repeat` method with the given repeat count and + *number* executions. + + +.. function:: timeit(stmt='pass', setup='pass', timer=, number=1000000) + + Create a :class:`Timer` instance with the given statement, setup code and timer + function and run its :meth:`timeit` method with *number* executions. + + +Command Line Interface ---------------------- When called as a program from the command line, the following form is used:: - python -m timeit [-n N] [-r N] [-u U] [-s S] [-t] [-c] [-h] [statement ...] + python -m timeit [-n N] [-r N] [-s S] [-t] [-c] [-h] [statement ...] Where the following options are understood: @@ -199,26 +137,13 @@ statement to be executed once initially (default ``pass``) -.. cmdoption:: -p, --process - - measure process time, not wallclock time, using :func:`time.process_time` - instead of :func:`time.perf_counter`, which is the default - - .. versionadded:: 3.3 - .. cmdoption:: -t, --time - use :func:`time.time` (deprecated) - -.. cmdoption:: -u, --unit=U - - specify a time unit for timer output; can select usec, msec, or sec - - .. versionadded:: 3.5 + use :func:`time.time` (default on all platforms but Windows) .. cmdoption:: -c, --clock - use :func:`time.clock` (deprecated) + use :func:`time.clock` (default on Windows) .. cmdoption:: -v, --verbose @@ -236,63 +161,40 @@ If :option:`-n` is not given, a suitable number of loops is calculated by trying successive powers of 10 until the total time is at least 0.2 seconds. -:func:`default_timer` measurements can be affected by other programs running on -the same machine, so the best thing to do when accurate timing is necessary is -to repeat the timing a few times and use the best time. The :option:`-r` -option is good for this; the default of 3 repetitions is probably enough in -most cases. You can use :func:`time.process_time` to measure CPU time. +The default timer function is platform dependent. On Windows, +:func:`time.clock` has microsecond granularity but :func:`time.time`'s +granularity is 1/60th of a second; on Unix, :func:`time.clock` has 1/100th of a +second granularity and :func:`time.time` is much more precise. On either +platform, the default timer functions measure wall clock time, not the CPU time. +This means that other processes running on the same computer may interfere with +the timing. The best thing to do when accurate timing is necessary is to repeat +the timing a few times and use the best time. The :option:`-r` option is good +for this; the default of 3 repetitions is probably enough in most cases. On +Unix, you can use :func:`time.clock` to measure CPU time. .. note:: There is a certain baseline overhead associated with executing a pass statement. The code here doesn't try to hide it, but you should be aware of it. The - baseline overhead can be measured by invoking the program without arguments, - and it might differ between Python versions. + baseline overhead can be measured by invoking the program without arguments. +The baseline overhead differs between Python versions! Also, to fairly compare +older Python versions to Python 2.3, you may want to use Python's :option:`-O` +option for the older versions to avoid timing ``SET_LINENO`` instructions. -.. _timeit-examples: Examples -------- -It is possible to provide a setup statement that is executed only once at the beginning: - -.. code-block:: sh - - $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' - 10000000 loops, best of 3: 0.0877 usec per loop - $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' - 1000000 loops, best of 3: 0.342 usec per loop - -:: - - >>> import timeit - >>> timeit.timeit('char in text', setup='text = "sample string"; char = "g"') - 0.41440500499993504 - >>> timeit.timeit('text.find(char)', setup='text = "sample string"; char = "g"') - 1.7246671520006203 - -The same can be done using the :class:`Timer` class and its methods:: - - >>> import timeit - >>> t = timeit.Timer('char in text', setup='text = "sample string"; char = "g"') - >>> t.timeit() - 0.3955516149999312 - >>> t.repeat() - [0.40193588800002544, 0.3960157959998014, 0.39594301399984033] - - -The following examples show how to time expressions that contain multiple lines. -Here we compare the cost of using :func:`hasattr` vs. :keyword:`try`/:keyword:`except` -to test for missing and present object attributes: - -.. code-block:: sh +Here are two example sessions (one using the command line, one using the module +interface) that compare the cost of using :func:`hasattr` vs. +:keyword:`try`/:keyword:`except` to test for missing and present object +attributes. :: $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' 100000 loops, best of 3: 15.7 usec per loop $ python -m timeit 'if hasattr(str, "__bool__"): pass' 100000 loops, best of 3: 4.26 usec per loop - $ python -m timeit 'try:' ' int.__bool__' 'except AttributeError:' ' pass' 1000000 loops, best of 3: 1.43 usec per loop $ python -m timeit 'if hasattr(int, "__bool__"): pass' @@ -301,54 +203,46 @@ :: >>> import timeit - >>> # attribute is missing >>> s = """\ ... try: ... str.__bool__ ... except AttributeError: ... pass ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.9138244460009446 - >>> s = "if hasattr(str, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.5829014980008651 - >>> - >>> # attribute is present + >>> t = timeit.Timer(stmt=s) + >>> print("%.2f usec/pass" % (1000000 * t.timeit(number=100000)/100000)) + 17.09 usec/pass + >>> s = """\ + ... if hasattr(str, '__bool__'): pass + ... """ + >>> t = timeit.Timer(stmt=s) + >>> print("%.2f usec/pass" % (1000000 * t.timeit(number=100000)/100000)) + 4.85 usec/pass >>> s = """\ ... try: ... int.__bool__ ... except AttributeError: ... pass ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.04215312199994514 - >>> s = "if hasattr(int, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.08588060699912603 - + >>> t = timeit.Timer(stmt=s) + >>> print("%.2f usec/pass" % (1000000 * t.timeit(number=100000)/100000)) + 1.97 usec/pass + >>> s = """\ + ... if hasattr(int, '__bool__'): pass + ... """ + >>> t = timeit.Timer(stmt=s) + >>> print("%.2f usec/pass" % (1000000 * t.timeit(number=100000)/100000)) + 3.15 usec/pass To give the :mod:`timeit` module access to functions you define, you can pass a -*setup* parameter which contains an import statement:: +``setup`` parameter which contains an import statement:: def test(): """Stupid test function""" L = [i for i in range(100)] if __name__ == '__main__': - import timeit - print(timeit.timeit("test()", setup="from __main__ import test")) + from timeit import Timer + t = Timer("test()", "from __main__ import test") + print(t.timeit()) -Another option is to pass :func:`globals` to the *globals* parameter, which will cause the code -to be executed within your current global namespace. This can be more convenient -than individually specifying imports:: - - def f(x): - return x**2 - def g(x): - return x**4 - def h(x): - return x**8 - - import timeit - print(timeit.timeit('[func(42) for func in (f,g,h)]', globals=globals())) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tkinter.rst --- a/Doc/library/tkinter.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tkinter.rst Mon Jan 25 17:05:13 2016 +0100 @@ -15,29 +15,17 @@ .. seealso:: - `Python Tkinter Resources `_ + `Python Tkinter Resources `_ The Python Tkinter Topic Guide provides a great deal of information on using Tk from Python and links to other sources of information on Tk. - `TKDocs `_ - Extensive tutorial plus friendlier widget pages for some of the widgets. + `An Introduction to Tkinter `_ + Fredrik Lundh's on-line reference material. - `Tkinter reference: a GUI for Python `_ + `Tkinter reference: a GUI for Python `_ On-line reference material. - `Tkinter docs from effbot `_ - Online reference for tkinter supported by effbot.org. - - `Tcl/Tk manual `_ - Official manual for the latest tcl/tk version. - - `Programming Python `_ - Book by Mark Lutz, has excellent coverage of Tkinter. - - `Modern Tkinter for Busy Python Developers `_ - Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. - - `Python and Tkinter Programming `_ + `Python and Tkinter Programming `_ The book by John Grayson (ISBN 1-884777-81-3). @@ -180,9 +168,9 @@ The Tk/Tcl development is largely taking place at ActiveState. `Tcl and the Tk Toolkit `_ - The book by John Ousterhout, the inventor of Tcl. + The book by John Ousterhout, the inventor of Tcl . - `Practical Programming in Tcl and Tk `_ + `Practical Programming in Tcl and Tk `_ Brent Welch's encyclopedic book. @@ -440,7 +428,7 @@ Example:: >>> print(fred.config()) - {'relief': ('relief', 'relief', 'Relief', 'raised', 'groove')} + {'relief' : ('relief', 'relief', 'Relief', 'raised', 'groove')} Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. @@ -613,7 +601,7 @@ preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. boolean - You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. + You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"`` . callback This is any Python function that takes no arguments. For example:: @@ -747,6 +735,22 @@ displayed. You can use these :mod:`tkinter` functions to access these special points in text widgets: + AtEnd() + refers to the last position in the text + + AtInsert() + refers to the point where the text cursor is + + AtSelFirst() + indicates the beginning point of the selected text + + AtSelLast() + denotes the last point of the selected text and finally + + At(x[, y]) + refers to the character at pixel location *x*, *y* (with *y* not used in the + case of a text entry widget, which contains a single line of text). + Text widget indexes The index notation for Text widgets is very rich and is best described in the Tk man pages. @@ -795,52 +799,3 @@ deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. - -.. _tkinter-file-handlers: - -File Handlers -------------- - -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: - - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) - -This feature is not available on Windows. - -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. - - -.. method:: Widget.tk.createfilehandler(file, mask, func) - - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: - - callback(file, mask) - - -.. method:: Widget.tk.deletefilehandler(file) - - Unregisters a file handler. - - -.. data:: READABLE - WRITABLE - EXCEPTION - - Constants used in the *mask* arguments. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tkinter.tix.rst --- a/Doc/library/tkinter.tix.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tkinter.tix.rst Mon Jan 25 17:05:13 2016 +0100 @@ -504,7 +504,7 @@ print(root.tix_configure()) -.. method:: tixCommand.tix_configure(cnf=None, **kw) +.. method:: tixCommand.tix_configure([cnf,] **kw) Query or modify the configuration options of the Tix application context. If no option is specified, returns a dictionary all of the available options. If diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tkinter.ttk.rst --- a/Doc/library/tkinter.ttk.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tkinter.ttk.rst Mon Jan 25 17:05:13 2016 +0100 @@ -102,17 +102,14 @@ All the :mod:`ttk` Widgets accepts the following options: - .. tabularcolumns:: |l|L| - +-----------+--------------------------------------------------------------+ | Option | Description | +===========+==============================================================+ | class | Specifies the window class. The class is used when querying | | | the option database for the window's other options, to | | | determine the default bindtags for the window, and to select | - | | the widget's default layout and style. This option is | - | | read-only, and may only be specified when the window is | - | | created. | + | | the widget's default layout and style. This is a read-only | + | | which may only be specified when the window is created | +-----------+--------------------------------------------------------------+ | cursor | Specifies the mouse cursor to be used for the widget. If set | | | to the empty string (the default), the cursor is inherited | @@ -137,10 +134,8 @@ The following options are supported by widgets that are controlled by a scrollbar. - .. tabularcolumns:: |l|L| - +----------------+---------------------------------------------------------+ - | Option | Description | + | option | description | +================+=========================================================+ | xscrollcommand | Used to communicate with horizontal scrollbars. | | | | @@ -163,10 +158,11 @@ The following options are supported by labels, buttons and other button-like widgets. - .. tabularcolumns:: |l|p{0.7\linewidth}| +.. tabularcolumns:: |p{0.2\textwidth}|p{0.7\textwidth}| +.. +--------------+-----------------------------------------------------------+ - | Option | Description | + | option | description | +==============+===========================================================+ | text | Specifies a text string to be displayed inside the widget.| +--------------+-----------------------------------------------------------+ @@ -206,10 +202,8 @@ Compatibility Options ^^^^^^^^^^^^^^^^^^^^^ - .. tabularcolumns:: |l|L| - +--------+----------------------------------------------------------------+ - | Option | Description | + | option | description | +========+================================================================+ | state | May be set to "normal" or "disabled" to control the "disabled" | | | state bit. This is a write-only option: setting it changes the | @@ -222,10 +216,8 @@ The widget state is a bitmap of independent state flags. - .. tabularcolumns:: |l|L| - +------------+-------------------------------------------------------------+ - | Flag | Description | + | flag | description | +============+=============================================================+ | active | The mouse cursor is over the widget and pressing a mouse | | | button will cause some action to occur | @@ -273,8 +265,8 @@ .. method:: instate(statespec, callback=None, *args, **kw) - Test the widget's state. If a callback is not specified, returns ``True`` - if the widget state matches *statespec* and ``False`` otherwise. If callback + Test the widget's state. If a callback is not specified, returns True + if the widget state matches *statespec* and False otherwise. If callback is specified then it is called with args if widget state matches *statespec*. @@ -309,10 +301,8 @@ This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - +-----------------+--------------------------------------------------------+ - | Option | Description | + | option | description | +=================+========================================================+ | exportselection | Boolean value. If set, the widget selection is linked | | | to the Window Manager selection (which can be returned | @@ -390,10 +380,8 @@ This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - +---------+----------------------------------------------------------------+ - | Option | Description | + | option | description | +=========+================================================================+ | height | If present and greater than zero, specifies the desired height | | | of the pane area (not including internal padding or tabs). | @@ -416,10 +404,8 @@ There are also specific options for tabs: - .. tabularcolumns:: |l|L| - +-----------+--------------------------------------------------------------+ - | Option | Description | + | option | description | +===========+==============================================================+ | state | Either "normal", "disabled" or "hidden". If "disabled", then | | | the tab is not selectable. If "hidden", then the tab is not | @@ -555,9 +541,9 @@ This will extend the bindings for the toplevel window containing the notebook as follows: - * :kbd:`Control-Tab`: selects the tab following the currently selected one. - * :kbd:`Shift-Control-Tab`: selects the tab preceding the currently selected one. - * :kbd:`Alt-K`: where *K* is the mnemonic (underlined) character of any tab, will + * Control-Tab: selects the tab following the currently selected one. + * Shift-Control-Tab: selects the tab preceding the currently selected one. + * Alt-K: where K is the mnemonic (underlined) character of any tab, will select that tab. Multiple notebooks in a single toplevel may be enabled for traversal, @@ -580,10 +566,8 @@ This widget accepts the following specific options: - .. tabularcolumns:: |l|L| - +----------+---------------------------------------------------------------+ - | Option | Description | + | option | description | +==========+===============================================================+ | orient | One of "horizontal" or "vertical". Specifies the orientation | | | of the progress bar. | @@ -651,10 +635,8 @@ This widget accepts the following specific option: - .. tabularcolumns:: |l|L| - +--------+----------------------------------------------------------------+ - | Option | Description | + | option | description | +========+================================================================+ | orient | One of "horizontal" or "vertical". Specifies the orientation of| | | the separator. | @@ -719,10 +701,11 @@ This widget accepts the following specific options: - .. tabularcolumns:: |l|p{0.7\linewidth}| +.. tabularcolumns:: |p{0.2\textwidth}|p{0.7\textwidth}| +.. +----------------+--------------------------------------------------------+ - | Option | Description | + | option | description | +================+========================================================+ | columns | A list of column identifiers, specifying the number of | | | columns and their names. | @@ -770,10 +753,8 @@ The following item options may be specified for items in the insert and item widget commands. - .. tabularcolumns:: |l|L| - +--------+---------------------------------------------------------------+ - | Option | Description | + | option | description | +========+===============================================================+ | text | The textual label to display for the item. | +--------+---------------------------------------------------------------+ @@ -798,10 +779,8 @@ The following options may be specified on tags: - .. tabularcolumns:: |l|L| - +------------+-----------------------------------------------------------+ - | Option | Description | + | option | description | +============+===========================================================+ | foreground | Specifies the text foreground color. | +------------+-----------------------------------------------------------+ @@ -843,10 +822,8 @@ The Treeview widget generates the following virtual events. - .. tabularcolumns:: |l|L| - +--------------------+--------------------------------------------------+ - | Event | Description | + | event | description | +====================+==================================================+ | <> | Generated whenever the selection changes. | +--------------------+--------------------------------------------------+ @@ -939,7 +916,7 @@ .. method:: exists(item) - Returns ``True`` if the specified *item* is present in the tree. + Returns True if the specified *item* is present in the tree. .. method:: focus(item=None) @@ -1085,7 +1062,7 @@ Ensure that *item* is visible. - Sets all of *item*'s ancestors open option to ``True``, and scrolls the + Sets all of *item*'s ancestors open option to True, and scrolls the widget if necessary so that *item* is within the visible portion of the tree. @@ -1168,7 +1145,7 @@ Each widget in :mod:`ttk` is assigned a style, which specifies the set of elements making up the widget and how they are arranged, along with dynamic and default settings for element options. By default the style name is the -same as the widget's class name, but it may be overridden by the widget's style +same as the widget's class name, but it may be overriden by the widget's style option. If you don't know the class name of a widget, use the method :meth:`Misc.winfo_class` (somewidget.winfo_class()). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/token.rst --- a/Doc/library/token.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/token.rst Mon Jan 25 17:05:13 2016 +0100 @@ -93,20 +93,13 @@ DOUBLESLASH DOUBLESLASHEQUAL AT - ATEQUAL RARROW ELLIPSIS OP - AWAIT - ASYNC ERRORTOKEN N_TOKENS NT_OFFSET - .. versionchanged:: 3.5 - Added :data:`AWAIT` and :data:`ASYNC` tokens. Starting with - Python 3.7, "async" and "await" will be tokenized as :data:`NAME` - tokens, and :data:`AWAIT` and :data:`ASYNC` will be removed. .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tokenize.rst --- a/Doc/library/tokenize.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/tokenize.rst Mon Jan 25 17:05:13 2016 +0100 @@ -27,7 +27,7 @@ .. function:: tokenize(readline) - The :func:`.tokenize` generator requires one argument, *readline*, which + The :func:`tokenize` generator requires one argument, *readline*, which must be a callable object which provides the same interface as the :meth:`io.IOBase.readline` method of file objects. Each call to the function should return one line of input as bytes. @@ -41,7 +41,7 @@ returned as a :term:`named tuple` with the field names: ``type string start end line``. - The returned :term:`named tuple` has an additional property named + The returned :term:`named tuple` has a additional property named ``exact_type`` that contains the exact operator type for :data:`token.OP` tokens. For all other token types ``exact_type`` equals the named tuple ``type`` field. @@ -52,7 +52,7 @@ .. versionchanged:: 3.3 Added support for ``exact_type``. - :func:`.tokenize` determines the source encoding of the file by looking for a + :func:`tokenize` determines the source encoding of the file by looking for a UTF-8 BOM or encoding cookie, according to :pep:`263`. @@ -74,7 +74,7 @@ .. data:: ENCODING Token value that indicates the encoding used to decode the source bytes - into text. The first token returned by :func:`.tokenize` will always be an + into text. The first token returned by :func:`tokenize` will always be an ENCODING token. @@ -96,17 +96,17 @@ positions) may change. It returns bytes, encoded using the ENCODING token, which is the first - token sequence output by :func:`.tokenize`. + token sequence output by :func:`tokenize`. -:func:`.tokenize` needs to detect the encoding of source files it tokenizes. The +:func:`tokenize` needs to detect the encoding of source files it tokenizes. The function it uses to do this is available: .. function:: detect_encoding(readline) The :func:`detect_encoding` function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, - readline, in the same way as the :func:`.tokenize` generator. + readline, in the same way as the :func:`tokenize` generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (not decoded from bytes) it has read @@ -120,7 +120,7 @@ If no encoding is specified, then the default of ``'utf-8'`` will be returned. - Use :func:`.open` to open Python source files: it uses + Use :func:`open` to open Python source files: it uses :func:`detect_encoding` to detect the file encoding. @@ -131,24 +131,6 @@ .. versionadded:: 3.2 -.. exception:: TokenError - - Raised when either a docstring or expression that may be split over several - lines is not completed anywhere in the file, for example:: - - """Beginning of - docstring - - or:: - - [1, - 2, - 3 - -Note that unclosed single-quoted strings do not cause an error to be -raised. They are tokenized as ``ERRORTOKEN``, followed by the tokenization of -their contents. - .. _tokenize-cli: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/trace.rst --- a/Doc/library/trace.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/trace.rst Mon Jan 25 17:05:13 2016 +0100 @@ -41,8 +41,8 @@ At least one of the following options must be specified when invoking :mod:`trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with -the :option:`--trace <-t>` and :option:`--count <-c>` options. When -:option:`--listfuncs <-l>` is provided, neither :option:`--count <-c>` nor +the :option:`--trace <-t>` and :option:`--counts <-c>` options . When +:option:`--listfuncs <-l>` is provided, neither :option:`--counts <-c>` nor :option:`--trace <-t>` are accepted, and vice versa. .. program:: trace @@ -201,7 +201,7 @@ # run the new command using the given tracer tracer.run('main()') - # make a report, placing output in the current directory + # make a report, placing output in /tmp r = tracer.results() - r.write_results(show_missing=True, coverdir=".") + r.write_results(show_missing=True, coverdir="/tmp") diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/traceback.rst --- a/Doc/library/traceback.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/traceback.rst Mon Jan 25 17:05:13 2016 +0100 @@ -20,33 +20,27 @@ The module defines the following functions: -.. function:: print_tb(tb, limit=None, file=None) +.. function:: print_tb(traceback, limit=None, file=None) - Print up to *limit* stack trace entries from traceback object *tb* (starting - from the caller's frame) if *limit* is positive. Otherwise, print the last - ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are - printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to - receive the output. + Print up to *limit* stack trace entries from *traceback*. If *limit* is omitted + or ``None``, all entries are printed. If *file* is omitted or ``None``, the + output goes to ``sys.stderr``; otherwise it should be an open file or file-like + object to receive the output. - .. versionchanged:: 3.5 - Added negative *limit* support. +.. function:: print_exception(type, value, traceback, limit=None, file=None, chain=True) -.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True) - - Print exception information and stack trace entries from traceback object - *tb* to *file*. This differs from :func:`print_tb` in the following + Print exception information and up to *limit* stack trace entries from + *traceback* to *file*. This differs from :func:`print_tb` in the following ways: - * if *tb* is not ``None``, it prints a header ``Traceback (most recent + * if *traceback* is not ``None``, it prints a header ``Traceback (most recent call last):`` - * it prints the exception *etype* and *value* after the stack trace - * if *etype* is :exc:`SyntaxError` and *value* has the appropriate format, it + * it prints the exception *type* and *value* after the stack trace + * if *type* is :exc:`SyntaxError` and *value* has the appropriate format, it prints the line where the syntax error occurred with a caret indicating the approximate position of the error. - The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be printed as well, like the interpreter itself does when printing an unhandled @@ -55,41 +49,33 @@ .. function:: print_exc(limit=None, file=None, chain=True) - This is a shorthand for ``print_exception(*sys.exc_info(), limit, file, - chain)``. + This is a shorthand for ``print_exception(*sys.exc_info())``. .. function:: print_last(limit=None, file=None, chain=True) This is a shorthand for ``print_exception(sys.last_type, sys.last_value, - sys.last_traceback, limit, file, chain)``. In general it will work only - after an exception has reached an interactive prompt (see - :data:`sys.last_type`). + sys.last_traceback, limit, file)``. In general it will work only after + an exception has reached an interactive prompt (see :data:`sys.last_type`). .. function:: print_stack(f=None, limit=None, file=None) - Print up to *limit* stack trace entries (starting from the invocation - point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` - entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame - to start. The optional *file* argument has the same meaning as for - :func:`print_tb`. + This function prints a stack trace from its invocation point. The optional *f* + argument can be used to specify an alternate stack frame to start. The optional + *limit* and *file* arguments have the same meaning as for + :func:`print_exception`. - .. versionchanged:: 3.5 - Added negative *limit* support. +.. function:: extract_tb(traceback, limit=None) -.. function:: extract_tb(tb, limit=None) - - Return a list of "pre-processed" stack trace entries extracted from the - traceback object *tb*. It is useful for alternate formatting of - stack traces. The optional *limit* argument has the same meaning as for - :func:`print_tb`. A "pre-processed" stack trace entry is a 4-tuple - (*filename*, *line number*, *function name*, *text*) representing the - information that is usually printed for a stack trace. The *text* is a - string with leading and trailing whitespace stripped; if the source is - not available it is ``None``. + Return a list of up to *limit* "pre-processed" stack trace entries extracted + from the traceback object *traceback*. It is useful for alternate formatting of + stack traces. If *limit* is omitted or ``None``, all entries are extracted. A + "pre-processed" stack trace entry is a quadruple (*filename*, *line number*, + *function name*, *text*) representing the information that is usually printed + for a stack trace. The *text* is a string with leading and trailing whitespace + stripped; if the source is not available it is ``None``. .. function:: extract_stack(f=None, limit=None) @@ -99,40 +85,39 @@ arguments have the same meaning as for :func:`print_stack`. -.. function:: format_list(extracted_list) +.. function:: format_list(list) Given a list of tuples as returned by :func:`extract_tb` or - :func:`extract_stack`, return a list of strings ready for printing. Each - string in the resulting list corresponds to the item with the same index in - the argument list. Each string ends in a newline; the strings may contain - internal newlines as well, for those items whose source text line is not - ``None``. + :func:`extract_stack`, return a list of strings ready for printing. Each string + in the resulting list corresponds to the item with the same index in the + argument list. Each string ends in a newline; the strings may contain internal + newlines as well, for those items whose source text line is not ``None``. -.. function:: format_exception_only(etype, value) +.. function:: format_exception_only(type, value) - Format the exception part of a traceback. The arguments are the exception - type and value such as given by ``sys.last_type`` and ``sys.last_value``. - The return value is a list of strings, each ending in a newline. Normally, - the list contains a single string; however, for :exc:`SyntaxError` - exceptions, it contains several lines that (when printed) display detailed - information about where the syntax error occurred. The message indicating - which exception occurred is the always last string in the list. + Format the exception part of a traceback. The arguments are the exception type + and value such as given by ``sys.last_type`` and ``sys.last_value``. The return + value is a list of strings, each ending in a newline. Normally, the list + contains a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information about + where the syntax error occurred. The message indicating which exception + occurred is the always last string in the list. -.. function:: format_exception(etype, value, tb, limit=None, chain=True) +.. function:: format_exception(type, value, tb, limit=None, chain=True) Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to :func:`print_exception`. The - return value is a list of strings, each ending in a newline and some - containing internal newlines. When these lines are concatenated and printed, - exactly the same text is printed as does :func:`print_exception`. + return value is a list of strings, each ending in a newline and some containing + internal newlines. When these lines are concatenated and printed, exactly the + same text is printed as does :func:`print_exception`. .. function:: format_exc(limit=None, chain=True) - This is like ``print_exc(limit)`` but returns a string instead of printing to - a file. + This is like ``print_exc(limit)`` but returns a string instead of printing to a + file. .. function:: format_tb(tb, limit=None) @@ -144,169 +129,6 @@ A shorthand for ``format_list(extract_stack(f, limit))``. -.. function:: clear_frames(tb) - - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. - - .. versionadded:: 3.4 - -.. function:: walk_stack(f) - - Walk a stack following ``f.f_back`` from the given frame, yielding the frame - and line number for each frame. If *f* is ``None``, the current stack is - used. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -.. function:: walk_tb(tb) - - Walk a traceback following ``tb_next`` yielding the frame and line number - for each frame. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -The module also defines the following classes: - -:class:`TracebackException` Objects ------------------------------------ - -.. versionadded:: 3.5 - -:class:`TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. - -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - Note that when locals are captured, they are also shown in the traceback. - - .. attribute:: __cause__ - - A :class:`TracebackException` of the original ``__cause__``. - - .. attribute:: __context__ - - A :class:`TracebackException` of the original ``__context__``. - - .. attribute:: __suppress_context__ - - The ``__suppress_context__`` value from the original exception. - - .. attribute:: stack - - A :class:`StackSummary` representing the traceback. - - .. attribute:: exc_type - - The class of the original traceback. - - .. attribute:: filename - - For syntax errors - the file name where the error occurred. - - .. attribute:: lineno - - For syntax errors - the line number where the error occurred. - - .. attribute:: text - - For syntax errors - the text where the error occurred. - - .. attribute:: offset - - For syntax errors - the offset into the text where the error occurred. - - .. attribute:: msg - - For syntax errors - the compiler error message. - - .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - Note that when locals are captured, they are also shown in the traceback. - - .. method:: format(*, chain=True) - - Format the exception. - - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. - - The return value is a generator of strings, each ending in a newline and - some containing internal newlines. :func:`~traceback.print_exception` - is a wrapper around this method which just prints the lines to a file. - - The message indicating which exception occurred is always the last - string in the output. - - .. method:: format_exception_only() - - Format the exception part of the traceback. - - The return value is a generator of strings, each ending in a newline. - - Normally, the generator emits a single string; however, for - :exc:`SyntaxError` exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. - - -:class:`StackSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -:class:`StackSummary` objects represent a call stack ready for formatting. - -.. class:: StackSummary - - .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - - Construct a :class:`StackSummary` object from a frame generator (such as - is returned by :func:`~traceback.walk_stack` or - :func:`~traceback.walk_tb`). - - If *limit* is supplied, only this many frames are taken from *frame_gen*. - If *lookup_lines* is ``False``, the returned :class:`FrameSummary` - objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it - may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object - representations. - - .. classmethod:: from_list(a_list) - - Construct a :class:`StackSummary` object from a supplied old-style list - of tuples. Each tuple should be a 4-tuple with filename, lineno, name, - line as the elements. - - -:class:`FrameSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -:class:`FrameSummary` objects represent a single frame in a traceback. - -.. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames - locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). - :attr:`~FrameSummary.line` may be directly provided, and will prevent line - lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the - summary for later display. .. _traceback-example: @@ -324,7 +146,7 @@ source = input(">>> ") try: exec(source, envdir) - except Exception: + except: print("Exception in user code:") print("-"*60) traceback.print_exc(file=sys.stdout) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tracemalloc.rst --- a/Doc/library/tracemalloc.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,637 +0,0 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== - -.. module:: tracemalloc - :synopsis: Trace memory allocations. - -.. versionadded:: 3.4 - -The tracemalloc module is a debug tool to trace memory blocks allocated by -Python. It provides the following information: - -* Traceback where an object was allocated -* Statistics on allocated memory blocks per filename and per line number: - total size, number and average size of allocated memory blocks -* Compute the differences between two snapshots to detect memory leaks - -To trace most memory blocks allocated by Python, the module should be started -as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment -variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line -option. The :func:`tracemalloc.start` function can be called at runtime to -start tracing Python memory allocations. - -By default, a trace of an allocated memory block only stores the most recent -frame (1 frame). To store 25 frames at startup: set the -:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the -:option:`-X` ``tracemalloc=25`` command line option. - - -Examples --------- - -Display the top 10 -^^^^^^^^^^^^^^^^^^ - -Display the 10 files allocating the most memory:: - - import tracemalloc - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('lineno') - - print("[ Top 10 ]") - for stat in top_stats[:10]: - print(stat) - - -Example of output of the Python test suite:: - - [ Top 10 ] - :716: size=4855 KiB, count=39328, average=126 B - :284: size=521 KiB, count=3199, average=167 B - /usr/lib/python3.4/collections/__init__.py:368: size=244 KiB, count=2315, average=108 B - /usr/lib/python3.4/unittest/case.py:381: size=185 KiB, count=779, average=243 B - /usr/lib/python3.4/unittest/case.py:402: size=154 KiB, count=378, average=416 B - /usr/lib/python3.4/abc.py:133: size=88.7 KiB, count=347, average=262 B - :1446: size=70.4 KiB, count=911, average=79 B - :1454: size=52.0 KiB, count=25, average=2131 B - :5: size=49.7 KiB, count=148, average=344 B - /usr/lib/python3.4/sysconfig.py:411: size=48.0 KiB, count=1, average=48.0 KiB - -We can see that Python loaded ``4.8 MiB`` data (bytecode and constants) from -modules and that the :mod:`collections` module allocated ``244 KiB`` to build -:class:`~collections.namedtuple` types. - -See :meth:`Snapshot.statistics` for more options. - - -Compute differences -^^^^^^^^^^^^^^^^^^^ - -Take two snapshots and display the differences:: - - import tracemalloc - tracemalloc.start() - # ... start your application ... - - snapshot1 = tracemalloc.take_snapshot() - # ... call the function leaking memory ... - snapshot2 = tracemalloc.take_snapshot() - - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - - print("[ Top 10 differences ]") - for stat in top_stats[:10]: - print(stat) - -Example of output before/after running some tests of the Python test suite:: - - [ Top 10 differences ] - :716: size=8173 KiB (+4428 KiB), count=71332 (+39369), average=117 B - /usr/lib/python3.4/linecache.py:127: size=940 KiB (+940 KiB), count=8106 (+8106), average=119 B - /usr/lib/python3.4/unittest/case.py:571: size=298 KiB (+298 KiB), count=589 (+589), average=519 B - :284: size=1005 KiB (+166 KiB), count=7423 (+1526), average=139 B - /usr/lib/python3.4/mimetypes.py:217: size=112 KiB (+112 KiB), count=1334 (+1334), average=86 B - /usr/lib/python3.4/http/server.py:848: size=96.0 KiB (+96.0 KiB), count=1 (+1), average=96.0 KiB - /usr/lib/python3.4/inspect.py:1465: size=83.5 KiB (+83.5 KiB), count=109 (+109), average=784 B - /usr/lib/python3.4/unittest/mock.py:491: size=77.7 KiB (+77.7 KiB), count=143 (+143), average=557 B - /usr/lib/python3.4/urllib/parse.py:476: size=71.8 KiB (+71.8 KiB), count=969 (+969), average=76 B - /usr/lib/python3.4/contextlib.py:38: size=67.2 KiB (+67.2 KiB), count=126 (+126), average=546 B - -We can see that Python has loaded ``8.2 MiB`` of module data (bytecode and -constants), and that this is ``4.4 MiB`` more than had been loaded before the -tests, when the previous snapshot was taken. Similarly, the :mod:`linecache` -module has cached ``940 KiB`` of Python source code to format tracebacks, all -of it since the previous snapshot. - -If the system has little free memory, snapshots can be written on disk using -the :meth:`Snapshot.dump` method to analyze the snapshot offline. Then use the -:meth:`Snapshot.load` method reload the snapshot. - - -Get the traceback of a memory block -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Code to display the traceback of the biggest memory block:: - - import tracemalloc - - # Store 25 frames - tracemalloc.start(25) - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('traceback') - - # pick the biggest memory block - stat = top_stats[0] - print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024)) - for line in stat.traceback.format(): - print(line) - -Example of output of the Python test suite (traceback limited to 25 frames):: - - 903 memory blocks: 870.1 KiB - File "", line 716 - File "", line 1036 - File "", line 934 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/doctest.py", line 101 - import pdb - File "", line 284 - File "", line 938 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/test/support/__init__.py", line 1728 - import doctest - File "/usr/lib/python3.4/test/test_pickletools.py", line 21 - support.run_doctest(pickletools) - File "/usr/lib/python3.4/test/regrtest.py", line 1276 - test_runner() - File "/usr/lib/python3.4/test/regrtest.py", line 976 - display_failure=not verbose) - File "/usr/lib/python3.4/test/regrtest.py", line 761 - match_tests=ns.match_tests) - File "/usr/lib/python3.4/test/regrtest.py", line 1563 - main() - File "/usr/lib/python3.4/test/__main__.py", line 3 - regrtest.main_in_temp_cwd() - File "/usr/lib/python3.4/runpy.py", line 73 - exec(code, run_globals) - File "/usr/lib/python3.4/runpy.py", line 160 - "__main__", fname, loader, pkg_name) - -We can see that the most memory was allocated in the :mod:`importlib` module to -load data (bytecode and constants) from modules: ``870 KiB``. The traceback is -where the :mod:`importlib` loaded data most recently: on the ``import pdb`` -line of the :mod:`doctest` module. The traceback may change if a new module is -loaded. - - -Pretty top -^^^^^^^^^^ - -Code to display the 10 lines allocating the most memory with a pretty output, -ignoring ```` and ```` files:: - - import linecache - import os - import tracemalloc - - def display_top(snapshot, group_by='lineno', limit=10): - snapshot = snapshot.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - )) - top_stats = snapshot.statistics(group_by) - - print("Top %s lines" % limit) - for index, stat in enumerate(top_stats[:limit], 1): - frame = stat.traceback[0] - # replace "/path/to/module/file.py" with "module/file.py" - filename = os.sep.join(frame.filename.split(os.sep)[-2:]) - print("#%s: %s:%s: %.1f KiB" - % (index, filename, frame.lineno, stat.size / 1024)) - line = linecache.getline(frame.filename, frame.lineno).strip() - if line: - print(' %s' % line) - - other = top_stats[limit:] - if other: - size = sum(stat.size for stat in other) - print("%s other: %.1f KiB" % (len(other), size / 1024)) - total = sum(stat.size for stat in top_stats) - print("Total allocated size: %.1f KiB" % (total / 1024)) - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - display_top(snapshot) - -Example of output of the Python test suite:: - - Top 10 lines - #1: Lib/base64.py:414: 419.8 KiB - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - #2: Lib/base64.py:306: 419.8 KiB - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - #3: collections/__init__.py:368: 293.6 KiB - exec(class_definition, namespace) - #4: Lib/abc.py:133: 115.2 KiB - cls = super().__new__(mcls, name, bases, namespace) - #5: unittest/case.py:574: 103.1 KiB - testMethod() - #6: Lib/linecache.py:127: 95.4 KiB - lines = fp.readlines() - #7: urllib/parse.py:476: 71.8 KiB - for a in _hexdig for b in _hexdig} - #8: :5: 62.0 KiB - #9: Lib/_weakrefset.py:37: 60.0 KiB - self.data = set() - #10: Lib/base64.py:142: 59.8 KiB - _b32tab2 = [a + b for a in _b32tab for b in _b32tab] - 6220 other: 3602.8 KiB - Total allocated size: 5303.1 KiB - -See :meth:`Snapshot.statistics` for more options. - - -API ---- - -Functions -^^^^^^^^^ - -.. function:: clear_traces() - - Clear traces of memory blocks allocated by Python. - - See also :func:`stop`. - - -.. function:: get_object_traceback(obj) - - Get the traceback where the Python object *obj* was allocated. - Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` - module is not tracing memory allocations or did not trace the allocation of - the object. - - See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. - - -.. function:: get_traceback_limit() - - Get the maximum number of frames stored in the traceback of a trace. - - The :mod:`tracemalloc` module must be tracing memory allocations to - get the limit, otherwise an exception is raised. - - The limit is set by the :func:`start` function. - - -.. function:: get_traced_memory() - - Get the current size and peak size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. - - -.. function:: get_tracemalloc_memory() - - Get the memory usage in bytes of the :mod:`tracemalloc` module used to store - traces of memory blocks. - Return an :class:`int`. - - -.. function:: is_tracing() - - ``True`` if the :mod:`tracemalloc` module is tracing Python memory - allocations, ``False`` otherwise. - - See also :func:`start` and :func:`stop` functions. - - -.. function:: start(nframe: int=1) - - Start tracing Python memory allocations: install hooks on Python memory - allocators. Collected tracebacks of traces will be limited to *nframe* - frames. By default, a trace of a memory block only stores the most recent - frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. - - Storing more than ``1`` frame is only useful to compute statistics grouped - by ``'traceback'`` or to compute cumulative statistics: see the - :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. - - Storing more frames increases the memory and CPU overhead of the - :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function - to measure how much memory is used by the :mod:`tracemalloc` module. - - The :envvar:`PYTHONTRACEMALLOC` environment variable - (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` - command line option can be used to start tracing at startup. - - See also :func:`stop`, :func:`is_tracing` and :func:`get_traceback_limit` - functions. - - -.. function:: stop() - - Stop tracing Python memory allocations: uninstall hooks on Python memory - allocators. Also clears all previously collected traces of memory blocks - allocated by Python. - - Call :func:`take_snapshot` function to take a snapshot of traces before - clearing them. - - See also :func:`start`, :func:`is_tracing` and :func:`clear_traces` - functions. - - -.. function:: take_snapshot() - - Take a snapshot of traces of memory blocks allocated by Python. Return a new - :class:`Snapshot` instance. - - The snapshot does not include memory blocks allocated before the - :mod:`tracemalloc` module started to trace memory allocations. - - Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use - the *nframe* parameter of the :func:`start` function to store more frames. - - The :mod:`tracemalloc` module must be tracing memory allocations to take a - snapshot, see the :func:`start` function. - - See also the :func:`get_object_traceback` function. - - -Filter -^^^^^^ - -.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False) - - Filter on traces of memory blocks. - - See the :func:`fnmatch.fnmatch` function for the syntax of - *filename_pattern*. The ``'.pyc'`` file extension is - replaced with ``'.py'``. - - Examples: - - * ``Filter(True, subprocess.__file__)`` only includes traces of the - :mod:`subprocess` module - * ``Filter(False, tracemalloc.__file__)`` excludes traces of the - :mod:`tracemalloc` module - * ``Filter(False, "")`` excludes empty tracebacks - - - .. versionchanged:: 3.5 - The ``'.pyo'`` file extension is no longer replaced with ``'.py'``. - - .. attribute:: inclusive - - If *inclusive* is ``True`` (include), only trace memory blocks allocated - in a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in - a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - .. attribute:: lineno - - Line number (``int``) of the filter. If *lineno* is ``None``, the filter - matches any line number. - - .. attribute:: filename_pattern - - Filename pattern of the filter (``str``). - - .. attribute:: all_frames - - If *all_frames* is ``True``, all frames of the traceback are checked. If - *all_frames* is ``False``, only the most recent frame is checked. - - This attribute has no effect if the traceback limit is ``1``. See the - :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit` - attribute. - - -Frame -^^^^^ - -.. class:: Frame - - Frame of a traceback. - - The :class:`Traceback` class is a sequence of :class:`Frame` instances. - - .. attribute:: filename - - Filename (``str``). - - .. attribute:: lineno - - Line number (``int``). - - -Snapshot -^^^^^^^^ - -.. class:: Snapshot - - Snapshot of traces of memory blocks allocated by Python. - - The :func:`take_snapshot` function creates a snapshot instance. - - .. method:: compare_to(old_snapshot: Snapshot, group_by: str, cumulative: bool=False) - - Compute the differences with an old snapshot. Get statistics as a sorted - list of :class:`StatisticDiff` instances grouped by *group_by*. - - See the :meth:`Snapshot.statistics` method for *group_by* and *cumulative* - parameters. - - The result is sorted from the biggest to the smallest by: absolute value - of :attr:`StatisticDiff.size_diff`, :attr:`StatisticDiff.size`, absolute - value of :attr:`StatisticDiff.count_diff`, :attr:`Statistic.count` and - then by :attr:`StatisticDiff.traceback`. - - - .. method:: dump(filename) - - Write the snapshot into a file. - - Use :meth:`load` to reload the snapshot. - - - .. method:: filter_traces(filters) - - Create a new :class:`Snapshot` instance with a filtered :attr:`traces` - sequence, *filters* is a list of :class:`Filter` instances. If *filters* - is an empty list, return a new :class:`Snapshot` instance with a copy of - the traces. - - All inclusive filters are applied at once, a trace is ignored if no - inclusive filters match it. A trace is ignored if at least one exclusive - filter matches it. - - - .. classmethod:: load(filename) - - Load a snapshot from a file. - - See also :meth:`dump`. - - - .. method:: statistics(group_by: str, cumulative: bool=False) - - Get statistics as a sorted list of :class:`Statistic` instances grouped - by *group_by*: - - ===================== ======================== - group_by description - ===================== ======================== - ``'filename'`` filename - ``'lineno'`` filename and line number - ``'traceback'`` traceback - ===================== ======================== - - If *cumulative* is ``True``, cumulate size and count of memory blocks of - all frames of the traceback of a trace, not only the most recent frame. - The cumulative mode can only be used with *group_by* equals to - ``'filename'`` and ``'lineno'``. - - The result is sorted from the biggest to the smallest by: - :attr:`Statistic.size`, :attr:`Statistic.count` and then by - :attr:`Statistic.traceback`. - - - .. attribute:: traceback_limit - - Maximum number of frames stored in the traceback of :attr:`traces`: - result of the :func:`get_traceback_limit` when the snapshot was taken. - - .. attribute:: traces - - Traces of all memory blocks allocated by Python: sequence of - :class:`Trace` instances. - - The sequence has an undefined order. Use the :meth:`Snapshot.statistics` - method to get a sorted list of statistics. - - -Statistic -^^^^^^^^^ - -.. class:: Statistic - - Statistic on memory allocations. - - :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances. - - See also the :class:`StatisticDiff` class. - - .. attribute:: count - - Number of memory blocks (``int``). - - .. attribute:: size - - Total size of memory blocks in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -StatisticDiff -^^^^^^^^^^^^^ - -.. class:: StatisticDiff - - Statistic difference on memory allocations between an old and a new - :class:`Snapshot` instance. - - :func:`Snapshot.compare_to` returns a list of :class:`StatisticDiff` - instances. See also the :class:`Statistic` class. - - .. attribute:: count - - Number of memory blocks in the new snapshot (``int``): ``0`` if - the memory blocks have been released in the new snapshot. - - .. attribute:: count_diff - - Difference of number of memory blocks between the old and the new - snapshots (``int``): ``0`` if the memory blocks have been allocated in - the new snapshot. - - .. attribute:: size - - Total size of memory blocks in bytes in the new snapshot (``int``): - ``0`` if the memory blocks have been released in the new snapshot. - - .. attribute:: size_diff - - Difference of total size of memory blocks in bytes between the old and - the new snapshots (``int``): ``0`` if the memory blocks have been - allocated in the new snapshot. - - .. attribute:: traceback - - Traceback where the memory blocks were allocated, :class:`Traceback` - instance. - - -Trace -^^^^^ - -.. class:: Trace - - Trace of a memory block. - - The :attr:`Snapshot.traces` attribute is a sequence of :class:`Trace` - instances. - - .. attribute:: size - - Size of the memory block in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -Traceback -^^^^^^^^^ - -.. class:: Traceback - - Sequence of :class:`Frame` instances sorted from the most recent frame to - the oldest frame. - - A traceback contains at least ``1`` frame. If the ``tracemalloc`` module - failed to get a frame, the filename ``""`` at line number ``0`` is - used. - - When a snapshot is taken, tracebacks of traces are limited to - :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. - - The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` - instance. - - .. method:: format(limit=None) - - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. If - *limit* is set, only format the *limit* most recent frames. - - Similar to the :func:`traceback.format_tb` function, except that - :meth:`format` does not include newlines. - - Example:: - - print("Traceback (most recent call first):") - for line in traceback: - print(line) - - Output:: - - Traceback (most recent call first): - File "test.py", line 9 - obj = Object() - File "test.py", line 12 - tb = tracemalloc.get_object_traceback(f()) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tulip_coro.dia Binary file Doc/library/tulip_coro.dia has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/tulip_coro.png Binary file Doc/library/tulip_coro.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/turtle.rst --- a/Doc/library/turtle.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/turtle.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1055,8 +1055,8 @@ Write text - the string representation of *arg* - at the current turtle position according to *align* ("left", "center" or right") and with the given - font. If *move* is true, the pen is moved to the bottom-right corner of the - text. By default, *move* is ``False``. + font. If *move* is True, the pen is moved to the bottom-right corner of the + text. By default, *move* is False. >>> turtle.write("Home = ", True, align="center") >>> turtle.write((0,0), True) @@ -1092,7 +1092,7 @@ .. function:: isvisible() - Return ``True`` if the Turtle is shown, ``False`` if it's hidden. + Return True if the Turtle is shown, False if it's hidden. >>> turtle.hideturtle() >>> turtle.isvisible() @@ -1809,7 +1809,7 @@ Pop up a dialog window for input of a number. title is the title of the dialog window, prompt is a text mostly describing what numerical information - to input. default: default value, minval: minimum value for input, + to input. default: default value, minval: minimum value for imput, maxval: maximum value for input The number input must be in the range minval .. maxval if these are given. If not, a hint is issued and the dialog remains open for @@ -1879,7 +1879,7 @@ >>> cv = screen.getcanvas() >>> cv - + .. function:: getshapes() @@ -1981,7 +1981,7 @@ :param startx: if positive, starting position in pixels from the left edge of the screen, if negative from the right edge, if None, center window horizontally - :param starty: if positive, starting position in pixels from the top + :param startx: if positive, starting position in pixels from the top edge of the screen, if negative from the bottom edge, if None, center window vertically @@ -2274,13 +2274,10 @@ not from within the demo-viewer). -:mod:`turtledemo` --- Demo scripts -================================== - -.. module:: turtledemo - :synopsis: A viewer for example turtle scripts - -The :mod:`turtledemo` package includes a set of demo scripts. These +Demo scripts +============ + +There is a set of demo scripts in the :mod:`turtledemo` package. These scripts can be run and viewed using the supplied demo viewer as follows:: python -m turtledemo @@ -2291,21 +2288,22 @@ The :mod:`turtledemo` package directory contains: -- A demo viewer :file:`__main__.py` which can be used to view the sourcecode - of the scripts and run them at the same time. -- Multiple scripts demonstrating different features of the :mod:`turtle` - module. Examples can be accessed via the Examples menu. They can also - be run standalone. -- A :file:`turtle.cfg` file which serves as an example of how to write - and use such files. +- a set of 15 demo scripts demonstrating different features of the new module + :mod:`turtle`; +- a demo viewer :file:`__main__.py` which can be used to view the sourcecode + of the scripts and run them at the same time. 14 of the examples can be + accessed via the Examples menu; all of them can also be run standalone. +- The example :mod:`turtledemo.two_canvases` demonstrates the simultaneous + use of two canvases with the turtle module. Therefore it only can be run + standalone. +- There is a :file:`turtle.cfg` file in this directory, which serves as an + example for how to write and use such files. The demo scripts are: -.. tabularcolumns:: |l|L|L| - +----------------+------------------------------+-----------------------+ | Name | Description | Features | -+================+==============================+=======================+ ++----------------+------------------------------+-----------------------+ | bytedesign | complex classical | :func:`tracer`, delay,| | | turtle graphics pattern | :func:`update` | +----------------+------------------------------+-----------------------+ @@ -2320,8 +2318,6 @@ +----------------+------------------------------+-----------------------+ | colormixer | experiment with r, g, b | :func:`ondrag` | +----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ | fractalcurves | Hilbert & Koch curves | recursion | +----------------+------------------------------+-----------------------+ | lindenmayer | ethnomathematics | L-System | @@ -2351,15 +2347,9 @@ | | pairwise in opposite | shapesize, tilt, | | | direction | get_shapepoly, update | +----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ | tree | a (graphical) breadth | :func:`clone` | | | first tree (using generators)| | +----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ | wikipedia | a pattern from the wikipedia | :func:`clone`, | | | article on turtle graphics | :func:`undo` | +----------------+------------------------------+-----------------------+ @@ -2405,7 +2395,7 @@ Accordingly the latter has got an alias: :meth:`Screen.onkeyrelease`. - The method :meth:`Screen.mainloop` has been added. So when working only - with Screen and Turtle objects one must not additionally import + with Screen and Turtle objects one must not additonally import :func:`mainloop` anymore. - Two input methods has been added :meth:`Screen.textinput` and diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/types.rst --- a/Doc/library/types.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/types.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,5 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== +:mod:`types` --- Names for built-in types +========================================= .. module:: types :synopsis: Names for built-in types. @@ -8,94 +8,26 @@ -------------- -This module defines utility function to assist in dynamic creation of -new types. +This module defines names for some object types that are used by the standard +Python interpreter, but not exposed as builtins like :class:`int` or +:class:`str` are. Also, it does not include some of the types that arise +transparently during processing such as the ``listiterator`` type. -It also defines names for some object types that are used by the standard -Python interpreter, but not exposed as builtins like :class:`int` or -:class:`str` are. +Typical use is for :func:`isinstance` or :func:`issubclass` checks. -Finally, it provides some additional type-related utility classes and functions -that are not fundamental enough to be builtins. - - -Dynamic Type Creation ---------------------- - -.. function:: new_class(name, bases=(), kwds=None, exec_body=None) - - Creates a class object dynamically using the appropriate metaclass. - - The first three arguments are the components that make up a class - definition header: the class name, the base classes (in order), the - keyword arguments (such as ``metaclass``). - - The *exec_body* argument is a callback that is used to populate the - freshly created class namespace. It should accept the class namespace - as its sole argument and update the namespace directly with the class - contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: ns``. - - .. versionadded:: 3.3 - -.. function:: prepare_class(name, bases=(), kwds=None) - - Calculates the appropriate metaclass and creates the class namespace. - - The arguments are the components that make up a class definition header: - the class name, the base classes (in order) and the keyword arguments - (such as ``metaclass``). - - The return value is a 3-tuple: ``metaclass, namespace, kwds`` - - *metaclass* is the appropriate metaclass, *namespace* is the - prepared class namespace and *kwds* is an updated copy of the passed - in *kwds* argument with any ``'metaclass'`` entry removed. If no *kwds* - argument is passed in, this will be an empty dict. - - .. versionadded:: 3.3 - -.. seealso:: - - :ref:`metaclasses` - Full details of the class creation process supported by these functions - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - - -Standard Interpreter Types --------------------------- - -This module provides names for many of the types that are required to -implement a Python interpreter. It deliberately avoids including some of -the types that arise only incidentally during processing such as the -``listiterator`` type. - -Typical use of these names is for :func:`isinstance` or -:func:`issubclass` checks. - -Standard names are defined for the following types: +The module defines the following names: .. data:: FunctionType LambdaType - The type of user-defined functions and functions created by - :keyword:`lambda` expressions. + The type of user-defined functions and functions created by :keyword:`lambda` + expressions. .. data:: GeneratorType - The type of :term:`generator`-iterator objects, created by - generator functions. - - -.. data:: CoroutineType - - The type of :term:`coroutine` objects, created by - :keyword:`async def` functions. - - .. versionadded:: 3.5 + The type of :term:`generator`-iterator objects, produced by calling a + generator function. .. data:: CodeType @@ -118,39 +50,9 @@ C".) -.. class:: ModuleType(name, doc=None) +.. data:: ModuleType - The type of :term:`modules `. Constructor takes the name of the - module to be created and optionally its :term:`docstring`. - - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. + The type of modules. .. data:: TracebackType @@ -183,121 +85,3 @@ In other implementations of Python, this type may be identical to ``GetSetDescriptorType``. - -.. class:: MappingProxyType(mapping) - - Read-only proxy of a mapping. It provides a dynamic view on the mapping's - entries, which means that when the mapping changes, the view reflects these - changes. - - .. versionadded:: 3.3 - - .. describe:: key in proxy - - Return ``True`` if the underlying mapping has a key *key*, else - ``False``. - - .. describe:: proxy[key] - - Return the item of the underlying mapping with key *key*. Raises a - :exc:`KeyError` if *key* is not in the underlying mapping. - - .. describe:: iter(proxy) - - Return an iterator over the keys of the underlying mapping. This is a - shortcut for ``iter(proxy.keys())``. - - .. describe:: len(proxy) - - Return the number of items in the underlying mapping. - - .. method:: copy() - - Return a shallow copy of the underlying mapping. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the underlying mapping, else - *default*. If *default* is not given, it defaults to ``None``, so that - this method never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the underlying mapping's items (``(key, value)`` - pairs). - - .. method:: keys() - - Return a new view of the underlying mapping's keys. - - .. method:: values() - - Return a new view of the underlying mapping's values. - - -Additional Utility Classes and Functions ----------------------------------------- - -.. class:: SimpleNamespace - - A simple :class:`object` subclass that provides attribute access to its - namespace, as well as a meaningful repr. - - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. - - The type is roughly equivalent to the following code:: - - class SimpleNamespace: - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - def __repr__(self): - keys = sorted(self.__dict__) - items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) - return "{}({})".format(type(self).__name__, ", ".join(items)) - def __eq__(self, other): - return self.__dict__ == other.__dict__ - - ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. - However, for a structured record type use :func:`~collections.namedtuple` - instead. - - .. versionadded:: 3.3 - - -.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) - - Route attribute access on a class to __getattr__. - - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see Enum for an example). - - .. versionadded:: 3.4 - - -Coroutine Utility Functions ---------------------------- - -.. function:: coroutine(gen_func) - - This function transforms a :term:`generator` function into a - :term:`coroutine function` which returns a generator-based coroutine. - The generator-based coroutine is still a :term:`generator iterator`, - but is also considered to be a :term:`coroutine` object and is - :term:`awaitable`. However, it may not necessarily implement - the :meth:`__await__` method. - - If *gen_func* is a generator function, it will be modified in-place. - - If *gen_func* is not a generator function, it will be wrapped. If it - returns an instance of :class:`collections.abc.Generator`, the instance - will be wrapped in an *awaitable* proxy object. All other types - of objects will be returned as is. - - .. versionadded:: 3.5 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/typing.rst --- a/Doc/library/typing.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,524 +0,0 @@ -:mod:`typing` --- Support for type hints -======================================== - -.. module:: typing - :synopsis: Support for type hints (see PEP 484). - -**Source code:** :source:`Lib/typing.py` - --------------- - -This module supports type hints as specified by :pep:`484`. The most -fundamental support consists of the type :class:`Any`, :class:`Union`, -:class:`Tuple`, :class:`Callable`, :class:`TypeVar`, and -:class:`Generic`. For full specification please see :pep:`484`. For -a simplified introduction to type hints see :pep:`483`. - - -The function below takes and returns a string and is annotated as follows:: - - def greeting(name: str) -> str: - return 'Hello ' + name - -In the function ``greeting``, the argument ``name`` is expected to by of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. - -Type aliases ------------- - -A type alias is defined by assigning the type to the alias:: - - Vector = List[float] - -Callable --------- - -Frameworks expecting callback functions of specific signatures might be -type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. - -For example:: - - from typing import Callable - - def feeder(get_next_item: Callable[[], str]) -> None: - # Body - - def async_query(on_success: Callable[[int], None], - on_error: Callable[[int, Exception], None]) -> None: - # Body - -It is possible to declare the return type of a callable without specifying -the call signature by substituting a literal ellipsis -for the list of arguments in the type hint: ``Callable[..., ReturnType]``. -``None`` as a type hint is a special case and is replaced by ``type(None)``. - -Generics --------- - -Since type information about objects kept in containers cannot be statically -inferred in a generic way, abstract base classes have been extended to support -subscription to denote expected types for container elements. - -:: - - from typing import Mapping, Sequence - - def notify_by_email(employees: Sequence[Employee], - overrides: Mapping[str, str]) -> None: ... - -Generics can be parametrized by using a new factory available in typing -called :class:`TypeVar`. - -:: - - from typing import Sequence, TypeVar - - T = TypeVar('T') # Declare type variable - - def first(l: Sequence[T]) -> T: # Generic function - return l[0] - - -User-defined generic types --------------------------- - -A user-defined class can be defined as a generic class. - -:: - - from typing import TypeVar, Generic - from logging import Logger - - T = TypeVar('T') - - class LoggedVar(Generic[T]): - def __init__(self, value: T, name: str, logger: Logger) -> None: - self.name = name - self.logger = logger - self.value = value - - def set(self, new: T) -> None: - self.log('Set ' + repr(self.value)) - self.value = new - - def get(self) -> T: - self.log('Get ' + repr(self.value)) - return self.value - - def log(self, message: str) -> None: - self.logger.info('{}: {}'.format(self.name, message)) - -``Generic[T]`` as a base class defines that the class ``LoggedVar`` takes a -single type parameter ``T`` . This also makes ``T`` valid as a type within the -class body. - -The :class:`Generic` base class uses a metaclass that defines -:meth:`__getitem__` so that ``LoggedVar[t]`` is valid as a type:: - - from typing import Iterable - - def zero_all_vars(vars: Iterable[LoggedVar[int]]) -> None: - for var in vars: - var.set(0) - -A generic type can have any number of type variables, and type variables may -be constrained:: - - from typing import TypeVar, Generic - ... - - T = TypeVar('T') - S = TypeVar('S', int, str) - - class StrangePair(Generic[T, S]): - ... - -Each type variable argument to :class:`Generic` must be distinct. -This is thus invalid:: - - from typing import TypeVar, Generic - ... - - T = TypeVar('T') - - class Pair(Generic[T, T]): # INVALID - ... - -You can use multiple inheritance with :class:`Generic`:: - - from typing import TypeVar, Generic, Sized - - T = TypeVar('T') - - class LinkedList(Sized, Generic[T]): - ... - -When inheriting from generic classes, some type variables could fixed:: - - from typing import TypeVar, Mapping - - T = TypeVar('T') - - class MyDict(Mapping[str, T]): - ... - -In this case ``MyDict`` has a single parameter, ``T``. - -Subclassing a generic class without specifying type parameters assumes -:class:`Any` for each position. In the following example, ``MyIterable`` is -not generic but implicitly inherits from ``Iterable[Any]``:: - - from typing import Iterable - - class MyIterable(Iterable): # Same as Iterable[Any] - -The metaclass used by :class:`Generic` is a subclass of :class:`abc.ABCMeta`. -A generic class can be an ABC by including abstract methods or properties, -and generic classes can also have ABCs as base classes without a metaclass -conflict. Generic metaclasses are not supported. - - -The :class:`Any` type ---------------------- - -A special kind of type is :class:`Any`. Every type is a subtype of -:class:`Any`. This is also true for the builtin type object. However, to the -static type checker these are completely different. - -When the type of a value is :class:`object`, the type checker will reject -almost all operations on it, and assigning it to a variable (or using it as a -return value) of a more specialized type is a type error. On the other hand, -when a value has type :class:`Any`, the type checker will allow all operations -on it, and a value of type :class:`Any` can be assigned to a variable (or used -as a return value) of a more constrained type. - - -Classes, functions, and decorators ----------------------------------- - -The module defines the following classes, functions and decorators: - -.. class:: Any - - Special type indicating an unconstrained type. - - * Any object is an instance of :class:`Any`. - * Any class is a subclass of :class:`Any`. - * As a special case, :class:`Any` and :class:`object` are subclasses of - each other. - -.. class:: TypeVar - - Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows:: - - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n - - def longest(x: A, y: A) -> A: - """Return the longest of two strings.""" - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note - that if the arguments are instances of some subclass of :class:`str`, - the return type is still plain :class:`str`. - - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, - :func:`isinstance` and :func:`issubclass` should not be used with types. - - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default type variables are invariant. Alternatively, - a type variable may specify an upper bound using ``bound=``. - This means that an actual type substituted (explicitly or implicitly) - for the type variable must be a subclass of the boundary type, - see :pep:`484`. - -.. class:: Union - - Union type; ``Union[X, Y]`` means either X or Y. - - To define a union, use e.g. ``Union[int, str]``. Details: - - * The arguments must be types and there must be at least one. - - * Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - * Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - * Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] - - * When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - * If :class:`Any` is present it is the sole survivor, e.g.:: - - Union[int, Any] == Any - - * You cannot subclass or instantiate a union. - - * You cannot write ``Union[X][Y]``. - - * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. - -.. class:: Optional - - Optional type. - - ``Optional[X]`` is equivalent to ``Union[X, type(None)]``. - -.. class:: Tuple - - Tuple type; ``Tuple[X, Y]`` is the is the type of a tuple of two items - with the first item of type X and the second of type Y. - - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. - - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. - -.. class:: Callable - - Callable type; ``Callable[[int], str]`` is a function of (int) -> str. - - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types; the return type must be a single type. - - There is no syntax to indicate optional or keyword arguments, - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` could be used to type hint a callable - taking any number of arguments and returning ``ReturnType``. - A plain :class:`Callable` is equivalent to ``Callable[..., Any]``. - -.. class:: Generic - - Abstract base class for generic types. - - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - X = TypeVar('X') - Y = TypeVar('Y') - - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: - try: - return mapping[key] - except KeyError: - return default - -.. class:: Iterable(Generic[T_co]) - - A generic version of the :class:`collections.abc.Iterable`. - -.. class:: Iterator(Iterable[T_co]) - - A generic version of the :class:`collections.abc.Iterator`. - -.. class:: SupportsInt - - An ABC with one abstract method ``__int__``. - -.. class:: SupportsFloat - - An ABC with one abstract method ``__float__``. - -.. class:: SupportsAbs - - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. - -.. class:: SupportsRound - - An ABC with one abstract method ``__round__`` - that is covariant in its return type. - -.. class:: Reversible - - An ABC with one abstract method ``__reversed__`` returning - an ``Iterator[T_co]``. - -.. class:: Container(Generic[T_co]) - - A generic version of :class:`collections.abc.Container`. - -.. class:: AbstractSet(Sized, Iterable[T_co], Container[T_co]) - - A generic version of :class:`collections.abc.Set`. - -.. class:: MutableSet(AbstractSet[T]) - - A generic version of :class:`collections.abc.MutableSet`. - -.. class:: Mapping(Sized, Iterable[KT], Container[KT], Generic[VT_co]) - - A generic version of :class:`collections.abc.Mapping`. - -.. class:: MutableMapping(Mapping[KT, VT]) - - A generic version of :class:`collections.abc.MutableMapping`. - -.. class:: Sequence(Sized, Iterable[T_co], Container[T_co]) - - A generic version of :class:`collections.abc.Sequence`. - -.. class:: MutableSequence(Sequence[T]) - - A generic version of :class:`collections.abc.MutableSequence`. - -.. class:: ByteString(Sequence[int]) - - A generic version of :class:`collections.abc.ByteString`. - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. - - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. - -.. class:: List(list, MutableSequence[T]) - - Generic version of :class:`list`. - Useful for annotating return types. To annotate arguments it is preferred - to use abstract collection types such as :class:`Mapping`, :class:`Sequence`, - or :class:`AbstractSet`. - - This type may be used as follows:: - - T = TypeVar('T', int, float) - - def vec2(x: T, y: T) -> List[T]: - return [x, y] - - def slice__to_4(vector: Sequence[T]) -> List[T]: - return vector[0:4] - -.. class:: AbstractSet(set, MutableSet[T]) - - A generic version of :class:`collections.abc.Set`. - -.. class:: MappingView(Sized, Iterable[T_co]) - - A generic version of :class:`collections.abc.MappingView`. - -.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) - - A generic version of :class:`collections.abc.KeysView`. - -.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) - - A generic version of :class:`collections.abc.ItemsView`. - -.. class:: ValuesView(MappingView[VT_co]) - - A generic version of :class:`collections.abc.ValuesView`. - -.. class:: Dict(dict, MutableMapping[KT, VT]) - - A generic version of :class:`dict`. - The usage of this type is as follows:: - - def get_position_in_index(word_list: Dict[str, int], word: str) -> int: - return word_list[word] - -.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) - -.. class:: io - - Wrapper namespace for I/O stream types. - - This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` - and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These representing the types of I/O streams such as returned by - :func:`open`. - -.. class:: re - - Wrapper namespace for regular expression matching types. - - This defines the type aliases ``Pattern`` and ``Match`` which - correspond to the return types from :func:`re.compile` and - :func:`re.match`. These types (and the corresponding functions) - are generic in ``AnyStr`` and can be made specific by writing - ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or - ``Match[bytes]``. - -.. function:: NamedTuple(typename, fields) - - Typed version of namedtuple. - - Usage:: - - Employee = typing.NamedTuple('Employee', [('name', str), ('id', int)]) - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - The resulting class has one extra attribute: _field_types, - giving a dict mapping field names to types. (The field names - are in the _fields attribute, which is part of the namedtuple - API.) - -.. function:: cast(typ, val) - - Cast a value to a type. - - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). - -.. function:: get_type_hints(obj) - - Return type hints for a function or method object. - - This is often the same as ``obj.__annotations__``, but it handles - forward references encoded as string literals, and if necessary - adds ``Optional[t]`` if a default value equal to None is set. - -.. decorator:: no_type_check(arg) - - Decorator to indicate that annotations are not type hints. - - The argument must be a class or function; if it is a class, it - applies recursively to all methods defined in that class (but not - to methods defined in its superclasses or subclasses). - - This mutates the function(s) in place. - -.. decorator:: no_type_check_decorator(decorator) - - Decorator to give another decorator the :func:`no_type_check` effect. - - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/undoc.rst --- a/Doc/library/undoc.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/undoc.rst Mon Jan 25 17:05:13 2016 +0100 @@ -20,7 +20,7 @@ documented beyond this mention. There's little need to document these. :mod:`ntpath` - --- Implementation of :mod:`os.path` on Win32, Win64, and WinCE platforms. + --- Implementation of :mod:`os.path` on Win32, Win64, WinCE, and OS/2 platforms. :mod:`posixpath` --- Implementation of :mod:`os.path` on POSIX. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unicodedata.rst --- a/Doc/library/unicodedata.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/unicodedata.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,8 +3,8 @@ .. module:: unicodedata :synopsis: Access the Unicode Database. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg +.. moduleauthor:: Marc-Andre Lemburg +.. sectionauthor:: Marc-Andre Lemburg .. sectionauthor:: Martin v. Löwis @@ -15,8 +15,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 8.0.0 -`_. +this database is compiled from the `UCD version 6.1.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -69,7 +69,7 @@ .. function:: bidirectional(chr) - Returns the bidirectional class assigned to the character *chr* as + Returns the bidirectional category assigned to the character *chr* as string. If no such value is defined, an empty string is returned. @@ -166,6 +166,6 @@ .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/6.1.0/ucd/NameAliases.txt -.. [#] http://www.unicode.org/Public/8.0.0/ucd/NamedSequences.txt +.. [#] http://www.unicode.org/Public/6.1.0/ucd/NamedSequences.txt diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock-examples.rst --- a/Doc/library/unittest.mock-examples.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/unittest.mock-examples.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,453 +1,30 @@ -:mod:`unittest.mock` --- getting started -======================================== +.. _further-examples: +:mod:`unittest.mock` --- further examples +========================================= + +.. module:: unittest.mock + :synopsis: Mock object library. .. moduleauthor:: Michael Foord .. currentmodule:: unittest.mock .. versionadded:: 3.3 -.. _getting-started: - -Using Mock ----------- - -Mock Patching Methods -~~~~~~~~~~~~~~~~~~~~~ - -Common uses for :class:`Mock` objects include: - -* Patching methods -* Recording method calls on objects - -You might want to replace a method on an object to check that -it is called with the correct arguments by another part of the system: - - >>> real = SomeClass() - >>> real.method = MagicMock(name='method') - >>> real.method(3, 4, 5, key='value') - - -Once our mock has been used (``real.method`` in this example) it has methods -and attributes that allow you to make assertions about how it has been used. - -.. note:: - - In most of these examples the :class:`Mock` and :class:`MagicMock` classes - are interchangeable. As the ``MagicMock`` is the more capable class it makes - a sensible one to use by default. - -Once the mock has been called its :attr:`~Mock.called` attribute is set to -``True``. More importantly we can use the :meth:`~Mock.assert_called_with` or -:meth:`~Mock.assert_called_once_with` method to check that it was called with -the correct arguments. - -This example tests that calling ``ProductionClass().method`` results in a call to -the ``something`` method: - - >>> class ProductionClass: - ... def method(self): - ... self.something(1, 2, 3) - ... def something(self, a, b, c): - ... pass - ... - >>> real = ProductionClass() - >>> real.something = MagicMock() - >>> real.method() - >>> real.something.assert_called_once_with(1, 2, 3) - - - -Mock for Method Calls on an Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the last example we patched a method directly on an object to check that it -was called correctly. Another common use case is to pass an object into a -method (or some part of the system under test) and then check that it is used -in the correct way. - -The simple ``ProductionClass`` below has a ``closer`` method. If it is called with -an object then it calls ``close`` on it. - - >>> class ProductionClass: - ... def closer(self, something): - ... something.close() - ... - -So to test it we need to pass in an object with a ``close`` method and check -that it was called correctly. - - >>> real = ProductionClass() - >>> mock = Mock() - >>> real.closer(mock) - >>> mock.close.assert_called_with() - -We don't have to do any work to provide the 'close' method on our mock. -Accessing close creates it. So, if 'close' hasn't already been called then -accessing it in the test will create it, but :meth:`~Mock.assert_called_with` -will raise a failure exception. - - -Mocking Classes -~~~~~~~~~~~~~~~ - -A common use case is to mock out classes instantiated by your code under test. -When you patch a class, then that class is replaced with a mock. Instances -are created by *calling the class*. This means you access the "mock instance" -by looking at the return value of the mocked class. - -In the example below we have a function ``some_function`` that instantiates ``Foo`` -and calls a method on it. The call to :func:`patch` replaces the class ``Foo`` with a -mock. The ``Foo`` instance is the result of calling the mock, so it is configured -by modifying the mock :attr:`~Mock.return_value`. - - >>> def some_function(): - ... instance = module.Foo() - ... return instance.method() - ... - >>> with patch('module.Foo') as mock: - ... instance = mock.return_value - ... instance.method.return_value = 'the result' - ... result = some_function() - ... assert result == 'the result' - - -Naming your mocks -~~~~~~~~~~~~~~~~~ - -It can be useful to give your mocks a name. The name is shown in the repr of -the mock and can be helpful when the mock appears in test failure messages. The -name is also propagated to attributes or methods of the mock: - - >>> mock = MagicMock(name='foo') - >>> mock - - >>> mock.method - - - -Tracking all Calls -~~~~~~~~~~~~~~~~~~ - -Often you want to track more than a single call to a method. The -:attr:`~Mock.mock_calls` attribute records all calls -to child attributes of the mock - and also to their children. - - >>> mock = MagicMock() - >>> mock.method() - - >>> mock.attribute.method(10, x=53) - - >>> mock.mock_calls - [call.method(), call.attribute.method(10, x=53)] - -If you make an assertion about ``mock_calls`` and any unexpected methods -have been called, then the assertion will fail. This is useful because as well -as asserting that the calls you expected have been made, you are also checking -that they were made in the right order and with no additional calls: - -You use the :data:`call` object to construct lists for comparing with -``mock_calls``: - - >>> expected = [call.method(), call.attribute.method(10, x=53)] - >>> mock.mock_calls == expected - True - - -Setting Return Values and Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Setting the return values on a mock object is trivially easy: - - >>> mock = Mock() - >>> mock.return_value = 3 - >>> mock() - 3 - -Of course you can do the same for methods on the mock: - - >>> mock = Mock() - >>> mock.method.return_value = 3 - >>> mock.method() - 3 - -The return value can also be set in the constructor: - - >>> mock = Mock(return_value=3) - >>> mock() - 3 - -If you need an attribute setting on your mock, just do it: - - >>> mock = Mock() - >>> mock.x = 3 - >>> mock.x - 3 - -Sometimes you want to mock up a more complex situation, like for example -``mock.connection.cursor().execute("SELECT 1")``. If we wanted this call to -return a list, then we have to configure the result of the nested call. - -We can use :data:`call` to construct the set of calls in a "chained call" like -this for easy assertion afterwards: - - >>> mock = Mock() - >>> cursor = mock.connection.cursor.return_value - >>> cursor.execute.return_value = ['foo'] - >>> mock.connection.cursor().execute("SELECT 1") - ['foo'] - >>> expected = call.connection.cursor().execute("SELECT 1").call_list() - >>> mock.mock_calls - [call.connection.cursor(), call.connection.cursor().execute('SELECT 1')] - >>> mock.mock_calls == expected - True - -It is the call to ``.call_list()`` that turns our call object into a list of -calls representing the chained calls. - - -Raising exceptions with mocks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A useful attribute is :attr:`~Mock.side_effect`. If you set this to an -exception class or instance then the exception will be raised when the mock -is called. - - >>> mock = Mock(side_effect=Exception('Boom!')) - >>> mock() - Traceback (most recent call last): - ... - Exception: Boom! - - -Side effect functions and iterables -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``side_effect`` can also be set to a function or an iterable. The use case for -``side_effect`` as an iterable is where your mock is going to be called several -times, and you want each call to return a different value. When you set -``side_effect`` to an iterable every call to the mock returns the next value -from the iterable: - - >>> mock = MagicMock(side_effect=[4, 5, 6]) - >>> mock() - 4 - >>> mock() - 5 - >>> mock() - 6 - - -For more advanced use cases, like dynamically varying the return values -depending on what the mock is called with, ``side_effect`` can be a function. -The function will be called with the same arguments as the mock. Whatever the -function returns is what the call returns: - - >>> vals = {(1, 2): 1, (2, 3): 2} - >>> def side_effect(*args): - ... return vals[args] - ... - >>> mock = MagicMock(side_effect=side_effect) - >>> mock(1, 2) - 1 - >>> mock(2, 3) - 2 - - -Creating a Mock from an Existing Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One problem with over use of mocking is that it couples your tests to the -implementation of your mocks rather than your real code. Suppose you have a -class that implements ``some_method``. In a test for another class, you -provide a mock of this object that *also* provides ``some_method``. If later -you refactor the first class, so that it no longer has ``some_method`` - then -your tests will continue to pass even though your code is now broken! - -:class:`Mock` allows you to provide an object as a specification for the mock, -using the *spec* keyword argument. Accessing methods / attributes on the -mock that don't exist on your specification object will immediately raise an -attribute error. If you change the implementation of your specification, then -tests that use that class will start failing immediately without you having to -instantiate the class in those tests. - - >>> mock = Mock(spec=SomeClass) - >>> mock.old_method() - Traceback (most recent call last): - ... - AttributeError: object has no attribute 'old_method' - -Using a specification also enables a smarter matching of calls made to the -mock, regardless of whether some parameters were passed as positional or -named arguments:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, 3) - - >>> mock.assert_called_with(a=1, b=2, c=3) - -If you want this smarter matching to also work with method calls on the mock, -you can use :ref:`auto-speccing `. - -If you want a stronger form of specification that prevents the setting -of arbitrary attributes as well as the getting of them then you can use -*spec_set* instead of *spec*. - - - -Patch Decorators ----------------- - -.. note:: - - With :func:`patch` it matters that you patch objects in the namespace where - they are looked up. This is normally straightforward, but for a quick guide - read :ref:`where to patch `. - - -A common need in tests is to patch a class attribute or a module attribute, -for example patching a builtin or patching a class in a module to test that it -is instantiated. Modules and classes are effectively global, so patching on -them has to be undone after the test or the patch will persist into other -tests and cause hard to diagnose problems. - -mock provides three convenient decorators for this: :func:`patch`, :func:`patch.object` and -:func:`patch.dict`. ``patch`` takes a single string, of the form -``package.module.Class.attribute`` to specify the attribute you are patching. It -also optionally takes a value that you want the attribute (or class or -whatever) to be replaced with. 'patch.object' takes an object and the name of -the attribute you would like patched, plus optionally the value to patch it -with. - -``patch.object``: - - >>> original = SomeClass.attribute - >>> @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test(): - ... assert SomeClass.attribute == sentinel.attribute - ... - >>> test() - >>> assert SomeClass.attribute == original - - >>> @patch('package.module.attribute', sentinel.attribute) - ... def test(): - ... from package.module import attribute - ... assert attribute is sentinel.attribute - ... - >>> test() - -If you are patching a module (including :mod:`builtins`) then use :func:`patch` -instead of :func:`patch.object`: - - >>> mock = MagicMock(return_value=sentinel.file_handle) - >>> with patch('builtins.open', mock): - ... handle = open('filename', 'r') - ... - >>> mock.assert_called_with('filename', 'r') - >>> assert handle == sentinel.file_handle, "incorrect file handle returned" - -The module name can be 'dotted', in the form ``package.module`` if needed: - - >>> @patch('package.module.ClassName.attribute', sentinel.attribute) - ... def test(): - ... from package.module import ClassName - ... assert ClassName.attribute == sentinel.attribute - ... - >>> test() - -A nice pattern is to actually decorate test methods themselves: - - >>> class MyTest(unittest2.TestCase): - ... @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test_something(self): - ... self.assertEqual(SomeClass.attribute, sentinel.attribute) - ... - >>> original = SomeClass.attribute - >>> MyTest('test_something').test_something() - >>> assert SomeClass.attribute == original - -If you want to patch with a Mock, you can use :func:`patch` with only one argument -(or :func:`patch.object` with two arguments). The mock will be created for you and -passed into the test function / method: - - >>> class MyTest(unittest2.TestCase): - ... @patch.object(SomeClass, 'static_method') - ... def test_something(self, mock_method): - ... SomeClass.static_method() - ... mock_method.assert_called_with() - ... - >>> MyTest('test_something').test_something() - -You can stack up multiple patch decorators using this pattern: - - >>> class MyTest(unittest2.TestCase): - ... @patch('package.module.ClassName1') - ... @patch('package.module.ClassName2') - ... def test_something(self, MockClass2, MockClass1): - ... self.assertIs(package.module.ClassName1, MockClass1) - ... self.assertIs(package.module.ClassName2, MockClass2) - ... - >>> MyTest('test_something').test_something() - -When you nest patch decorators the mocks are passed in to the decorated -function in the same order they applied (the normal *python* order that -decorators are applied). This means from the bottom up, so in the example -above the mock for ``test_module.ClassName2`` is passed in first. - -There is also :func:`patch.dict` for setting values in a dictionary just -during a scope and restoring the dictionary to its original state when the test -ends: - - >>> foo = {'key': 'value'} - >>> original = foo.copy() - >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == original - -``patch``, ``patch.object`` and ``patch.dict`` can all be used as context managers. - -Where you use :func:`patch` to create a mock for you, you can get a reference to the -mock using the "as" form of the with statement: - - >>> class ProductionClass: - ... def method(self): - ... pass - ... - >>> with patch.object(ProductionClass, 'method') as mock_method: - ... mock_method.return_value = None - ... real = ProductionClass() - ... real.method(1, 2, 3) - ... - >>> mock_method.assert_called_with(1, 2, 3) - - -As an alternative ``patch``, ``patch.object`` and ``patch.dict`` can be used as -class decorators. When used in this way it is the same as applying the -decorator individually to every method whose name starts with "test". - - -.. _further-examples: - -Further Examples ----------------- - - -Here are some more examples for some slightly more advanced scenarios. +Here are some more examples for some slightly more advanced scenarios than in +the :ref:`getting started ` guide. Mocking chained calls -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Mocking chained calls is actually straightforward with mock once you understand the :attr:`~Mock.return_value` attribute. When a mock is called for -the first time, or you fetch its ``return_value`` before it has been called, a -new :class:`Mock` is created. +the first time, or you fetch its `return_value` before it has been called, a +new `Mock` is created. This means that you can see how the object returned from a call to a mocked -object has been used by interrogating the ``return_value`` mock: +object has been used by interrogating the `return_value` mock: >>> mock = Mock() >>> mock().foo(a=2, b=3) @@ -460,35 +37,35 @@ So, suppose we have some code that looks a little bit like this: - >>> class Something: + >>> class Something(object): ... def __init__(self): ... self.backend = BackendProvider() ... def method(self): ... response = self.backend.get_endpoint('foobar').create_call('spam', 'eggs').start_call() ... # more code -Assuming that ``BackendProvider`` is already well tested, how do we test -``method()``? Specifically, we want to test that the code section ``# more -code`` uses the response object in the correct way. +Assuming that `BackendProvider` is already well tested, how do we test +`method()`? Specifically, we want to test that the code section `# more +code` uses the response object in the correct way. As this chain of calls is made from an instance attribute we can monkey patch -the ``backend`` attribute on a ``Something`` instance. In this particular case +the `backend` attribute on a `Something` instance. In this particular case we are only interested in the return value from the final call to -``start_call`` so we don't have much configuration to do. Let's assume the +`start_call` so we don't have much configuration to do. Let's assume the object it returns is 'file-like', so we'll ensure that our response object -uses the builtin :func:`open` as its ``spec``. +uses the builtin `file` as its `spec`. To do this we create a mock instance as our mock backend and create a mock response object for it. To set the response as the return value for that final -``start_call`` we could do this:: +`start_call` we could do this: - mock_backend.get_endpoint.return_value.create_call.return_value.start_call.return_value = mock_response + `mock_backend.get_endpoint.return_value.create_call.return_value.start_call.return_value = mock_response`. We can do that in a slightly nicer way using the :meth:`~Mock.configure_mock` method to directly set the return value for us: >>> something = Something() - >>> mock_response = Mock(spec=open) + >>> mock_response = Mock(spec=file) >>> mock_backend = Mock() >>> config = {'get_endpoint.return_value.create_call.return_value.start_call.return_value': mock_response} >>> mock_backend.configure_mock(**config) @@ -501,7 +78,7 @@ Using :attr:`~Mock.mock_calls` we can check the chained call with a single assert. A chained call is several calls in one line of code, so there will be -several entries in ``mock_calls``. We can use :meth:`call.call_list` to create +several entries in `mock_calls`. We can use :meth:`call.call_list` to create this list of calls for us: >>> chained = call.get_endpoint('foobar').create_call('spam', 'eggs').start_call() @@ -510,22 +87,23 @@ Partial mocking -~~~~~~~~~~~~~~~ +--------------- -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`date.today` method. +In some tests I wanted to mock out a call to `datetime.date.today() +`_ to return +a known date, but I didn't want to prevent the code under test from +creating new date objects. Unfortunately `datetime.date` is written in C, and +so I couldn't just monkey-patch out the static `date.today` method. I found a simple way of doing this that involved effectively wrapping the date class with a mock, but passing through calls to the constructor to the real class (and returning real instances). The :func:`patch decorator ` is used here to -mock out the ``date`` class in the module under test. The :attr:`side_effect` +mock out the `date` class in the module under test. The :attr:`side_effect` attribute on the mock date class is then set to a lambda function that returns a real date. When the mock date class is called a real date will be -constructed and returned by ``side_effect``. +constructed and returned by `side_effect`. >>> from datetime import date >>> with patch('mymodule.date') as mock_date: @@ -536,36 +114,38 @@ ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) ... -Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the +Note that we don't patch `datetime.date` globally, we patch `date` in the module that *uses* it. See :ref:`where to patch `. -When ``date.today()`` is called a known date is returned, but calls to the -``date(...)`` constructor still return normal dates. Without this you can find +When `date.today()` is called a known date is returned, but calls to the +`date(...)` constructor still return normal dates. Without this you can find yourself having to calculate an expected result using exactly the same algorithm as the code under test, which is a classic testing anti-pattern. -Calls to the date constructor are recorded in the ``mock_date`` attributes -(``call_count`` and friends) which may also be useful for your tests. +Calls to the date constructor are recorded in the `mock_date` attributes +(`call_count` and friends) which may also be useful for your tests. An alternative way of dealing with mocking dates, or other builtin classes, is discussed in `this blog entry -`_. +`_. Mocking a Generator Method -~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------- -A Python generator is a function or method that uses the :keyword:`yield` statement -to return a series of values when iterated over [#]_. +A Python generator is a function or method that uses the `yield statement +`_ to +return a series of values when iterated over [#]_. A generator method / function is called to return the generator object. It is the generator object that is then iterated over. The protocol method for -iteration is :meth:`~container.__iter__`, so we can -mock this using a :class:`MagicMock`. +iteration is `__iter__ +`_, so we can +mock this using a `MagicMock`. Here's an example class with an "iter" method implemented as a generator: - >>> class Foo: + >>> class Foo(object): ... def iter(self): ... for i in [1, 2, 3]: ... yield i @@ -578,7 +158,7 @@ How would we mock this class, and in particular its "iter" method? To configure the values returned from the iteration (implicit in the call to -:class:`list`), we need to configure the object returned by the call to ``foo.iter()``. +`list`), we need to configure the object returned by the call to `foo.iter()`. >>> mock_foo = MagicMock() >>> mock_foo.iter.return_value = iter([1, 2, 3]) @@ -593,23 +173,23 @@ Applying the same patch to every test method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------- If you want several patches in place for multiple test methods the obvious way is to apply the patch decorators to every method. This can feel like unnecessary -repetition. For Python 2.6 or more recent you can use :func:`patch` (in all its +repetition. For Python 2.6 or more recent you can use `patch` (in all its various forms) as a class decorator. This applies the patches to all test methods on the class. A test method is identified by methods whose names start -with ``test``: +with `test`: >>> @patch('mymodule.SomeClass') ... class MyTest(TestCase): ... ... def test_one(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) + ... self.assertTrue(mymodule.SomeClass is MockSomeClass) ... ... def test_two(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) + ... self.assertTrue(mymodule.SomeClass is MockSomeClass) ... ... def not_a_test(self): ... return 'something' @@ -620,7 +200,7 @@ 'something' An alternative way of managing patches is to use the :ref:`start-and-stop`. -These allow you to move the patching into your ``setUp`` and ``tearDown`` methods. +These allow you to move the patching into your `setUp` and `tearDown` methods. >>> class MyTest(TestCase): ... def setUp(self): @@ -628,7 +208,7 @@ ... self.mock_foo = self.patcher.start() ... ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) + ... self.assertTrue(mymodule.foo is self.mock_foo) ... ... def tearDown(self): ... self.patcher.stop() @@ -636,7 +216,7 @@ >>> MyTest('test_foo').run() If you use this technique you must ensure that the patching is "undone" by -calling ``stop``. This can be fiddlier than you might think, because if an +calling `stop`. This can be fiddlier than you might think, because if an exception is raised in the setUp then tearDown is not called. :meth:`unittest.TestCase.addCleanup` makes this easier: @@ -647,13 +227,13 @@ ... self.mock_foo = patcher.start() ... ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) + ... self.assertTrue(mymodule.foo is self.mock_foo) ... >>> MyTest('test_foo').run() Mocking Unbound Methods -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Whilst writing tests today I needed to patch an *unbound method* (patching the method on the class rather than on the instance). I needed self to be passed @@ -666,16 +246,16 @@ patch out methods with a mock that having to create a real function becomes a nuisance. -If you pass ``autospec=True`` to patch then it does the patching with a +If you pass `autospec=True` to patch then it does the patching with a *real* function object. This function object has the same signature as the one it is replacing, but delegates to a mock under the hood. You still get your mock auto-created in exactly the same way as before. What it means though, is that if you use it to patch out an unbound method on a class the mocked function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I +It will have `self` passed in as the first argument, which is exactly what I wanted: - >>> class Foo: + >>> class Foo(object): ... def foo(self): ... pass ... @@ -687,12 +267,12 @@ 'foo' >>> mock_foo.assert_called_once_with(foo) -If we don't use ``autospec=True`` then the unbound method is patched out -with a Mock instance instead, and isn't called with ``self``. +If we don't use `autospec=True` then the unbound method is patched out +with a Mock instance instead, and isn't called with `self`. Checking multiple calls with mock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- mock has a nice API for making assertions about how your mock objects are used. @@ -712,7 +292,7 @@ ... AssertionError: Expected to be called once. Called 2 times. -Both ``assert_called_with`` and ``assert_called_once_with`` make assertions about +Both `assert_called_with` and `assert_called_once_with` make assertions about the *most recent* call. If your mock is going to be called several times, and you want to make assertions about *all* those calls you can use :attr:`~Mock.call_args_list`: @@ -725,8 +305,8 @@ [call(1, 2, 3), call(4, 5, 6), call()] The :data:`call` helper makes it easy to make assertions about these calls. You -can build up a list of expected calls and compare it to ``call_args_list``. This -looks remarkably similar to the repr of the ``call_args_list``: +can build up a list of expected calls and compare it to `call_args_list`. This +looks remarkably similar to the repr of the `call_args_list`: >>> expected = [call(1, 2, 3), call(4, 5, 6), call()] >>> mock.call_args_list == expected @@ -734,10 +314,10 @@ Coping with mutable arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- Another situation is rare, but can bite you, is when your mock is called with -mutable arguments. ``call_args`` and ``call_args_list`` store *references* to the +mutable arguments. `call_args` and `call_args_list` store *references* to the arguments. If the arguments are mutated by the code under test then you can no longer make assertions about what the values were when the mock was called. @@ -752,28 +332,28 @@ frob(val) val.clear() -When we try to test that ``grob`` calls ``frob`` with the correct argument look +When we try to test that `grob` calls `frob` with the correct argument look what happens: >>> with patch('mymodule.frob') as mock_frob: - ... val = {6} + ... val = set([6]) ... mymodule.grob(val) ... >>> val - set() - >>> mock_frob.assert_called_with({6}) + set([]) + >>> mock_frob.assert_called_with(set([6])) Traceback (most recent call last): ... - AssertionError: Expected: (({6},), {}) - Called with: ((set(),), {}) + AssertionError: Expected: ((set([6]),), {}) + Called with: ((set([]),), {}) One possibility would be for mock to copy the arguments you pass in. This could then cause problems if you do assertions that rely on object identity for equality. Here's one solution that uses the :attr:`side_effect` -functionality. If you provide a ``side_effect`` function for a mock then -``side_effect`` will be called with the same args as the mock. This gives us an +functionality. If you provide a `side_effect` function for a mock then +`side_effect` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this example I'm using *another* mock to store the arguments so that I can use the mock methods for doing the assertion. Again a helper function sets this up for @@ -793,35 +373,35 @@ ... >>> with patch('mymodule.frob') as mock_frob: ... new_mock = copy_call_args(mock_frob) - ... val = {6} + ... val = set([6]) ... mymodule.grob(val) ... - >>> new_mock.assert_called_with({6}) + >>> new_mock.assert_called_with(set([6])) >>> new_mock.call_args - call({6}) + call(set([6])) -``copy_call_args`` is called with the mock that will be called. It returns a new -mock that we do the assertion on. The ``side_effect`` function makes a copy of -the args and calls our ``new_mock`` with the copy. +`copy_call_args` is called with the mock that will be called. It returns a new +mock that we do the assertion on. The `side_effect` function makes a copy of +the args and calls our `new_mock` with the copy. .. note:: If your mock is only going to be used once there is an easier way of checking arguments at the point they are called. You can simply do the - checking inside a ``side_effect`` function. + checking inside a `side_effect` function. >>> def side_effect(arg): - ... assert arg == {6} + ... assert arg == set([6]) ... >>> mock = Mock(side_effect=side_effect) - >>> mock({6}) + >>> mock(set([6])) >>> mock(set()) Traceback (most recent call last): ... AssertionError -An alternative approach is to create a subclass of :class:`Mock` or -:class:`MagicMock` that copies (using :func:`copy.deepcopy`) the arguments. +An alternative approach is to create a subclass of `Mock` or `MagicMock` that +copies (using :func:`copy.deepcopy`) the arguments. Here's an example implementation: >>> from copy import deepcopy @@ -839,18 +419,68 @@ >>> c.assert_called_with(arg) Traceback (most recent call last): ... - AssertionError: Expected call: mock({1}) - Actual call: mock(set()) + AssertionError: Expected call: mock(set([1])) + Actual call: mock(set([])) >>> c.foo -When you subclass ``Mock`` or ``MagicMock`` all dynamically created attributes, -and the ``return_value`` will use your subclass automatically. That means all -children of a ``CopyingMock`` will also have the type ``CopyingMock``. +When you subclass `Mock` or `MagicMock` all dynamically created attributes, +and the `return_value` will use your subclass automatically. That means all +children of a `CopyingMock` will also have the type `CopyingMock`. + + +Multiple calls with different effects +------------------------------------- + +Handling code that needs to behave differently on subsequent calls during the +test can be tricky. For example you may have a function that needs to raise +an exception the first time it is called but returns a response on the second +call (testing retry behaviour). + +One approach is to use a :attr:`side_effect` function that replaces itself. The +first time it is called the `side_effect` sets a new `side_effect` that will +be used for the second call. It then raises an exception: + + >>> def side_effect(*args): + ... def second_call(*args): + ... return 'response' + ... mock.side_effect = second_call + ... raise Exception('boom') + ... + >>> mock = Mock(side_effect=side_effect) + >>> mock('first') + Traceback (most recent call last): + ... + Exception: boom + >>> mock('second') + 'response' + >>> mock.assert_called_with('second') + +Another perfectly valid way would be to pop return values from a list. If the +return value is an exception, raise it instead of returning it: + + >>> returns = [Exception('boom'), 'response'] + >>> def side_effect(*args): + ... result = returns.pop(0) + ... if isinstance(result, Exception): + ... raise result + ... return result + ... + >>> mock = Mock(side_effect=side_effect) + >>> mock('first') + Traceback (most recent call last): + ... + Exception: boom + >>> mock('second') + 'response' + >>> mock.assert_called_with('second') + +Which approach you prefer is a matter of taste. The first approach is actually +a line shorter but maybe the second approach is more readable. Nesting Patches -~~~~~~~~~~~~~~~ +--------------- Using patch as a context manager is nice, but if you do multiple patches you can end up with nested with statements indenting further and further to the @@ -870,9 +500,9 @@ >>> MyTest('test_foo').test_foo() >>> assert mymodule.Foo is original -With unittest ``cleanup`` functions and the :ref:`start-and-stop` we can +With unittest `cleanup` functions and the :ref:`start-and-stop` we can achieve the same effect without the nested indentation. A simple helper -method, ``create_patch``, puts the patch in place and returns the created mock +method, `create_patch`, puts the patch in place and returns the created mock for us: >>> class MyTest(TestCase): @@ -898,7 +528,7 @@ Mocking a dictionary with MagicMock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------- You may want to mock a dictionary, or other container object, recording all access to it whilst having it still behave like a dictionary. @@ -907,11 +537,11 @@ and using :data:`~Mock.side_effect` to delegate dictionary access to a real underlying dictionary that is under our control. -When the :meth:`__getitem__` and :meth:`__setitem__` methods of our ``MagicMock`` are called -(normal dictionary access) then ``side_effect`` is called with the key (and in -the case of ``__setitem__`` the value too). We can also control what is returned. +When the `__getitem__` and `__setitem__` methods of our `MagicMock` are called +(normal dictionary access) then `side_effect` is called with the key (and in +the case of `__setitem__` the value too). We can also control what is returned. -After the ``MagicMock`` has been used we can use attributes like +After the `MagicMock` has been used we can use attributes like :data:`~Mock.call_args_list` to assert about how the dictionary was used: >>> my_dict = {'a': 1, 'b': 2, 'c': 3} @@ -927,23 +557,23 @@ .. note:: - An alternative to using ``MagicMock`` is to use ``Mock`` and *only* provide + An alternative to using `MagicMock` is to use `Mock` and *only* provide the magic methods you specifically want: >>> mock = Mock() - >>> mock.__getitem__ = Mock(side_effect=getitem) - >>> mock.__setitem__ = Mock(side_effect=setitem) + >>> mock.__setitem__ = Mock(side_effect=getitem) + >>> mock.__getitem__ = Mock(side_effect=setitem) - A *third* option is to use ``MagicMock`` but passing in ``dict`` as the *spec* - (or *spec_set*) argument so that the ``MagicMock`` created only has + A *third* option is to use `MagicMock` but passing in `dict` as the `spec` + (or `spec_set`) argument so that the `MagicMock` created only has dictionary magic methods available: >>> mock = MagicMock(spec_set=dict) >>> mock.__getitem__.side_effect = getitem >>> mock.__setitem__.side_effect = setitem -With these side effect functions in place, the ``mock`` will behave like a normal -dictionary but recording the access. It even raises a :exc:`KeyError` if you try +With these side effect functions in place, the `mock` will behave like a normal +dictionary but recording the access. It even raises a `KeyError` if you try to access a key that doesn't exist. >>> mock['a'] @@ -973,10 +603,10 @@ Mock subclasses and their attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------ -There are various reasons why you might want to subclass :class:`Mock`. One -reason might be to add helper methods. Here's a silly example: +There are various reasons why you might want to subclass `Mock`. One reason +might be to add helper methods. Here's a silly example: >>> class MyMock(MagicMock): ... def has_been_called(self): @@ -991,9 +621,9 @@ >>> mymock.has_been_called() True -The standard behaviour for ``Mock`` instances is that attributes and the return +The standard behaviour for `Mock` instances is that attributes and the return value mocks are of the same type as the mock they are accessed on. This ensures -that ``Mock`` attributes are ``Mocks`` and ``MagicMock`` attributes are ``MagicMocks`` +that `Mock` attributes are `Mocks` and `MagicMock` attributes are `MagicMocks` [#]_. So if you're subclassing to add helper methods then they'll also be available on the attributes and return value mock of instances of your subclass. @@ -1013,10 +643,10 @@ `_. Having this applied to attributes too actually causes errors. -``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create +`Mock` (in all its flavours) uses a method called `_get_child_mock` to create these "sub-mocks" for attributes and return values. You can prevent your subclass being used for attributes by overriding this method. The signature is -that it takes arbitrary keyword arguments (``**kwargs``) which are then passed +that it takes arbitrary keyword arguments (`**kwargs`) which are then passed onto the mock constructor: >>> class Subclass(MagicMock): @@ -1036,7 +666,7 @@ Mocking imports with patch.dict -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- One situation where mocking can be hard is where you have a local import inside a function. These are harder to mock because they aren't using an object from @@ -1049,17 +679,17 @@ import (store the module as a class or module attribute and only do the import on first use). -That aside there is a way to use ``mock`` to affect the results of an import. -Importing fetches an *object* from the :data:`sys.modules` dictionary. Note that it +That aside there is a way to use `mock` to affect the results of an import. +Importing fetches an *object* from the `sys.modules` dictionary. Note that it fetches an *object*, which need not be a module. Importing a module for the first time results in a module object being put in `sys.modules`, so usually when you import something you get a module back. This need not be the case however. This means you can use :func:`patch.dict` to *temporarily* put a mock in place -in :data:`sys.modules`. Any imports whilst this patch is active will fetch the mock. +in `sys.modules`. Any imports whilst this patch is active will fetch the mock. When the patch is complete (the decorated function exits, the with statement -body is complete or ``patcher.stop()`` is called) then whatever was there +body is complete or `patcher.stop()` is called) then whatever was there previously will be restored safely. Here's an example that mocks out the 'fooble' module. @@ -1073,10 +703,10 @@ >>> assert 'fooble' not in sys.modules >>> mock.blob.assert_called_once_with() -As you can see the ``import fooble`` succeeds, but on exit there is no 'fooble' -left in :data:`sys.modules`. +As you can see the `import fooble` succeeds, but on exit there is no 'fooble' +left in `sys.modules`. -This also works for the ``from module import name`` form: +This also works for the `from module import name` form: >>> mock = Mock() >>> with patch.dict('sys.modules', {'fooble': mock}): @@ -1099,17 +729,17 @@ Tracking order of calls and less verbose call assertions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------------- The :class:`Mock` class allows you to track the *order* of method calls on your mock objects through the :attr:`~Mock.method_calls` attribute. This doesn't allow you to track the order of calls between separate mock objects, however we can use :attr:`~Mock.mock_calls` to achieve the same effect. -Because mocks track calls to child mocks in ``mock_calls``, and accessing an +Because mocks track calls to child mocks in `mock_calls`, and accessing an arbitrary attribute of a mock creates a child mock, we can create our separate mocks from a parent one. Calls to those child mock will then all be recorded, -in order, in the ``mock_calls`` of the parent: +in order, in the `mock_calls` of the parent: >>> manager = Mock() >>> mock_foo = manager.foo @@ -1124,15 +754,15 @@ [call.foo.something(), call.bar.other.thing()] We can then assert about the calls, including the order, by comparing with -the ``mock_calls`` attribute on the manager mock: +the `mock_calls` attribute on the manager mock: >>> expected_calls = [call.foo.something(), call.bar.other.thing()] >>> manager.mock_calls == expected_calls True -If ``patch`` is creating, and putting in place, your mocks then you can attach +If `patch` is creating, and putting in place, your mocks then you can attach them to a manager mock using the :meth:`~Mock.attach_mock` method. After -attaching calls will be recorded in ``mock_calls`` of the manager. +attaching calls will be recorded in `mock_calls` of the manager. >>> manager = MagicMock() >>> with patch('mymodule.Class1') as MockClass1: @@ -1164,12 +794,12 @@ >>> calls = call.one().two().three().call_list() >>> m.assert_has_calls(calls) -Even though the chained call ``m.one().two().three()`` aren't the only calls that +Even though the chained call `m.one().two().three()` aren't the only calls that have been made to the mock, the assert still succeeds. Sometimes a mock may have several calls made to it, and you are only interested in asserting about *some* of those calls. You may not even care about the -order. In this case you can pass ``any_order=True`` to ``assert_has_calls``: +order. In this case you can pass `any_order=True` to `assert_has_calls`: >>> m = MagicMock() >>> m(1), m.two(2, 3), m.seven(7), m.fifty('50') @@ -1179,7 +809,7 @@ More complex argument matching -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Using the same basic concept as :data:`ANY` we can implement matchers to do more complex assertions on objects used as arguments to mocks. @@ -1191,10 +821,10 @@ of this object then we can create a matcher that will check these attributes for us. -You can see in this example how a 'standard' call to ``assert_called_with`` isn't +You can see in this example how a 'standard' call to `assert_called_with` isn't sufficient: - >>> class Foo: + >>> class Foo(object): ... def __init__(self, a, b): ... self.a, self.b = a, b ... @@ -1206,7 +836,7 @@ AssertionError: Expected: call(<__main__.Foo object at 0x...>) Actual call: call(<__main__.Foo object at 0x...>) -A comparison function for our ``Foo`` class might look something like this: +A comparison function for our `Foo` class might look something like this: >>> def compare(self, other): ... if not type(self) == type(other): @@ -1221,7 +851,7 @@ And a matcher object that can use comparison functions like this for its equality operation would look something like this: - >>> class Matcher: + >>> class Matcher(object): ... def __init__(self, compare, some_obj): ... self.compare = compare ... self.some_obj = some_obj @@ -1234,11 +864,11 @@ >>> match_foo = Matcher(compare, Foo(1, 2)) >>> mock.assert_called_with(match_foo) -The ``Matcher`` is instantiated with our compare function and the ``Foo`` object -we want to compare against. In ``assert_called_with`` the ``Matcher`` equality +The `Matcher` is instantiated with our compare function and the `Foo` object +we want to compare against. In `assert_called_with` the `Matcher` equality method will be called, which compares the object the mock was called with against the one we created our matcher with. If they match then -``assert_called_with`` passes, and if they don't an :exc:`AssertionError` is raised: +`assert_called_with` passes, and if they don't an `AssertionError` is raised: >>> match_wrong = Matcher(compare, Foo(3, 4)) >>> mock.assert_called_with(match_wrong) @@ -1248,10 +878,10 @@ Called with: ((,), {}) With a bit of tweaking you could have the comparison function raise the -:exc:`AssertionError` directly and provide a more useful failure message. +`AssertionError` directly and provide a more useful failure message. As of version 1.5, the Python testing library `PyHamcrest -`_ provides similar functionality, +`_ provides similar functionality, that may be useful here, in the form of its equality matcher (`hamcrest.library.integration.match_equality -`_). +`_). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock-getting-started.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/unittest.mock-getting-started.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,419 @@ +:mod:`unittest.mock` --- getting started +======================================== + +.. module:: unittest.mock + :synopsis: Mock object library. +.. moduleauthor:: Michael Foord +.. currentmodule:: unittest.mock + +.. versionadded:: 3.3 + + +.. _getting-started: + +Using Mock +---------- + +Mock Patching Methods +~~~~~~~~~~~~~~~~~~~~~ + +Common uses for :class:`Mock` objects include: + +* Patching methods +* Recording method calls on objects + +You might want to replace a method on an object to check that +it is called with the correct arguments by another part of the system: + + >>> real = SomeClass() + >>> real.method = MagicMock(name='method') + >>> real.method(3, 4, 5, key='value') + + +Once our mock has been used (`real.method` in this example) it has methods +and attributes that allow you to make assertions about how it has been used. + +.. note:: + + In most of these examples the :class:`Mock` and :class:`MagicMock` classes + are interchangeable. As the `MagicMock` is the more capable class it makes + a sensible one to use by default. + +Once the mock has been called its :attr:`~Mock.called` attribute is set to +`True`. More importantly we can use the :meth:`~Mock.assert_called_with` or +:meth`~Mock.assert_called_once_with` method to check that it was called with +the correct arguments. + +This example tests that calling `ProductionClass().method` results in a call to +the `something` method: + + >>> class ProductionClass(object): + ... def method(self): + ... self.something(1, 2, 3) + ... def something(self, a, b, c): + ... pass + ... + >>> real = ProductionClass() + >>> real.something = MagicMock() + >>> real.method() + >>> real.something.assert_called_once_with(1, 2, 3) + + + +Mock for Method Calls on an Object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the last example we patched a method directly on an object to check that it +was called correctly. Another common use case is to pass an object into a +method (or some part of the system under test) and then check that it is used +in the correct way. + +The simple `ProductionClass` below has a `closer` method. If it is called with +an object then it calls `close` on it. + + >>> class ProductionClass(object): + ... def closer(self, something): + ... something.close() + ... + +So to test it we need to pass in an object with a `close` method and check +that it was called correctly. + + >>> real = ProductionClass() + >>> mock = Mock() + >>> real.closer(mock) + >>> mock.close.assert_called_with() + +We don't have to do any work to provide the 'close' method on our mock. +Accessing close creates it. So, if 'close' hasn't already been called then +accessing it in the test will create it, but :meth:`~Mock.assert_called_with` +will raise a failure exception. + + +Mocking Classes +~~~~~~~~~~~~~~~ + +A common use case is to mock out classes instantiated by your code under test. +When you patch a class, then that class is replaced with a mock. Instances +are created by *calling the class*. This means you access the "mock instance" +by looking at the return value of the mocked class. + +In the example below we have a function `some_function` that instantiates `Foo` +and calls a method on it. The call to `patch` replaces the class `Foo` with a +mock. The `Foo` instance is the result of calling the mock, so it is configured +by modify the mock :attr:`~Mock.return_value`. + + >>> def some_function(): + ... instance = module.Foo() + ... return instance.method() + ... + >>> with patch('module.Foo') as mock: + ... instance = mock.return_value + ... instance.method.return_value = 'the result' + ... result = some_function() + ... assert result == 'the result' + + +Naming your mocks +~~~~~~~~~~~~~~~~~ + +It can be useful to give your mocks a name. The name is shown in the repr of +the mock and can be helpful when the mock appears in test failure messages. The +name is also propagated to attributes or methods of the mock: + + >>> mock = MagicMock(name='foo') + >>> mock + + >>> mock.method + + + +Tracking all Calls +~~~~~~~~~~~~~~~~~~ + +Often you want to track more than a single call to a method. The +:attr:`~Mock.mock_calls` attribute records all calls +to child attributes of the mock - and also to their children. + + >>> mock = MagicMock() + >>> mock.method() + + >>> mock.attribute.method(10, x=53) + + >>> mock.mock_calls + [call.method(), call.attribute.method(10, x=53)] + +If you make an assertion about `mock_calls` and any unexpected methods +have been called, then the assertion will fail. This is useful because as well +as asserting that the calls you expected have been made, you are also checking +that they were made in the right order and with no additional calls: + +You use the :data:`call` object to construct lists for comparing with +`mock_calls`: + + >>> expected = [call.method(), call.attribute.method(10, x=53)] + >>> mock.mock_calls == expected + True + + +Setting Return Values and Attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Setting the return values on a mock object is trivially easy: + + >>> mock = Mock() + >>> mock.return_value = 3 + >>> mock() + 3 + +Of course you can do the same for methods on the mock: + + >>> mock = Mock() + >>> mock.method.return_value = 3 + >>> mock.method() + 3 + +The return value can also be set in the constructor: + + >>> mock = Mock(return_value=3) + >>> mock() + 3 + +If you need an attribute setting on your mock, just do it: + + >>> mock = Mock() + >>> mock.x = 3 + >>> mock.x + 3 + +Sometimes you want to mock up a more complex situation, like for example +`mock.connection.cursor().execute("SELECT 1")`. If we wanted this call to +return a list, then we have to configure the result of the nested call. + +We can use :data:`call` to construct the set of calls in a "chained call" like +this for easy assertion afterwards: + + >>> mock = Mock() + >>> cursor = mock.connection.cursor.return_value + >>> cursor.execute.return_value = ['foo'] + >>> mock.connection.cursor().execute("SELECT 1") + ['foo'] + >>> expected = call.connection.cursor().execute("SELECT 1").call_list() + >>> mock.mock_calls + [call.connection.cursor(), call.connection.cursor().execute('SELECT 1')] + >>> mock.mock_calls == expected + True + +It is the call to `.call_list()` that turns our call object into a list of +calls representing the chained calls. + + +Raising exceptions with mocks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A useful attribute is :attr:`~Mock.side_effect`. If you set this to an +exception class or instance then the exception will be raised when the mock +is called. + + >>> mock = Mock(side_effect=Exception('Boom!')) + >>> mock() + Traceback (most recent call last): + ... + Exception: Boom! + + +Side effect functions and iterables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`side_effect` can also be set to a function or an iterable. The use case for +`side_effect` as an iterable is where your mock is going to be called several +times, and you want each call to return a different value. When you set +`side_effect` to an iterable every call to the mock returns the next value +from the iterable: + + >>> mock = MagicMock(side_effect=[4, 5, 6]) + >>> mock() + 4 + >>> mock() + 5 + >>> mock() + 6 + + +For more advanced use cases, like dynamically varying the return values +depending on what the mock is called with, `side_effect` can be a function. +The function will be called with the same arguments as the mock. Whatever the +function returns is what the call returns: + + >>> vals = {(1, 2): 1, (2, 3): 2} + >>> def side_effect(*args): + ... return vals[args] + ... + >>> mock = MagicMock(side_effect=side_effect) + >>> mock(1, 2) + 1 + >>> mock(2, 3) + 2 + + +Creating a Mock from an Existing Object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +One problem with over use of mocking is that it couples your tests to the +implementation of your mocks rather than your real code. Suppose you have a +class that implements `some_method`. In a test for another class, you +provide a mock of this object that *also* provides `some_method`. If later +you refactor the first class, so that it no longer has `some_method` - then +your tests will continue to pass even though your code is now broken! + +`Mock` allows you to provide an object as a specification for the mock, +using the `spec` keyword argument. Accessing methods / attributes on the +mock that don't exist on your specification object will immediately raise an +attribute error. If you change the implementation of your specification, then +tests that use that class will start failing immediately without you having to +instantiate the class in those tests. + + >>> mock = Mock(spec=SomeClass) + >>> mock.old_method() + Traceback (most recent call last): + ... + AttributeError: object has no attribute 'old_method' + +If you want a stronger form of specification that prevents the setting +of arbitrary attributes as well as the getting of them then you can use +`spec_set` instead of `spec`. + + + +Patch Decorators +---------------- + +.. note:: + + With `patch` it matters that you patch objects in the namespace where they + are looked up. This is normally straightforward, but for a quick guide + read :ref:`where to patch `. + + +A common need in tests is to patch a class attribute or a module attribute, +for example patching a builtin or patching a class in a module to test that it +is instantiated. Modules and classes are effectively global, so patching on +them has to be undone after the test or the patch will persist into other +tests and cause hard to diagnose problems. + +mock provides three convenient decorators for this: `patch`, `patch.object` and +`patch.dict`. `patch` takes a single string, of the form +`package.module.Class.attribute` to specify the attribute you are patching. It +also optionally takes a value that you want the attribute (or class or +whatever) to be replaced with. 'patch.object' takes an object and the name of +the attribute you would like patched, plus optionally the value to patch it +with. + +`patch.object`: + + >>> original = SomeClass.attribute + >>> @patch.object(SomeClass, 'attribute', sentinel.attribute) + ... def test(): + ... assert SomeClass.attribute == sentinel.attribute + ... + >>> test() + >>> assert SomeClass.attribute == original + + >>> @patch('package.module.attribute', sentinel.attribute) + ... def test(): + ... from package.module import attribute + ... assert attribute is sentinel.attribute + ... + >>> test() + +If you are patching a module (including `__builtin__`) then use `patch` +instead of `patch.object`: + + >>> mock = MagicMock(return_value = sentinel.file_handle) + >>> with patch('__builtin__.open', mock): + ... handle = open('filename', 'r') + ... + >>> mock.assert_called_with('filename', 'r') + >>> assert handle == sentinel.file_handle, "incorrect file handle returned" + +The module name can be 'dotted', in the form `package.module` if needed: + + >>> @patch('package.module.ClassName.attribute', sentinel.attribute) + ... def test(): + ... from package.module import ClassName + ... assert ClassName.attribute == sentinel.attribute + ... + >>> test() + +A nice pattern is to actually decorate test methods themselves: + + >>> class MyTest(unittest2.TestCase): + ... @patch.object(SomeClass, 'attribute', sentinel.attribute) + ... def test_something(self): + ... self.assertEqual(SomeClass.attribute, sentinel.attribute) + ... + >>> original = SomeClass.attribute + >>> MyTest('test_something').test_something() + >>> assert SomeClass.attribute == original + +If you want to patch with a Mock, you can use `patch` with only one argument +(or `patch.object` with two arguments). The mock will be created for you and +passed into the test function / method: + + >>> class MyTest(unittest2.TestCase): + ... @patch.object(SomeClass, 'static_method') + ... def test_something(self, mock_method): + ... SomeClass.static_method() + ... mock_method.assert_called_with() + ... + >>> MyTest('test_something').test_something() + +You can stack up multiple patch decorators using this pattern: + + >>> class MyTest(unittest2.TestCase): + ... @patch('package.module.ClassName1') + ... @patch('package.module.ClassName2') + ... def test_something(self, MockClass2, MockClass1): + ... self.assertTrue(package.module.ClassName1 is MockClass1) + ... self.assertTrue(package.module.ClassName2 is MockClass2) + ... + >>> MyTest('test_something').test_something() + +When you nest patch decorators the mocks are passed in to the decorated +function in the same order they applied (the normal *python* order that +decorators are applied). This means from the bottom up, so in the example +above the mock for `test_module.ClassName2` is passed in first. + +There is also :func:`patch.dict` for setting values in a dictionary just +during a scope and restoring the dictionary to its original state when the test +ends: + + >>> foo = {'key': 'value'} + >>> original = foo.copy() + >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): + ... assert foo == {'newkey': 'newvalue'} + ... + >>> assert foo == original + +`patch`, `patch.object` and `patch.dict` can all be used as context managers. + +Where you use `patch` to create a mock for you, you can get a reference to the +mock using the "as" form of the with statement: + + >>> class ProductionClass(object): + ... def method(self): + ... pass + ... + >>> with patch.object(ProductionClass, 'method') as mock_method: + ... mock_method.return_value = None + ... real = ProductionClass() + ... real.method(1, 2, 3) + ... + >>> mock_method.assert_called_with(1, 2, 3) + + +As an alternative `patch`, `patch.object` and `patch.dict` can be used as +class decorators. When used in this way it is the same as applying the +decorator indvidually to every method whose name starts with "test". + +For some more advanced examples, see the :ref:`further-examples` page. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock-helpers.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/unittest.mock-helpers.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,537 @@ +:mod:`unittest.mock` --- helpers +================================ + +.. module:: unittest.mock + :synopsis: Mock object library. +.. moduleauthor:: Michael Foord +.. currentmodule:: unittest.mock + +.. versionadded:: 3.3 + + +sentinel +-------- + +.. data:: sentinel + + The ``sentinel`` object provides a convenient way of providing unique + objects for your tests. + + Attributes are created on demand when you access them by name. Accessing + the same attribute will always return the same object. The objects + returned have a sensible repr so that test failure messages are readable. + +Sometimes when testing you need to test that a specific object is passed as an +argument to another method, or returned. It can be common to create named +sentinel objects to test this. `sentinel` provides a convenient way of +creating and testing the identity of objects like this. + +In this example we monkey patch `method` to return `sentinel.some_object`: + + >>> real = ProductionClass() + >>> real.method = Mock(name="method") + >>> real.method.return_value = sentinel.some_object + >>> result = real.method() + >>> assert result is sentinel.some_object + >>> sentinel.some_object + sentinel.some_object + + +DEFAULT +------- + + +.. data:: DEFAULT + + The `DEFAULT` object is a pre-created sentinel (actually + `sentinel.DEFAULT`). It can be used by :attr:`~Mock.side_effect` + functions to indicate that the normal return value should be used. + + + +call +---- + +.. function:: call(*args, **kwargs) + + `call` is a helper object for making simpler assertions, for comparing + with :attr:`~Mock.call_args`, :attr:`~Mock.call_args_list`, + :attr:`~Mock.mock_calls` and:attr: `~Mock.method_calls`. `call` can also be + used with :meth:`~Mock.assert_has_calls`. + + >>> m = MagicMock(return_value=None) + >>> m(1, 2, a='foo', b='bar') + >>> m() + >>> m.call_args_list == [call(1, 2, a='foo', b='bar'), call()] + True + +.. method:: call.call_list() + + For a call object that represents multiple calls, `call_list` + returns a list of all the intermediate calls as well as the + final call. + +`call_list` is particularly useful for making assertions on "chained calls". A +chained call is multiple calls on a single line of code. This results in +multiple entries in :attr:`~Mock.mock_calls` on a mock. Manually constructing +the sequence of calls can be tedious. + +:meth:`~call.call_list` can construct the sequence of calls from the same +chained call: + + >>> m = MagicMock() + >>> m(1).method(arg='foo').other('bar')(2.0) + + >>> kall = call(1).method(arg='foo').other('bar')(2.0) + >>> kall.call_list() + [call(1), + call().method(arg='foo'), + call().method().other('bar'), + call().method().other()(2.0)] + >>> m.mock_calls == kall.call_list() + True + +.. _calls-as-tuples: + +A `call` object is either a tuple of (positional args, keyword args) or +(name, positional args, keyword args) depending on how it was constructed. When +you construct them yourself this isn't particularly interesting, but the `call` +objects that are in the :attr:`Mock.call_args`, :attr:`Mock.call_args_list` and +:attr:`Mock.mock_calls` attributes can be introspected to get at the individual +arguments they contain. + +The `call` objects in :attr:`Mock.call_args` and :attr:`Mock.call_args_list` +are two-tuples of (positional args, keyword args) whereas the `call` objects +in :attr:`Mock.mock_calls`, along with ones you construct yourself, are +three-tuples of (name, positional args, keyword args). + +You can use their "tupleness" to pull out the individual arguments for more +complex introspection and assertions. The positional arguments are a tuple +(an empty tuple if there are no positional arguments) and the keyword +arguments are a dictionary: + + >>> m = MagicMock(return_value=None) + >>> m(1, 2, 3, arg='one', arg2='two') + >>> kall = m.call_args + >>> args, kwargs = kall + >>> args + (1, 2, 3) + >>> kwargs + {'arg2': 'two', 'arg': 'one'} + >>> args is kall[0] + True + >>> kwargs is kall[1] + True + + >>> m = MagicMock() + >>> m.foo(4, 5, 6, arg='two', arg2='three') + + >>> kall = m.mock_calls[0] + >>> name, args, kwargs = kall + >>> name + 'foo' + >>> args + (4, 5, 6) + >>> kwargs + {'arg2': 'three', 'arg': 'two'} + >>> name is m.mock_calls[0][0] + True + + +create_autospec +--------------- + +.. function:: create_autospec(spec, spec_set=False, instance=False, **kwargs) + + Create a mock object using another object as a spec. Attributes on the + mock will use the corresponding attribute on the `spec` object as their + spec. + + Functions or methods being mocked will have their arguments checked to + ensure that they are called with the correct signature. + + If `spec_set` is `True` then attempting to set attributes that don't exist + on the spec object will raise an `AttributeError`. + + If a class is used as a spec then the return value of the mock (the + instance of the class) will have the same spec. You can use a class as the + spec for an instance object by passing `instance=True`. The returned mock + will only be callable if instances of the mock are callable. + + `create_autospec` also takes arbitrary keyword arguments that are passed to + the constructor of the created mock. + +See :ref:`auto-speccing` for examples of how to use auto-speccing with +`create_autospec` and the `autospec` argument to :func:`patch`. + + +ANY +--- + +.. data:: ANY + +Sometimes you may need to make assertions about *some* of the arguments in a +call to mock, but either not care about some of the arguments or want to pull +them individually out of :attr:`~Mock.call_args` and make more complex +assertions on them. + +To ignore certain arguments you can pass in objects that compare equal to +*everything*. Calls to :meth:`~Mock.assert_called_with` and +:meth:`~Mock.assert_called_once_with` will then succeed no matter what was +passed in. + + >>> mock = Mock(return_value=None) + >>> mock('foo', bar=object()) + >>> mock.assert_called_once_with('foo', bar=ANY) + +`ANY` can also be used in comparisons with call lists like +:attr:`~Mock.mock_calls`: + + >>> m = MagicMock(return_value=None) + >>> m(1) + >>> m(1, 2) + >>> m(object()) + >>> m.mock_calls == [call(1), call(1, 2), ANY] + True + + + +FILTER_DIR +---------- + +.. data:: FILTER_DIR + +`FILTER_DIR` is a module level variable that controls the way mock objects +respond to `dir` (only for Python 2.6 or more recent). The default is `True`, +which uses the filtering described below, to only show useful members. If you +dislike this filtering, or need to switch it off for diagnostic purposes, then +set `mock.FILTER_DIR = False`. + +With filtering on, `dir(some_mock)` shows only useful attributes and will +include any dynamically created attributes that wouldn't normally be shown. +If the mock was created with a `spec` (or `autospec` of course) then all the +attributes from the original are shown, even if they haven't been accessed +yet: + + >>> dir(Mock()) + ['assert_any_call', + 'assert_called_once_with', + 'assert_called_with', + 'assert_has_calls', + 'attach_mock', + ... + >>> from urllib import request + >>> dir(Mock(spec=request)) + ['AbstractBasicAuthHandler', + 'AbstractDigestAuthHandler', + 'AbstractHTTPHandler', + 'BaseHandler', + ... + +Many of the not-very-useful (private to `Mock` rather than the thing being +mocked) underscore and double underscore prefixed attributes have been +filtered from the result of calling `dir` on a `Mock`. If you dislike this +behaviour you can switch it off by setting the module level switch +`FILTER_DIR`: + + >>> from unittest import mock + >>> mock.FILTER_DIR = False + >>> dir(mock.Mock()) + ['_NonCallableMock__get_return_value', + '_NonCallableMock__get_side_effect', + '_NonCallableMock__return_value_doc', + '_NonCallableMock__set_return_value', + '_NonCallableMock__set_side_effect', + '__call__', + '__class__', + ... + +Alternatively you can just use `vars(my_mock)` (instance members) and +`dir(type(my_mock))` (type members) to bypass the filtering irrespective of +`mock.FILTER_DIR`. + + +mock_open +--------- + +.. function:: mock_open(mock=None, read_data=None) + + A helper function to create a mock to replace the use of `open`. It works + for `open` called directly or used as a context manager. + + The `mock` argument is the mock object to configure. If `None` (the + default) then a `MagicMock` will be created for you, with the API limited + to methods or attributes available on standard file handles. + + `read_data` is a string for the `read` method of the file handle to return. + This is an empty string by default. + +Using `open` as a context manager is a great way to ensure your file handles +are closed properly and is becoming common:: + + with open('/some/path', 'w') as f: + f.write('something') + +The issue is that even if you mock out the call to `open` it is the +*returned object* that is used as a context manager (and has `__enter__` and +`__exit__` called). + +Mocking context managers with a :class:`MagicMock` is common enough and fiddly +enough that a helper function is useful. + + >>> m = mock_open() + >>> with patch('__main__.open', m, create=True): + ... with open('foo', 'w') as h: + ... h.write('some stuff') + ... + >>> m.mock_calls + [call('foo', 'w'), + call().__enter__(), + call().write('some stuff'), + call().__exit__(None, None, None)] + >>> m.assert_called_once_with('foo', 'w') + >>> handle = m() + >>> handle.write.assert_called_once_with('some stuff') + +And for reading files: + + >>> with patch('__main__.open', mock_open(read_data='bibble'), create=True) as m: + ... with open('foo') as h: + ... result = h.read() + ... + >>> m.assert_called_once_with('foo') + >>> assert result == 'bibble' + + +.. _auto-speccing: + +Autospeccing +------------ + +Autospeccing is based on the existing `spec` feature of mock. It limits the +api of mocks to the api of an original object (the spec), but it is recursive +(implemented lazily) so that attributes of mocks only have the same api as +the attributes of the spec. In addition mocked functions / methods have the +same call signature as the original so they raise a `TypeError` if they are +called incorrectly. + +Before I explain how auto-speccing works, here's why it is needed. + +`Mock` is a very powerful and flexible object, but it suffers from two flaws +when used to mock out objects from a system under test. One of these flaws is +specific to the `Mock` api and the other is a more general problem with using +mock objects. + +First the problem specific to `Mock`. `Mock` has two assert methods that are +extremely handy: :meth:`~Mock.assert_called_with` and +:meth:`~Mock.assert_called_once_with`. + + >>> mock = Mock(name='Thing', return_value=None) + >>> mock(1, 2, 3) + >>> mock.assert_called_once_with(1, 2, 3) + >>> mock(1, 2, 3) + >>> mock.assert_called_once_with(1, 2, 3) + Traceback (most recent call last): + ... + AssertionError: Expected to be called once. Called 2 times. + +Because mocks auto-create attributes on demand, and allow you to call them +with arbitrary arguments, if you misspell one of these assert methods then +your assertion is gone: + +.. code-block:: pycon + + >>> mock = Mock(name='Thing', return_value=None) + >>> mock(1, 2, 3) + >>> mock.assret_called_once_with(4, 5, 6) + +Your tests can pass silently and incorrectly because of the typo. + +The second issue is more general to mocking. If you refactor some of your +code, rename members and so on, any tests for code that is still using the +*old api* but uses mocks instead of the real objects will still pass. This +means your tests can all pass even though your code is broken. + +Note that this is another reason why you need integration tests as well as +unit tests. Testing everything in isolation is all fine and dandy, but if you +don't test how your units are "wired together" there is still lots of room +for bugs that tests might have caught. + +`mock` already provides a feature to help with this, called speccing. If you +use a class or instance as the `spec` for a mock then you can only access +attributes on the mock that exist on the real class: + + >>> from urllib import request + >>> mock = Mock(spec=request.Request) + >>> mock.assret_called_with + Traceback (most recent call last): + ... + AttributeError: Mock object has no attribute 'assret_called_with' + +The spec only applies to the mock itself, so we still have the same issue +with any methods on the mock: + +.. code-block:: pycon + + >>> mock.has_data() + + >>> mock.has_data.assret_called_with() + +Auto-speccing solves this problem. You can either pass `autospec=True` to +`patch` / `patch.object` or use the `create_autospec` function to create a +mock with a spec. If you use the `autospec=True` argument to `patch` then the +object that is being replaced will be used as the spec object. Because the +speccing is done "lazily" (the spec is created as attributes on the mock are +accessed) you can use it with very complex or deeply nested objects (like +modules that import modules that import modules) without a big performance +hit. + +Here's an example of it in use: + + >>> from urllib import request + >>> patcher = patch('__main__.request', autospec=True) + >>> mock_request = patcher.start() + >>> request is mock_request + True + >>> mock_request.Request + + +You can see that `request.Request` has a spec. `request.Request` takes two +arguments in the constructor (one of which is `self`). Here's what happens if +we try to call it incorrectly: + + >>> req = request.Request() + Traceback (most recent call last): + ... + TypeError: () takes at least 2 arguments (1 given) + +The spec also applies to instantiated classes (i.e. the return value of +specced mocks): + + >>> req = request.Request('foo') + >>> req + + +`Request` objects are not callable, so the return value of instantiating our +mocked out `request.Request` is a non-callable mock. With the spec in place +any typos in our asserts will raise the correct error: + + >>> req.add_header('spam', 'eggs') + + >>> req.add_header.assret_called_with + Traceback (most recent call last): + ... + AttributeError: Mock object has no attribute 'assret_called_with' + >>> req.add_header.assert_called_with('spam', 'eggs') + +In many cases you will just be able to add `autospec=True` to your existing +`patch` calls and then be protected against bugs due to typos and api +changes. + +As well as using `autospec` through `patch` there is a +:func:`create_autospec` for creating autospecced mocks directly: + + >>> from urllib import request + >>> mock_request = create_autospec(request) + >>> mock_request.Request('foo', 'bar') + + +This isn't without caveats and limitations however, which is why it is not +the default behaviour. In order to know what attributes are available on the +spec object, autospec has to introspect (access attributes) the spec. As you +traverse attributes on the mock a corresponding traversal of the original +object is happening under the hood. If any of your specced objects have +properties or descriptors that can trigger code execution then you may not be +able to use autospec. On the other hand it is much better to design your +objects so that introspection is safe [#]_. + +A more serious problem is that it is common for instance attributes to be +created in the `__init__` method and not to exist on the class at all. +`autospec` can't know about any dynamically created attributes and restricts +the api to visible attributes. + + >>> class Something(object): + ... def __init__(self): + ... self.a = 33 + ... + >>> with patch('__main__.Something', autospec=True): + ... thing = Something() + ... thing.a + ... + Traceback (most recent call last): + ... + AttributeError: Mock object has no attribute 'a' + +There are a few different ways of resolving this problem. The easiest, but +not necessarily the least annoying, way is to simply set the required +attributes on the mock after creation. Just because `autospec` doesn't allow +you to fetch attributes that don't exist on the spec it doesn't prevent you +setting them: + + >>> with patch('__main__.Something', autospec=True): + ... thing = Something() + ... thing.a = 33 + ... + +There is a more aggressive version of both `spec` and `autospec` that *does* +prevent you setting non-existent attributes. This is useful if you want to +ensure your code only *sets* valid attributes too, but obviously it prevents +this particular scenario: + + >>> with patch('__main__.Something', autospec=True, spec_set=True): + ... thing = Something() + ... thing.a = 33 + ... + Traceback (most recent call last): + ... + AttributeError: Mock object has no attribute 'a' + +Probably the best way of solving the problem is to add class attributes as +default values for instance members initialised in `__init__`. Note that if +you are only setting default attributes in `__init__` then providing them via +class attributes (shared between instances of course) is faster too. e.g. + +.. code-block:: python + + class Something(object): + a = 33 + +This brings up another issue. It is relatively common to provide a default +value of `None` for members that will later be an object of a different type. +`None` would be useless as a spec because it wouldn't let you access *any* +attributes or methods on it. As `None` is *never* going to be useful as a +spec, and probably indicates a member that will normally of some other type, +`autospec` doesn't use a spec for members that are set to `None`. These will +just be ordinary mocks (well - `MagicMocks`): + + >>> class Something(object): + ... member = None + ... + >>> mock = create_autospec(Something) + >>> mock.member.foo.bar.baz() + + +If modifying your production classes to add defaults isn't to your liking +then there are more options. One of these is simply to use an instance as the +spec rather than the class. The other is to create a subclass of the +production class and add the defaults to the subclass without affecting the +production class. Both of these require you to use an alternative object as +the spec. Thankfully `patch` supports this - you can simply pass the +alternative object as the `autospec` argument: + + >>> class Something(object): + ... def __init__(self): + ... self.a = 33 + ... + >>> class SomethingForTest(Something): + ... a = 33 + ... + >>> p = patch('__main__.Something', autospec=SomethingForTest) + >>> mock = p.start() + >>> mock.a + + + +.. [#] This only applies to classes or already instantiated objects. Calling + a mocked class to create a mock instance *does not* create a real instance. + It is only attribute lookups - along with calls to `dir` - that are done. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock-magicmethods.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/unittest.mock-magicmethods.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,226 @@ +:mod:`unittest.mock` --- MagicMock and magic method support +=========================================================== + +.. module:: unittest.mock + :synopsis: Mock object library. +.. moduleauthor:: Michael Foord +.. currentmodule:: unittest.mock + +.. versionadded:: 3.3 + + +.. _magic-methods: + +Mocking Magic Methods +--------------------- + +:class:`Mock` supports mocking the Python protocol methods, also known as +"magic methods". This allows mock objects to replace containers or other +objects that implement Python protocols. + +Because magic methods are looked up differently from normal methods [#]_, this +support has been specially implemented. This means that only specific magic +methods are supported. The supported list includes *almost* all of them. If +there are any missing that you need please let us know. + +You mock magic methods by setting the method you are interested in to a function +or a mock instance. If you are using a function then it *must* take ``self`` as +the first argument [#]_. + + >>> def __str__(self): + ... return 'fooble' + ... + >>> mock = Mock() + >>> mock.__str__ = __str__ + >>> str(mock) + 'fooble' + + >>> mock = Mock() + >>> mock.__str__ = Mock() + >>> mock.__str__.return_value = 'fooble' + >>> str(mock) + 'fooble' + + >>> mock = Mock() + >>> mock.__iter__ = Mock(return_value=iter([])) + >>> list(mock) + [] + +One use case for this is for mocking objects used as context managers in a +`with` statement: + + >>> mock = Mock() + >>> mock.__enter__ = Mock(return_value='foo') + >>> mock.__exit__ = Mock(return_value=False) + >>> with mock as m: + ... assert m == 'foo' + ... + >>> mock.__enter__.assert_called_with() + >>> mock.__exit__.assert_called_with(None, None, None) + +Calls to magic methods do not appear in :attr:`~Mock.method_calls`, but they +are recorded in :attr:`~Mock.mock_calls`. + +.. note:: + + If you use the `spec` keyword argument to create a mock then attempting to + set a magic method that isn't in the spec will raise an `AttributeError`. + +The full list of supported magic methods is: + +* ``__hash__``, ``__sizeof__``, ``__repr__`` and ``__str__`` +* ``__dir__``, ``__format__`` and ``__subclasses__`` +* ``__floor__``, ``__trunc__`` and ``__ceil__`` +* Comparisons: ``__cmp__``, ``__lt__``, ``__gt__``, ``__le__``, ``__ge__``, + ``__eq__`` and ``__ne__`` +* Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, + ``__contains__``, ``__len__``, ``__iter__``, ``__getslice__``, + ``__setslice__``, ``__reversed__`` and ``__missing__`` +* Context manager: ``__enter__`` and ``__exit__`` +* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` +* The numeric methods (including right hand and in-place variants): + ``__add__``, ``__sub__``, ``__mul__``, ``__div__``, + ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``, + ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__`` +* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__``, + ``__index__`` and ``__coerce__`` +* Descriptor methods: ``__get__``, ``__set__`` and ``__delete__`` +* Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, + ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` + + +The following methods exist but are *not* supported as they are either in use +by mock, can't be set dynamically, or can cause problems: + +* ``__getattr__``, ``__setattr__``, ``__init__`` and ``__new__`` +* ``__prepare__``, ``__instancecheck__``, ``__subclasscheck__``, ``__del__`` + + + +Magic Mock +---------- + +There are two `MagicMock` variants: `MagicMock` and `NonCallableMagicMock`. + + +.. class:: MagicMock(*args, **kw) + + ``MagicMock`` is a subclass of :class:`Mock` with default implementations + of most of the magic methods. You can use ``MagicMock`` without having to + configure the magic methods yourself. + + The constructor parameters have the same meaning as for :class:`Mock`. + + If you use the `spec` or `spec_set` arguments then *only* magic methods + that exist in the spec will be created. + + +.. class:: NonCallableMagicMock(*args, **kw) + + A non-callable version of `MagicMock`. + + The constructor parameters have the same meaning as for + :class:`MagicMock`, with the exception of `return_value` and + `side_effect` which have no meaning on a non-callable mock. + +The magic methods are setup with `MagicMock` objects, so you can configure them +and use them in the usual way: + + >>> mock = MagicMock() + >>> mock[3] = 'fish' + >>> mock.__setitem__.assert_called_with(3, 'fish') + >>> mock.__getitem__.return_value = 'result' + >>> mock[2] + 'result' + +By default many of the protocol methods are required to return objects of a +specific type. These methods are preconfigured with a default return value, so +that they can be used without you having to do anything if you aren't interested +in the return value. You can still *set* the return value manually if you want +to change the default. + +Methods and their defaults: + +* ``__lt__``: NotImplemented +* ``__gt__``: NotImplemented +* ``__le__``: NotImplemented +* ``__ge__``: NotImplemented +* ``__int__`` : 1 +* ``__contains__`` : False +* ``__len__`` : 1 +* ``__iter__`` : iter([]) +* ``__exit__`` : False +* ``__complex__`` : 1j +* ``__float__`` : 1.0 +* ``__bool__`` : True +* ``__index__`` : 1 +* ``__hash__`` : default hash for the mock +* ``__str__`` : default str for the mock +* ``__sizeof__``: default sizeof for the mock + +For example: + + >>> mock = MagicMock() + >>> int(mock) + 1 + >>> len(mock) + 0 + >>> list(mock) + [] + >>> object() in mock + False + +The two equality method, `__eq__` and `__ne__`, are special. +They do the default equality comparison on identity, using a side +effect, unless you change their return value to return something else: + + >>> MagicMock() == 3 + False + >>> MagicMock() != 3 + True + >>> mock = MagicMock() + >>> mock.__eq__.return_value = True + >>> mock == 3 + True + +The return value of `MagicMock.__iter__` can be any iterable object and isn't +required to be an iterator: + + >>> mock = MagicMock() + >>> mock.__iter__.return_value = ['a', 'b', 'c'] + >>> list(mock) + ['a', 'b', 'c'] + >>> list(mock) + ['a', 'b', 'c'] + +If the return value *is* an iterator, then iterating over it once will consume +it and subsequent iterations will result in an empty list: + + >>> mock.__iter__.return_value = iter(['a', 'b', 'c']) + >>> list(mock) + ['a', 'b', 'c'] + >>> list(mock) + [] + +``MagicMock`` has all of the supported magic methods configured except for some +of the obscure and obsolete ones. You can still set these up if you want. + +Magic methods that are supported but not setup by default in ``MagicMock`` are: + +* ``__subclasses__`` +* ``__dir__`` +* ``__format__`` +* ``__get__``, ``__set__`` and ``__delete__`` +* ``__reversed__`` and ``__missing__`` +* ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, + ``__getstate__`` and ``__setstate__`` +* ``__getformat__`` and ``__setformat__`` + + + +.. [#] Magic methods *should* be looked up on the class rather than the + instance. Different versions of Python are inconsistent about applying this + rule. The supported protocol methods should work with all supported versions + of Python. +.. [#] The function is basically hooked up to the class, but each ``Mock`` + instance is kept isolated from the others. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock-patch.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/unittest.mock-patch.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,538 @@ +:mod:`unittest.mock` --- the patchers +===================================== + +.. module:: unittest.mock + :synopsis: Mock object library. +.. moduleauthor:: Michael Foord +.. currentmodule:: unittest.mock + +.. versionadded:: 3.3 + +The patch decorators are used for patching objects only within the scope of +the function they decorate. They automatically handle the unpatching for you, +even if exceptions are raised. All of these functions can also be used in with +statements or as class decorators. + + +patch +----- + +.. note:: + + `patch` is straightforward to use. The key is to do the patching in the + right namespace. See the section `where to patch`_. + +.. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) + + `patch` acts as a function decorator, class decorator or a context + manager. Inside the body of the function or with statement, the `target` + (specified in the form `'package.module.ClassName'`) is patched + with a `new` object. When the function/with statement exits the patch is + undone. + + The `target` is imported and the specified attribute patched with the new + object, so it must be importable from the environment you are calling the + decorator from. The target is imported when the decorated function is + executed, not at decoration time. + + If `new` is omitted, then a new `MagicMock` is created and passed in as an + extra argument to the decorated function. + + The `spec` and `spec_set` keyword arguments are passed to the `MagicMock` + if patch is creating one for you. + + In addition you can pass `spec=True` or `spec_set=True`, which causes + patch to pass in the object being mocked as the spec/spec_set object. + + `new_callable` allows you to specify a different class, or callable object, + that will be called to create the `new` object. By default `MagicMock` is + used. + + A more powerful form of `spec` is `autospec`. If you set `autospec=True` + then the mock with be created with a spec from the object being replaced. + All attributes of the mock will also have the spec of the corresponding + attribute of the object being replaced. Methods and functions being mocked + will have their arguments checked and will raise a `TypeError` if they are + called with the wrong signature. For mocks + replacing a class, their return value (the 'instance') will have the same + spec as the class. See the :func:`create_autospec` function and + :ref:`auto-speccing`. + + Instead of `autospec=True` you can pass `autospec=some_object` to use an + arbitrary object as the spec instead of the one being replaced. + + By default `patch` will fail to replace attributes that don't exist. If + you pass in `create=True`, and the attribute doesn't exist, patch will + create the attribute for you when the patched function is called, and + delete it again afterwards. This is useful for writing tests against + attributes that your production code creates at runtime. It is off by by + default because it can be dangerous. With it switched on you can write + passing tests against APIs that don't actually exist! + + Patch can be used as a `TestCase` class decorator. It works by + decorating each test method in the class. This reduces the boilerplate + code when your test methods share a common patchings set. `patch` finds + tests by looking for method names that start with `patch.TEST_PREFIX`. + By default this is `test`, which matches the way `unittest` finds tests. + You can specify an alternative prefix by setting `patch.TEST_PREFIX`. + + Patch can be used as a context manager, with the with statement. Here the + patching applies to the indented block after the with statement. If you + use "as" then the patched object will be bound to the name after the + "as"; very useful if `patch` is creating a mock object for you. + + `patch` takes arbitrary keyword arguments. These will be passed to + the `Mock` (or `new_callable`) on construction. + + `patch.dict(...)`, `patch.multiple(...)` and `patch.object(...)` are + available for alternate use-cases. + + +Patching a class replaces the class with a `MagicMock` *instance*. If the +class is instantiated in the code under test then it will be the +:attr:`~Mock.return_value` of the mock that will be used. + +If the class is instantiated multiple times you could use +:attr:`~Mock.side_effect` to return a new mock each time. Alternatively you +can set the `return_value` to be anything you want. + +To configure return values on methods of *instances* on the patched class +you must do this on the `return_value`. For example: + + >>> class Class(object): + ... def method(self): + ... pass + ... + >>> with patch('__main__.Class') as MockClass: + ... instance = MockClass.return_value + ... instance.method.return_value = 'foo' + ... assert Class() is instance + ... assert Class().method() == 'foo' + ... + +If you use `spec` or `spec_set` and `patch` is replacing a *class*, then the +return value of the created mock will have the same spec. + + >>> Original = Class + >>> patcher = patch('__main__.Class', spec=True) + >>> MockClass = patcher.start() + >>> instance = MockClass() + >>> assert isinstance(instance, Original) + >>> patcher.stop() + +The `new_callable` argument is useful where you want to use an alternative +class to the default :class:`MagicMock` for the created mock. For example, if +you wanted a :class:`NonCallableMock` to be used: + + >>> thing = object() + >>> with patch('__main__.thing', new_callable=NonCallableMock) as mock_thing: + ... assert thing is mock_thing + ... thing() + ... + Traceback (most recent call last): + ... + TypeError: 'NonCallableMock' object is not callable + +Another use case might be to replace an object with a `StringIO` instance: + + >>> from StringIO import StringIO + >>> def foo(): + ... print 'Something' + ... + >>> @patch('sys.stdout', new_callable=StringIO) + ... def test(mock_stdout): + ... foo() + ... assert mock_stdout.getvalue() == 'Something\n' + ... + >>> test() + +When `patch` is creating a mock for you, it is common that the first thing +you need to do is to configure the mock. Some of that configuration can be done +in the call to patch. Any arbitrary keywords you pass into the call will be +used to set attributes on the created mock: + + >>> patcher = patch('__main__.thing', first='one', second='two') + >>> mock_thing = patcher.start() + >>> mock_thing.first + 'one' + >>> mock_thing.second + 'two' + +As well as attributes on the created mock attributes, like the +:attr:`~Mock.return_value` and :attr:`~Mock.side_effect`, of child mocks can +also be configured. These aren't syntactically valid to pass in directly as +keyword arguments, but a dictionary with these as keys can still be expanded +into a `patch` call using `**`: + + >>> config = {'method.return_value': 3, 'other.side_effect': KeyError} + >>> patcher = patch('__main__.thing', **config) + >>> mock_thing = patcher.start() + >>> mock_thing.method() + 3 + >>> mock_thing.other() + Traceback (most recent call last): + ... + KeyError + + +patch.object +------------ + +.. function:: patch.object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) + + patch the named member (`attribute`) on an object (`target`) with a mock + object. + + `patch.object` can be used as a decorator, class decorator or a context + manager. Arguments `new`, `spec`, `create`, `spec_set`, `autospec` and + `new_callable` have the same meaning as for `patch`. Like `patch`, + `patch.object` takes arbitrary keyword arguments for configuring the mock + object it creates. + + When used as a class decorator `patch.object` honours `patch.TEST_PREFIX` + for choosing which methods to wrap. + +You can either call `patch.object` with three arguments or two arguments. The +three argument form takes the object to be patched, the attribute name and the +object to replace the attribute with. + +When calling with the two argument form you omit the replacement object, and a +mock is created for you and passed in as an extra argument to the decorated +function: + + >>> @patch.object(SomeClass, 'class_method') + ... def test(mock_method): + ... SomeClass.class_method(3) + ... mock_method.assert_called_with(3) + ... + >>> test() + +`spec`, `create` and the other arguments to `patch.object` have the same +meaning as they do for `patch`. + + +patch.dict +---------- + +.. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) + + Patch a dictionary, or dictionary like object, and restore the dictionary + to its original state after the test. + + `in_dict` can be a dictionary or a mapping like container. If it is a + mapping then it must at least support getting, setting and deleting items + plus iterating over keys. + + `in_dict` can also be a string specifying the name of the dictionary, which + will then be fetched by importing it. + + `values` can be a dictionary of values to set in the dictionary. `values` + can also be an iterable of `(key, value)` pairs. + + If `clear` is True then the dictionary will be cleared before the new + values are set. + + `patch.dict` can also be called with arbitrary keyword arguments to set + values in the dictionary. + + `patch.dict` can be used as a context manager, decorator or class + decorator. When used as a class decorator `patch.dict` honours + `patch.TEST_PREFIX` for choosing which methods to wrap. + +`patch.dict` can be used to add members to a dictionary, or simply let a test +change a dictionary, and ensure the dictionary is restored when the test +ends. + + >>> foo = {} + >>> with patch.dict(foo, {'newkey': 'newvalue'}): + ... assert foo == {'newkey': 'newvalue'} + ... + >>> assert foo == {} + + >>> import os + >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): + ... print os.environ['newkey'] + ... + newvalue + >>> assert 'newkey' not in os.environ + +Keywords can be used in the `patch.dict` call to set values in the dictionary: + + >>> mymodule = MagicMock() + >>> mymodule.function.return_value = 'fish' + >>> with patch.dict('sys.modules', mymodule=mymodule): + ... import mymodule + ... mymodule.function('some', 'args') + ... + 'fish' + +`patch.dict` can be used with dictionary like objects that aren't actually +dictionaries. At the very minimum they must support item getting, setting, +deleting and either iteration or membership test. This corresponds to the +magic methods `__getitem__`, `__setitem__`, `__delitem__` and either +`__iter__` or `__contains__`. + + >>> class Container(object): + ... def __init__(self): + ... self.values = {} + ... def __getitem__(self, name): + ... return self.values[name] + ... def __setitem__(self, name, value): + ... self.values[name] = value + ... def __delitem__(self, name): + ... del self.values[name] + ... def __iter__(self): + ... return iter(self.values) + ... + >>> thing = Container() + >>> thing['one'] = 1 + >>> with patch.dict(thing, one=2, two=3): + ... assert thing['one'] == 2 + ... assert thing['two'] == 3 + ... + >>> assert thing['one'] == 1 + >>> assert list(thing) == ['one'] + + +patch.multiple +-------------- + +.. function:: patch.multiple(target, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) + + Perform multiple patches in a single call. It takes the object to be + patched (either as an object or a string to fetch the object by importing) + and keyword arguments for the patches:: + + with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'): + ... + + Use :data:`DEFAULT` as the value if you want `patch.multiple` to create + mocks for you. In this case the created mocks are passed into a decorated + function by keyword, and a dictionary is returned when `patch.multiple` is + used as a context manager. + + `patch.multiple` can be used as a decorator, class decorator or a context + manager. The arguments `spec`, `spec_set`, `create`, `autospec` and + `new_callable` have the same meaning as for `patch`. These arguments will + be applied to *all* patches done by `patch.multiple`. + + When used as a class decorator `patch.multiple` honours `patch.TEST_PREFIX` + for choosing which methods to wrap. + +If you want `patch.multiple` to create mocks for you, then you can use +:data:`DEFAULT` as the value. If you use `patch.multiple` as a decorator +then the created mocks are passed into the decorated function by keyword. + + >>> thing = object() + >>> other = object() + + >>> @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) + ... def test_function(thing, other): + ... assert isinstance(thing, MagicMock) + ... assert isinstance(other, MagicMock) + ... + >>> test_function() + +`patch.multiple` can be nested with other `patch` decorators, but put arguments +passed by keyword *after* any of the standard arguments created by `patch`: + + >>> @patch('sys.exit') + ... @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) + ... def test_function(mock_exit, other, thing): + ... assert 'other' in repr(other) + ... assert 'thing' in repr(thing) + ... assert 'exit' in repr(mock_exit) + ... + >>> test_function() + +If `patch.multiple` is used as a context manager, the value returned by the +context manger is a dictionary where created mocks are keyed by name: + + >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values: + ... assert 'other' in repr(values['other']) + ... assert 'thing' in repr(values['thing']) + ... assert values['thing'] is thing + ... assert values['other'] is other + ... + + +.. _start-and-stop: + +patch methods: start and stop +----------------------------- + +All the patchers have `start` and `stop` methods. These make it simpler to do +patching in `setUp` methods or where you want to do multiple patches without +nesting decorators or with statements. + +To use them call `patch`, `patch.object` or `patch.dict` as normal and keep a +reference to the returned `patcher` object. You can then call `start` to put +the patch in place and `stop` to undo it. + +If you are using `patch` to create a mock for you then it will be returned by +the call to `patcher.start`. + + >>> patcher = patch('package.module.ClassName') + >>> from package import module + >>> original = module.ClassName + >>> new_mock = patcher.start() + >>> assert module.ClassName is not original + >>> assert module.ClassName is new_mock + >>> patcher.stop() + >>> assert module.ClassName is original + >>> assert module.ClassName is not new_mock + + +A typical use case for this might be for doing multiple patches in the `setUp` +method of a `TestCase`: + + >>> class MyTest(TestCase): + ... def setUp(self): + ... self.patcher1 = patch('package.module.Class1') + ... self.patcher2 = patch('package.module.Class2') + ... self.MockClass1 = self.patcher1.start() + ... self.MockClass2 = self.patcher2.start() + ... + ... def tearDown(self): + ... self.patcher1.stop() + ... self.patcher2.stop() + ... + ... def test_something(self): + ... assert package.module.Class1 is self.MockClass1 + ... assert package.module.Class2 is self.MockClass2 + ... + >>> MyTest('test_something').run() + +.. caution:: + + If you use this technique you must ensure that the patching is "undone" by + calling `stop`. This can be fiddlier than you might think, because if an + exception is raised in the ``setUp`` then ``tearDown`` is not called. + :meth:`unittest.TestCase.addCleanup` makes this easier: + + >>> class MyTest(TestCase): + ... def setUp(self): + ... patcher = patch('package.module.Class') + ... self.MockClass = patcher.start() + ... self.addCleanup(patcher.stop) + ... + ... def test_something(self): + ... assert package.module.Class is self.MockClass + ... + + As an added bonus you no longer need to keep a reference to the `patcher` + object. + +In fact `start` and `stop` are just aliases for the context manager +`__enter__` and `__exit__` methods. + + +TEST_PREFIX +----------- + +All of the patchers can be used as class decorators. When used in this way +they wrap every test method on the class. The patchers recognise methods that +start with `test` as being test methods. This is the same way that the +:class:`unittest.TestLoader` finds test methods by default. + +It is possible that you want to use a different prefix for your tests. You can +inform the patchers of the different prefix by setting `patch.TEST_PREFIX`: + + >>> patch.TEST_PREFIX = 'foo' + >>> value = 3 + >>> + >>> @patch('__main__.value', 'not three') + ... class Thing(object): + ... def foo_one(self): + ... print value + ... def foo_two(self): + ... print value + ... + >>> + >>> Thing().foo_one() + not three + >>> Thing().foo_two() + not three + >>> value + 3 + + +Nesting Patch Decorators +------------------------ + +If you want to perform multiple patches then you can simply stack up the +decorators. + +You can stack up multiple patch decorators using this pattern: + + >>> @patch.object(SomeClass, 'class_method') + ... @patch.object(SomeClass, 'static_method') + ... def test(mock1, mock2): + ... assert SomeClass.static_method is mock1 + ... assert SomeClass.class_method is mock2 + ... SomeClass.static_method('foo') + ... SomeClass.class_method('bar') + ... return mock1, mock2 + ... + >>> mock1, mock2 = test() + >>> mock1.assert_called_once_with('foo') + >>> mock2.assert_called_once_with('bar') + + +Note that the decorators are applied from the bottom upwards. This is the +standard way that Python applies decorators. The order of the created mocks +passed into your test function matches this order. + + +.. _where-to-patch: + +Where to patch +-------------- + +`patch` works by (temporarily) changing the object that a *name* points to with +another one. There can be many names pointing to any individual object, so +for patching to work you must ensure that you patch the name used by the system +under test. + +The basic principle is that you patch where an object is *looked up*, which +is not necessarily the same place as where it is defined. A couple of +examples will help to clarify this. + +Imagine we have a project that we want to test with the following structure:: + + a.py + -> Defines SomeClass + + b.py + -> from a import SomeClass + -> some_function instantiates SomeClass + +Now we want to test `some_function` but we want to mock out `SomeClass` using +`patch`. The problem is that when we import module b, which we will have to +do then it imports `SomeClass` from module a. If we use `patch` to mock out +`a.SomeClass` then it will have no effect on our test; module b already has a +reference to the *real* `SomeClass` and it looks like our patching had no +effect. + +The key is to patch out `SomeClass` where it is used (or where it is looked up +). In this case `some_function` will actually look up `SomeClass` in module b, +where we have imported it. The patching should look like:: + + @patch('b.SomeClass') + +However, consider the alternative scenario where instead of `from a import +SomeClass` module b does `import a` and `some_function` uses `a.SomeClass`. Both +of these import forms are common. In this case the class we want to patch is +being looked up on the a module and so we have to patch `a.SomeClass` instead:: + + @patch('a.SomeClass') + + +Patching Descriptors and Proxy Objects +-------------------------------------- + +Both patch_ and patch.object_ correctly patch and restore descriptors: class +methods, static methods and properties. You should patch these on the *class* +rather than an instance. They also work with *some* objects +that proxy attribute access, like the `django setttings object +`_. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.mock.rst --- a/Doc/library/unittest.mock.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/unittest.mock.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,3 @@ - :mod:`unittest.mock` --- mock object library ============================================ @@ -13,7 +12,7 @@ replace parts of your system under test with mock objects and make assertions about how they have been used. -:mod:`unittest.mock` provides a core :class:`Mock` class removing the need to +`unittest.mock` provides a core :class:`Mock` class removing the need to create a host of stubs throughout your test suite. After performing an action, you can make assertions about which methods / attributes were used and arguments they were called with. You can also specify return values and @@ -26,11 +25,11 @@ :func:`patch`. Mock is very easy to use and is designed for use with :mod:`unittest`. Mock -is based on the 'action -> assertion' pattern instead of 'record -> replay' +is based on the 'action -> assertion' pattern instead of `'record -> replay'` used by many mocking frameworks. -There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI `_. +There is a backport of `unittest.mock` for earlier versions of Python, +available as `mock on PyPI `_. **Source code:** :source:`Lib/unittest/mock.py` @@ -71,9 +70,9 @@ (5, 4, 3) Mock has many other ways you can configure it and control its behaviour. For -example the *spec* argument configures the mock to take its specification +example the `spec` argument configures the mock to take its specification from another object. Attempting to access attributes or methods on the mock -that don't exist on the spec will fail with an :exc:`AttributeError`. +that don't exist on the spec will fail with an `AttributeError`. The :func:`patch` decorator / context manager makes it easy to mock classes or objects in a module under test. The object you specify will be replaced with a @@ -85,6 +84,7 @@ ... def test(MockClass1, MockClass2): ... module.ClassName1() ... module.ClassName2() + ... assert MockClass1 is module.ClassName1 ... assert MockClass2 is module.ClassName2 ... assert MockClass1.called @@ -97,13 +97,13 @@ When you nest patch decorators the mocks are passed in to the decorated function in the same order they applied (the normal *python* order that decorators are applied). This means from the bottom up, so in the example - above the mock for ``module.ClassName1`` is passed in first. + above the mock for `module.ClassName1` is passed in first. - With :func:`patch` it matters that you patch objects in the namespace where they + With `patch` it matters that you patch objects in the namespace where they are looked up. This is normally straightforward, but for a quick guide read :ref:`where to patch `. -As well as a decorator :func:`patch` can be used as a context manager in a with +As well as a decorator `patch` can be used as a context manager in a with statement: >>> with patch.object(ProductionClass, 'method', return_value=None) as mock_method: @@ -135,7 +135,7 @@ >>> mock.__str__.assert_called_with() Mock allows you to assign functions (or other Mock instances) to magic methods -and they will be called appropriately. The :class:`MagicMock` class is just a Mock +and they will be called appropriately. The `MagicMock` class is just a Mock variant that has all of the magic methods pre-created for you (well, all the useful ones anyway). @@ -149,7 +149,7 @@ For ensuring that the mock objects in your tests have the same api as the objects they are replacing, you can use :ref:`auto-speccing `. -Auto-speccing can be done through the *autospec* argument to patch, or the +Auto-speccing can be done through the `autospec` argument to patch, or the :func:`create_autospec` function. Auto-speccing creates mock objects that have the same attributes and methods as the objects they are replacing, and any functions and methods (including constructors) have the same call @@ -171,9 +171,9 @@ ... TypeError: () takes exactly 3 arguments (1 given) -:func:`create_autospec` can also be used on classes, where it copies the signature of -the ``__init__`` method, and on callable objects where it copies the signature of -the ``__call__`` method. +`create_autospec` can also be used on classes, where it copies the signature of +the `__init__` method, and on callable objects where it copies the signature of +the `__call__` method. @@ -181,77 +181,71 @@ -------------- -:class:`Mock` is a flexible mock object intended to replace the use of stubs and +`Mock` is a flexible mock object intended to replace the use of stubs and test doubles throughout your code. Mocks are callable and create attributes as new mocks when you access them [#]_. Accessing the same attribute will always return the same mock. Mocks record how you use them, allowing you to make assertions about what your code has done to them. -:class:`MagicMock` is a subclass of :class:`Mock` with all the magic methods +:class:`MagicMock` is a subclass of `Mock` with all the magic methods pre-created and ready to use. There are also non-callable variants, useful when you are mocking out objects that aren't callable: :class:`NonCallableMock` and :class:`NonCallableMagicMock` The :func:`patch` decorators makes it easy to temporarily replace classes -in a particular module with a :class:`Mock` object. By default :func:`patch` will create -a :class:`MagicMock` for you. You can specify an alternative class of :class:`Mock` using -the *new_callable* argument to :func:`patch`. +in a particular module with a `Mock` object. By default `patch` will create +a `MagicMock` for you. You can specify an alternative class of `Mock` using +the `new_callable` argument to `patch`. -.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) +.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, **kwargs) - Create a new :class:`Mock` object. :class:`Mock` takes several optional arguments + Create a new `Mock` object. `Mock` takes several optional arguments that specify the behaviour of the Mock object: - * *spec*: This can be either a list of strings or an existing object (a + * `spec`: This can be either a list of strings or an existing object (a class or instance) that acts as the specification for the mock object. If you pass in an object then a list of strings is formed by calling dir on the object (excluding unsupported magic attributes and methods). - Accessing any attribute not in this list will raise an :exc:`AttributeError`. + Accessing any attribute not in this list will raise an `AttributeError`. - If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This - allows mocks to pass :func:`isinstance` tests. + If `spec` is an object (rather than a list of strings) then + :attr:`__class__` returns the class of the spec object. This allows mocks + to pass `isinstance` tests. - * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* + * `spec_set`: A stricter variant of `spec`. If used, attempting to *set* or get an attribute on the mock that isn't on the object passed as - *spec_set* will raise an :exc:`AttributeError`. + `spec_set` will raise an `AttributeError`. - * *side_effect*: A function to be called whenever the Mock is called. See + * `side_effect`: A function to be called whenever the Mock is called. See the :attr:`~Mock.side_effect` attribute. Useful for raising exceptions or dynamically changing return values. The function is called with the same arguments as the mock, and unless it returns :data:`DEFAULT`, the return value of this function is used as the return value. - Alternatively *side_effect* can be an exception class or instance. In + Alternatively `side_effect` can be an exception class or instance. In this case the exception will be raised when the mock is called. - If *side_effect* is an iterable then each call to the mock will return + If `side_effect` is an iterable then each call to the mock will return the next value from the iterable. - A *side_effect* can be cleared by setting it to ``None``. + A `side_effect` can be cleared by setting it to `None`. - * *return_value*: The value returned when the mock is called. By default + * `return_value`: The value returned when the mock is called. By default this is a new Mock (created on first access). See the :attr:`return_value` attribute. - * *unsafe*: By default if any attribute starts with *assert* or - *assret* will raise an :exc:`AttributeError`. Passing ``unsafe=True`` - will allow access to these attributes. + * `wraps`: Item for the mock object to wrap. If `wraps` is not None then + calling the Mock will pass the call through to the wrapped object + (returning the real result and ignoring `return_value`). Attribute access + on the mock will return a Mock object that wraps the corresponding + attribute of the wrapped object (so attempting to access an attribute + that doesn't exist will raise an `AttributeError`). - .. versionadded:: 3.5 + If the mock has an explicit `return_value` set then calls are not passed + to the wrapped object and the `return_value` is returned instead. - * *wraps*: Item for the mock object to wrap. If *wraps* is not None then - calling the Mock will pass the call through to the wrapped object - (returning the real result). Attribute access on the mock will return a - Mock object that wraps the corresponding attribute of the wrapped - object (so attempting to access an attribute that doesn't exist will - raise an :exc:`AttributeError`). - - If the mock has an explicit *return_value* set then calls are not passed - to the wrapped object and the *return_value* is returned instead. - - * *name*: If the mock has a name then it will be used in the repr of the + * `name`: If the mock has a name then it will be used in the repr of the mock. This can be useful for debugging. The name is propagated to child mocks. @@ -270,6 +264,7 @@ >>> mock.method.assert_called_with(1, 2, 3, test='wow') + .. method:: assert_called_once_with(*args, **kwargs) Assert that the mock was called exactly once and with the specified @@ -282,7 +277,7 @@ >>> mock.assert_called_once_with('foo', bar='baz') Traceback (most recent call last): ... - AssertionError: Expected 'mock' to be called once. Called 2 times. + AssertionError: Expected to be called once. Called 2 times. .. method:: assert_any_call(*args, **kwargs) @@ -302,13 +297,13 @@ .. method:: assert_has_calls(calls, any_order=False) assert the mock has been called with the specified calls. - The :attr:`mock_calls` list is checked for the calls. + The `mock_calls` list is checked for the calls. - If *any_order* is false (the default) then the calls must be + If `any_order` is False (the default) then the calls must be sequential. There can be extra calls before or after the specified calls. - If *any_order* is true then the calls can be in any order, but + If `any_order` is True then the calls can be in any order, but they must all appear in :attr:`mock_calls`. >>> mock = Mock(return_value=None) @@ -321,20 +316,6 @@ >>> calls = [call(4), call(2), call(3)] >>> mock.assert_has_calls(calls, any_order=True) - .. method:: assert_not_called(*args, **kwargs) - - Assert the mock was never called. - - >>> m = Mock() - >>> m.hello.assert_not_called() - >>> obj = m.hello() - >>> m.hello.assert_not_called() - Traceback (most recent call last): - ... - AssertionError: Expected 'hello' to not have been called. Called 1 times. - - .. versionadded:: 3.5 - .. method:: reset_mock() @@ -349,7 +330,7 @@ False This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the + reuse the same object. Note that `reset_mock` *doesn't* clear the return value, :attr:`side_effect` or any child attributes you have set using normal assignment. Child mocks and the return value mock (if any) are reset as well. @@ -357,11 +338,11 @@ .. method:: mock_add_spec(spec, spec_set=False) - Add a spec to a mock. *spec* can either be an object or a - list of strings. Only attributes on the *spec* can be fetched as + Add a spec to a mock. `spec` can either be an object or a + list of strings. Only attributes on the `spec` can be fetched as attributes from the mock. - If *spec_set* is true then only attributes on the spec can be set. + If `spec_set` is `True` then only attributes on the spec can be set. .. method:: attach_mock(mock, attribute) @@ -402,14 +383,14 @@ ... KeyError - :meth:`configure_mock` exists to make it easier to do configuration + `configure_mock` exists to make it easier to do configuration after the mock has been created. .. method:: __dir__() - :class:`Mock` objects limit the results of ``dir(some_mock)`` to useful results. - For mocks with a *spec* this includes all the permitted attributes + `Mock` objects limit the results of `dir(some_mock)` to useful results. + For mocks with a `spec` this includes all the permitted attributes for the mock. See :data:`FILTER_DIR` for what this filtering does, and how to @@ -469,7 +450,7 @@ >>> mock.return_value.assert_called_with() - :attr:`return_value` can also be set in the constructor: + `return_value` can also be set in the constructor: >>> mock = Mock(return_value=3) >>> mock.return_value @@ -481,18 +462,13 @@ .. attribute:: side_effect This can either be a function to be called when the mock is called, - an iterable or an exception (class or instance) to be raised. + or an exception (class or instance) to be raised. If you pass in a function it will be called with same arguments as the mock and unless the function returns the :data:`DEFAULT` singleton the call to the mock will then return whatever the function returns. If the function returns :data:`DEFAULT` then the mock will return its normal - value (from the :attr:`return_value`). - - If you pass in an iterable, it is used to retrieve an iterator which - must yield a value on every call. This value can either be an exception - instance to be raised, or a value to be returned from the call to the - mock (:data:`DEFAULT` handling is identical to the function case). + value (from the :attr:`return_value`. An example of a mock that raises an exception (to test exception handling of an API): @@ -504,14 +480,18 @@ ... Exception: Boom! - Using :attr:`side_effect` to return a sequence of values: + Using `side_effect` to return a sequence of values: >>> mock = Mock() >>> mock.side_effect = [3, 2, 1] >>> mock(), mock(), mock() (3, 2, 1) - Using a callable: + The `side_effect` function is called with the same arguments as the + mock (so it is wise for it to take arbitrary args and keyword + arguments) and whatever it returns is used as the return value for + the call. The exception is if `side_effect` returns :data:`DEFAULT`, + in which case the normal :attr:`return_value` is used. >>> mock = Mock(return_value=3) >>> def side_effect(*args, **kwargs): @@ -521,7 +501,7 @@ >>> mock() 3 - :attr:`side_effect` can be set in the constructor. Here's an example that + `side_effect` can be set in the constructor. Here's an example that adds one to the value the mock is called with and returns it: >>> side_effect = lambda value: value + 1 @@ -531,7 +511,7 @@ >>> mock(-8) -7 - Setting :attr:`side_effect` to ``None`` clears it: + Setting `side_effect` to `None` clears it: >>> m = Mock(side_effect=KeyError, return_value=3) >>> m() @@ -545,14 +525,14 @@ .. attribute:: call_args - This is either ``None`` (if the mock hasn't been called), or the + This is either `None` (if the mock hasn't been called), or the arguments that the mock was last called with. This will be in the form of a tuple: the first member is any ordered arguments the mock was called with (or an empty tuple) and the second member is any keyword arguments (or an empty dictionary). >>> mock = Mock(return_value=None) - >>> print(mock.call_args) + >>> print mock.call_args None >>> mock() >>> mock.call_args @@ -568,7 +548,7 @@ >>> mock.call_args call(3, 4, 5, key='fish', next='w00t!') - :attr:`call_args`, along with members of the lists :attr:`call_args_list`, + `call_args`, along with members of the lists :attr:`call_args_list`, :attr:`method_calls` and :attr:`mock_calls` are :data:`call` objects. These are tuples, so they can be unpacked to get at the individual arguments and make more complex assertions. See @@ -581,7 +561,7 @@ (so the length of the list is the number of times it has been called). Before any calls have been made it is an empty list. The :data:`call` object can be used for conveniently constructing lists of - calls to compare with :attr:`call_args_list`. + calls to compare with `call_args_list`. >>> mock = Mock(return_value=None) >>> mock() @@ -593,7 +573,7 @@ >>> mock.call_args_list == expected True - Members of :attr:`call_args_list` are :data:`call` objects. These can be + Members of `call_args_list` are :data:`call` objects. These can be unpacked as tuples to get at the individual arguments. See :ref:`calls as tuples `. @@ -611,15 +591,15 @@ >>> mock.method_calls [call.method(), call.property.method.attribute()] - Members of :attr:`method_calls` are :data:`call` objects. These can be + Members of `method_calls` are :data:`call` objects. These can be unpacked as tuples to get at the individual arguments. See :ref:`calls as tuples `. .. attribute:: mock_calls - :attr:`mock_calls` records *all* calls to the mock object, its methods, - magic methods *and* return value mocks. + `mock_calls` records *all* calls to the mock object, its methods, magic + methods *and* return value mocks. >>> mock = MagicMock() >>> result = mock(1, 2, 3) @@ -636,24 +616,24 @@ >>> mock.mock_calls == expected True - Members of :attr:`mock_calls` are :data:`call` objects. These can be + Members of `mock_calls` are :data:`call` objects. These can be unpacked as tuples to get at the individual arguments. See :ref:`calls as tuples `. .. attribute:: __class__ - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class - instead. This allows mock objects to pass :func:`isinstance` tests for the + Normally the `__class__` attribute of an object will return its type. + For a mock object with a `spec` `__class__` returns the spec class + instead. This allows mock objects to pass `isinstance` tests for the object they are replacing / masquerading as: >>> mock = Mock(spec=3) >>> isinstance(mock, int) True - :attr:`__class__` is assignable to, this allows a mock to pass an - :func:`isinstance` check without forcing you to use a spec: + `__class__` is assignable to, this allows a mock to pass an + `isinstance` check without forcing you to use a spec: >>> mock = Mock() >>> mock.__class__ = dict @@ -662,12 +642,12 @@ .. class:: NonCallableMock(spec=None, wraps=None, name=None, spec_set=None, **kwargs) - A non-callable version of :class:`Mock`. The constructor parameters have the same - meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* + A non-callable version of `Mock`. The constructor parameters have the same + meaning of `Mock`, with the exception of `return_value` and `side_effect` which have no meaning on a non-callable mock. -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: +Mock objects that use a class or an instance as a `spec` or `spec_set` are able +to pass `isintance` tests: >>> mock = Mock(spec=SomeClass) >>> isinstance(mock, SomeClass) @@ -676,11 +656,11 @@ >>> isinstance(mock, SomeClass) True -The :class:`Mock` classes have support for mocking magic methods. See :ref:`magic +The `Mock` classes have support for mocking magic methods. See :ref:`magic methods ` for the full details. The mock classes and the :func:`patch` decorators all take arbitrary keyword -arguments for configuration. For the :func:`patch` decorators the keywords are +arguments for configuration. For the `patch` decorators the keywords are passed to the constructor of the mock being created. The keyword arguments are for configuring attributes of the mock: @@ -692,7 +672,7 @@ The return value and side effect of child mocks can be set in the same way, using dotted notation. As you can't use dotted names directly in a call you -have to create a dictionary and unpack it using ``**``: +have to create a dictionary and unpack it using `**`: >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} >>> mock = Mock(some_attribute='eggs', **attrs) @@ -705,38 +685,17 @@ ... KeyError -A callable mock which was created with a *spec* (or a *spec_set*) will -introspect the specification object's signature when matching calls to -the mock. Therefore, it can match the actual call's arguments regardless -of whether they were passed positionally or by name:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, c=3) - - >>> mock.assert_called_with(1, 2, 3) - >>> mock.assert_called_with(a=1, b=2, c=3) - -This applies to :meth:`~Mock.assert_called_with`, -:meth:`~Mock.assert_called_once_with`, :meth:`~Mock.assert_has_calls` and -:meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also -apply to method calls on the mock object. - - .. versionchanged:: 3.4 - Added signature introspection on specced and autospecced mock objects. - .. class:: PropertyMock(*args, **kwargs) A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods - so you can specify a return value when it is fetched. + `PropertyMock` provides `__get__` and `__set__` methods so you can specify + a return value when it is fetched. - Fetching a :class:`PropertyMock` instance from an object calls the mock, with + Fetching a `PropertyMock` instance from an object calls the mock, with no args. Setting it calls the mock with the value being set. - >>> class Foo: + >>> class Foo(object): ... @property ... def foo(self): ... return 'something' @@ -747,24 +706,13 @@ >>> with patch('__main__.Foo.foo', new_callable=PropertyMock) as mock_foo: ... mock_foo.return_value = 'mockity-mock' ... this_foo = Foo() - ... print(this_foo.foo) + ... print this_foo.foo ... this_foo.foo = 6 ... mockity-mock >>> mock_foo.mock_calls [call(), call(6)] -Because of the way mock attributes are stored you can't directly attach a -:class:`PropertyMock` to a mock object. Instead you can attach it to the mock type -object:: - - >>> m = MagicMock() - >>> p = PropertyMock(return_value=3) - >>> type(m).foo = p - >>> m.foo - 3 - >>> p.assert_called_once_with() - Calling ~~~~~~~ @@ -779,7 +727,7 @@ like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still +been recorded, so if `side_effect` raises an exception the call is still recorded. The simplest way to make a mock raise an exception when called is to make @@ -800,8 +748,8 @@ >>> m.mock_calls [call(1, 2, 3), call('two', 'three', 'four')] -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the +If `side_effect` is a function then whatever that function returns is what +calls to the mock return. The `side_effect` function is called with the same arguments as the mock. This allows you to vary the return value of the call dynamically, based on the input: @@ -818,7 +766,7 @@ If you want the mock to still return the default return value (a new mock), or any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: +`mock.return_value` from inside `side_effect`, or return :data:`DEFAULT`: >>> m = MagicMock() >>> def side_effect(*args, **kwargs): @@ -835,8 +783,8 @@ >>> m() 3 -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: +To remove a `side_effect`, and return to the default behaviour, set the +`side_effect` to `None`: >>> m = MagicMock(return_value=6) >>> def side_effect(*args, **kwargs): @@ -849,9 +797,9 @@ >>> m() 6 -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock +The `side_effect` can also be any iterable object. Repeated calls to the mock will return values from the iterable (until the iterable is exhausted and -a :exc:`StopIteration` is raised): +a `StopIteration` is raised): >>> m = MagicMock(side_effect=[1, 2, 3]) >>> m() @@ -865,20 +813,6 @@ ... StopIteration -If any members of the iterable are exceptions they will be raised instead of -returned:: - - >>> iterable = (33, ValueError, 66) - >>> m = MagicMock(side_effect=iterable) - >>> m() - 33 - >>> m() - Traceback (most recent call last): - ... - ValueError - >>> m() - 66 - .. _deleting-attributes: @@ -888,12 +822,12 @@ Mock objects create attributes on demand. This allows them to pretend to be objects of any type. -You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an -:exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. +You may want a mock object to return `False` to a `hasattr` call, or raise an +`AttributeError` when an attribute is fetched. You can do this by providing +an object as a `spec` for a mock, but that isn't always convenient. You "block" attributes by deleting them. Once deleted, accessing an attribute -will raise an :exc:`AttributeError`. +will raise an `AttributeError`. >>> mock = MagicMock() >>> hasattr(mock, 'm') @@ -908,25 +842,6 @@ AttributeError: f -Mock names and the name attribute -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since "name" is an argument to the :class:`Mock` constructor, if you want your -mock object to have a "name" attribute you can't just pass it in at creation -time. There are two alternatives. One option is to use -:meth:`~Mock.configure_mock`:: - - >>> mock = MagicMock() - >>> mock.configure_mock(name='my_name') - >>> mock.name - 'my_name' - -A simpler option is to simply set the "name" attribute after mock creation:: - - >>> mock = MagicMock() - >>> mock.name = "foo" - - Attaching Mocks as Attributes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -979,1345 +894,7 @@ .. [#] The only exceptions are magic methods and attributes (those that have leading and trailing double underscores). Mock doesn't create these but - instead raises an :exc:`AttributeError`. This is because the interpreter + instead of raises an ``AttributeError``. This is because the interpreter will often implicitly request these methods, and gets *very* confused to get a new Mock object when it expects a magic method. If you need magic method support see :ref:`magic methods `. - - -The patchers ------------- - -The patch decorators are used for patching objects only within the scope of -the function they decorate. They automatically handle the unpatching for you, -even if exceptions are raised. All of these functions can also be used in with -statements or as class decorators. - - -patch -~~~~~ - -.. note:: - - :func:`patch` is straightforward to use. The key is to do the patching in the - right namespace. See the section `where to patch`_. - -.. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - :func:`patch` acts as a function decorator, class decorator or a context - manager. Inside the body of the function or with statement, the *target* - is patched with a *new* object. When the function/with statement exits - the patch is undone. - - If *new* is omitted, then the target is replaced with a - :class:`MagicMock`. If :func:`patch` is used as a decorator and *new* is - omitted, the created mock is passed in as an extra argument to the - decorated function. If :func:`patch` is used as a context manager the created - mock is returned by the context manager. - - *target* should be a string in the form ``'package.module.ClassName'``. The - *target* is imported and the specified object replaced with the *new* - object, so the *target* must be importable from the environment you are - calling :func:`patch` from. The target is imported when the decorated function - is executed, not at decoration time. - - The *spec* and *spec_set* keyword arguments are passed to the :class:`MagicMock` - if patch is creating one for you. - - In addition you can pass ``spec=True`` or ``spec_set=True``, which causes - patch to pass in the object being mocked as the spec/spec_set object. - - *new_callable* allows you to specify a different class, or callable object, - that will be called to create the *new* object. By default :class:`MagicMock` is - used. - - A more powerful form of *spec* is *autospec*. If you set ``autospec=True`` - then the mock will be created with a spec from the object being replaced. - All attributes of the mock will also have the spec of the corresponding - attribute of the object being replaced. Methods and functions being mocked - will have their arguments checked and will raise a :exc:`TypeError` if they are - called with the wrong signature. For mocks - replacing a class, their return value (the 'instance') will have the same - spec as the class. See the :func:`create_autospec` function and - :ref:`auto-speccing`. - - Instead of ``autospec=True`` you can pass ``autospec=some_object`` to use an - arbitrary object as the spec instead of the one being replaced. - - By default :func:`patch` will fail to replace attributes that don't exist. If - you pass in ``create=True``, and the attribute doesn't exist, patch will - create the attribute for you when the patched function is called, and - delete it again afterwards. This is useful for writing tests against - attributes that your production code creates at runtime. It is off by - default because it can be dangerous. With it switched on you can write - passing tests against APIs that don't actually exist! - - .. note:: - - .. versionchanged:: 3.5 - If you are patching builtins in a module then you don't - need to pass ``create=True``, it will be added by default. - - Patch can be used as a :class:`TestCase` class decorator. It works by - decorating each test method in the class. This reduces the boilerplate - code when your test methods share a common patchings set. :func:`patch` finds - tests by looking for method names that start with ``patch.TEST_PREFIX``. - By default this is ``'test'``, which matches the way :mod:`unittest` finds tests. - You can specify an alternative prefix by setting ``patch.TEST_PREFIX``. - - Patch can be used as a context manager, with the with statement. Here the - patching applies to the indented block after the with statement. If you - use "as" then the patched object will be bound to the name after the - "as"; very useful if :func:`patch` is creating a mock object for you. - - :func:`patch` takes arbitrary keyword arguments. These will be passed to - the :class:`Mock` (or *new_callable*) on construction. - - ``patch.dict(...)``, ``patch.multiple(...)`` and ``patch.object(...)`` are - available for alternate use-cases. - -:func:`patch` as function decorator, creating the mock for you and passing it into -the decorated function: - - >>> @patch('__main__.SomeClass') - ... def function(normal_argument, mock_class): - ... print(mock_class is SomeClass) - ... - >>> function(None) - True - -Patching a class replaces the class with a :class:`MagicMock` *instance*. If the -class is instantiated in the code under test then it will be the -:attr:`~Mock.return_value` of the mock that will be used. - -If the class is instantiated multiple times you could use -:attr:`~Mock.side_effect` to return a new mock each time. Alternatively you -can set the *return_value* to be anything you want. - -To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example: - - >>> class Class: - ... def method(self): - ... pass - ... - >>> with patch('__main__.Class') as MockClass: - ... instance = MockClass.return_value - ... instance.method.return_value = 'foo' - ... assert Class() is instance - ... assert Class().method() == 'foo' - ... - -If you use *spec* or *spec_set* and :func:`patch` is replacing a *class*, then the -return value of the created mock will have the same spec. - - >>> Original = Class - >>> patcher = patch('__main__.Class', spec=True) - >>> MockClass = patcher.start() - >>> instance = MockClass() - >>> assert isinstance(instance, Original) - >>> patcher.stop() - -The *new_callable* argument is useful where you want to use an alternative -class to the default :class:`MagicMock` for the created mock. For example, if -you wanted a :class:`NonCallableMock` to be used: - - >>> thing = object() - >>> with patch('__main__.thing', new_callable=NonCallableMock) as mock_thing: - ... assert thing is mock_thing - ... thing() - ... - Traceback (most recent call last): - ... - TypeError: 'NonCallableMock' object is not callable - -Another use case might be to replace an object with an :class:`io.StringIO` instance: - - >>> from io import StringIO - >>> def foo(): - ... print('Something') - ... - >>> @patch('sys.stdout', new_callable=StringIO) - ... def test(mock_stdout): - ... foo() - ... assert mock_stdout.getvalue() == 'Something\n' - ... - >>> test() - -When :func:`patch` is creating a mock for you, it is common that the first thing -you need to do is to configure the mock. Some of that configuration can be done -in the call to patch. Any arbitrary keywords you pass into the call will be -used to set attributes on the created mock: - - >>> patcher = patch('__main__.thing', first='one', second='two') - >>> mock_thing = patcher.start() - >>> mock_thing.first - 'one' - >>> mock_thing.second - 'two' - -As well as attributes on the created mock attributes, like the -:attr:`~Mock.return_value` and :attr:`~Mock.side_effect`, of child mocks can -also be configured. These aren't syntactically valid to pass in directly as -keyword arguments, but a dictionary with these as keys can still be expanded -into a :func:`patch` call using ``**``: - - >>> config = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> patcher = patch('__main__.thing', **config) - >>> mock_thing = patcher.start() - >>> mock_thing.method() - 3 - >>> mock_thing.other() - Traceback (most recent call last): - ... - KeyError - - -patch.object -~~~~~~~~~~~~ - -.. function:: patch.object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - patch the named member (*attribute*) on an object (*target*) with a mock - object. - - :func:`patch.object` can be used as a decorator, class decorator or a context - manager. Arguments *new*, *spec*, *create*, *spec_set*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. Like :func:`patch`, - :func:`patch.object` takes arbitrary keyword arguments for configuring the mock - object it creates. - - When used as a class decorator :func:`patch.object` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -You can either call :func:`patch.object` with three arguments or two arguments. The -three argument form takes the object to be patched, the attribute name and the -object to replace the attribute with. - -When calling with the two argument form you omit the replacement object, and a -mock is created for you and passed in as an extra argument to the decorated -function: - - >>> @patch.object(SomeClass, 'class_method') - ... def test(mock_method): - ... SomeClass.class_method(3) - ... mock_method.assert_called_with(3) - ... - >>> test() - -*spec*, *create* and the other arguments to :func:`patch.object` have the same -meaning as they do for :func:`patch`. - - -patch.dict -~~~~~~~~~~ - -.. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) - - Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. - - *in_dict* can be a dictionary or a mapping like container. If it is a - mapping then it must at least support getting, setting and deleting items - plus iterating over keys. - - *in_dict* can also be a string specifying the name of the dictionary, which - will then be fetched by importing it. - - *values* can be a dictionary of values to set in the dictionary. *values* - can also be an iterable of ``(key, value)`` pairs. - - If *clear* is true then the dictionary will be cleared before the new - values are set. - - :func:`patch.dict` can also be called with arbitrary keyword arguments to set - values in the dictionary. - - :func:`patch.dict` can be used as a context manager, decorator or class - decorator. When used as a class decorator :func:`patch.dict` honours - ``patch.TEST_PREFIX`` for choosing which methods to wrap. - -:func:`patch.dict` can be used to add members to a dictionary, or simply let a test -change a dictionary, and ensure the dictionary is restored when the test -ends. - - >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == {} - - >>> import os - >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): - ... print(os.environ['newkey']) - ... - newvalue - >>> assert 'newkey' not in os.environ - -Keywords can be used in the :func:`patch.dict` call to set values in the dictionary: - - >>> mymodule = MagicMock() - >>> mymodule.function.return_value = 'fish' - >>> with patch.dict('sys.modules', mymodule=mymodule): - ... import mymodule - ... mymodule.function('some', 'args') - ... - 'fish' - -:func:`patch.dict` can be used with dictionary like objects that aren't actually -dictionaries. At the very minimum they must support item getting, setting, -deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. - - >>> class Container: - ... def __init__(self): - ... self.values = {} - ... def __getitem__(self, name): - ... return self.values[name] - ... def __setitem__(self, name, value): - ... self.values[name] = value - ... def __delitem__(self, name): - ... del self.values[name] - ... def __iter__(self): - ... return iter(self.values) - ... - >>> thing = Container() - >>> thing['one'] = 1 - >>> with patch.dict(thing, one=2, two=3): - ... assert thing['one'] == 2 - ... assert thing['two'] == 3 - ... - >>> assert thing['one'] == 1 - >>> assert list(thing) == ['one'] - - -patch.multiple -~~~~~~~~~~~~~~ - -.. function:: patch.multiple(target, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - Perform multiple patches in a single call. It takes the object to be - patched (either as an object or a string to fetch the object by importing) - and keyword arguments for the patches:: - - with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'): - ... - - Use :data:`DEFAULT` as the value if you want :func:`patch.multiple` to create - mocks for you. In this case the created mocks are passed into a decorated - function by keyword, and a dictionary is returned when :func:`patch.multiple` is - used as a context manager. - - :func:`patch.multiple` can be used as a decorator, class decorator or a context - manager. The arguments *spec*, *spec_set*, *create*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. These arguments will - be applied to *all* patches done by :func:`patch.multiple`. - - When used as a class decorator :func:`patch.multiple` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -If you want :func:`patch.multiple` to create mocks for you, then you can use -:data:`DEFAULT` as the value. If you use :func:`patch.multiple` as a decorator -then the created mocks are passed into the decorated function by keyword. - - >>> thing = object() - >>> other = object() - - >>> @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(thing, other): - ... assert isinstance(thing, MagicMock) - ... assert isinstance(other, MagicMock) - ... - >>> test_function() - -:func:`patch.multiple` can be nested with other ``patch`` decorators, but put arguments -passed by keyword *after* any of the standard arguments created by :func:`patch`: - - >>> @patch('sys.exit') - ... @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(mock_exit, other, thing): - ... assert 'other' in repr(other) - ... assert 'thing' in repr(thing) - ... assert 'exit' in repr(mock_exit) - ... - >>> test_function() - -If :func:`patch.multiple` is used as a context manager, the value returned by the -context manger is a dictionary where created mocks are keyed by name: - - >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values: - ... assert 'other' in repr(values['other']) - ... assert 'thing' in repr(values['thing']) - ... assert values['thing'] is thing - ... assert values['other'] is other - ... - - -.. _start-and-stop: - -patch methods: start and stop -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do -patching in ``setUp`` methods or where you want to do multiple patches without -nesting decorators or with statements. - -To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as -normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. - -If you are using :func:`patch` to create a mock for you then it will be returned by -the call to ``patcher.start``. - - >>> patcher = patch('package.module.ClassName') - >>> from package import module - >>> original = module.ClassName - >>> new_mock = patcher.start() - >>> assert module.ClassName is not original - >>> assert module.ClassName is new_mock - >>> patcher.stop() - >>> assert module.ClassName is original - >>> assert module.ClassName is not new_mock - - -A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`: - - >>> class MyTest(TestCase): - ... def setUp(self): - ... self.patcher1 = patch('package.module.Class1') - ... self.patcher2 = patch('package.module.Class2') - ... self.MockClass1 = self.patcher1.start() - ... self.MockClass2 = self.patcher2.start() - ... - ... def tearDown(self): - ... self.patcher1.stop() - ... self.patcher2.stop() - ... - ... def test_something(self): - ... assert package.module.Class1 is self.MockClass1 - ... assert package.module.Class2 is self.MockClass2 - ... - >>> MyTest('test_something').run() - -.. caution:: - - If you use this technique you must ensure that the patching is "undone" by - calling ``stop``. This can be fiddlier than you might think, because if an - exception is raised in the ``setUp`` then ``tearDown`` is not called. - :meth:`unittest.TestCase.addCleanup` makes this easier: - - >>> class MyTest(TestCase): - ... def setUp(self): - ... patcher = patch('package.module.Class') - ... self.MockClass = patcher.start() - ... self.addCleanup(patcher.stop) - ... - ... def test_something(self): - ... assert package.module.Class is self.MockClass - ... - - As an added bonus you no longer need to keep a reference to the ``patcher`` - object. - -It is also possible to stop all patches which have been started by using -:func:`patch.stopall`. - -.. function:: patch.stopall - - Stop all active patches. Only stops patches started with ``start``. - - -.. _patch-builtins: - -patch builtins -~~~~~~~~~~~~~~ -You can patch any builtins within a module. The following example patches -builtin :func:`ord`: - - >>> @patch('__main__.ord') - ... def test(mock_ord): - ... mock_ord.return_value = 101 - ... print(ord('c')) - ... - >>> test() - 101 - - -TEST_PREFIX -~~~~~~~~~~~ - -All of the patchers can be used as class decorators. When used in this way -they wrap every test method on the class. The patchers recognise methods that -start with ``'test'`` as being test methods. This is the same way that the -:class:`unittest.TestLoader` finds test methods by default. - -It is possible that you want to use a different prefix for your tests. You can -inform the patchers of the different prefix by setting ``patch.TEST_PREFIX``: - - >>> patch.TEST_PREFIX = 'foo' - >>> value = 3 - >>> - >>> @patch('__main__.value', 'not three') - ... class Thing: - ... def foo_one(self): - ... print(value) - ... def foo_two(self): - ... print(value) - ... - >>> - >>> Thing().foo_one() - not three - >>> Thing().foo_two() - not three - >>> value - 3 - - -Nesting Patch Decorators -~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to perform multiple patches then you can simply stack up the -decorators. - -You can stack up multiple patch decorators using this pattern: - - >>> @patch.object(SomeClass, 'class_method') - ... @patch.object(SomeClass, 'static_method') - ... def test(mock1, mock2): - ... assert SomeClass.static_method is mock1 - ... assert SomeClass.class_method is mock2 - ... SomeClass.static_method('foo') - ... SomeClass.class_method('bar') - ... return mock1, mock2 - ... - >>> mock1, mock2 = test() - >>> mock1.assert_called_once_with('foo') - >>> mock2.assert_called_once_with('bar') - - -Note that the decorators are applied from the bottom upwards. This is the -standard way that Python applies decorators. The order of the created mocks -passed into your test function matches this order. - - -.. _where-to-patch: - -Where to patch -~~~~~~~~~~~~~~ - -:func:`patch` works by (temporarily) changing the object that a *name* points to with -another one. There can be many names pointing to any individual object, so -for patching to work you must ensure that you patch the name used by the system -under test. - -The basic principle is that you patch where an object is *looked up*, which -is not necessarily the same place as where it is defined. A couple of -examples will help to clarify this. - -Imagine we have a project that we want to test with the following structure:: - - a.py - -> Defines SomeClass - - b.py - -> from a import SomeClass - -> some_function instantiates SomeClass - -Now we want to test ``some_function`` but we want to mock out ``SomeClass`` using -:func:`patch`. The problem is that when we import module b, which we will have to -do then it imports ``SomeClass`` from module a. If we use :func:`patch` to mock out -``a.SomeClass`` then it will have no effect on our test; module b already has a -reference to the *real* ``SomeClass`` and it looks like our patching had no -effect. - -The key is to patch out ``SomeClass`` where it is used (or where it is looked up -). In this case ``some_function`` will actually look up ``SomeClass`` in module b, -where we have imported it. The patching should look like:: - - @patch('b.SomeClass') - -However, consider the alternative scenario where instead of ``from a import -SomeClass`` module b does ``import a`` and ``some_function`` uses ``a.SomeClass``. Both -of these import forms are common. In this case the class we want to patch is -being looked up in the module and so we have to patch ``a.SomeClass`` instead:: - - @patch('a.SomeClass') - - -Patching Descriptors and Proxy Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Both patch_ and patch.object_ correctly patch and restore descriptors: class -methods, static methods and properties. You should patch these on the *class* -rather than an instance. They also work with *some* objects -that proxy attribute access, like the `django settings object -`_. - - -MagicMock and magic method support ----------------------------------- - -.. _magic-methods: - -Mocking Magic Methods -~~~~~~~~~~~~~~~~~~~~~ - -:class:`Mock` supports mocking the Python protocol methods, also known as -"magic methods". This allows mock objects to replace containers or other -objects that implement Python protocols. - -Because magic methods are looked up differently from normal methods [#]_, this -support has been specially implemented. This means that only specific magic -methods are supported. The supported list includes *almost* all of them. If -there are any missing that you need please let us know. - -You mock magic methods by setting the method you are interested in to a function -or a mock instance. If you are using a function then it *must* take ``self`` as -the first argument [#]_. - - >>> def __str__(self): - ... return 'fooble' - ... - >>> mock = Mock() - >>> mock.__str__ = __str__ - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__str__ = Mock() - >>> mock.__str__.return_value = 'fooble' - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__iter__ = Mock(return_value=iter([])) - >>> list(mock) - [] - -One use case for this is for mocking objects used as context managers in a -:keyword:`with` statement: - - >>> mock = Mock() - >>> mock.__enter__ = Mock(return_value='foo') - >>> mock.__exit__ = Mock(return_value=False) - >>> with mock as m: - ... assert m == 'foo' - ... - >>> mock.__enter__.assert_called_with() - >>> mock.__exit__.assert_called_with(None, None, None) - -Calls to magic methods do not appear in :attr:`~Mock.method_calls`, but they -are recorded in :attr:`~Mock.mock_calls`. - -.. note:: - - If you use the *spec* keyword argument to create a mock then attempting to - set a magic method that isn't in the spec will raise an :exc:`AttributeError`. - -The full list of supported magic methods is: - -* ``__hash__``, ``__sizeof__``, ``__repr__`` and ``__str__`` -* ``__dir__``, ``__format__`` and ``__subclasses__`` -* ``__floor__``, ``__trunc__`` and ``__ceil__`` -* Comparisons: ``__lt__``, ``__gt__``, ``__le__``, ``__ge__``, - ``__eq__`` and ``__ne__`` -* Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, - ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` - and ``__missing__`` -* Context manager: ``__enter__`` and ``__exit__`` -* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` -* The numeric methods (including right hand and in-place variants): - ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, - ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``, - ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__`` -* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__`` - and ``__index__`` -* Descriptor methods: ``__get__``, ``__set__`` and ``__delete__`` -* Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, - ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` - - -The following methods exist but are *not* supported as they are either in use -by mock, can't be set dynamically, or can cause problems: - -* ``__getattr__``, ``__setattr__``, ``__init__`` and ``__new__`` -* ``__prepare__``, ``__instancecheck__``, ``__subclasscheck__``, ``__del__`` - - - -Magic Mock -~~~~~~~~~~ - -There are two ``MagicMock`` variants: :class:`MagicMock` and :class:`NonCallableMagicMock`. - - -.. class:: MagicMock(*args, **kw) - - ``MagicMock`` is a subclass of :class:`Mock` with default implementations - of most of the magic methods. You can use ``MagicMock`` without having to - configure the magic methods yourself. - - The constructor parameters have the same meaning as for :class:`Mock`. - - If you use the *spec* or *spec_set* arguments then *only* magic methods - that exist in the spec will be created. - - -.. class:: NonCallableMagicMock(*args, **kw) - - A non-callable version of :class:`MagicMock`. - - The constructor parameters have the same meaning as for - :class:`MagicMock`, with the exception of *return_value* and - *side_effect* which have no meaning on a non-callable mock. - -The magic methods are setup with :class:`MagicMock` objects, so you can configure them -and use them in the usual way: - - >>> mock = MagicMock() - >>> mock[3] = 'fish' - >>> mock.__setitem__.assert_called_with(3, 'fish') - >>> mock.__getitem__.return_value = 'result' - >>> mock[2] - 'result' - -By default many of the protocol methods are required to return objects of a -specific type. These methods are preconfigured with a default return value, so -that they can be used without you having to do anything if you aren't interested -in the return value. You can still *set* the return value manually if you want -to change the default. - -Methods and their defaults: - -* ``__lt__``: NotImplemented -* ``__gt__``: NotImplemented -* ``__le__``: NotImplemented -* ``__ge__``: NotImplemented -* ``__int__``: 1 -* ``__contains__``: False -* ``__len__``: 1 -* ``__iter__``: iter([]) -* ``__exit__``: False -* ``__complex__``: 1j -* ``__float__``: 1.0 -* ``__bool__``: True -* ``__index__``: 1 -* ``__hash__``: default hash for the mock -* ``__str__``: default str for the mock -* ``__sizeof__``: default sizeof for the mock - -For example: - - >>> mock = MagicMock() - >>> int(mock) - 1 - >>> len(mock) - 0 - >>> list(mock) - [] - >>> object() in mock - False - -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. -They do the default equality comparison on identity, using the -:attr:`~Mock.side_effect` attribute, unless you change their return value to -return something else:: - - >>> MagicMock() == 3 - False - >>> MagicMock() != 3 - True - >>> mock = MagicMock() - >>> mock.__eq__.return_value = True - >>> mock == 3 - True - -The return value of :meth:`MagicMock.__iter__` can be any iterable object and isn't -required to be an iterator: - - >>> mock = MagicMock() - >>> mock.__iter__.return_value = ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - -If the return value *is* an iterator, then iterating over it once will consume -it and subsequent iterations will result in an empty list: - - >>> mock.__iter__.return_value = iter(['a', 'b', 'c']) - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - [] - -``MagicMock`` has all of the supported magic methods configured except for some -of the obscure and obsolete ones. You can still set these up if you want. - -Magic methods that are supported but not setup by default in ``MagicMock`` are: - -* ``__subclasses__`` -* ``__dir__`` -* ``__format__`` -* ``__get__``, ``__set__`` and ``__delete__`` -* ``__reversed__`` and ``__missing__`` -* ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, - ``__getstate__`` and ``__setstate__`` -* ``__getformat__`` and ``__setformat__`` - - - -.. [#] Magic methods *should* be looked up on the class rather than the - instance. Different versions of Python are inconsistent about applying this - rule. The supported protocol methods should work with all supported versions - of Python. -.. [#] The function is basically hooked up to the class, but each ``Mock`` - instance is kept isolated from the others. - - -Helpers -------- - -sentinel -~~~~~~~~ - -.. data:: sentinel - - The ``sentinel`` object provides a convenient way of providing unique - objects for your tests. - - Attributes are created on demand when you access them by name. Accessing - the same attribute will always return the same object. The objects - returned have a sensible repr so that test failure messages are readable. - -Sometimes when testing you need to test that a specific object is passed as an -argument to another method, or returned. It can be common to create named -sentinel objects to test this. :data:`sentinel` provides a convenient way of -creating and testing the identity of objects like this. - -In this example we monkey patch ``method`` to return ``sentinel.some_object``: - - >>> real = ProductionClass() - >>> real.method = Mock(name="method") - >>> real.method.return_value = sentinel.some_object - >>> result = real.method() - >>> assert result is sentinel.some_object - >>> sentinel.some_object - sentinel.some_object - - -DEFAULT -~~~~~~~ - - -.. data:: DEFAULT - - The :data:`DEFAULT` object is a pre-created sentinel (actually - ``sentinel.DEFAULT``). It can be used by :attr:`~Mock.side_effect` - functions to indicate that the normal return value should be used. - - -call -~~~~ - -.. function:: call(*args, **kwargs) - - :func:`call` is a helper object for making simpler assertions, for comparing with - :attr:`~Mock.call_args`, :attr:`~Mock.call_args_list`, - :attr:`~Mock.mock_calls` and :attr:`~Mock.method_calls`. :func:`call` can also be - used with :meth:`~Mock.assert_has_calls`. - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, a='foo', b='bar') - >>> m() - >>> m.call_args_list == [call(1, 2, a='foo', b='bar'), call()] - True - -.. method:: call.call_list() - - For a call object that represents multiple calls, :meth:`call_list` - returns a list of all the intermediate calls as well as the - final call. - -``call_list`` is particularly useful for making assertions on "chained calls". A -chained call is multiple calls on a single line of code. This results in -multiple entries in :attr:`~Mock.mock_calls` on a mock. Manually constructing -the sequence of calls can be tedious. - -:meth:`~call.call_list` can construct the sequence of calls from the same -chained call: - - >>> m = MagicMock() - >>> m(1).method(arg='foo').other('bar')(2.0) - - >>> kall = call(1).method(arg='foo').other('bar')(2.0) - >>> kall.call_list() - [call(1), - call().method(arg='foo'), - call().method().other('bar'), - call().method().other()(2.0)] - >>> m.mock_calls == kall.call_list() - True - -.. _calls-as-tuples: - -A ``call`` object is either a tuple of (positional args, keyword args) or -(name, positional args, keyword args) depending on how it was constructed. When -you construct them yourself this isn't particularly interesting, but the ``call`` -objects that are in the :attr:`Mock.call_args`, :attr:`Mock.call_args_list` and -:attr:`Mock.mock_calls` attributes can be introspected to get at the individual -arguments they contain. - -The ``call`` objects in :attr:`Mock.call_args` and :attr:`Mock.call_args_list` -are two-tuples of (positional args, keyword args) whereas the ``call`` objects -in :attr:`Mock.mock_calls`, along with ones you construct yourself, are -three-tuples of (name, positional args, keyword args). - -You can use their "tupleness" to pull out the individual arguments for more -complex introspection and assertions. The positional arguments are a tuple -(an empty tuple if there are no positional arguments) and the keyword -arguments are a dictionary: - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, 3, arg='one', arg2='two') - >>> kall = m.call_args - >>> args, kwargs = kall - >>> args - (1, 2, 3) - >>> kwargs - {'arg2': 'two', 'arg': 'one'} - >>> args is kall[0] - True - >>> kwargs is kall[1] - True - - >>> m = MagicMock() - >>> m.foo(4, 5, 6, arg='two', arg2='three') - - >>> kall = m.mock_calls[0] - >>> name, args, kwargs = kall - >>> name - 'foo' - >>> args - (4, 5, 6) - >>> kwargs - {'arg2': 'three', 'arg': 'two'} - >>> name is m.mock_calls[0][0] - True - - -create_autospec -~~~~~~~~~~~~~~~ - -.. function:: create_autospec(spec, spec_set=False, instance=False, **kwargs) - - Create a mock object using another object as a spec. Attributes on the - mock will use the corresponding attribute on the *spec* object as their - spec. - - Functions or methods being mocked will have their arguments checked to - ensure that they are called with the correct signature. - - If *spec_set* is ``True`` then attempting to set attributes that don't exist - on the spec object will raise an :exc:`AttributeError`. - - If a class is used as a spec then the return value of the mock (the - instance of the class) will have the same spec. You can use a class as the - spec for an instance object by passing ``instance=True``. The returned mock - will only be callable if instances of the mock are callable. - - :func:`create_autospec` also takes arbitrary keyword arguments that are passed to - the constructor of the created mock. - -See :ref:`auto-speccing` for examples of how to use auto-speccing with -:func:`create_autospec` and the *autospec* argument to :func:`patch`. - - -ANY -~~~ - -.. data:: ANY - -Sometimes you may need to make assertions about *some* of the arguments in a -call to mock, but either not care about some of the arguments or want to pull -them individually out of :attr:`~Mock.call_args` and make more complex -assertions on them. - -To ignore certain arguments you can pass in objects that compare equal to -*everything*. Calls to :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with` will then succeed no matter what was -passed in. - - >>> mock = Mock(return_value=None) - >>> mock('foo', bar=object()) - >>> mock.assert_called_once_with('foo', bar=ANY) - -:data:`ANY` can also be used in comparisons with call lists like -:attr:`~Mock.mock_calls`: - - >>> m = MagicMock(return_value=None) - >>> m(1) - >>> m(1, 2) - >>> m(object()) - >>> m.mock_calls == [call(1), call(1, 2), ANY] - True - - - -FILTER_DIR -~~~~~~~~~~ - -.. data:: FILTER_DIR - -:data:`FILTER_DIR` is a module level variable that controls the way mock objects -respond to :func:`dir` (only for Python 2.6 or more recent). The default is ``True``, -which uses the filtering described below, to only show useful members. If you -dislike this filtering, or need to switch it off for diagnostic purposes, then -set ``mock.FILTER_DIR = False``. - -With filtering on, ``dir(some_mock)`` shows only useful attributes and will -include any dynamically created attributes that wouldn't normally be shown. -If the mock was created with a *spec* (or *autospec* of course) then all the -attributes from the original are shown, even if they haven't been accessed -yet: - - >>> dir(Mock()) - ['assert_any_call', - 'assert_called_once_with', - 'assert_called_with', - 'assert_has_calls', - 'attach_mock', - ... - >>> from urllib import request - >>> dir(Mock(spec=request)) - ['AbstractBasicAuthHandler', - 'AbstractDigestAuthHandler', - 'AbstractHTTPHandler', - 'BaseHandler', - ... - -Many of the not-very-useful (private to :class:`Mock` rather than the thing being -mocked) underscore and double underscore prefixed attributes have been -filtered from the result of calling :func:`dir` on a :class:`Mock`. If you dislike this -behaviour you can switch it off by setting the module level switch -:data:`FILTER_DIR`: - - >>> from unittest import mock - >>> mock.FILTER_DIR = False - >>> dir(mock.Mock()) - ['_NonCallableMock__get_return_value', - '_NonCallableMock__get_side_effect', - '_NonCallableMock__return_value_doc', - '_NonCallableMock__set_return_value', - '_NonCallableMock__set_side_effect', - '__call__', - '__class__', - ... - -Alternatively you can just use ``vars(my_mock)`` (instance members) and -``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:data:`mock.FILTER_DIR`. - - -mock_open -~~~~~~~~~ - -.. function:: mock_open(mock=None, read_data=None) - - A helper function to create a mock to replace the use of :func:`open`. It works - for :func:`open` called directly or used as a context manager. - - The *mock* argument is the mock object to configure. If ``None`` (the - default) then a :class:`MagicMock` will be created for you, with the API limited - to methods or attributes available on standard file handles. - - *read_data* is a string for the :meth:`~io.IOBase.read`, - :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods - of the file handle to return. Calls to those methods will take data from - *read_data* until it is depleted. The mock of these methods is pretty - simplistic: every time the *mock* is called, the *read_data* is rewound to - the start. If you need more control over the data that you are feeding to - the tested code you will need to customize this mock for yourself. When that - is insufficient, one of the in-memory filesystem packages on `PyPI - `_ can offer a realistic filesystem for testing. - - .. versionchanged:: 3.4 - Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather - than returning it on each call. - - .. versionchanged:: 3.5 - *read_data* is now reset on each call to the *mock*. - -Using :func:`open` as a context manager is a great way to ensure your file handles -are closed properly and is becoming common:: - - with open('/some/path', 'w') as f: - f.write('something') - -The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`__enter__` and -:meth:`__exit__` called). - -Mocking context managers with a :class:`MagicMock` is common enough and fiddly -enough that a helper function is useful. - - >>> m = mock_open() - >>> with patch('__main__.open', m): - ... with open('foo', 'w') as h: - ... h.write('some stuff') - ... - >>> m.mock_calls - [call('foo', 'w'), - call().__enter__(), - call().write('some stuff'), - call().__exit__(None, None, None)] - >>> m.assert_called_once_with('foo', 'w') - >>> handle = m() - >>> handle.write.assert_called_once_with('some stuff') - -And for reading files: - - >>> with patch('__main__.open', mock_open(read_data='bibble')) as m: - ... with open('foo') as h: - ... result = h.read() - ... - >>> m.assert_called_once_with('foo') - >>> assert result == 'bibble' - - -.. _auto-speccing: - -Autospeccing -~~~~~~~~~~~~ - -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the -api of mocks to the api of an original object (the spec), but it is recursive -(implemented lazily) so that attributes of mocks only have the same api as -the attributes of the spec. In addition mocked functions / methods have the -same call signature as the original so they raise a :exc:`TypeError` if they are -called incorrectly. - -Before I explain how auto-speccing works, here's why it is needed. - -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) - -Your tests can pass silently and incorrectly because of the typo. - -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. - -Note that this is another reason why you need integration tests as well as -unit tests. Testing everything in isolation is all fine and dandy, but if you -don't test how your units are "wired together" there is still lots of room -for bugs that tests might have caught. - -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access -attributes on the mock that exist on the real class: - - >>> from urllib import request - >>> mock = Mock(spec=request.Request) - >>> mock.assret_called_with - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - -The spec only applies to the mock itself, so we still have the same issue -with any methods on the mock: - -.. code-block:: pycon - - >>> mock.has_data() - - >>> mock.has_data.assret_called_with() - -Auto-speccing solves this problem. You can either pass ``autospec=True`` to -:func:`patch` / :func:`patch.object` or use the :func:`create_autospec` function to create a -mock with a spec. If you use the ``autospec=True`` argument to :func:`patch` then the -object that is being replaced will be used as the spec object. Because the -speccing is done "lazily" (the spec is created as attributes on the mock are -accessed) you can use it with very complex or deeply nested objects (like -modules that import modules that import modules) without a big performance -hit. - -Here's an example of it in use: - - >>> from urllib import request - >>> patcher = patch('__main__.request', autospec=True) - >>> mock_request = patcher.start() - >>> request is mock_request - True - >>> mock_request.Request - - -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two -arguments in the constructor (one of which is *self*). Here's what happens if -we try to call it incorrectly: - - >>> req = request.Request() - Traceback (most recent call last): - ... - TypeError: () takes at least 2 arguments (1 given) - -The spec also applies to instantiated classes (i.e. the return value of -specced mocks): - - >>> req = request.Request('foo') - >>> req - - -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place -any typos in our asserts will raise the correct error: - - >>> req.add_header('spam', 'eggs') - - >>> req.add_header.assret_called_with - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - >>> req.add_header.assert_called_with('spam', 'eggs') - -In many cases you will just be able to add ``autospec=True`` to your existing -:func:`patch` calls and then be protected against bugs due to typos and api -changes. - -As well as using *autospec* through :func:`patch` there is a -:func:`create_autospec` for creating autospecced mocks directly: - - >>> from urllib import request - >>> mock_request = create_autospec(request) - >>> mock_request.Request('foo', 'bar') - - -This isn't without caveats and limitations however, which is why it is not -the default behaviour. In order to know what attributes are available on the -spec object, autospec has to introspect (access attributes) the spec. As you -traverse attributes on the mock a corresponding traversal of the original -object is happening under the hood. If any of your specced objects have -properties or descriptors that can trigger code execution then you may not be -able to use autospec. On the other hand it is much better to design your -objects so that introspection is safe [#]_. - -A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. -*autospec* can't know about any dynamically created attributes and restricts -the api to visible attributes. - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -There are a few different ways of resolving this problem. The easiest, but -not necessarily the least annoying, way is to simply set the required -attributes on the mock after creation. Just because *autospec* doesn't allow -you to fetch attributes that don't exist on the spec it doesn't prevent you -setting them: - - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a = 33 - ... - -There is a more aggressive version of both *spec* and *autospec* that *does* -prevent you setting non-existent attributes. This is useful if you want to -ensure your code only *sets* valid attributes too, but obviously it prevents -this particular scenario: - - >>> with patch('__main__.Something', autospec=True, spec_set=True): - ... thing = Something() - ... thing.a = 33 - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via -class attributes (shared between instances of course) is faster too. e.g. - -.. code-block:: python - - class Something: - a = 33 - -This brings up another issue. It is relatively common to provide a default -value of ``None`` for members that will later be an object of a different type. -``None`` would be useless as a spec because it wouldn't let you access *any* -attributes or methods on it. As ``None`` is *never* going to be useful as a -spec, and probably indicates a member that will normally of some other type, -autospec doesn't use a spec for members that are set to ``None``. These will -just be ordinary mocks (well - MagicMocks): - - >>> class Something: - ... member = None - ... - >>> mock = create_autospec(Something) - >>> mock.member.foo.bar.baz() - - -If modifying your production classes to add defaults isn't to your liking -then there are more options. One of these is simply to use an instance as the -spec rather than the class. The other is to create a subclass of the -production class and add the defaults to the subclass without affecting the -production class. Both of these require you to use an alternative object as -the spec. Thankfully :func:`patch` supports this - you can simply pass the -alternative object as the *autospec* argument: - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> class SomethingForTest(Something): - ... a = 33 - ... - >>> p = patch('__main__.Something', autospec=SomethingForTest) - >>> mock = p.start() - >>> mock.a - - - -.. [#] This only applies to classes or already instantiated objects. Calling - a mocked class to create a mock instance *does not* create a real instance. - It is only attribute lookups - along with calls to :func:`dir` - that are done. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/unittest.rst --- a/Doc/library/unittest.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/unittest.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,14 +11,17 @@ (If you are already familiar with the basic concepts of testing, you might want to skip to :ref:`the list of assert methods `.) -The :mod:`unittest` unit testing framework was originally inspired by JUnit -and has a similar flavor as major unit testing frameworks in other -languages. It supports test automation, sharing of setup and shutdown code -for tests, aggregation of tests into collections, and independence of the -tests from the reporting framework. - -To achieve this, :mod:`unittest` supports some important concepts in an -object-oriented way: +The Python unit testing framework, sometimes referred to as "PyUnit," is a +Python language version of JUnit, by Kent Beck and Erich Gamma. JUnit is, in +turn, a Java version of Kent's Smalltalk testing framework. Each is the de +facto standard unit testing framework for its respective language. + +:mod:`unittest` supports test automation, sharing of setup and shutdown code for +tests, aggregation of tests into collections, and independence of the tests from +the reporting framework. The :mod:`unittest` module provides classes that make +it easy to support these qualities for a set of tests. + +To achieve this, :mod:`unittest` supports some important concepts: test fixture A :dfn:`test fixture` represents the preparation needed to perform one or more @@ -27,7 +30,7 @@ process. test case - A :dfn:`test case` is the individual unit of testing. It checks for a specific + A :dfn:`test case` is the smallest unit of testing. It checks for a specific response to a particular set of inputs. :mod:`unittest` provides a base class, :class:`TestCase`, which may be used to create new test cases. @@ -41,21 +44,52 @@ a textual interface, or return a special value to indicate the results of executing the tests. +The test case and test fixture concepts are supported through the +:class:`TestCase` and :class:`FunctionTestCase` classes; the former should be +used when creating new tests, and the latter can be used when integrating +existing test code with a :mod:`unittest`\ -driven framework. When building test +fixtures using :class:`TestCase`, the :meth:`~TestCase.setUp` and +:meth:`~TestCase.tearDown` methods can be overridden to provide initialization +and cleanup for the fixture. With :class:`FunctionTestCase`, existing functions +can be passed to the constructor for these purposes. When the test is run, the +fixture initialization is run first; if it succeeds, the cleanup method is run +after the test has been executed, regardless of the outcome of the test. Each +instance of the :class:`TestCase` will only be used to run a single test method, +so a new fixture is created for each test. + +Test suites are implemented by the :class:`TestSuite` class. This class allows +individual tests and test suites to be aggregated; when the suite is executed, +all tests added directly to the suite and in "child" test suites are run. + +A test runner is an object that provides a single method, +:meth:`~TestRunner.run`, which accepts a :class:`TestCase` or :class:`TestSuite` +object as a parameter, and returns a result object. The class +:class:`TestResult` is provided for use as the result object. :mod:`unittest` +provides the :class:`TextTestRunner` as an example test runner which reports +test results on the standard error stream by default. Alternate runners can be +implemented for other environments (such as graphical environments) without any +need to derive from a specific class. + .. seealso:: Module :mod:`doctest` Another test-support module with a very different flavor. - `Simple Smalltalk Testing: With Patterns `_ + `unittest2: A backport of new unittest features for Python 2.4-2.6 `_ + Many new features were added to unittest in Python 2.7, including test + discovery. unittest2 allows you to use these features with earlier + versions of Python. + + `Simple Smalltalk Testing: With Patterns `_ Kent Beck's original paper on testing frameworks using the pattern shared by :mod:`unittest`. - `Nose `_ and `py.test `_ + `Nose `_ and `py.test `_ Third-party unittest frameworks with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. - `The Python Testing Tools Taxonomy `_ + `The Python Testing Tools Taxonomy `_ An extensive list of Python testing tools including functional testing frameworks and mock object libraries. @@ -65,10 +99,9 @@ The script :file:`Tools/unittestgui/unittestgui.py` in the Python source distribution is a GUI tool for test discovery and execution. This is intended largely for ease of use - for those new to unit testing. For production environments it is - recommended that tests be driven by a continuous integration system such as - `Buildbot `_, `Jenkins `_ - or `Hudson `_. + for those new to unit testing. For production environments it is recommended that + tests be driven by a continuous integration system such as `Hudson `_ + or `Buildbot `_. .. _unittest-minimal-example: @@ -80,29 +113,37 @@ running tests. This section demonstrates that a small subset of the tools suffice to meet the needs of most users. -Here is a short script to test three string methods:: - - import unittest - - class TestStringMethods(unittest.TestCase): - - def test_upper(self): - self.assertEqual('foo'.upper(), 'FOO') - - def test_isupper(self): - self.assertTrue('FOO'.isupper()) - self.assertFalse('Foo'.isupper()) - - def test_split(self): - s = 'hello world' - self.assertEqual(s.split(), ['hello', 'world']) - # check that s.split fails when the separator is not a string - with self.assertRaises(TypeError): - s.split(2) - - if __name__ == '__main__': - unittest.main() - +Here is a short script to test three functions from the :mod:`random` module:: + + import random + import unittest + + class TestSequenceFunctions(unittest.TestCase): + + def setUp(self): + self.seq = list(range(10)) + + def test_shuffle(self): + # make sure the shuffled sequence does not lose any elements + random.shuffle(self.seq) + self.seq.sort() + self.assertEqual(self.seq, list(range(10))) + + # should raise an exception for an immutable sequence + self.assertRaises(TypeError, random.shuffle, (1,2,3)) + + def test_choice(self): + element = random.choice(self.seq) + self.assertTrue(element in self.seq) + + def test_sample(self): + with self.assertRaises(ValueError): + random.sample(self.seq, 20) + for element in random.sample(self.seq, 5): + self.assertTrue(element in self.seq) + + if __name__ == '__main__': + unittest.main() A testcase is created by subclassing :class:`unittest.TestCase`. The three individual tests are defined with methods whose names start with the letters @@ -110,15 +151,16 @@ represent tests. The crux of each test is a call to :meth:`~TestCase.assertEqual` to check for an -expected result; :meth:`~TestCase.assertTrue` or :meth:`~TestCase.assertFalse` -to verify a condition; or :meth:`~TestCase.assertRaises` to verify that a -specific exception gets raised. These methods are used instead of the -:keyword:`assert` statement so the test runner can accumulate all test results -and produce a report. - -The :meth:`~TestCase.setUp` and :meth:`~TestCase.tearDown` methods allow you -to define instructions that will be executed before and after each test method. -They are covered in more detail in the section :ref:`organizing-tests`. +expected result; :meth:`~TestCase.assertTrue` to verify a condition; or +:meth:`~TestCase.assertRaises` to verify that an expected exception gets raised. +These methods are used instead of the :keyword:`assert` statement so the test +runner can accumulate all test results and produce a report. + +When a :meth:`~TestCase.setUp` method is defined, the test runner will run that +method prior to each test. Likewise, if a :meth:`~TestCase.tearDown` method is +defined, the test runner will invoke that method after each test. In the +example, :meth:`~TestCase.setUp` was used to create a fresh sequence for each +test. The final block shows a simple way to run the tests. :func:`unittest.main` provides a command-line interface to the test script. When run from the command @@ -130,15 +172,22 @@ OK -Passing the ``-v`` option to your test script will instruct :func:`unittest.main` -to enable a higher level of verbosity, and produce the following output:: - - test_isupper (__main__.TestStringMethods) ... ok - test_split (__main__.TestStringMethods) ... ok - test_upper (__main__.TestStringMethods) ... ok +Instead of :func:`unittest.main`, there are other ways to run the tests with a +finer level of control, less terse output, and no requirement to be run from the +command line. For example, the last two lines may be replaced with:: + + suite = unittest.TestLoader().loadTestsFromTestCase(TestSequenceFunctions) + unittest.TextTestRunner(verbosity=2).run(suite) + +Running the revised script from the interpreter or another script produces the +following output:: + + test_choice (__main__.TestSequenceFunctions) ... ok + test_sample (__main__.TestSequenceFunctions) ... ok + test_shuffle (__main__.TestSequenceFunctions) ... ok ---------------------------------------------------------------------- - Ran 3 tests in 0.001s + Ran 3 tests in 0.110s OK @@ -204,8 +253,8 @@ .. cmdoption:: -c, --catch - :kbd:`Control-C` during the test run waits for the current test to end and then - reports all the results so far. A second :kbd:`Control-C` raises the normal + Control-C during the test run waits for the current test to end and then + reports all the results so far. A second control-C raises the normal :exc:`KeyboardInterrupt` exception. See `Signal Handling`_ for the functions that provide this functionality. @@ -214,16 +263,9 @@ Stop the test run on the first error or failure. -.. cmdoption:: --locals - - Show local variables in tracebacks. - .. versionadded:: 3.2 The command-line options ``-b``, ``-c`` and ``-f`` were added. -.. versionadded:: 3.5 - The command-line option ``--locals``. - The command line can also be used for test discovery, for running all of the tests in a project or just a subset. @@ -237,10 +279,9 @@ Unittest supports simple test discovery. In order to be compatible with test discovery, all of the test files must be :ref:`modules ` or -:ref:`packages ` (including :term:`namespace packages -`) importable from the top-level directory of -the project (this means that their filenames must be valid :ref:`identifiers -`). +:ref:`packages ` importable from the top-level directory of +the project (this means that their filenames must be valid +:ref:`identifiers `). Test discovery is implemented in :meth:`TestLoader.discover`, but can also be used from the command line. The basic command-line usage is:: @@ -262,15 +303,15 @@ Verbose output -.. cmdoption:: -s, --start-directory directory +.. cmdoption:: -s directory Directory to start discovery (``.`` default) -.. cmdoption:: -p, --pattern pattern +.. cmdoption:: -p pattern Pattern to match test files (``test*.py`` default) -.. cmdoption:: -t, --top-level-directory directory +.. cmdoption:: -t directory Top level directory of project (defaults to start directory) @@ -278,8 +319,8 @@ as positional arguments in that order. The following two command lines are equivalent:: - python -m unittest discover -s project_directory -p "*_test.py" - python -m unittest discover project_directory "*_test.py" + python -m unittest discover -s project_directory -p '*_test.py' + python -m unittest discover project_directory '*_test.py' As well as being a path it is possible to pass a package name, for example ``myproject.subpackage.test``, as the start directory. The package name you @@ -305,9 +346,6 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. -.. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages `. - .. _organizing-tests: @@ -320,30 +358,45 @@ To make your own test cases you must write subclasses of :class:`TestCase` or use :class:`FunctionTestCase`. +An instance of a :class:`TestCase`\ -derived class is an object that can +completely run a single test method, together with optional set-up and tidy-up +code. + The testing code of a :class:`TestCase` instance should be entirely self contained, such that it can be run either in isolation or in arbitrary combination with any number of other test cases. -The simplest :class:`TestCase` subclass will simply implement a test method -(i.e. a method whose name starts with ``test``) in order to perform specific -testing code:: +The simplest :class:`TestCase` subclass will simply override the +:meth:`~TestCase.runTest` method in order to perform specific testing code:: import unittest class DefaultWidgetSizeTestCase(unittest.TestCase): - def test_default_widget_size(self): + def runTest(self): widget = Widget('The widget') - self.assertEqual(widget.size(), (50, 50)) + self.assertEqual(widget.size(), (50, 50), 'incorrect default size') Note that in order to test something, we use one of the :meth:`assert\*` methods provided by the :class:`TestCase` base class. If the test fails, an exception will be raised, and :mod:`unittest` will identify the test case as a -:dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. - -Tests can be numerous, and their set-up can be repetitive. Luckily, we -can factor out set-up code by implementing a method called -:meth:`~TestCase.setUp`, which the testing framework will automatically -call for every single test we run:: +:dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. This +helps you identify where the problem is: :dfn:`failures` are caused by incorrect +results - a 5 where you expected a 6. :dfn:`Errors` are caused by incorrect +code - e.g., a :exc:`TypeError` caused by an incorrect function call. + +The way to run a test case will be described later. For now, note that to +construct an instance of such a test case, we call its constructor without +arguments:: + + testCase = DefaultWidgetSizeTestCase() + +Now, such test cases can be numerous, and their set-up can be repetitive. In +the above case, constructing a :class:`Widget` in each of 100 Widget test case +subclasses would mean unsightly duplication. + +Luckily, we can factor out such set-up code by implementing a method called +:meth:`~TestCase.setUp`, which the testing framework will automatically call for +us when we run the test:: import unittest @@ -351,26 +404,23 @@ def setUp(self): self.widget = Widget('The widget') - def test_default_widget_size(self): + class DefaultWidgetSizeTestCase(SimpleWidgetTestCase): + def runTest(self): self.assertEqual(self.widget.size(), (50,50), 'incorrect default size') - def test_widget_resize(self): + class WidgetResizeTestCase(SimpleWidgetTestCase): + def runTest(self): self.widget.resize(100,150) self.assertEqual(self.widget.size(), (100,150), 'wrong size after resize') -.. note:: - The order in which the various tests will be run is determined - by sorting the test method names with respect to the built-in - ordering for strings. - If the :meth:`~TestCase.setUp` method raises an exception while the test is -running, the framework will consider the test to have suffered an error, and -the test method will not be executed. +running, the framework will consider the test to have suffered an error, and the +:meth:`~TestCase.runTest` method will not be executed. Similarly, we can provide a :meth:`~TestCase.tearDown` method that tidies up -after the test method has been run:: +after the :meth:`~TestCase.runTest` method has been run:: import unittest @@ -380,20 +430,59 @@ def tearDown(self): self.widget.dispose() - -If :meth:`~TestCase.setUp` succeeded, :meth:`~TestCase.tearDown` will be -run whether the test method succeeded or not. + self.widget = None + +If :meth:`~TestCase.setUp` succeeded, the :meth:`~TestCase.tearDown` method will +be run whether :meth:`~TestCase.runTest` succeeded or not. Such a working environment for the testing code is called a :dfn:`fixture`. +Often, many small test cases will use the same fixture. In this case, we would +end up subclassing :class:`SimpleWidgetTestCase` into many small one-method +classes such as :class:`DefaultWidgetSizeTestCase`. This is time-consuming and +discouraging, so in the same vein as JUnit, :mod:`unittest` provides a simpler +mechanism:: + + import unittest + + class WidgetTestCase(unittest.TestCase): + def setUp(self): + self.widget = Widget('The widget') + + def tearDown(self): + self.widget.dispose() + self.widget = None + + def test_default_size(self): + self.assertEqual(self.widget.size(), (50,50), + 'incorrect default size') + + def test_resize(self): + self.widget.resize(100,150) + self.assertEqual(self.widget.size(), (100,150), + 'wrong size after resize') + +Here we have not provided a :meth:`~TestCase.runTest` method, but have instead +provided two different test methods. Class instances will now each run one of +the :meth:`test_\*` methods, with ``self.widget`` created and destroyed +separately for each instance. When creating an instance we must specify the +test method it is to run. We do this by passing the method name in the +constructor:: + + defaultSizeTestCase = WidgetTestCase('test_default_size') + resizeTestCase = WidgetTestCase('test_resize') + Test case instances are grouped together according to the features they test. :mod:`unittest` provides a mechanism for this: the :dfn:`test suite`, -represented by :mod:`unittest`'s :class:`TestSuite` class. In most cases, -calling :func:`unittest.main` will do the right thing and collect all the -module's test cases for you, and then execute them. - -However, should you want to customize the building of your test suite, -you can do it yourself:: +represented by :mod:`unittest`'s :class:`TestSuite` class:: + + widgetTestSuite = unittest.TestSuite() + widgetTestSuite.addTest(WidgetTestCase('test_default_size')) + widgetTestSuite.addTest(WidgetTestCase('test_resize')) + +For the ease of running tests, as we will see later, it is a good idea to +provide in each test module a callable object that returns a pre-built test +suite:: def suite(): suite = unittest.TestSuite() @@ -401,6 +490,37 @@ suite.addTest(WidgetTestCase('test_resize')) return suite +or even:: + + def suite(): + tests = ['test_default_size', 'test_resize'] + + return unittest.TestSuite(map(WidgetTestCase, tests)) + +Since it is a common pattern to create a :class:`TestCase` subclass with many +similarly named test functions, :mod:`unittest` provides a :class:`TestLoader` +class that can be used to automate the process of creating a test suite and +populating it with individual tests. For example, :: + + suite = unittest.TestLoader().loadTestsFromTestCase(WidgetTestCase) + +will create a test suite that will run ``WidgetTestCase.test_default_size()`` and +``WidgetTestCase.test_resize``. :class:`TestLoader` uses the ``'test'`` method +name prefix to identify test methods automatically. + +Note that the order in which the various test cases will be run is +determined by sorting the test function names with respect to the +built-in ordering for strings. + +Often it is desirable to group suites of test cases together, so as to run tests +for the whole system at once. This is easy, since :class:`TestSuite` instances +can be added to a :class:`TestSuite` just as :class:`TestCase` instances can be +added to a :class:`TestSuite`:: + + suite1 = module1.TheTestSuite() + suite2 = module2.TheTestSuite() + alltests = unittest.TestSuite([suite1, suite2]) + You can place the definitions of test cases and test suites in the same modules as the code they are to test (such as :file:`widget.py`), but there are several advantages to placing the test code in a separate module, such as @@ -443,13 +563,23 @@ assert something.name is not None # ... -one can create an equivalent test case instance as follows, with optional -set-up and tear-down methods:: +one can create an equivalent test case instance as follows:: + + testcase = unittest.FunctionTestCase(testSomething) + +If there are additional set-up and tear-down methods that should be called as +part of the test case's operation, they can also be provided like so:: testcase = unittest.FunctionTestCase(testSomething, setUp=makeSomethingDB, tearDown=deleteSomethingDB) +To make migrating existing test suites easier, :mod:`unittest` supports tests +raising :exc:`AssertionError` to indicate test failure. However, it is +recommended that you use the explicit :meth:`TestCase.fail\*` and +:meth:`TestCase.assert\*` methods instead, as future versions of :mod:`unittest` +may treat :exc:`AssertionError` differently. + .. note:: Even though :class:`FunctionTestCase` can be used to quickly convert an @@ -471,14 +601,14 @@ .. versionadded:: 3.1 Unittest supports skipping individual test methods and even whole classes of -tests. In addition, it supports marking a test as an "expected failure," a test +tests. In addition, it supports marking a test as a "expected failure," a test that is broken and will fail, but shouldn't be counted as a failure on a :class:`TestResult`. Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` or one of its conditional variants. -Basic skipping looks like this:: +Basic skipping looks like this: :: class MyTestCase(unittest.TestCase): @@ -497,7 +627,7 @@ # windows specific testing code pass -This is the output of running the example above in verbose mode:: +This is the output of running the example above in verbose mode: :: test_format (__main__.MyTestCase) ... skipped 'not supported in this library version' test_nothing (__main__.MyTestCase) ... skipped 'demonstrating skipping' @@ -508,9 +638,9 @@ OK (skipped=3) -Classes can be skipped just like methods:: - - @unittest.skip("showing class skipping") +Classes can be skipped just like methods: :: + + @skip("showing class skipping") class MySkippedTestCase(unittest.TestCase): def test_not_run(self): pass @@ -527,12 +657,12 @@ It's easy to roll your own skipping decorators by making a decorator that calls :func:`skip` on the test when it wants it to be skipped. This decorator skips -the test unless the passed object has a certain attribute:: +the test unless the passed object has a certain attribute: :: def skipUnlessHasattr(obj, attr): if hasattr(obj, attr): return lambda func: func - return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) + return unittest.skip("{0!r} doesn't have {1!r}".format(obj, attr)) The following decorators implement test skipping and expected failures: @@ -554,78 +684,8 @@ Mark the test as an expected failure. If the test fails when run, the test is not counted as a failure. -.. exception:: SkipTest(reason) - - This exception is raised to skip a test. - - Usually you can use :meth:`TestCase.skipTest` or one of the skipping - decorators instead of raising this directly. - -Skipped tests will not have :meth:`~TestCase.setUp` or :meth:`~TestCase.tearDown` run around them. -Skipped classes will not have :meth:`~TestCase.setUpClass` or :meth:`~TestCase.tearDownClass` run. -Skipped modules will not have :func:`setUpModule` or :func:`tearDownModule` run. - - -.. _subtests: - -Distinguishing test iterations using subtests ---------------------------------------------- - -.. versionadded:: 3.4 - -When some of your tests differ only by a some very small differences, for -instance some parameters, unittest allows you to distinguish them inside -the body of a test method using the :meth:`~TestCase.subTest` context manager. - -For example, the following test:: - - class NumbersTest(unittest.TestCase): - - def test_even(self): - """ - Test that numbers between 0 and 5 are all even. - """ - for i in range(0, 6): - with self.subTest(i=i): - self.assertEqual(i % 2, 0) - -will produce the following output:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=1) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=3) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) (i=5) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - -Without using a subtest, execution would stop after the first failure, -and the error would be less easy to diagnose because the value of ``i`` -wouldn't be displayed:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 +Skipped tests will not have :meth:`setUp` or :meth:`tearDown` run around them. +Skipped classes will not have :meth:`setUpClass` or :meth:`tearDownClass` run. .. _unittest-contents: @@ -643,22 +703,32 @@ .. class:: TestCase(methodName='runTest') - Instances of the :class:`TestCase` class represent the logical test units + Instances of the :class:`TestCase` class represent the smallest testable units in the :mod:`unittest` universe. This class is intended to be used as a base class, with specific tests being implemented by concrete subclasses. This class implements the interface needed by the test runner to allow it to drive the - tests, and methods that the test code can use to check for and report various + test, and methods that the test code can use to check for and report various kinds of failure. - Each instance of :class:`TestCase` will run a single base method: the method - named *methodName*. - In most uses of :class:`TestCase`, you will neither change - the *methodName* nor reimplement the default ``runTest()`` method. + Each instance of :class:`TestCase` will run a single test method: the method + named *methodName*. If you remember, we had an earlier example that went + something like this:: + + def suite(): + suite = unittest.TestSuite() + suite.addTest(WidgetTestCase('test_default_size')) + suite.addTest(WidgetTestCase('test_resize')) + return suite + + Here, we create two instances of :class:`WidgetTestCase`, each of which runs a + single test. .. versionchanged:: 3.2 - :class:`TestCase` can be instantiated successfully without providing a - *methodName*. This makes it easier to experiment with :class:`TestCase` - from the interactive interpreter. + :class:`TestCase` can be instantiated successfully without providing a method + name. This makes it easier to experiment with :class:`TestCase` from the + interactive interpreter. + + *methodName* defaults to :meth:`runTest`. :class:`TestCase` instances provide three groups of methods: one group used to run the test, another used by the test implementation to check conditions @@ -667,12 +737,13 @@ Methods in the first group (running the test) are: + .. method:: setUp() Method called to prepare the test fixture. This is called immediately - before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, - any exception raised by this method will be considered an error rather than - a test failure. The default implementation does nothing. + before calling the test method; any exception raised by this method will + be considered an error rather than a test failure. The default + implementation does nothing. .. method:: tearDown() @@ -680,10 +751,10 @@ Method called immediately after the test method has been called and the result recorded. This is called even if the test method raised an exception, so the implementation in subclasses may need to be particularly - careful about checking internal state. Any exception, other than :exc:`AssertionError` - or :exc:`SkipTest`, raised by this method will be considered an error rather than a - test failure. This method will only be called if the :meth:`setUp` succeeds, - regardless of the outcome of the test method. The default implementation does nothing. + careful about checking internal state. Any exception raised by this + method will be considered an error rather than a test failure. This + method will only be called if the :meth:`setUp` succeeds, regardless of + the outcome of the test method. The default implementation does nothing. .. method:: setUpClass() @@ -718,11 +789,10 @@ .. method:: run(result=None) - Run the test, collecting the result into the :class:`TestResult` object - passed as *result*. If *result* is omitted or ``None``, a temporary - result object is created (by calling the :meth:`defaultTestResult` - method) and used. The result object is returned to :meth:`run`'s - caller. + Run the test, collecting the result into the test result object passed as + *result*. If *result* is omitted or ``None``, a temporary result + object is created (by calling the :meth:`defaultTestResult` method) and + used. The result object is returned to :meth:`run`'s caller. The same effect may be had by simply calling the :class:`TestCase` instance. @@ -739,21 +809,6 @@ .. versionadded:: 3.1 - .. method:: subTest(msg=None, **params) - - Return a context manager which executes the enclosed code block as a - subtest. *msg* and *params* are optional, arbitrary values which are - displayed whenever a subtest fails, allowing you to identify them - clearly. - - A test case can contain any number of subtest declarations, and - they can be arbitrarily nested. - - See :ref:`subtests` for more information. - - .. versionadded:: 3.4 - - .. method:: debug() Run the test without collecting the result. This allows exceptions raised @@ -762,9 +817,8 @@ .. _assert-methods: - The :class:`TestCase` class provides several assert methods to check for and - report failures. The following table lists the most commonly used methods - (see the tables below for more assert methods): + The :class:`TestCase` class provides a number of methods to check for and + report failures, such as: +-----------------------------------------+-----------------------------+---------------+ | Method | Checks that | New in | @@ -885,8 +939,8 @@ - It is also possible to check the production of exceptions, warnings, and - log messages using the following methods: + It is also possible to check that exceptions and warnings are raised using + the following methods: +---------------------------------------------------------+--------------------------------------+------------+ | Method | Checks that | New in | @@ -894,17 +948,14 @@ | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegex(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | - | ` | and the message matches regex *r* | | + | :meth:`assertRaisesRegex(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | + | ` | and the message matches *re* | | +---------------------------------------------------------+--------------------------------------+------------+ | :meth:`assertWarns(warn, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | and the message matches regex *r* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 | - | ` | with minimum *level* | | + | :meth:`assertWarnsRegex(warn, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | + | ` | and the message matches *re* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -955,7 +1006,7 @@ a regular expression object or a string containing a regular expression suitable for use by :func:`re.search`. Examples:: - self.assertRaisesRegex(ValueError, "invalid literal for.*XYZ'$", + self.assertRaisesRegex(ValueError, 'invalid literal for.*XYZ$', int, 'XYZ') or:: @@ -979,25 +1030,25 @@ Test that a warning is triggered when *callable* is called with any positional or keyword arguments that are also passed to :meth:`assertWarns`. The test passes if *warning* is triggered and - fails if it isn't. Any exception is an error. + fails if it isn't. Also, any unexpected exception is an error. To catch any of a group of warnings, a tuple containing the warning classes may be passed as *warnings*. If only the *warning* and possibly the *msg* arguments are given, - return a context manager so that the code under test can be written + returns a context manager so that the code under test can be written inline rather than as a function:: with self.assertWarns(SomeWarning): do_something() - When used as a context manager, :meth:`assertWarns` accepts the + When used as a context manager, :meth:`assertRaises` accepts the additional keyword argument *msg*. The context manager will store the caught warning object in its :attr:`warning` attribute, and the source line which triggered the warnings in the :attr:`filename` and :attr:`lineno` attributes. This can be useful if the intention is to perform additional checks - on the warning caught:: + on the exception raised:: with self.assertWarns(SomeWarning) as cm: do_something() @@ -1036,47 +1087,6 @@ .. versionchanged:: 3.3 Added the *msg* keyword argument when used as a context manager. - .. method:: assertLogs(logger=None, level=None) - - A context manager to test that at least one message is logged on - the *logger* or one of its children, with at least the given - *level*. - - If given, *logger* should be a :class:`logging.Logger` object or a - :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages. - - If given, *level* should be either a numeric logging level or - its string equivalent (for example either ``"ERROR"`` or - :attr:`logging.ERROR`). The default is :attr:`logging.INFO`. - - The test passes if at least one message emitted inside the ``with`` - block matches the *logger* and *level* conditions, otherwise it fails. - - The object returned by the context manager is a recording helper - which keeps tracks of the matching log messages. It has two - attributes: - - .. attribute:: records - - A list of :class:`logging.LogRecord` objects of the matching - log messages. - - .. attribute:: output - - A list of :class:`str` objects with the formatted output of - matching messages. - - Example:: - - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - - .. versionadded:: 3.4 - There are also other methods used to perform more specific checks, such as: @@ -1101,10 +1111,10 @@ | :meth:`assertLessEqual(a, b) | ``a <= b`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegex(s, r) | ``r.search(s)`` | 3.1 | + | :meth:`assertRegex(s, re) | ``regex.search(s)`` | 3.1 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | + | :meth:`assertNotRegex(s, re) | ``not regex.search(s)`` | 3.2 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | @@ -1123,7 +1133,7 @@ like the :func:`round` function) and not *significant digits*. If *delta* is supplied instead of *places* then the difference - between *first* and *second* must be less or equal to (or greater than) *delta*. + between *first* and *second* must be less (or more) than *delta*. Supplying both *delta* and *places* raises a ``TypeError``. @@ -1255,7 +1265,7 @@ .. method:: assertListEqual(first, second, msg=None) assertTupleEqual(first, second, msg=None) - Tests that two lists or tuples are equal. If not, an error message is + Tests that two lists or tuples are equal. If not an error message is constructed that shows only the differences between the two. An error is also raised if either of the parameters are of the wrong type. These methods are used by default when comparing lists or tuples with @@ -1516,24 +1526,15 @@ Tests grouped by a :class:`TestSuite` are always accessed by iteration. Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note - that this method may be called several times on a single suite (for - example when counting tests or comparing for equality) so the tests - returned by repeated iterations before :meth:`TestSuite.run` must be the - same for each call iteration. After :meth:`TestSuite.run`, callers should - not rely on the tests returned by this method unless the caller uses a - subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve - test references. + that this method maybe called several times on a single suite + (for example when counting tests or comparing for equality) + so the tests returned must be the same for repeated iterations. .. versionchanged:: 3.2 In earlier versions the :class:`TestSuite` accessed tests directly rather than through iteration, so overriding :meth:`__iter__` wasn't sufficient for providing tests. - .. versionchanged:: 3.4 - In earlier versions the :class:`TestSuite` held references to each - :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore - that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. - In the typical usage of a :class:`TestSuite` object, the :meth:`run` method is invoked by a :class:`TestRunner` rather than by the end-user test harness. @@ -1546,22 +1547,8 @@ The :class:`TestLoader` class is used to create test suites from classes and modules. Normally, there is no need to create an instance of this class; the :mod:`unittest` module provides an instance that can be shared as - :data:`unittest.defaultTestLoader`. Using a subclass or instance, however, - allows customization of some configurable properties. - - :class:`TestLoader` objects have the following attributes: - - - .. attribute:: errors - - A list of the non-fatal errors encountered while loading tests. Not reset - by the loader at any point. Fatal errors are signalled by the relevant - a method raising an exception to the caller. Non-fatal errors are also - indicated by a synthetic test that will raise the original error when - run. - - .. versionadded:: 3.5 - + ``unittest.defaultTestLoader``. Using a subclass or instance, however, allows + customization of some configurable properties. :class:`TestLoader` objects have the following methods: @@ -1571,14 +1558,8 @@ Return a suite of all tests cases contained in the :class:`TestCase`\ -derived :class:`testCaseClass`. - A test case instance is created for each method named by - :meth:`getTestCaseNames`. By default these are the method names - beginning with ``test``. If :meth:`getTestCaseNames` returns no - methods, but the :meth:`runTest` method is implemented, a single test - case is created for that method instead. - - - .. method:: loadTestsFromModule(module, pattern=None) + + .. method:: loadTestsFromModule(module) Return a suite of all tests cases contained in the given module. This method searches *module* for classes derived from :class:`TestCase` and @@ -1595,18 +1576,11 @@ If a module provides a ``load_tests`` function it will be called to load the tests. This allows modules to customize test loading. - This is the `load_tests protocol`_. The *pattern* argument is passed as - the third argument to ``load_tests``. + This is the `load_tests protocol`_. .. versionchanged:: 3.2 Support for ``load_tests`` added. - .. versionchanged:: 3.5 - The undocumented and unofficial *use_load_tests* default argument is - deprecated and ignored, although it is still accepted for backward - compatibility. The method also now accepts a keyword-only argument - *pattern* which is passed to ``load_tests`` as the third argument. - .. method:: loadTestsFromName(name, module=None) @@ -1632,12 +1606,6 @@ The method optionally resolves *name* relative to the given *module*. - .. versionchanged:: 3.5 - If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing - *name* then a synthetic test that raises that error when run will be - returned. These errors are included in the errors accumulated by - self.errors. - .. method:: loadTestsFromNames(names, module=None) @@ -1654,32 +1622,26 @@ .. method:: discover(start_dir, pattern='test*.py', top_level_dir=None) - Find all the test modules by recursing into subdirectories from the - specified start directory, and return a TestSuite object containing them. - Only test files that match *pattern* will be loaded. (Using shell style - pattern matching.) Only module names that are importable (i.e. are valid - Python identifiers) will be loaded. + Find and return all test modules from the specified start directory, + recursing into subdirectories to find them. Only test files that match + *pattern* will be loaded. (Using shell style pattern matching.) Only + module names that are importable (i.e. are valid Python identifiers) will + be loaded. All test modules must be importable from the top level of the project. If the start directory is not the top level directory then the top level directory must be specified separately. - If importing a module fails, for example due to a syntax error, then - this will be recorded as a single error and discovery will continue. If - the import failure is due to :exc:`SkipTest` being raised, it will be - recorded as a skip instead of an error. - - If a package (a directory containing a file named :file:`__init__.py`) is - found, the package will be checked for a ``load_tests`` function. If this - exists then it will be called - ``package.load_tests(loader, tests, pattern)``. Test discovery takes care - to ensure that a package is only checked for tests once during an - invocation, even if the load_tests function itself calls - ``loader.discover``. - - If ``load_tests`` exists then discovery does *not* recurse into the - package, ``load_tests`` is responsible for loading all tests in the - package. + If importing a module fails, for example due to a syntax error, then this + will be recorded as a single error and discovery will continue. + + If a test package name (directory with :file:`__init__.py`) matches the + pattern then the package will be checked for a ``load_tests`` + function. If this exists then it will be called with *loader*, *tests*, + *pattern*. + + If load_tests exists then discovery does *not* recurse into the package, + ``load_tests`` is responsible for loading all tests in the package. The pattern is deliberately not stored as a loader attribute so that packages can continue discovery themselves. *top_level_dir* is stored so @@ -1690,19 +1652,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - Discovery works for :term:`namespace packages `. - Paths are sorted before being imported so that execution order is - the same even if the underlying file system's ordering is not - dependent on file name. - - .. versionchanged:: 3.5 - Found packages are now checked for ``load_tests`` regardless of - whether their path matches *pattern*, because it is impossible for - a package name to match the default pattern. - The following attributes of a :class:`TestLoader` can be configured either by subclassing or assignment on an instance: @@ -1761,7 +1710,8 @@ A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.fail\*` or + :meth:`TestCase.assert\*` methods. .. attribute:: skipped @@ -1785,10 +1735,12 @@ Set to ``True`` when the execution of tests should stop by :meth:`stop`. + .. attribute:: testsRun The total number of tests run so far. + .. attribute:: buffer If set to true, ``sys.stdout`` and ``sys.stderr`` will be buffered in between @@ -1798,6 +1750,7 @@ .. versionadded:: 3.2 + .. attribute:: failfast If set to true :meth:`stop` will be called on the first failure or error, @@ -1805,20 +1758,12 @@ .. versionadded:: 3.2 - .. attribute:: tb_locals - - If set to true then local variables will be shown in tracebacks. - - .. versionadded:: 3.5 .. method:: wasSuccessful() Return ``True`` if all tests run so far have passed, otherwise returns ``False``. - .. versionchanged:: 3.4 - Returns ``False`` if there were any :attr:`unexpectedSuccesses` - from tests marked with the :func:`expectedFailure` decorator. .. method:: stop() @@ -1847,14 +1792,14 @@ Called after the test case *test* has been executed, regardless of the outcome. - .. method:: startTestRun() + .. method:: startTestRun(test) Called once before any tests are executed. .. versionadded:: 3.1 - .. method:: stopTestRun() + .. method:: stopTestRun(test) Called once after all tests are executed. @@ -1863,7 +1808,7 @@ .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception. *err* is a + Called when the test case *test* raises an unexpected exception *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. @@ -1917,22 +1862,6 @@ :attr:`unexpectedSuccesses` attribute. - .. method:: addSubTest(test, subtest, outcome) - - Called when a subtest finishes. *test* is the test case - corresponding to the test method. *subtest* is a custom - :class:`TestCase` instance describing the subtest. - - If *outcome* is :const:`None`, the subtest succeeded. Otherwise, - it failed with an exception where *outcome* is a tuple of the form - returned by :func:`sys.exc_info`: ``(type, value, traceback)``. - - The default implementation does nothing when the outcome is a - success, and records subtest failures as normal failures. - - .. versionadded:: 3.4 - - .. class:: TextTestResult(stream, descriptions, verbosity) A concrete implementation of :class:`TestResult` used by the @@ -1950,25 +1879,21 @@ instead of repeatedly creating new instances. -.. class:: TextTestRunner(stream=None, descriptions=True, verbosity=1, failfast=False, \ - buffer=False, resultclass=None, warnings=None, *, tb_locals=False) +.. class:: TextTestRunner(stream=None, descriptions=True, verbosity=1, runnerclass=None, warnings=None) A basic test runner implementation that outputs results to a stream. If *stream* is ``None``, the default, :data:`sys.stderr` is used as the output stream. This class has a few configurable parameters, but is essentially very simple. Graphical - applications which run test suites should provide alternate implementations. Such - implementations should accept ``**kwargs`` as the interface to construct runners - changes when features are added to unittest. + applications which run test suites should provide alternate implementations. By default this runner shows :exc:`DeprecationWarning`, - :exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and - :exc:`ImportWarning` even if they are :ref:`ignored by default - `. Deprecation warnings caused by :ref:`deprecated unittest - methods ` are also special-cased and, when the warning - filters are ``'default'`` or ``'always'``, they will appear only once - per-module, in order to avoid too many warning messages. This behavior can - be overridden using the :option:`-Wd` or :option:`-Wa` options and leaving - *warnings* to ``None``. + :exc:`PendingDeprecationWarning`, and :exc:`ImportWarning` even if they are + :ref:`ignored by default `. Deprecation warnings caused by + :ref:`deprecated unittest methods ` are also + special-cased and, when the warning filters are ``'default'`` or ``'always'``, + they will appear only once per-module, in order to avoid too many warning + messages. This behavior can be overridden using the :option:`-Wd` or + :option:`-Wa` options and leaving *warnings* to ``None``. .. versionchanged:: 3.2 Added the ``warnings`` argument. @@ -1977,9 +1902,6 @@ The default stream is set to :data:`sys.stderr` at instantiation time rather than import time. - .. versionchanged:: 3.5 - Added the tb_locals parameter. - .. method:: _makeResult() This method returns the instance of ``TestResult`` used by :meth:`run`. @@ -1993,23 +1915,14 @@ stream, descriptions, verbosity - .. method:: run(test) - - This method is the main public interface to the `TextTestRunner`. This - method takes a :class:`TestSuite` or :class:`TestCase` instance. A - :class:`TestResult` is created by calling - :func:`_makeResult` and the test(s) are run and the - results printed to stdout. - .. function:: main(module='__main__', defaultTest=None, argv=None, testRunner=None, \ testLoader=unittest.defaultTestLoader, exit=True, verbosity=1, \ failfast=None, catchbreak=None, buffer=None, warnings=None) - A command-line program that loads a set of tests from *module* and runs them; - this is primarily for making test modules conveniently executable. - The simplest use for this function is to include the following line at the - end of a test script:: + A command-line program that runs a set of tests; this is primarily for making + test modules conveniently executable. The simplest use for this function is to + include the following line at the end of a test script:: if __name__ == '__main__': unittest.main() @@ -2020,22 +1933,10 @@ if __name__ == '__main__': unittest.main(verbosity=2) - The *defaultTest* argument is either the name of a single test or an - iterable of test names to run if no test names are specified via *argv*. If - not specified or ``None`` and no test names are provided via *argv*, all - tests found in *module* are run. - - The *argv* argument can be a list of options passed to the program, with the - first element being the program name. If not specified or ``None``, - the values of :data:`sys.argv` are used. - The *testRunner* argument can either be a test runner class or an already created instance of it. By default ``main`` calls :func:`sys.exit` with an exit code indicating success or failure of the tests run. - The *testLoader* argument has to be a :class:`TestLoader` instance, - and defaults to :data:`defaultTestLoader`. - ``main`` supports being used from the interactive interpreter by passing in the argument ``exit=False``. This displays the result on standard output without calling :func:`sys.exit`:: @@ -2043,7 +1944,7 @@ >>> from unittest import main >>> main(module='test_module', exit=False) - The *failfast*, *catchbreak* and *buffer* parameters have the same + The ``failfast``, ``catchbreak`` and ``buffer`` parameters have the same effect as the same-name `command-line options`_. The *warning* argument specifies the :ref:`warning filter ` @@ -2055,15 +1956,11 @@ This stores the result of the tests run as the ``result`` attribute. .. versionchanged:: 3.1 - The *exit* parameter was added. + The ``exit`` parameter was added. .. versionchanged:: 3.2 - The *verbosity*, *failfast*, *catchbreak*, *buffer* - and *warnings* parameters were added. - - .. versionchanged:: 3.4 - The *defaultTest* parameter was changed to also accept an iterable of - test names. + The ``verbosity``, ``failfast``, ``catchbreak``, ``buffer`` + and ``warnings`` parameters were added. load_tests Protocol @@ -2077,10 +1974,7 @@ If a test module defines ``load_tests`` it will be called by :meth:`TestLoader.loadTestsFromModule` with the following arguments:: - load_tests(loader, standard_tests, pattern) - -where *pattern* is passed straight through from ``loadTestsFromModule``. It -defaults to ``None``. + load_tests(loader, standard_tests, None) It should return a :class:`TestSuite`. @@ -2102,12 +1996,21 @@ suite.addTests(tests) return suite -If discovery is started in a directory containing a package, either from the -command line or by calling :meth:`TestLoader.discover`, then the package -:file:`__init__.py` will be checked for ``load_tests``. If that function does -not exist, discovery will recurse into the package as though it were just -another directory. Otherwise, discovery of the package's tests will be left up -to ``load_tests`` which is called with the following arguments:: +If discovery is started, either from the command line or by calling +:meth:`TestLoader.discover`, with a pattern that matches a package +name then the package :file:`__init__.py` will be checked for ``load_tests``. + +.. note:: + + The default pattern is 'test*.py'. This matches all Python files + that start with 'test' but *won't* match any test directories. + + A pattern like 'test*' will match test packages as well as + modules. + +If the package :file:`__init__.py` defines ``load_tests`` then it will be +called and discovery not continued into the package. ``load_tests`` +is called with the following arguments:: load_tests(loader, standard_tests, pattern) @@ -2126,11 +2029,6 @@ standard_tests.addTests(package_tests) return standard_tests -.. versionchanged:: 3.5 - Discovery no longer checks package names for matching *pattern* due to the - impossibility of package names matching the default pattern. - - Class and Module Fixtures ------------------------- @@ -2192,7 +2090,7 @@ If an exception is raised during a ``setUpClass`` then the tests in the class are not run and the ``tearDownClass`` is not run. Skipped classes will not have ``setUpClass`` or ``tearDownClass`` run. If the exception is a -:exc:`SkipTest` exception then the class will be reported as having been skipped +``SkipTest`` exception then the class will be reported as having been skipped instead of as an error. @@ -2209,7 +2107,7 @@ If an exception is raised in a ``setUpModule`` then none of the tests in the module will be run and the ``tearDownModule`` will not be run. If the exception is a -:exc:`SkipTest` exception then the module will be reported as having been skipped +``SkipTest`` exception then the module will be reported as having been skipped instead of as an error. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/urllib.error.rst --- a/Doc/library/urllib.error.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/urllib.error.rst Mon Jan 25 17:05:13 2016 +0100 @@ -31,9 +31,8 @@ Though being an exception (a subclass of :exc:`URLError`), an :exc:`HTTPError` can also function as a non-exceptional file-like return - value (the same thing that :func:`~urllib.request.urlopen` returns). This - is useful when handling exotic HTTP errors, such as requests for - authentication. + value (the same thing that :func:`urlopen` returns). This is useful when + handling exotic HTTP errors, such as requests for authentication. .. attribute:: code @@ -42,21 +41,9 @@ to a value found in the dictionary of codes as found in :attr:`http.server.BaseHTTPRequestHandler.responses`. - .. attribute:: reason - - This is usually a string explaining the reason for this error. - - .. attribute:: headers - - The HTTP response headers for the HTTP request that caused the - :exc:`HTTPError`. - - .. versionadded:: 3.4 - .. exception:: ContentTooShortError(msg, content) - This exception is raised when the :func:`~urllib.request.urlretrieve` - function detects that + This exception is raised when the :func:`urlretrieve` function detects that the amount of the downloaded data is less than the expected amount (given by the *Content-Length* header). The :attr:`content` attribute stores the downloaded (and supposedly truncated) data. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/urllib.parse.rst --- a/Doc/library/urllib.parse.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/urllib.parse.rst Mon Jan 25 17:05:13 2016 +0100 @@ -22,11 +22,11 @@ to an absolute URL given a "base URL." The module has been designed to match the Internet RFC on Relative Uniform -Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, -``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtspu``, ``sftp``, -``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, ``telnet``, -``wais``. +Resource Locators (and discovered a bug in an earlier draft!). It supports the +following URL schemes: ``file``, ``ftp``, ``gopher``, ``hdl``, ``http``, +``https``, ``imap``, ``mailto``, ``mms``, ``news``, ``nntp``, ``prospero``, +``rsync``, ``rtsp``, ``rtspu``, ``sftp``, ``shttp``, ``sip``, ``sips``, +``snews``, ``svn``, ``svn+ssh``, ``telnet``, ``wais``. The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in @@ -69,22 +69,20 @@ >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + >>> urlparse('www.cwi.nl:80/%7Eguido/Python.html') + ParseResult(scheme='', netloc='', path='www.cwi.nl:80/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('help/Python.html') ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') - The *scheme* argument gives the default addressing scheme, to be - used only if the URL does not specify one. It should be the same type - (text or bytes) as *urlstring*, except that the default value ``''`` is - always allowed, and is automatically converted to ``b''`` if appropriate. + If the *scheme* argument is specified, it gives the default addressing + scheme, to be used only if the URL does not specify one. The default value for + this argument is the empty string. If the *allow_fragments* argument is false, fragment identifiers are not - recognized. Instead, they are parsed as part of the path, parameters - or query component, and :attr:`fragment` is set to the empty string in - the return value. + allowed, even if the URL's addressing scheme normally does support them. The + default value for this argument is :const:`True`. The return value is actually an instance of a subclass of :class:`tuple`. This class has the following additional read-only convenience attributes: @@ -92,7 +90,7 @@ +------------------+-------+--------------------------+----------------------+ | Attribute | Index | Value | Value if not present | +==================+=======+==========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | + | :attr:`scheme` | 0 | URL scheme specifier | empty string | +------------------+-------+--------------------------+----------------------+ | :attr:`netloc` | 1 | Network location part | empty string | +------------------+-------+--------------------------+----------------------+ @@ -115,22 +113,12 @@ | | | if present | | +------------------+-------+--------------------------+----------------------+ - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. + See section :ref:`urlparse-result-object` for more information on the result + object. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. - .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is - false), in accordance with :rfc:`3986`. Previously, a whitelist of - schemes that support fragments existed. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace') @@ -153,9 +141,8 @@ percent-encoded sequences into Unicode characters, as accepted by the :meth:`bytes.decode` method. - Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` - parameter set to ``True``) to convert such dictionaries into query - strings. + Use the :func:`urllib.parse.urlencode` function to convert such + dictionaries into query strings. .. versionchanged:: 3.2 @@ -213,7 +200,7 @@ +------------------+-------+-------------------------+----------------------+ | Attribute | Index | Value | Value if not present | +==================+=======+=========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | + | :attr:`scheme` | 0 | URL scheme specifier | empty string | +------------------+-------+-------------------------+----------------------+ | :attr:`netloc` | 1 | Network location part | empty string | +------------------+-------+-------------------------+----------------------+ @@ -233,13 +220,8 @@ | | | if present | | +------------------+-------+-------------------------+----------------------+ - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. + See section :ref:`urlparse-result-object` for more information on the result + object. .. function:: urlunsplit(parts) @@ -280,11 +262,6 @@ :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. - .. versionchanged:: 3.5 - - Behaviour updated to match the semantics defined in :rfc:`3986`. - - .. function:: urldefrag(url) If *url* contains a fragment identifier, return a modified version of *url* @@ -531,27 +508,17 @@ Example: ``unquote_to_bytes('a%26%EF')`` yields ``b'a&\xef'``. -.. function:: urlencode(query, doseq=False, safe='', encoding=None, \ - errors=None, quote_via=quote_plus) +.. function:: urlencode(query, doseq=False, safe='', encoding=None, errors=None) Convert a mapping object or a sequence of two-element tuples, which may - contain :class:`str` or :class:`bytes` objects, to a percent-encoded ASCII - text string. If the resultant string is to be used as a *data* for POST - operation with the :func:`~urllib.request.urlopen` function, then - it should be encoded to bytes, otherwise it would result in a - :exc:`TypeError`. + either be a :class:`str` or a :class:`bytes`, to a "percent-encoded" + string. If the resultant string is to be used as a *data* for POST + operation with :func:`urlopen` function, then it should be properly encoded + to bytes, otherwise it would result in a :exc:`TypeError`. The resulting string is a series of ``key=value`` pairs separated by ``'&'`` - characters, where both *key* and *value* are quoted using the *quote_via* - function. By default, :func:`quote_plus` is used to quote the values, which - means spaces are quoted as a ``'+'`` character and '/' characters are - encoded as ``%2F``, which follows the standard for GET requests - (``application/x-www-form-urlencoded``). An alternate function that can be - passed as *quote_via* is :func:`quote`, which will encode spaces as ``%20`` - and not encode '/' characters. For maximum control of what is quoted, use - ``quote`` and specify a value for *safe*. - - When a sequence of two-element tuples is used as the *query* + characters, where both *key* and *value* are quoted using :func:`quote_plus` + above. When a sequence of two-element tuples is used as the *query* argument, the first element of each tuple is a key and the second is a value. The value element in itself can be a sequence and in that case, if the optional parameter *doseq* is evaluates to *True*, individual @@ -559,9 +526,8 @@ the value sequence for the key. The order of parameters in the encoded string will match the order of parameter tuples in the sequence. - The *safe*, *encoding*, and *errors* parameters are passed down to - *quote_via* (the *encoding* and *errors* parameters are only passed - when a query element is a :class:`str`). + When *query* parameter is a :class:`str`, the *safe*, *encoding* and *error* + parameters are passed down to :func:`quote_plus` for encoding. To reverse this encoding process, :func:`parse_qs` and :func:`parse_qsl` are provided in this module to parse query strings into Python data structures. @@ -572,9 +538,6 @@ .. versionchanged:: 3.2 Query parameter supports bytes and string objects. - .. versionadded:: 3.5 - *quote_via* parameter. - .. seealso:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/urllib.request.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,16 +12,11 @@ opening URLs (mostly HTTP) in a complex world --- basic and digest authentication, redirections, cookies and more. -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level http client interface. - The :mod:`urllib.request` module defines the following functions: -.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False, context=None) +.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None) Open the URL *url*, which can be either a string or a :class:`Request` object. @@ -36,8 +31,13 @@ *data* should be a buffer in the standard :mimetype:`application/x-www-form-urlencoded` format. The :func:`urllib.parse.urlencode` function takes a mapping or sequence of - 2-tuples and returns an ASCII text string in this format. It should - be encoded to bytes before being used as the *data* parameter. + 2-tuples and returns a string in this format. It should be encoded to bytes + before being used as the *data* parameter. The charset parameter in + ``Content-Type`` header may be used to specify the encoding. If charset + parameter is not sent with the Content-Type header, the server following the + HTTP 1.1 recommendation may assume that the data is encoded in ISO-8859-1 + encoding. It is advisable to use charset parameter with encoding used in + ``Content-Type`` header with the :class:`Request`. urllib.request module uses HTTP/1.1 and includes ``Connection:close`` header in its HTTP requests. @@ -47,52 +47,34 @@ the global default timeout setting will be used). This actually only works for HTTP, HTTPS and FTP connections. - If *context* is specified, it must be a :class:`ssl.SSLContext` instance - describing the various SSL options. See :class:`~http.client.HTTPSConnection` - for more details. - The optional *cafile* and *capath* parameters specify a set of trusted CA certificates for HTTPS requests. *cafile* should point to a single file containing a bundle of CA certificates, whereas *capath* should point to a directory of hashed certificate files. More information can be found in :meth:`ssl.SSLContext.load_verify_locations`. - The *cadefault* parameter is ignored. + .. warning:: + If neither *cafile* nor *capath* is specified, an HTTPS request + will not do any verification of the server's certificate. - This function always returns an object which can work as - :term:`context manager` and has methods such as + This function returns a file-like object that works as a :term:`context manager`, + with two additional methods from the :mod:`urllib.response` module - * :meth:`~urllib.response.addinfourl.geturl` --- return the URL of the resource retrieved, + * :meth:`geturl` --- return the URL of the resource retrieved, commonly used to determine if a redirect was followed - * :meth:`~urllib.response.addinfourl.info` --- return the meta-information of the page, such as headers, + * :meth:`info` --- return the meta-information of the page, such as headers, in the form of an :func:`email.message_from_string` instance (see `Quick Reference to HTTP Headers `_) - * :meth:`~urllib.response.addinfourl.getcode` -- return the HTTP status code of the response. - - For http and https urls, this function returns a - :class:`http.client.HTTPResponse` object slightly modified. In addition - to the three new methods above, the msg attribute contains the - same information as the :attr:`~http.client.HTTPResponse.reason` - attribute --- the reason phrase returned by server --- instead of - the response headers as it is specified in the documentation for - :class:`~http.client.HTTPResponse`. - - For ftp, file, and data urls and requests explicitly handled by legacy - :class:`URLopener` and :class:`FancyURLopener` classes, this function - returns a :class:`urllib.response.addinfourl` object. - - Raises :exc:`~urllib.error.URLError` on errors. + Raises :exc:`URLError` on errors. Note that ``None`` may be returned if no handler handles the request (though the default installed global :class:`OpenerDirector` uses :class:`UnknownHandler` to ensure this never happens). - In addition, if proxy settings are detected (for example, when a ``*_proxy`` - environment variable like :envvar:`http_proxy` is set), - :class:`ProxyHandler` is default installed and makes sure the requests are - handled through the proxy. + In addition, default installed :class:`ProxyHandler` makes sure the requests + are handled through the proxy when they are set. The legacy ``urllib.urlopen`` function from Python 2.6 and earlier has been discontinued; :func:`urllib.request.urlopen` corresponds to the old @@ -110,13 +92,6 @@ .. versionadded:: 3.2 *data* can be an iterable object. - .. versionchanged:: 3.3 - *cadefault* was added. - - .. versionchanged:: 3.4.3 - *context* was added. - - .. function:: install_opener(opener) Install an :class:`OpenerDirector` instance as the default global opener. @@ -134,10 +109,10 @@ subclasses of :class:`BaseHandler` (in which case it must be possible to call the constructor without any parameters). Instances of the following classes will be in front of the *handler*\s, unless the *handler*\s contain them, - instances of them or subclasses of them: :class:`ProxyHandler` (if proxy - settings are detected), :class:`UnknownHandler`, :class:`HTTPHandler`, - :class:`HTTPDefaultErrorHandler`, :class:`HTTPRedirectHandler`, - :class:`FTPHandler`, :class:`FileHandler`, :class:`HTTPErrorProcessor`. + instances of them or subclasses of them: :class:`ProxyHandler`, + :class:`UnknownHandler`, :class:`HTTPHandler`, :class:`HTTPDefaultErrorHandler`, + :class:`HTTPRedirectHandler`, :class:`FTPHandler`, :class:`FileHandler`, + :class:`HTTPErrorProcessor`. If the Python installation has SSL support (i.e., if the :mod:`ssl` module can be imported), :class:`HTTPSHandler` will also be added. @@ -150,14 +125,14 @@ Convert the pathname *path* from the local syntax for a path to the form used in the path component of a URL. This does not produce a complete URL. The return - value will already be quoted using the :func:`~urllib.parse.quote` function. + value will already be quoted using the :func:`quote` function. .. function:: url2pathname(path) Convert the path component *path* from a percent-encoded URL to the local syntax for a - path. This does not accept a complete URL. This function uses - :func:`~urllib.parse.unquote` to decode *path*. + path. This does not accept a complete URL. This function uses :func:`unquote` + to decode *path*. .. function:: getproxies() @@ -181,9 +156,16 @@ the only ones that use *data*; the HTTP request will be a POST instead of a GET when the *data* parameter is provided. *data* should be a buffer in the standard :mimetype:`application/x-www-form-urlencoded` format. + The :func:`urllib.parse.urlencode` function takes a mapping or sequence of - 2-tuples and returns an ASCII string in this format. It should be - encoded to bytes before being used as the *data* parameter. + 2-tuples and returns a string in this format. It should be encoded to bytes + before being used as the *data* parameter. The charset parameter in + ``Content-Type`` header may be used to specify the encoding. If charset + parameter is not sent with the Content-Type header, the server following the + HTTP 1.1 recommendation may assume that the data is encoded in ISO-8859-1 + encoding. It is advisable to use charset parameter with encoding used in + ``Content-Type`` header with the :class:`Request`. + *headers* should be a dictionary, and will be treated as if :meth:`add_header` was called with each key and value as arguments. @@ -196,7 +178,7 @@ ``"Python-urllib/2.6"`` (on Python 2.6). An example of using ``Content-Type`` header with *data* argument would be - sending a dictionary like ``{"Content-Type": "application/x-www-form-urlencoded"}``. + sending a dictionary like ``{"Content-Type":" application/x-www-form-urlencoded;charset=utf-8"}`` The final two arguments are only of interest for correct handling of third-party HTTP cookies: @@ -210,23 +192,18 @@ containing the image. *unverifiable* should indicate whether the request is unverifiable, - as defined by RFC 2965. It defaults to ``False``. An unverifiable + as defined by RFC 2965. It defaults to False. An unverifiable request is one whose URL the user did not have the option to approve. For example, if the request is for an image in an HTML document, and the user had no option to approve the automatic fetching of the image, this should be true. *method* should be a string that indicates the HTTP request method that - will be used (e.g. ``'HEAD'``). If provided, its value is stored in the + will be used (e.g. ``'HEAD'``). Its value is stored in the :attr:`~Request.method` attribute and is used by :meth:`get_method()`. - Subclasses may indicate a default method by setting the - :attr:`~Request.method` attribute in the class itself. .. versionchanged:: 3.3 - :attr:`Request.method` argument is added to the Request class. - - .. versionchanged:: 3.4 - Default :attr:`Request.method` may be indicated at the class level. + :attr:`Request.method` argument is added to the Request class. .. class:: OpenerDirector() @@ -244,7 +221,7 @@ .. class:: HTTPDefaultErrorHandler() A class which defines a default handler for HTTP error responses; all responses - are turned into :exc:`~urllib.error.HTTPError` exceptions. + are turned into :exc:`HTTPError` exceptions. .. class:: HTTPRedirectHandler() @@ -260,12 +237,12 @@ .. class:: ProxyHandler(proxies=None) Cause requests to go through a proxy. If *proxies* is given, it must be a - dictionary mapping protocol names to URLs of proxies. The default is to read - the list of proxies from the environment variables - :envvar:`_proxy`. If no proxy environment variables are set, then - in a Windows environment proxy settings are obtained from the registry's - Internet Settings section, and in a Mac OS X environment proxy information - is retrieved from the OS X System Configuration Framework. + dictionary mapping protocol names to URLs of proxies. The default is to read the + list of proxies from the environment variables :envvar:`_proxy`. + If no proxy environment variables are set, in a Windows environment, proxy + settings are obtained from the registry's Internet Settings section and in a + Mac OS X environment, proxy information is retrieved from the OS X System + Configuration Framework. To disable autodetected proxy pass an empty dictionary. @@ -282,37 +259,13 @@ fits. -.. class:: HTTPPasswordMgrWithPriorAuth() - - A variant of :class:`HTTPPasswordMgrWithDefaultRealm` that also has a - database of ``uri -> is_authenticated`` mappings. Can be used by a - BasicAuth handler to determine when to send authentication credentials - immediately instead of waiting for a ``401`` response first. - - .. versionadded:: 3.5 - - .. class:: AbstractBasicAuthHandler(password_mgr=None) This is a mixin class that helps with HTTP authentication, both to the remote host and to a proxy. *password_mgr*, if given, should be something that is compatible with :class:`HTTPPasswordMgr`; refer to section :ref:`http-password-mgr` for information on the interface that must be - supported. If *passwd_mgr* also provides ``is_authenticated`` and - ``update_authenticated`` methods (see - :ref:`http-password-mgr-with-prior-auth`), then the handler will use the - ``is_authenticated`` result for a given URI to determine whether or not to - send authentication credentials with the request. If ``is_authenticated`` - returns ``True`` for the URI, credentials are sent. If ``is_authenticated`` - is ``False``, credentials are not sent, and then if a ``401`` response is - received the request is re-sent with the authentication credentials. If - authentication succeeds, ``update_authenticated`` is called to set - ``is_authenticated`` ``True`` for the URI, so that subsequent requests to - the URI or any of its super-URIs will automatically include the - authentication credentials. - - .. versionadded:: 3.5 - Added ``is_authenticated`` support. + supported. .. class:: HTTPBasicAuthHandler(password_mgr=None) @@ -384,11 +337,6 @@ Open local files. -.. class:: DataHandler() - - Open data URLs. - - .. versionadded:: 3.4 .. class:: FTPHandler() @@ -424,12 +372,6 @@ The original URL passed to the constructor. - .. versionchanged:: 3.4 - - Request.full_url is a property with setter, getter and a deleter. Getting - :attr:`~Request.full_url` returns the original request URL with the - fragment, if it was present. - .. attribute:: Request.type The URI scheme. @@ -452,10 +394,6 @@ The entity body for the request, or None if not specified. - .. versionchanged:: 3.4 - Changing value of :attr:`Request.data` now deletes "Content-Length" - header if it was previously set or calculated. - .. attribute:: Request.unverifiable boolean, indicates whether the request is unverifiable as defined @@ -463,20 +401,13 @@ .. attribute:: Request.method - The HTTP request method to use. By default its value is :const:`None`, - which means that :meth:`~Request.get_method` will do its normal computation - of the method to be used. Its value can be set (thus overriding the default - computation in :meth:`~Request.get_method`) either by providing a default - value by setting it at the class level in a :class:`Request` subclass, or by - passing a value in to the :class:`Request` constructor via the *method* - argument. + The HTTP request method to use. This value is used by + :meth:`~Request.get_method` to override the computed HTTP request + method that would otherwise be returned. This attribute is initialized with + the value of the *method* argument passed to the constructor. .. versionadded:: 3.3 - .. versionchanged:: 3.4 - A default value can now be set in subclasses; previously it could only - be set via the constructor argument. - .. method:: Request.get_method() @@ -511,22 +442,10 @@ unredirected). -.. method:: Request.remove_header(header) - - Remove named header from the request instance (both from regular and - unredirected headers). - - .. versionadded:: 3.4 - - .. method:: Request.get_full_url() Return the URL given in the constructor. - .. versionchanged:: 3.4 - - Returns :attr:`Request.full_url` - .. method:: Request.set_proxy(host, type) @@ -535,20 +454,71 @@ URL given in the constructor. -.. method:: Request.get_header(header_name, default=None) +.. method:: Request.add_data(data) - Return the value of the given header. If the header is not present, return - the default value. + Set the :class:`Request` data to *data*. This is ignored by all handlers except + HTTP handlers --- and there it should be a byte string, and will change the + request to be ``POST`` rather than ``GET``. Deprecated in 3.3, use + :attr:`Request.data`. + .. deprecated:: 3.3 -.. method:: Request.header_items() - Return a list of tuples (header_name, header_value) of the Request headers. +.. method:: Request.has_data() -.. versionchanged:: 3.4 - The request methods add_data, has_data, get_data, get_type, get_host, - get_selector, get_origin_req_host and is_unverifiable that were deprecated - since 3.3 have been removed. + Return whether the instance has a non-\ ``None`` data. Deprecated in 3.3, + use :attr:`Request.data`. + + .. deprecated:: 3.3 + + +.. method:: Request.get_data() + + Return the instance's data. Deprecated in 3.3, use :attr:`Request.data`. + + .. deprecated:: 3.3 + + +.. method:: Request.get_type() + + Return the type of the URL --- also known as the scheme. Deprecated in 3.3, + use :attr:`Request.type`. + + .. deprecated:: 3.3 + + +.. method:: Request.get_host() + + Return the host to which a connection will be made. Deprecated in 3.3, use + :attr:`Request.host`. + + .. deprecated:: 3.3 + + +.. method:: Request.get_selector() + + Return the selector --- the part of the URL that is sent to the server. + Deprecated in 3.3, use :attr:`Request.selector`. + + .. deprecated:: 3.3 + + +.. method:: Request.get_origin_req_host() + + Return the request-host of the origin transaction, as defined by + :rfc:`2965`. See the documentation for the :class:`Request` constructor. + Deprecated in 3.3, use :attr:`Request.origin_req_host`. + + .. deprecated:: 3.3 + + +.. method:: Request.is_unverifiable() + + Return whether the request is unverifiable, as defined by RFC 2965. See the + documentation for the :class:`Request` constructor. Deprecated in 3.3, use + :attr:`Request.is_unverifiable`. + + .. deprecated:: 3.3 .. _opener-director-objects: @@ -613,8 +583,8 @@ #. Handlers with a method named like :meth:`protocol_open` are called to handle the request. This stage ends when a handler either returns a non-\ :const:`None` - value (ie. a response), or raises an exception (usually - :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. + value (ie. a response), or raises an exception (usually :exc:`URLError`). + Exceptions are allowed to propagate. In fact, the above algorithm is first tried for methods named :meth:`default_open`. If all such methods return :const:`None`, the algorithm @@ -673,9 +643,8 @@ This method, if implemented, will be called by the parent :class:`OpenerDirector`. It should return a file-like object as described in the return value of the :meth:`open` of :class:`OpenerDirector`, or ``None``. - It should raise :exc:`~urllib.error.URLError`, unless a truly exceptional - thing happens (for example, :exc:`MemoryError` should not be mapped to - :exc:`URLError`). + It should raise :exc:`URLError`, unless a truly exceptional thing happens (for + example, :exc:`MemoryError` should not be mapped to :exc:`URLError`). This method will be called before any protocol-specific open method. @@ -761,8 +730,8 @@ .. note:: Some HTTP redirections require action from this module's client code. If this - is the case, :exc:`~urllib.error.HTTPError` is raised. See :rfc:`2616` for - details of the precise meanings of the various redirection codes. + is the case, :exc:`HTTPError` is raised. See :rfc:`2616` for details of the + precise meanings of the various redirection codes. An :class:`HTTPError` exception raised as a security consideration if the HTTPRedirectHandler is presented with a redirected url which is not an HTTP, @@ -775,9 +744,9 @@ by the default implementations of the :meth:`http_error_30\*` methods when a redirection is received from the server. If a redirection should take place, return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the - redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if - no other handler should try to handle this URL, or return ``None`` if you - can't but another handler might. + redirect to *newurl*. Otherwise, raise :exc:`HTTPError` if no other handler + should try to handle this URL, or return ``None`` if you can't but another + handler might. .. note:: @@ -864,42 +833,6 @@ searched if the given *realm* has no matching user/password. -.. _http-password-mgr-with-prior-auth: - -HTTPPasswordMgrWithPriorAuth Objects ------------------------------------- - -This password manager extends :class:`HTTPPasswordMgrWithDefaultRealm` to support -tracking URIs for which authentication credentials should always be sent. - - -.. method:: HTTPPasswordMgrWithPriorAuth.add_password(realm, uri, user, \ - passwd, is_authenticated=False) - - *realm*, *uri*, *user*, *passwd* are as for - :meth:`HTTPPasswordMgr.add_password`. *is_authenticated* sets the initial - value of the ``is_authenticated`` flag for the given URI or list of URIs. - If *is_authenticated* is specified as ``True``, *realm* is ignored. - - -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) - - Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects - - -.. method:: HTTPPasswordMgrWithPriorAuth.update_authenticated(self, uri, \ - is_authenticated=False) - - Update the ``is_authenticated`` flag for the given *uri* or list - of URIs. - - -.. method:: HTTPPasswordMgrWithPriorAuth.is_authenticated(self, authuri) - - Returns the current state of the ``is_authenticated`` flag for - the given URI. - - .. _abstract-basic-auth-handler: AbstractBasicAuthHandler Objects @@ -1013,24 +946,10 @@ Open the file locally, if there is no host name, or the host name is ``'localhost'``. - .. versionchanged:: 3.2 - This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. + This method is applicable only for local hostnames. When a remote hostname + is given, an :exc:`URLError` is raised. - -.. _data-handler-objects: - -DataHandler Objects -------------------- - -.. method:: DataHandler.data_open(req) - - Read a data URL. This kind of URL contains the content encoded in the URL - itself. The data URL syntax is specified in :rfc:`2397`. This implementation - ignores white spaces in base64 encoded data URLs so the URL may be wrapped - in whatever source file it comes from. But even though some browsers don't - mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. +.. versionchanged:: 3.2 .. _ftp-handler-objects: @@ -1072,7 +991,7 @@ .. method:: UnknownHandler.unknown_open() - Raise a :exc:`~urllib.error.URLError` exception. + Raise a :exc:`URLError` exception. .. _http-error-processor-objects: @@ -1089,7 +1008,7 @@ For non-200 error codes, this simply passes the job on to the :meth:`protocol_error_code` handler methods, via :meth:`OpenerDirector.error`. Eventually, :class:`HTTPDefaultErrorHandler` will raise an - :exc:`~urllib.error.HTTPError` if no other handler handles the error. + :exc:`HTTPError` if no other handler handles the error. .. method:: HTTPErrorProcessor.https_response() @@ -1108,9 +1027,8 @@ it. :: >>> import urllib.request - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(300)) - ... + >>> f = urllib.request.urlopen('http://www.python.org/') + >>> print(f.read(300)) b'\n\n\n\n\n\n @@ -1123,11 +1041,11 @@ the returned bytes object to string once it determines or guesses the appropriate encoding. -The following W3C document, http://www.w3.org/International/O-charset\ , lists +The following W3C document, http://www.w3.org/International/O-charset , lists the various ways in which a (X)HTML or a XML document could have specified its encoding information. -As the python.org website uses *utf-8* encoding as specified in its meta tag, we +As the python.org website uses *utf-8* encoding as specified in it's meta tag, we will use the same for decoding the bytes object. :: >>> with urllib.request.urlopen('http://www.python.org/') as f: @@ -1152,9 +1070,8 @@ >>> import urllib.request >>> req = urllib.request.Request(url='https://localhost/cgi-bin/test.cgi', ... data=b'This data is passed to stdin of the CGI') - >>> with urllib.request.urlopen(req) as f: - ... print(f.read().decode('utf-8')) - ... + >>> f = urllib.request.urlopen(req) + >>> print(f.read().decode('utf-8')) Got Data: "This data is passed to stdin of the CGI" The code for the sample CGI used in the above example is:: @@ -1162,17 +1079,7 @@ #!/usr/bin/env python import sys data = sys.stdin.read() - print('Content-type: text/plain\n\nGot Data: "%s"' % data) - -Here is an example of doing a ``PUT`` request using :class:`Request`:: - - import urllib.request - DATA=b'some data' - req = urllib.request.Request(url='http://localhost:8080', data=DATA,method='PUT') - with urllib.request.urlopen(req) as f: - pass - print(f.status) - print(f.reason) + print('Content-type: text-plain\n\nGot Data: "%s"' % data) Use of Basic HTTP Authentication:: @@ -1224,7 +1131,7 @@ opener.open('http://www.example.com/') Also, remember that a few standard headers (:mailheader:`Content-Length`, -:mailheader:`Content-Type` and :mailheader:`Host`) +:mailheader:`Content-Type` without charset parameter and :mailheader:`Host`) are added when the :class:`Request` is passed to :func:`urlopen` (or :meth:`OpenerDirector.open`). @@ -1236,10 +1143,8 @@ >>> import urllib.request >>> import urllib.parse >>> params = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> url = "http://www.musi-cal.com/cgi-bin/query?%s" % params - >>> with urllib.request.urlopen(url) as f: - ... print(f.read().decode('utf-8')) - ... + >>> f = urllib.request.urlopen("http://www.musi-cal.com/cgi-bin/query?%s" % params) + >>> print(f.read().decode('utf-8')) The following example uses the ``POST`` method instead. Note that params output from urlencode is encoded to bytes before it is sent to urlopen as data:: @@ -1247,10 +1152,12 @@ >>> import urllib.request >>> import urllib.parse >>> data = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> data = data.encode('ascii') - >>> with urllib.request.urlopen("http://requestb.in/xrbl82xr", data) as f: - ... print(f.read().decode('utf-8')) - ... + >>> data = data.encode('utf-8') + >>> request = urllib.request.Request("http://requestb.in/xrbl82xr") + >>> # adding charset parameter to the Content-Type header. + >>> request.add_header("Content-Type","application/x-www-form-urlencoded;charset=utf-8") + >>> f = urllib.request.urlopen(request, data) + >>> print(f.read().decode('utf-8')) The following example uses an explicitly specified HTTP proxy, overriding environment settings:: @@ -1258,17 +1165,15 @@ >>> import urllib.request >>> proxies = {'http': 'http://proxy.example.com:8080/'} >>> opener = urllib.request.FancyURLopener(proxies) - >>> with opener.open("http://www.python.org") as f: - ... f.read().decode('utf-8') - ... + >>> f = opener.open("http://www.python.org") + >>> f.read().decode('utf-8') The following example uses no proxies at all, overriding environment settings:: >>> import urllib.request >>> opener = urllib.request.FancyURLopener({}) - >>> with opener.open("http://www.python.org/") as f: - ... f.read().decode('utf-8') - ... + >>> f = opener.open("http://www.python.org/") + >>> f.read().decode('utf-8') Legacy interface @@ -1307,7 +1212,7 @@ argument may be given to specify a ``POST`` request (normally the request type is ``GET``). The *data* argument must be a bytes object in standard :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. + :func:`urlencode` function below. :func:`urlretrieve` will raise :exc:`ContentTooShortError` when it detects that the amount of data available was less than the expected amount (which is the @@ -1332,8 +1237,6 @@ .. class:: URLopener(proxies=None, **x509) - .. deprecated:: 3.3 - Base class for opening and reading URLs. Unless you need to support opening objects using schemes other than :file:`http:`, :file:`ftp:`, or :file:`file:`, you probably want to use :class:`FancyURLopener`. @@ -1374,7 +1277,7 @@ .. method:: retrieve(url, filename=None, reporthook=None, data=None) Retrieves the contents of *url* and places it in *filename*. The return value - is a tuple consisting of a local filename and either an + is a tuple consisting of a local filename and either a :class:`email.message.Message` object containing the response headers (for remote URLs) or ``None`` (for local URLs). The caller must then open and read the contents of *filename*. If *filename* is not given and the URL refers to a @@ -1382,15 +1285,14 @@ *filename* is not given, the filename is the output of :func:`tempfile.mktemp` with a suffix that matches the suffix of the last path component of the input URL. If *reporthook* is given, it must be a function accepting three numeric - parameters: A chunk number, the maximum size chunks are read in and the total size of the download - (-1 if unknown). It will be called once at the start and after each chunk of data is read from the + parameters. It will be called after each chunk of data is read from the network. *reporthook* is ignored for local URLs. If the *url* uses the :file:`http:` scheme identifier, the optional *data* argument may be given to specify a ``POST`` request (normally the request type is ``GET``). The *data* argument must in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. + :mimetype:`application/x-www-form-urlencoded` format; see the :func:`urlencode` + function below. .. attribute:: version @@ -1403,8 +1305,6 @@ .. class:: FancyURLopener(...) - .. deprecated:: 3.3 - :class:`FancyURLopener` subclasses :class:`URLopener` providing default handling for the following HTTP response codes: 301, 302, 303, 307 and 401. For the 30x response codes listed above, the :mailheader:`Location` header is used to fetch @@ -1453,9 +1353,7 @@ pair: FTP; protocol * Currently, only the following protocols are supported: HTTP (versions 0.9 and - 1.0), FTP, local files, and data URLs. - - .. versionchanged:: 3.4 Added support for data URLs. + 1.0), FTP, and local files. * The caching feature of :func:`urlretrieve` has been disabled until someone finds the time to hack proper processing of Expiration time headers. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/urllib.robotparser.rst --- a/Doc/library/urllib.robotparser.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/urllib.robotparser.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,10 +19,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. -.. class:: RobotFileParser(url='') +.. class:: RobotFileParser() - This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. + This class provides a set of methods to read, parse and answer questions + about a single :file:`robots.txt` file. .. method:: set_url(url) @@ -53,41 +53,15 @@ Sets the time the ``robots.txt`` file was last fetched to the current time. - .. method:: crawl_delay(useragent) - Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` - for the *useragent* in question. If there is no such parameter or it - doesn't apply to the *useragent* specified or the ``robots.txt`` entry - for this parameter has invalid syntax, return ``None``. - - .. versionadded:: 3.6 - - .. method:: request_rate(useragent) - - Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. - - .. versionadded:: 3.6 - - -The following example demonstrates basic use of the :class:`RobotFileParser` -class:: +The following example demonstrates basic use of the RobotFileParser class. >>> import urllib.robotparser >>> rp = urllib.robotparser.RobotFileParser() >>> rp.set_url("http://www.musi-cal.com/robots.txt") >>> rp.read() - >>> rrate = rp.request_rate("*") - >>> rrate.requests - 3 - >>> rrate.seconds - 20 - >>> rp.crawl_delay("*") - 6 >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") False >>> rp.can_fetch("*", "http://www.musi-cal.com/") True + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/urllib.rst --- a/Doc/library/urllib.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,11 +0,0 @@ -:mod:`urllib` --- URL handling modules -====================================== - -.. module:: urllib - -``urllib`` is a package that collects several modules for working with URLs: - -* :mod:`urllib.request` for opening and reading URLs -* :mod:`urllib.error` containing the exceptions raised by :mod:`urllib.request` -* :mod:`urllib.parse` for parsing URLs -* :mod:`urllib.robotparser` for parsing ``robots.txt`` files diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/venv.rst --- a/Doc/library/venv.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,442 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - - -.. index:: pair: Environments; virtual - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv` - --------------- - -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (allowing -creation of environments with various Python versions) and can have its own -independent set of installed Python packages in its site directories. - -See :pep:`405` for more information about Python virtual environments. - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - - -.. _venv-def: - -.. note:: A virtual environment (also called a ``venv``) is a Python - environment such that the Python interpreter, libraries and scripts - installed into it are isolated from those installed in other virtual - environments, and (by default) any libraries installed in a "system" Python, - i.e. one which is installed as part of your operating system. - - A venv is a directory tree which contains Python executable files and - other files which indicate that it is a venv. - - Common installation tools such as ``Setuptools`` and ``pip`` work as - expected with venvs - i.e. when a venv is active, they install Python - packages into the venv without needing to be told to do so explicitly. - Of course, you need to install them into the venv first: this could be - done by running ``ez_setup.py`` with the venv activated, - followed by running ``easy_install pip``. Alternatively, you could download - the source tarballs and run ``python setup.py install`` after unpacking, - with the venv activated. - - When a venv is active (i.e. the venv's Python interpreter is running), the - attributes :attr:`sys.prefix` and :attr:`sys.exec_prefix` point to the base - directory of the venv, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-venv Python installation - which was used to create the venv. If a venv is not active, then - :attr:`sys.prefix` is the same as :attr:`sys.base_prefix` and - :attr:`sys.exec_prefix` is the same as :attr:`sys.base_exec_prefix` (they - all point to a non-venv Python installation). - - When a venv is active, any options that change the installation path will be - ignored from all distutils configuration files to prevent projects being - inadvertently installed outside of the virtual environment. - - When working in a command shell, users can make a venv active by running an - ``activate`` script in the venv's executables directory (the precise filename - is shell-dependent), which prepends the venv's directory for executables to - the ``PATH`` environment variable for the running shell. There should be no - need in other circumstances to activate a venv -- scripts installed into - venvs have a shebang line which points to the venv's Python interpreter. This - means that the script will run with that interpreter regardless of the value - of ``PATH``. On Windows, shebang line processing is supported if you have the - Python Launcher for Windows installed (this was added to Python in 3.3 - see - :pep:`397` for more details). Thus, double-clicking an installed script in - a Windows Explorer window should run the script with the correct interpreter - without there needing to be any reference to its venv in ``PATH``. - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary (and any necessary DLLs or other binaries, - e.g. ``pythonw.exe``), rather than copying. Defaults to ``True`` on Linux and - Unix systems, but ``False`` on Windows. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - - Creators of third-party virtual environment tools will be free to use the - provided ``EnvBuilder`` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - This method takes as required argument the path (absolute or relative to - the current directory) of the target directory which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the ``EnvBuilder`` class illustrates the hooks - available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary directories, and - returns a context object. This is just a holder for attributes (such as - paths), for use by the other methods. The directories are allowed to - exist already, as long as either ``clear`` or ``upgrade`` were - specified to allow operating on an existing environment directory. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy of the Python executable (and, under Windows, DLLs) in - the environment. On a POSIX system, if a specific executable - ``python3.x`` was used, symlinks to ``python`` and ``python3`` will be - created pointing to that executable, unless files with those names - already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created venv:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created environment. - - :param nodist: If True, setuptools and pip are not installed into the - created environment. - :param nopip: If True, pip is not installed into the created - environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - environment being created. - - :param context: The information for the environment creation request - being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the env's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the env - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the environment. - - :param context: The information for the environment creation request - being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the environment. - - :param context: The information for the environment creation request - being processed. - """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -`_. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/warnings.rst --- a/Doc/library/warnings.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/warnings.rst Mon Jan 25 17:05:13 2016 +0100 @@ -54,8 +54,6 @@ This categorization is useful to be able to filter out groups of warnings. The following warnings category classes are currently defined: -.. tabularcolumns:: |l|p{0.6\linewidth}| - +----------------------------------+-----------------------------------------------+ | Class | Description | +==================================+===============================================+ @@ -89,7 +87,7 @@ | | Unicode. | +----------------------------------+-----------------------------------------------+ | :exc:`BytesWarning` | Base category for warnings related to | -| | :class:`bytes` and :class:`bytearray`. | +| | :class:`bytes` and :class:`buffer`. | +----------------------------------+-----------------------------------------------+ | :exc:`ResourceWarning` | Base category for warnings related to | | | resource usage. | diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/wave.rst --- a/Doc/library/wave.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/wave.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,32 +19,26 @@ .. function:: open(file, mode=None) If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + seekable file-like object. *mode* can be any of - ``'rb'`` + ``'r'``, ``'rb'`` Read only mode. - ``'wb'`` + ``'w'``, ``'wb'`` Write only mode. Note that it does not allow read/write WAV files. - A *mode* of ``'rb'`` returns a :class:`Wave_read` object, while a *mode* of - ``'wb'`` returns a :class:`Wave_write` object. If *mode* is omitted and a - file-like object is passed as *file*, ``file.mode`` is used as the default - value for *mode*. + A *mode* of ``'r'`` or ``'rb'`` returns a :class:`Wave_read` object, while a + *mode* of ``'w'`` or ``'wb'`` returns a :class:`Wave_write` object. If + *mode* is omitted and a file-like object is passed as *file*, ``file.mode`` + is used as the default value for *mode* (the ``'b'`` flag is still added if + necessary). If you pass in a file-like object, the wave object will not close it when its :meth:`close` method is called; it is the caller's responsibility to close the file object. - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`with` block completes, the :meth:`Wave_read.close() - ` or :meth:`Wave_write.close() - ` method is called. - - .. versionchanged:: 3.4 - Added support for unseekable files. .. function:: openfp(file, mode) @@ -104,9 +98,8 @@ .. method:: Wave_read.getparams() - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. + Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype, + compname)``, equivalent to output of the :meth:`get\*` methods. .. method:: Wave_read.readframes(n) @@ -150,30 +143,13 @@ Wave_write Objects ------------------ -For seekable output streams, the ``wave`` header will automatically be updated -to reflect the number of frames actually written. For unseekable streams, the -*nframes* value must be accurate when the first frame data is written. An -accurate *nframes* value can be achieved either by calling -:meth:`~Wave_write.setnframes` or :meth:`~Wave_write.setparams` with the number -of frames that will be written before :meth:`~Wave_write.close` is called and -then using :meth:`~Wave_write.writeframesraw` to write the frame data, or by -calling :meth:`~Wave_write.writeframes` with all of the frame data to be -written. In the latter case :meth:`~Wave_write.writeframes` will calculate -the number of frames in the data and set *nframes* accordingly before writing -the frame data. - Wave_write objects, as returned by :func:`.open`, have the following methods: -.. versionchanged:: 3.4 - Added support for unseekable files. - .. method:: Wave_write.close() Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise - an exception if the output stream is not seekable and *nframes* does not - match the number of frames actually written. + :mod:`wave`. This method is called upon object collection. .. method:: Wave_write.setnchannels(n) @@ -197,9 +173,8 @@ .. method:: Wave_write.setnframes(n) - Set the number of frames to *n*. This will be changed later if the number - of frames actually written is different (this update attempt will - raise an error if the output stream is not seekable). + Set the number of frames to *n*. This will be changed later if more frames are + written. .. method:: Wave_write.setcomptype(type, name) @@ -225,19 +200,10 @@ Write audio frames, without correcting *nframes*. - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - .. method:: Wave_write.writeframes(data) - Write audio frames and make sure *nframes* is correct. It will raise an - error if the output stream is not seekable and the total number of frames - that have been written after *data* has been written does not match the - previously set value for *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. + Write audio frames and make sure *nframes* is correct. Note that it is invalid to set any parameters after calling :meth:`writeframes` diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/weakref.rst --- a/Doc/library/weakref.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/weakref.rst Mon Jan 25 17:05:13 2016 +0100 @@ -24,10 +24,7 @@ A weak reference to an object is not enough to keep the object alive: when the only remaining references to a referent are weak references, :term:`garbage collection` is free to destroy the referent and reuse its memory -for something else. However, until the object is actually destroyed the weak -reference may return the object even if there are no strong references to it. - -A primary use for weak references is to implement caches or +for something else. A primary use for weak references is to implement caches or mappings holding large objects, where it's desired that a large object not be kept alive solely because it appears in a cache or mapping. @@ -51,16 +48,16 @@ but keeps weak references to its elements, just like a :class:`WeakKeyDictionary` does. -:class:`finalize` provides a straight forward way to register a -cleanup function to be called when an object is garbage collected. -This is simpler to use than setting up a callback function on a raw -weak reference, since the module automatically ensures that the finalizer -remains alive until the object is collected. +Most programs should find that using one of these weak container types is all +they need -- it's not usually necessary to create your own weak references +directly. The low-level machinery used by the weak dictionary implementations +is exposed by the :mod:`weakref` module for the benefit of advanced uses. -Most programs should find that using one of these weak container types -or :class:`finalize` is all they need -- it's not usually necessary to -create your own weak references directly. The low-level machinery is -exposed by the :mod:`weakref` module for the benefit of advanced uses. +.. note:: + + Weak references to an object are cleared before the object's :meth:`__del__` + is called, to ensure that the weak reference callback (if any) finds the + object still alive. Not all objects can be weakly referenced; those objects which can include class instances, functions written in Python (but not in C), instance methods, sets, @@ -117,15 +114,6 @@ This is a subclassable type rather than a factory function. - .. attribute:: __callback__ - - This read-only attribute returns the callback currently associated to the - weakref. If there is no callback or if the referent of the weakref is - no longer alive then this attribute will have value ``None``. - - .. versionchanged:: 3.4 - Added the :attr:`__callback__` attribute. - .. function:: proxy(object[, callback]) @@ -207,98 +195,6 @@ discarded when no strong reference to it exists any more. -.. class:: WeakMethod(method) - - A custom :class:`ref` subclass which simulates a weak reference to a bound - method (i.e., a method defined on a class and looked up on an instance). - Since a bound method is ephemeral, a standard weak reference cannot keep - hold of it. :class:`WeakMethod` has special code to recreate the bound - method until either the object or the original function dies:: - - >>> class C: - ... def method(self): - ... print("method called!") - ... - >>> c = C() - >>> r = weakref.ref(c.method) - >>> r() - >>> r = weakref.WeakMethod(c.method) - >>> r() - > - >>> r()() - method called! - >>> del c - >>> gc.collect() - 0 - >>> r() - >>> - - .. versionadded:: 3.4 - -.. class:: finalize(obj, func, *args, **kwargs) - - Return a callable finalizer object which will be called when *obj* - is garbage collected. Unlike an ordinary weak reference, a finalizer - will always survive until the reference object is collected, greatly - simplifying lifecycle management. - - A finalizer is considered *alive* until it is called (either explicitly - or at garbage collection), and after that it is *dead*. Calling a live - finalizer returns the result of evaluating ``func(*arg, **kwargs)``, - whereas calling a dead finalizer returns :const:`None`. - - Exceptions raised by finalizer callbacks during garbage collection - will be shown on the standard error output, but cannot be - propagated. They are handled in the same way as exceptions raised - from an object's :meth:`__del__` method or a weak reference's - callback. - - When the program exits, each remaining live finalizer is called - unless its :attr:`atexit` attribute has been set to false. They - are called in reverse order of creation. - - A finalizer will never invoke its callback during the later part of - the :term:`interpreter shutdown` when module globals are liable to have - been replaced by :const:`None`. - - .. method:: __call__() - - If *self* is alive then mark it as dead and return the result of - calling ``func(*args, **kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: detach() - - If *self* is alive then mark it as dead and return the tuple - ``(obj, func, args, kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: peek() - - If *self* is alive then return the tuple ``(obj, func, args, - kwargs)``. If *self* is dead then return :const:`None`. - - .. attribute:: alive - - Property which is true if the finalizer is alive, false otherwise. - - .. attribute:: atexit - - A writable boolean property which by default is true. When the - program exits, it calls all remaining live finalizers for which - :attr:`.atexit` is true. They are called in reverse order of - creation. - - .. note:: - - It is important to ensure that *func*, *args* and *kwargs* do - not own any references to *obj*, either directly or indirectly, - since otherwise *obj* will never be garbage collected. In - particular, *func* should not be a bound method of *obj*. - - .. versionadded:: 3.4 - - .. data:: ReferenceType The type object for weak references objects. @@ -339,9 +235,8 @@ Weak Reference Objects ---------------------- -Weak reference objects have no methods and no attributes besides -:attr:`ref.__callback__`. A weak reference object allows the referent to be -obtained, if it still exists, by calling it: +Weak reference objects have no attributes or methods, but do allow the referent +to be obtained, if it still exists, by calling it: >>> import weakref >>> class Object: @@ -434,140 +329,3 @@ def id2obj(oid): return _id2obj_dict[oid] - -.. _finalize-examples: - -Finalizer Objects ------------------ - -The main benefit of using :class:`finalize` is that it makes it simple -to register a callback without needing to preserve the returned finalizer -object. For instance - - >>> import weakref - >>> class Object: - ... pass - ... - >>> kenny = Object() - >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS - - >>> del kenny - You killed Kenny! - -The finalizer can be called directly as well. However the finalizer -will invoke the callback at most once. - - >>> def callback(x, y, z): - ... print("CALLBACK") - ... return x + y + z - ... - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> assert f.alive - >>> assert f() == 6 - CALLBACK - >>> assert not f.alive - >>> f() # callback not called because finalizer dead - >>> del obj # callback not called because finalizer dead - -You can unregister a finalizer using its :meth:`~finalize.detach` -method. This kills the finalizer and returns the arguments passed to -the constructor when it was created. - - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> f.detach() #doctest:+ELLIPSIS - (<__main__.Object object ...>, , (1, 2), {'z': 3}) - >>> newobj, func, args, kwargs = _ - >>> assert not f.alive - >>> assert newobj is obj - >>> assert func(*args, **kwargs) == 6 - CALLBACK - -Unless you set the :attr:`~finalize.atexit` attribute to -:const:`False`, a finalizer will be called when the program exits if it -is still alive. For instance - - >>> obj = Object() - >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS - - >>> exit() #doctest:+SKIP - obj dead or exiting - - -Comparing finalizers with :meth:`__del__` methods -------------------------------------------------- - -Suppose we want to create a class whose instances represent temporary -directories. The directories should be deleted with their contents -when the first of the following events occurs: - -* the object is garbage collected, -* the object's :meth:`remove` method is called, or -* the program exits. - -We might try to implement the class using a :meth:`__del__` method as -follows:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - - def remove(self): - if self.name is not None: - shutil.rmtree(self.name) - self.name = None - - @property - def removed(self): - return self.name is None - - def __del__(self): - self.remove() - -Starting with Python 3.4, :meth:`__del__` methods no longer prevent -reference cycles from being garbage collected, and module globals are -no longer forced to :const:`None` during :term:`interpreter shutdown`. -So this code should work without any issues on CPython. - -However, handling of :meth:`__del__` methods is notoriously implementation -specific, since it depends on internal details of the interpreter's garbage -collector implementation. - -A more robust alternative can be to define a finalizer which only references -the specific functions and objects that it needs, rather than having access -to the full state of the object:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - self._finalizer = weakref.finalize(self, shutil.rmtree, self.name) - - def remove(self): - self._finalizer() - - @property - def removed(self): - return not self._finalizer.alive - -Defined like this, our finalizer only receives a reference to the details -it needs to clean up the directory appropriately. If the object never gets -garbage collected the finalizer will still be called at exit. - -The other advantage of weakref based finalizers is that they can be used to -register finalizers for classes where the definition is controlled by a -third party, such as running code when a module is unloaded:: - - import weakref, sys - def unloading_module(): - # implicit reference to the module globals from the function body - weakref.finalize(sys.modules[__name__], unloading_module) - - -.. note:: - - If you create a finalizer object in a daemonic thread just as the program - exits then there is the possibility that the finalizer - does not get called at exit. However, in a daemonic thread - :func:`atexit.register`, ``try: ... finally: ...`` and ``with: ...`` - do not guarantee that cleanup occurs either. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/webbrowser.rst --- a/Doc/library/webbrowser.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/webbrowser.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,12 +19,12 @@ available. If text-mode browsers are used, the calling process will block until the user exits the browser. -If the environment variable :envvar:`BROWSER` exists, it is interpreted as the -:data:`os.pathsep`-separated list of browsers to try ahead of the platform -defaults. When the value of a list part contains the string ``%s``, then it is -interpreted as a literal browser command line to be used with the argument URL -substituted for ``%s``; if the part does not contain ``%s``, it is simply -interpreted as the name of the browser to launch. [1]_ +If the environment variable :envvar:`BROWSER` exists, it is interpreted to +override the platform default list of browsers, as a :data:`os.pathsep`-separated +list of browsers to try in order. When the value of a list part contains the +string ``%s``, then it is interpreted as a literal browser command line to be +used with the argument URL substituted for ``%s``; if the part does not contain +``%s``, it is simply interpreted as the name of the browser to launch. [1]_ For non-Unix platforms, or when a remote browser is available on Unix, the controlling process will not wait for the user to finish with the browser, but @@ -36,9 +36,7 @@ module. It accepts an URL as the argument. It accepts the following optional parameters: ``-n`` opens the URL in a new browser window, if possible; ``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: - - python -m webbrowser -t "http://www.python.org" +naturally, mutually exclusive. The following exception is defined: @@ -135,9 +133,9 @@ +------------------------+-----------------------------------------+-------+ | ``'windows-default'`` | :class:`WindowsDefault` | \(2) | +------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | :class:`MacOSX('default')` | \(3) | +| ``'internet-config'`` | :class:`InternetConfig` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'safari'`` | :class:`MacOSX('safari')` | \(3) | +| ``'macosx'`` | :class:`MacOSX('default')` | \(4) | +------------------------+-----------------------------------------+-------+ | ``'google-chrome'`` | :class:`Chrome('google-chrome')` | | +------------------------+-----------------------------------------+-------+ @@ -161,6 +159,9 @@ Only on Windows platforms. (3) + Only on Mac OS platforms; requires the standard MacPython :mod:`ic` module. + +(4) Only on Mac OS X platform. .. versionadded:: 3.3 diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/winreg.rst --- a/Doc/library/winreg.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/winreg.rst Mon Jan 25 17:05:13 2016 +0100 @@ -12,17 +12,6 @@ to ensure that the handles are closed correctly, even if the programmer neglects to explicitly close them. -.. _exception-changed: - -.. versionchanged:: 3.3 - Several functions in this module used to raise a - :exc:`WindowsError`, which is now an alias of :exc:`OSError`. - -.. _functions: - -Functions ------------------- - This module offers the following functions: @@ -48,11 +37,12 @@ *key* is the predefined handle to connect to. - The return value is the handle of the opened key. If the function fails, an + The return value is the handle of the opened key. If the function fails, a :exc:`OSError` exception is raised. .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. .. function:: CreateKey(key, sub_key) @@ -70,14 +60,15 @@ If the key already exists, this function opens the existing key. - The return value is the handle of the opened key. If the function fails, an + The return value is the handle of the opened key. If the function fails, a :exc:`OSError` exception is raised. .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. -.. function:: CreateKeyEx(key, sub_key, reserved=0, access=KEY_WRITE) +.. function:: CreateKeyEx(key, sub_key, reserved=0, access=KEY_ALL_ACCESS) Creates or opens the specified key, returning a :ref:`handle object `. @@ -87,10 +78,10 @@ *sub_key* is a string that names the key this method opens or creates. - *reserved* is a reserved integer, and must be zero. The default is zero. + *res* is a reserved integer, and must be zero. The default is zero. - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_WRITE`. See + *sam* is an integer that specifies an access mask that describes the desired + security access for the key. Default is :const:`KEY_ALL_ACCESS`. See :ref:`Access Rights ` for other allowed values. If *key* is one of the predefined keys, *sub_key* may be ``None``. In that @@ -98,13 +89,14 @@ If the key already exists, this function opens the existing key. - The return value is the handle of the opened key. If the function fails, an + The return value is the handle of the opened key. If the function fails, a :exc:`OSError` exception is raised. .. versionadded:: 3.2 .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. .. function:: DeleteKey(key, sub_key) @@ -120,13 +112,14 @@ *This method can not delete keys with subkeys.* If the method succeeds, the entire key, including all of its values, is removed. - If the method fails, an :exc:`OSError` exception is raised. + If the method fails, a :exc:`OSError` exception is raised. .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. -.. function:: DeleteKeyEx(key, sub_key, access=KEY_WOW64_64KEY, reserved=0) +.. function:: DeleteKeyEx(key, sub_key, access=KEY_ALL_ACCESS, reserved=0) Deletes the specified key. @@ -143,23 +136,24 @@ *key* parameter. This value must not be ``None``, and the key may not have subkeys. - *reserved* is a reserved integer, and must be zero. The default is zero. + *res* is a reserved integer, and must be zero. The default is zero. - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_WOW64_64KEY`. See + *sam* is an integer that specifies an access mask that describes the desired + security access for the key. Default is :const:`KEY_ALL_ACCESS`. See :ref:`Access Rights ` for other allowed values. *This method can not delete keys with subkeys.* If the method succeeds, the entire key, including all of its values, is - removed. If the method fails, an :exc:`OSError` exception is raised. + removed. If the method fails, a :exc:`OSError` exception is raised. On unsupported Windows versions, :exc:`NotImplementedError` is raised. .. versionadded:: 3.2 .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. .. function:: DeleteValue(key, value) @@ -182,11 +176,12 @@ *index* is an integer that identifies the index of the key to retrieve. The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly until an :exc:`OSError` exception is + typically called repeatedly until a :exc:`OSError` exception is raised, indicating, no more values are available. .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. .. function:: EnumValue(key, index) @@ -199,7 +194,7 @@ *index* is an integer that identifies the index of the value to retrieve. The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly, until an :exc:`OSError` exception is + typically called repeatedly, until a :exc:`OSError` exception is raised, indicating no more values. The result is a tuple of 3 items: @@ -219,7 +214,8 @@ +-------+--------------------------------------------+ .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. .. function:: ExpandEnvironmentStrings(str) @@ -275,8 +271,7 @@ specified in *file_name* is relative to the remote computer. -.. function:: OpenKey(key, sub_key, reserved=0, access=KEY_READ) - OpenKeyEx(key, sub_key, reserved=0, access=KEY_READ) +.. function:: OpenKey(key, sub_key, reserved=0, access=KEY_ALL_ACCESS) Opens the specified key, returning a :ref:`handle object `. @@ -285,9 +280,9 @@ *sub_key* is a string that identifies the sub_key to open. - *reserved* is a reserved integer, and must be zero. The default is zero. + *res* is a reserved integer, and must be zero. The default is zero. - *access* is an integer that specifies an access mask that describes the desired + *sam* is an integer that specifies an access mask that describes the desired security access for the key. Default is :const:`KEY_READ`. See :ref:`Access Rights ` for other allowed values. @@ -295,11 +290,17 @@ If the function fails, :exc:`OSError` is raised. - .. versionchanged:: 3.2 - Allow the use of named arguments. + .. versionchanged:: 3.2 Allow the use of named arguments. .. versionchanged:: 3.3 - See :ref:`above `. + This function used to raise a :exc:`WindowsError`, which is now an + alias of :exc:`OSError`. + + +.. function:: OpenKeyEx() + + The functionality of :func:`OpenKeyEx` is provided via :func:`OpenKey`, + by the use of default arguments. .. function:: QueryInfoKey(key) @@ -322,7 +323,7 @@ +-------+---------------------------------------------+ | ``2`` | An integer giving when the key was last | | | modified (if available) as 100's of | - | | nanoseconds since Jan 1, 1601. | + | | nanoseconds since Jan 1, 1600. | +-------+---------------------------------------------+ @@ -425,11 +426,11 @@ *value_name* is a string that names the subkey with which the value is associated. - *reserved* can be anything -- zero is always passed to the API. - *type* is an integer that specifies the type of the data. See :ref:`Value Types ` for the available types. + *reserved* can be anything -- zero is always passed to the API. + *value* is a string that specifies the new value. This method can also set additional value and type information for the specified diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/winsound.rst --- a/Doc/library/winsound.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/winsound.rst Mon Jan 25 17:05:13 2016 +0100 @@ -126,10 +126,6 @@ Return immediately if the sound driver is busy. - .. note:: - - This flag is not supported on modern Windows platforms. - .. data:: MB_ICONASTERISK diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/wsgiref.rst --- a/Doc/library/wsgiref.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/wsgiref.rst Mon Jan 25 17:05:13 2016 +0100 @@ -184,11 +184,10 @@ manipulation of WSGI response headers using a mapping-like interface. -.. class:: Headers([headers]) +.. class:: Headers(headers) Create a mapping-like object wrapping *headers*, which must be a list of header - name/value tuples as described in :pep:`3333`. The default value of *headers* is - an empty list. + name/value tuples as described in :pep:`3333`. :class:`Headers` objects support typical mapping operations including :meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, @@ -252,10 +251,6 @@ Content-Disposition: attachment; filename="bud.gif" - .. versionchanged:: 3.5 - *headers* parameter is optional. - - :mod:`wsgiref.simple_server` -- a simple WSGI HTTP server --------------------------------------------------------- @@ -506,7 +501,7 @@ Similar to :class:`BaseCGIHandler`, but designed for use with HTTP origin servers. If you are writing an HTTP server implementation, you will probably - want to subclass this instead of :class:`BaseCGIHandler`. + want to subclass this instead of :class:`BaseCGIHandler` This class is a subclass of :class:`BaseHandler`. It overrides the :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, @@ -614,9 +609,6 @@ as :class:`BaseCGIHandler` and :class:`CGIHandler`) that are not HTTP origin servers. - .. versionchanged:: 3.3 - The term "Python" is replaced with implementation specific term like - "CPython", "Jython" etc. .. method:: BaseHandler.get_scheme() @@ -759,7 +751,7 @@ # object that accepts two arguments. For that purpose, we're going to # use a function (note that you're not limited to a function, you can # use a class for example). The first argument passed to the function - # is a dictionary containing CGI-style environment variables and the + # is a dictionary containing CGI-style envrironment variables and the # second variable is the callable object (see PEP 333). def hello_world_app(environ, start_response): status = '200 OK' # HTTP Status diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.dom.minidom.rst --- a/Doc/library/xml.dom.minidom.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.dom.minidom.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,8 +1,8 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== +:mod:`xml.dom.minidom` --- Lightweight DOM implementation +========================================================= .. module:: xml.dom.minidom - :synopsis: Minimal Document Object Model (DOM) implementation. + :synopsis: Lightweight Document Object Model (DOM) implementation. .. moduleauthor:: Paul Prescod .. sectionauthor:: Paul Prescod .. sectionauthor:: Martin v. Löwis @@ -11,19 +11,17 @@ -------------- -:mod:`xml.dom.minidom` is a minimal implementation of the Document Object -Model interface, with an API similar to that in other languages. It is intended -to be simpler than the full DOM and also significantly smaller. Users who are -not already proficient with the DOM should consider using the -:mod:`xml.etree.ElementTree` module for their XML processing instead. +:mod:`xml.dom.minidom` is a light-weight implementation of the Document Object +Model interface. It is intended to be simpler than the full DOM and also +significantly smaller. +.. note:: -.. warning:: - - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - + The :mod:`xml.dom.minidom` module provides an implementation of the W3C-DOM, + with an API similar to that in other programming languages. Users who are + unfamiliar with the W3C-DOM interface or who would like to write less code + for processing XML files should consider using the + :mod:`xml.etree.ElementTree` module instead. DOM applications typically start by parsing some XML into a DOM. With :mod:`xml.dom.minidom`, this is done through the parse functions:: @@ -54,8 +52,8 @@ .. function:: parseString(string, parser=None) - Return a :class:`Document` that represents the *string*. This method creates an - :class:`io.StringIO` object for the string and passes that on to :func:`parse`. + Return a :class:`Document` that represents the *string*. This method creates a + :class:`StringIO` object for the string and passes that on to :func:`parse`. Both functions return a :class:`Document` object representing the content of the document. @@ -149,7 +147,12 @@ the DOM node. With an explicit *encoding* [1]_ argument, the result is a byte - string in the specified encoding. + string in the specified encoding. It is recommended that you + always specify an encoding; you may use any encoding you like, but + an argument of "utf-8" is the most common choice, avoiding + :exc:`UnicodeError` exceptions in case of unrepresentable text + data. + With no *encoding* argument, the result is a Unicode string, and the XML declaration in the resulting string does not specify an encoding. Encoding this string in an encoding other than UTF-8 is @@ -252,4 +255,4 @@ "UTF8" is not valid in an XML document's declaration, even though Python accepts it as an encoding name. See http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and http://www.iana.org/assignments/character-sets/character-sets.xhtml. + and http://www.iana.org/assignments/character-sets . diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.dom.pulldom.rst --- a/Doc/library/xml.dom.pulldom.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.dom.pulldom.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,14 +17,6 @@ responsible for explicitly pulling events from the stream, looping over those events until either processing is finished or an error condition occurs. - -.. warning:: - - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - - Example:: from xml.dom import pulldom @@ -47,7 +39,7 @@ * :data:`PROCESSING_INSTRUCTION` * :data:`IGNORABLE_WHITESPACE` -``node`` is an object of type :class:`xml.dom.minidom.Document`, +``node`` is a object of type :class:`xml.dom.minidom.Document`, :class:`xml.dom.minidom.Element` or :class:`xml.dom.minidom.Text`. Since the document is treated as a "flat" stream of events, the document "tree" @@ -74,8 +66,7 @@ Return a :class:`DOMEventStream` from the given input. *stream_or_string* may be either a file name, or a file-like object. *parser*, if given, must be a - :class:`~xml.sax.xmlreader.XMLReader` object. This function will change the - document handler of the + :class:`XmlReader` object. This function will change the document handler of the parser and activate namespace support; other parser configuration (like setting an entity resolver) must have been done in advance. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.dom.rst --- a/Doc/library/xml.dom.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.dom.rst Mon Jan 25 17:05:13 2016 +0100 @@ -304,7 +304,7 @@ .. attribute:: Node.prefix The part of the :attr:`tagName` preceding the colon if there is one, else the - empty string. The value is a string, or ``None``. + empty string. The value is a string, or ``None`` .. attribute:: Node.namespaceURI @@ -357,7 +357,7 @@ Add a new child node to this node at the end of the list of children, returning *newChild*. If the node was already in - the tree, it is removed first. + in the tree, it is removed first. .. method:: Node.insertBefore(newChild, refChild) @@ -412,7 +412,7 @@ .. method:: NodeList.item(i) Return the *i*'th item from the sequence, if there is one, or ``None``. The - index *i* is not allowed to be less than zero or greater than or equal to the + index *i* is not allowed to be less then zero or greater than or equal to the length of the sequence. @@ -422,15 +422,14 @@ In addition, the Python DOM interface requires that some additional support is provided to allow :class:`NodeList` objects to be used as Python sequences. All -:class:`NodeList` implementations must include support for -:meth:`~object.__len__` and -:meth:`~object.__getitem__`; this allows iteration over the :class:`NodeList` in +:class:`NodeList` implementations must include support for :meth:`__len__` and +:meth:`__getitem__`; this allows iteration over the :class:`NodeList` in :keyword:`for` statements and proper support for the :func:`len` built-in function. If a DOM implementation supports modification of the document, the -:class:`NodeList` implementation must also support the -:meth:`~object.__setitem__` and :meth:`~object.__delitem__` methods. +:class:`NodeList` implementation must also support the :meth:`__setitem__` and +:meth:`__delitem__` methods. .. _dom-documenttype-objects: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.etree.elementtree.rst --- a/Doc/library/xml.etree.elementtree.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.etree.elementtree.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,455 +5,50 @@ :synopsis: Implementation of the ElementTree API. .. moduleauthor:: Fredrik Lundh -The :mod:`xml.etree.ElementTree` module implements a simple and efficient API -for parsing and creating XML data. +**Source code:** :source:`Lib/xml/etree/ElementTree.py` + +-------------- + +The :class:`Element` type is a flexible container object, designed to store +hierarchical data structures in memory. The type can be described as a cross +between a list and a dictionary. + +Each element has a number of properties associated with it: + +* a tag which is a string identifying what kind of data this element represents + (the element type, in other words). + +* a number of attributes, stored in a Python dictionary. + +* a text string. + +* an optional tail string. + +* a number of child elements, stored in a Python sequence + +To create an element instance, use the :class:`Element` constructor or the +:func:`SubElement` factory function. + +The :class:`ElementTree` class can be used to wrap an element structure, and +convert it from and to XML. + +See http://effbot.org/zone/element-index.htm for tutorials and links to other +docs. + +.. versionchanged:: 3.2 + The ElementTree API is updated to 1.3. For more information, see + `Introducing ElementTree 1.3 + `_. .. versionchanged:: 3.3 This module will use a fast implementation whenever available. The :mod:`xml.etree.cElementTree` module is deprecated. -.. warning:: - - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - -Tutorial --------- - -This is a short tutorial for using :mod:`xml.etree.ElementTree` (``ET`` in -short). The goal is to demonstrate some of the building blocks and basic -concepts of the module. - -XML tree and elements -^^^^^^^^^^^^^^^^^^^^^ - -XML is an inherently hierarchical data format, and the most natural way to -represent it is with a tree. ``ET`` has two classes for this purpose - -:class:`ElementTree` represents the whole XML document as a tree, and -:class:`Element` represents a single node in this tree. Interactions with -the whole document (reading and writing to/from files) are usually done -on the :class:`ElementTree` level. Interactions with a single XML element -and its sub-elements are done on the :class:`Element` level. - -.. _elementtree-parsing-xml: - -Parsing XML -^^^^^^^^^^^ - -We'll be using the following XML document as the sample data for this section: - -.. code-block:: xml - - - - - 1 - 2008 - 141100 - - - - - 4 - 2011 - 59900 - - - - 68 - 2011 - 13600 - - - - - -We can import this data by reading from a file:: - - import xml.etree.ElementTree as ET - tree = ET.parse('country_data.xml') - root = tree.getroot() - -Or directly from a string:: - - root = ET.fromstring(country_data_as_string) - -:func:`fromstring` parses XML from a string directly into an :class:`Element`, -which is the root element of the parsed tree. Other parsing functions may -create an :class:`ElementTree`. Check the documentation to be sure. - -As an :class:`Element`, ``root`` has a tag and a dictionary of attributes:: - - >>> root.tag - 'data' - >>> root.attrib - {} - -It also has children nodes over which we can iterate:: - - >>> for child in root: - ... print(child.tag, child.attrib) - ... - country {'name': 'Liechtenstein'} - country {'name': 'Singapore'} - country {'name': 'Panama'} - -Children are nested, and we can access specific child nodes by index:: - - >>> root[0][1].text - '2008' - - -.. note:: - - Not all elements of the XML input will end up as elements of the - parsed tree. Currently, this module skips over any XML comments, - processing instructions, and document type declarations in the - input. Nevertheless, trees built using this module's API rather - than parsing from XML text can have comments and processing - instructions in them; they will be included when generating XML - output. A document type declaration may be accessed by passing a - custom :class:`TreeBuilder` instance to the :class:`XMLParser` - constructor. - - -.. _elementtree-pull-parsing: - -Pull API for non-blocking parsing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most parsing functions provided by this module require the whole document -to be read at once before returning any result. It is possible to use an -:class:`XMLParser` and feed data into it incrementally, but it is a push API that -calls methods on a callback target, which is too low-level and inconvenient for -most needs. Sometimes what the user really wants is to be able to parse XML -incrementally, without blocking operations, while enjoying the convenience of -fully constructed :class:`Element` objects. - -The most powerful tool for doing this is :class:`XMLPullParser`. It does not -require a blocking read to obtain the XML data, and is instead fed with data -incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML -elements, call :meth:`XMLPullParser.read_events`. Here is an example:: - - >>> parser = ET.XMLPullParser(['start', 'end']) - >>> parser.feed('sometext') - >>> list(parser.read_events()) - [('start', )] - >>> parser.feed(' more text') - >>> for event, elem in parser.read_events(): - ... print(event) - ... print(elem.tag, 'text=', elem.text) - ... - end - -The obvious use case is applications that operate in a non-blocking fashion -where the XML data is being received from a socket or read incrementally from -some storage device. In such cases, blocking reads are unacceptable. - -Because it's so flexible, :class:`XMLPullParser` can be inconvenient to use for -simpler use-cases. If you don't mind your application blocking on reading XML -data but would still like to have incremental parsing capabilities, take a look -at :func:`iterparse`. It can be useful when you're reading a large XML document -and don't want to hold it wholly in memory. - -Finding interesting elements -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`Element` has some useful methods that help iterate recursively over all -the sub-tree below it (its children, their children, and so on). For example, -:meth:`Element.iter`:: - - >>> for neighbor in root.iter('neighbor'): - ... print(neighbor.attrib) - ... - {'name': 'Austria', 'direction': 'E'} - {'name': 'Switzerland', 'direction': 'W'} - {'name': 'Malaysia', 'direction': 'N'} - {'name': 'Costa Rica', 'direction': 'W'} - {'name': 'Colombia', 'direction': 'E'} - -:meth:`Element.findall` finds only elements with a tag which are direct -children of the current element. :meth:`Element.find` finds the *first* child -with a particular tag, and :attr:`Element.text` accesses the element's text -content. :meth:`Element.get` accesses the element's attributes:: - - >>> for country in root.findall('country'): - ... rank = country.find('rank').text - ... name = country.get('name') - ... print(name, rank) - ... - Liechtenstein 1 - Singapore 4 - Panama 68 - -More sophisticated specification of which elements to look for is possible by -using :ref:`XPath `. - -Modifying an XML File -^^^^^^^^^^^^^^^^^^^^^ - -:class:`ElementTree` provides a simple way to build XML documents and write them to files. -The :meth:`ElementTree.write` method serves this purpose. - -Once created, an :class:`Element` object may be manipulated by directly changing -its fields (such as :attr:`Element.text`), adding and modifying attributes -(:meth:`Element.set` method), as well as adding new children (for example -with :meth:`Element.append`). - -Let's say we want to add one to each country's rank, and add an ``updated`` -attribute to the rank element:: - - >>> for rank in root.iter('rank'): - ... new_rank = int(rank.text) + 1 - ... rank.text = str(new_rank) - ... rank.set('updated', 'yes') - ... - >>> tree.write('output.xml') - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - 69 - 2011 - 13600 - - - - - -We can remove elements using :meth:`Element.remove`. Let's say we want to -remove all countries with a rank higher than 50:: - - >>> for country in root.findall('country'): - ... rank = int(country.find('rank').text) - ... if rank > 50: - ... root.remove(country) - ... - >>> tree.write('output.xml') - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - -Building XML documents -^^^^^^^^^^^^^^^^^^^^^^ - -The :func:`SubElement` function also provides a convenient way to create new -sub-elements for a given element:: - - >>> a = ET.Element('a') - >>> b = ET.SubElement(a, 'b') - >>> c = ET.SubElement(a, 'c') - >>> d = ET.SubElement(c, 'd') - >>> ET.dump(a) - - -Parsing XML with Namespaces -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If the XML input has `namespaces -`__, tags and attributes -with prefixes in the form ``prefix:sometag`` get expanded to -``{uri}sometag`` where the *prefix* is replaced by the full *URI*. -Also, if there is a `default namespace -`__, -that full URI gets prepended to all of the non-prefixed tags. - -Here is an XML example that incorporates two namespaces, one with the -prefix "fictional" and the other serving as the default namespace: - -.. code-block:: xml - - - - - John Cleese - Lancelot - Archie Leach - - - Eric Idle - Sir Robin - Gunther - Commander Clement - - - -One way to search and explore this XML example is to manually add the -URI to every tag or attribute in the xpath of a -:meth:`~Element.find` or :meth:`~Element.findall`:: - - root = fromstring(xml_text) - for actor in root.findall('{http://people.example.com}actor'): - name = actor.find('{http://people.example.com}name') - print(name.text) - for char in actor.findall('{http://characters.example.com}character'): - print(' |-->', char.text) - -A better way to search the namespaced XML example is to create a -dictionary with your own prefixes and use those in the search functions:: - - ns = {'real_person': 'http://people.example.com', - 'role': 'http://characters.example.com'} - - for actor in root.findall('real_person:actor', ns): - name = actor.find('real_person:name', ns) - print(name.text) - for char in actor.findall('role:character', ns): - print(' |-->', char.text) - -These two approaches both output:: - - John Cleese - |--> Lancelot - |--> Archie Leach - Eric Idle - |--> Sir Robin - |--> Gunther - |--> Commander Clement - - -Additional resources -^^^^^^^^^^^^^^^^^^^^ - -See http://effbot.org/zone/element-index.htm for tutorials and links to other -docs. - - -.. _elementtree-xpath: - -XPath support -------------- - -This module provides limited support for -`XPath expressions `_ for locating elements in a -tree. The goal is to support a small subset of the abbreviated syntax; a full -XPath engine is outside the scope of the module. - -Example -^^^^^^^ - -Here's an example that demonstrates some of the XPath capabilities of the -module. We'll be using the ``countrydata`` XML document from the -:ref:`Parsing XML ` section:: - - import xml.etree.ElementTree as ET - - root = ET.fromstring(countrydata) - - # Top-level elements - root.findall(".") - - # All 'neighbor' grand-children of 'country' children of the top-level - # elements - root.findall("./country/neighbor") - - # Nodes with name='Singapore' that have a 'year' child - root.findall(".//year/..[@name='Singapore']") - - # 'year' nodes that are children of nodes with name='Singapore' - root.findall(".//*[@name='Singapore']/year") - - # All 'neighbor' nodes that are the second child of their parent - root.findall(".//neighbor[2]") - -Supported XPath syntax -^^^^^^^^^^^^^^^^^^^^^^ - -.. tabularcolumns:: |l|L| - -+-----------------------+------------------------------------------------------+ -| Syntax | Meaning | -+=======================+======================================================+ -| ``tag`` | Selects all child elements with the given tag. | -| | For example, ``spam`` selects all child elements | -| | named ``spam``, and ``spam/egg`` selects all | -| | grandchildren named ``egg`` in all children named | -| | ``spam``. | -+-----------------------+------------------------------------------------------+ -| ``*`` | Selects all child elements. For example, ``*/egg`` | -| | selects all grandchildren named ``egg``. | -+-----------------------+------------------------------------------------------+ -| ``.`` | Selects the current node. This is mostly useful | -| | at the beginning of the path, to indicate that it's | -| | a relative path. | -+-----------------------+------------------------------------------------------+ -| ``//`` | Selects all subelements, on all levels beneath the | -| | current element. For example, ``.//egg`` selects | -| | all ``egg`` elements in the entire tree. | -+-----------------------+------------------------------------------------------+ -| ``..`` | Selects the parent element. Returns ``None`` if the | -| | path attempts to reach the ancestors of the start | -| | element (the element ``find`` was called on). | -+-----------------------+------------------------------------------------------+ -| ``[@attrib]`` | Selects all elements that have the given attribute. | -+-----------------------+------------------------------------------------------+ -| ``[@attrib='value']`` | Selects all elements for which the given attribute | -| | has the given value. The value cannot contain | -| | quotes. | -+-----------------------+------------------------------------------------------+ -| ``[tag]`` | Selects all elements that have a child named | -| | ``tag``. Only immediate children are supported. | -+-----------------------+------------------------------------------------------+ -| ``[tag='text']`` | Selects all elements that have a child named | -| | ``tag`` whose complete text content, including | -| | descendants, equals the given ``text``. | -+-----------------------+------------------------------------------------------+ -| ``[position]`` | Selects all elements that are located at the given | -| | position. The position can be either an integer | -| | (1 is the first position), the expression ``last()`` | -| | (for the last position), or a position relative to | -| | the last position (e.g. ``last()-1``). | -+-----------------------+------------------------------------------------------+ - -Predicates (expressions within square brackets) must be preceded by a tag -name, an asterisk, or another predicate. ``position`` predicates must be -preceded by a tag name. - -Reference ---------- - .. _elementtree-functions: Functions -^^^^^^^^^ +--------- .. function:: Comment(text=None) @@ -464,10 +59,6 @@ string containing the comment string. Returns an element instance representing a comment. - Note that :class:`XMLParser` skips over comments in the input - instead of creating comment objects for them. An :class:`ElementTree` will - only contain comment nodes if they have been inserted into to - the tree using one of the :class:`Element` methods. .. function:: dump(elem) @@ -506,32 +97,24 @@ Parses an XML section into an element tree incrementally, and reports what's going on to the user. *source* is a filename or :term:`file object` - containing XML data. *events* is a sequence of events to report back. The - supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and - ``"end-ns"`` (the "ns" events are used to get detailed namespace + containing XML data. *events* is a list of events to report back. The + supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` + and ``"end-ns"`` (the "ns" events are used to get detailed namespace information). If *events* is omitted, only ``"end"`` events are reported. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. *parser* must be a subclass of - :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a - target. Returns an :term:`iterator` providing ``(event, elem)`` pairs. - - Note that while :func:`iterparse` builds the tree incrementally, it issues - blocking reads on *source* (or the file it names). As such, it's unsuitable - for applications where blocking reads can't be made. For fully non-blocking - parsing, see :class:`XMLPullParser`. + :class:`XMLParser` parser is used. Returns an :term:`iterator` providing + ``(event, elem)`` pairs. .. note:: - :func:`iterparse` only guarantees that it has seen the ">" character of a - starting tag when it emits a "start" event, so the attributes are defined, - but the contents of the text and tail attributes are undefined at that - point. The same applies to the element children; they may or may not be - present. + :func:`iterparse` only guarantees that it has seen the ">" + character of a starting tag when it emits a "start" event, so the + attributes are defined, but the contents of the text and tail attributes + are undefined at that point. The same applies to the element children; + they may or may not be present. If you need a fully populated element, look for "end" events instead. - .. deprecated:: 3.4 - The *parser* argument. .. function:: parse(source, parser=None) @@ -548,11 +131,6 @@ containing the PI target. *text* is a string containing the PI contents, if given. Returns an element instance, representing a processing instruction. - Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An - :class:`ElementTree` will only contain processing instruction nodes if - they have been inserted into to the tree using one of the - :class:`Element` methods. .. function:: register_namespace(prefix, uri) @@ -577,39 +155,29 @@ arguments. Returns an element instance. -.. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - short_empty_elements=True) +.. function:: tostring(element, encoding="us-ascii", method="xml") Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. - Returns an (optionally) encoded string containing the XML data. + generate a Unicode string. *method* is either ``"xml"``, + ``"html"`` or ``"text"`` (default is ``"xml"``). Returns an (optionally) + encoded string containing the XML data. - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - -.. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - short_empty_elements=True) +.. function:: tostringlist(element, encoding="us-ascii", method="xml") Generates a string representation of an XML element, including all subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`. - Returns a list of (optionally) encoded strings containing the XML data. - It does not guarantee any specific sequence, except that - ``b"".join(tostringlist(element)) == tostring(element)``. + generate a Unicode string. *method* is either ``"xml"``, + ``"html"`` or ``"text"`` (default is ``"xml"``). Returns a list of + (optionally) encoded strings containing the XML data. It does not guarantee + any specific sequence, except that ``"".join(tostringlist(element)) == + tostring(element)``. .. versionadded:: 3.2 - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - .. function:: XML(text, parser=None) @@ -631,7 +199,7 @@ .. _elementtree-element-objects: Element Objects -^^^^^^^^^^^^^^^ +--------------- .. class:: Element(tag, attrib={}, **extra) @@ -651,29 +219,21 @@ .. attribute:: text - tail - These attributes can be used to hold additional data associated with - the element. Their values are usually strings but may be any - application-specific object. If the element is created from - an XML file, the *text* attribute holds either the text between - the element's start tag and its first child or end tag, or ``None``, and - the *tail* attribute holds either the text between the element's - end tag and the next tag, or ``None``. For the XML data + The *text* attribute can be used to hold additional data associated with + the element. As the name implies this attribute is usually a string but + may be any application-specific object. If the element is created from + an XML file the attribute will contain any text found between the element + tags. - .. code-block:: xml - 1234 + .. attribute:: tail - the *a* element has ``None`` for both *text* and *tail* attributes, - the *b* element has *text* ``"1"`` and *tail* ``"4"``, - the *c* element has *text* ``"2"`` and *tail* ``None``, - and the *d* element has *text* ``None`` and *tail* ``"3"``. - - To collect the inner text of an element, see :meth:`itertext`, for - example ``"".join(element.itertext())``. - - Applications may store arbitrary objects in these attributes. + The *tail* attribute can be used to hold additional data associated with + the element. This attribute is usually a string but may be any + application-specific object. If the element is created from an XML file + the attribute will contain any text found after the element's end tag and + before the next tag. .. attribute:: attrib @@ -690,7 +250,7 @@ .. method:: clear() Resets an element. This function removes all subelements, clears all - attributes, and sets the text and tail attributes to ``None``. + attributes, and sets the text and tail attributes to None. .. method:: get(key, default=None) @@ -734,30 +294,24 @@ .. versionadded:: 3.2 - .. method:: find(match, namespaces=None) + .. method:: find(match) Finds the first subelement matching *match*. *match* may be a tag name - or a :ref:`path `. Returns an element instance - or ``None``. *namespaces* is an optional mapping from namespace prefix - to full name. + or path. Returns an element instance or ``None``. - .. method:: findall(match, namespaces=None) + .. method:: findall(match) - Finds all matching subelements, by tag name or - :ref:`path `. Returns a list containing all matching - elements in document order. *namespaces* is an optional mapping from - namespace prefix to full name. + Finds all matching subelements, by tag name or path. Returns a list + containing all matching elements in document order. - .. method:: findtext(match, default=None, namespaces=None) + .. method:: findtext(match, default=None) Finds text for the first subelement matching *match*. *match* may be - a tag name or a :ref:`path `. Returns the text content - of the first matching element, or *default* if no element was found. - Note that if the matching element has no text content an empty string - is returned. *namespaces* is an optional mapping from namespace prefix - to full name. + a tag name or path. Returns the text content of the first matching + element, or *default* if no element was found. Note that if the matching + element has no text content an empty string is returned. .. method:: getchildren() @@ -789,13 +343,10 @@ .. versionadded:: 3.2 - .. method:: iterfind(match, namespaces=None) + .. method:: iterfind(match) - Finds all matching subelements, by tag name or - :ref:`path `. Returns an iterable yielding all - matching elements in document order. *namespaces* is an optional mapping - from namespace prefix to full name. - + Finds all matching subelements, by tag name or path. Returns an iterable + yielding all matching elements in document order. .. versionadded:: 3.2 @@ -821,9 +372,8 @@ or contents. :class:`Element` objects also support the following sequence type methods - for working with subelements: :meth:`~object.__delitem__`, - :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, - :meth:`~object.__len__`. + for working with subelements: :meth:`__delitem__`, :meth:`__getitem__`, + :meth:`__setitem__`, :meth:`__len__`. Caution: Elements with no subelements will test as ``False``. This behavior will change in future versions. Use specific ``len(elem)`` or ``elem is @@ -841,7 +391,7 @@ .. _elementtree-elementtree-objects: ElementTree Objects -^^^^^^^^^^^^^^^^^^^ +------------------- .. class:: ElementTree(element=None, file=None) @@ -861,19 +411,28 @@ care. *element* is an element instance. - .. method:: find(match, namespaces=None) + .. method:: find(match) - Same as :meth:`Element.find`, starting at the root of the tree. + Finds the first toplevel element matching *match*. *match* may be a tag + name or path. Same as getroot().find(match). Returns the first matching + element, or ``None`` if no element was found. - .. method:: findall(match, namespaces=None) + .. method:: findall(match) - Same as :meth:`Element.findall`, starting at the root of the tree. + Finds all matching subelements, by tag name or path. Same as + getroot().findall(match). *match* may be a tag name or path. Returns a + list containing all matching elements, in document order. - .. method:: findtext(match, default=None, namespaces=None) + .. method:: findtext(match, default=None) - Same as :meth:`Element.findtext`, starting at the root of the tree. + Finds the element text for the first toplevel element with given tag. + Same as getroot().findtext(match). *match* may be a tag name or path. + *default* is the value to return if the element was not found. Returns + the text content of the first matching element, or the default value no + element was found. Note that if the element is found, but has no text + content, this method returns an empty string. .. method:: getiterator(tag=None) @@ -891,12 +450,14 @@ Creates and returns a tree iterator for the root element. The iterator loops over all elements in this tree, in section order. *tag* is the tag - to look for (default is to return all elements). + to look for (default is to return all elements) - .. method:: iterfind(match, namespaces=None) + .. method:: iterfind(match) - Same as :meth:`Element.iterfind`, starting at the root of the tree. + Finds all matching subelements, by tag name or path. Same as + getroot().iterfind(match). Returns an iterable yielding all matching + elements in document order. .. versionadded:: 3.2 @@ -905,38 +466,20 @@ Loads an external XML section into this element tree. *source* is a file name or :term:`file object`. *parser* is an optional parser instance. - If not given, the standard :class:`XMLParser` parser is used. Returns the - section root element. + If not given, the standard XMLParser parser is used. Returns the section + root element. - .. method:: write(file, encoding="us-ascii", xml_declaration=None, \ - default_namespace=None, method="xml", *, \ - short_empty_elements=True) + .. method:: write(file, encoding="us-ascii", xml_declaration=None, method="xml") Writes the element tree to a file, as XML. *file* is a file name, or a - :term:`file object` opened for writing. *encoding* [1]_ is the output - encoding (default is US-ASCII). - *xml_declaration* controls if an XML declaration should be added to the - file. Use ``False`` for never, ``True`` for always, ``None`` - for only if not US-ASCII or UTF-8 or Unicode (default is ``None``). - *default_namespace* sets the default XML namespace (for "xmlns"). - *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is - ``"xml"``). - The keyword-only *short_empty_elements* parameter controls the formatting - of elements that contain no content. If *True* (the default), they are - emitted as a single self-closed tag, otherwise they are emitted as a pair - of start/end tags. - - The output is either a string (:class:`str`) or binary (:class:`bytes`). - This is controlled by the *encoding* argument. If *encoding* is - ``"unicode"``, the output is a string; otherwise, it's binary. Note that - this may conflict with the type of *file* if it's an open - :term:`file object`; make sure you do not try to write a string to a - binary stream and vice versa. - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - + :term:`file object` opened for writing. *encoding* [1]_ is the output encoding + (default is US-ASCII). Use ``encoding="unicode"`` to write a Unicode string. + *xml_declaration* controls if an XML declaration + should be added to the file. Use False for never, True for always, None + for only if not US-ASCII or UTF-8 or Unicode (default is None). *method* is + either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). + Returns an (optionally) encoded string. This is the XML file that is going to be manipulated:: @@ -969,7 +512,7 @@ .. _elementtree-qname-objects: QName Objects -^^^^^^^^^^^^^ +------------- .. class:: QName(text_or_uri, tag=None) @@ -982,11 +525,10 @@ :class:`QName` instances are opaque. - .. _elementtree-treebuilder-objects: TreeBuilder Objects -^^^^^^^^^^^^^^^^^^^ +------------------- .. class:: TreeBuilder(element_factory=None) @@ -994,9 +536,9 @@ Generic element structure builder. This builder converts a sequence of start, data, and end method calls to a well-formed element structure. You can use this class to build an element structure using a custom XML parser, - or a parser for some other XML-like format. *element_factory*, when given, - must be a callable accepting two positional arguments: a tag and - a dict of attributes. It is expected to return a new element instance. + or a parser for some other XML-like format. The *element_factory* is called + to create new :class:`Element` instances when given. + .. method:: close() @@ -1037,29 +579,22 @@ .. _elementtree-xmlparser-objects: XMLParser Objects -^^^^^^^^^^^^^^^^^ +----------------- .. class:: XMLParser(html=0, target=None, encoding=None) - This class is the low-level building block of the module. It uses - :mod:`xml.parsers.expat` for efficient, event-based parsing of XML. It can - be fed XML data incrementall with the :meth:`feed` method, and parsing events - are translated to a push API - by invoking callbacks on the *target* object. - If *target* is omitted, the standard :class:`TreeBuilder` is used. The - *html* argument was historically used for backwards compatibility and is now - deprecated. If *encoding* [1]_ is given, the value overrides the encoding - specified in the XML file. + :class:`Element` structure builder for XML source data, based on the expat + parser. *html* are predefined HTML entities. This flag is not supported by + the current implementation. *target* is the target object. If omitted, the + builder uses an instance of the standard TreeBuilder class. *encoding* [1]_ + is optional. If given, the value overrides the encoding specified in the + XML file. - .. deprecated:: 3.4 - The *html* argument. The remaining arguments should be passed via - keywword to prepare for the removal of the *html* argument. .. method:: close() - Finishes feeding data to the parser. Returns the result of calling the - ``close()`` method of the *target* passed during construction; by default, - this is the toplevel document element. + Finishes feeding data to the parser. Returns an element structure. .. method:: doctype(name, pubid, system) @@ -1073,12 +608,12 @@ Feeds data to the parser. *data* is encoded data. - :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method - for each opening tag, its ``end(tag)`` method for each closing tag, and data - is processed by method ``data(data)``. :meth:`XMLParser.close` calls - *target*\'s method ``close()``. :class:`XMLParser` can be used not only for - building a tree structure. This is an example of counting the maximum depth - of an XML file:: +:meth:`XMLParser.feed` calls *target*\'s :meth:`start` method +for each opening tag, its :meth:`end` method for each closing tag, +and data is processed by method :meth:`data`. :meth:`XMLParser.close` +calls *target*\'s method :meth:`close`. +:class:`XMLParser` can be used not only for building a tree structure. +This is an example of counting the maximum depth of an XML file:: >>> from xml.etree.ElementTree import XMLParser >>> class MaxDepth: # The target object of the parser @@ -1112,62 +647,8 @@ >>> parser.close() 4 - -.. _elementtree-xmlpullparser-objects: - -XMLPullParser Objects -^^^^^^^^^^^^^^^^^^^^^ - -.. class:: XMLPullParser(events=None) - - A pull parser suitable for non-blocking applications. Its input-side API is - similar to that of :class:`XMLParser`, but instead of pushing calls to a - callback target, :class:`XMLPullParser` collects an internal list of parsing - events and lets the user read from it. *events* is a sequence of events to - report back. The supported events are the strings ``"start"``, ``"end"``, - ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed - namespace information). If *events* is omitted, only ``"end"`` events are - reported. - - .. method:: feed(data) - - Feed the given bytes data to the parser. - - .. method:: close() - - Signal the parser that the data stream is terminated. Unlike - :meth:`XMLParser.close`, this method always returns :const:`None`. - Any events not yet retrieved when the parser is closed can still be - read with :meth:`read_events`. - - .. method:: read_events() - - Return an iterator over the events which have been encountered in the - data fed to the - parser. The iterator yields ``(event, elem)`` pairs, where *event* is a - string representing the type of event (e.g. ``"end"``) and *elem* is the - encountered :class:`Element` object. - - Events provided in a previous call to :meth:`read_events` will not be - yielded again. Events are consumed from the internal queue only when - they are retrieved from the iterator, so multiple readers iterating in - parallel over iterators obtained from :meth:`read_events` will have - unpredictable results. - - .. note:: - - :class:`XMLPullParser` only guarantees that it has seen the ">" - character of a starting tag when it emits a "start" event, so the - attributes are defined, but the contents of the text and tail attributes - are undefined at that point. The same applies to the element children; - they may or may not be present. - - If you need a fully populated element, look for "end" events instead. - - .. versionadded:: 3.4 - Exceptions -^^^^^^^^^^ +---------- .. class:: ParseError @@ -1190,4 +671,4 @@ .. [#] The encoding string included in XML output should conform to the appropriate standards. For example, "UTF-8" is valid, but "UTF8" is not. See http://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and http://www.iana.org/assignments/character-sets/character-sets.xhtml. + and http://www.iana.org/assignments/character-sets. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.rst --- a/Doc/library/xml.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,133 +0,0 @@ -.. _xml: - -XML Processing Modules -====================== - -.. module:: xml - :synopsis: Package containing XML processing modules -.. sectionauthor:: Christian Heimes -.. sectionauthor:: Georg Brandl - - -Python's interfaces for processing XML are grouped in the ``xml`` package. - -.. warning:: - - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defused-packages` sections. - -It is important to note that modules in the :mod:`xml` package require that -there be at least one SAX-compliant XML parser available. The Expat parser is -included with Python, so the :mod:`xml.parsers.expat` module will always be -available. - -The documentation for the :mod:`xml.dom` and :mod:`xml.sax` packages are the -definition of the Python bindings for the DOM and SAX interfaces. - -The XML handling submodules are: - -* :mod:`xml.etree.ElementTree`: the ElementTree API, a simple and lightweight - XML processor - -.. - -* :mod:`xml.dom`: the DOM API definition -* :mod:`xml.dom.minidom`: a minimal DOM implementation -* :mod:`xml.dom.pulldom`: support for building partial DOM trees - -.. - -* :mod:`xml.sax`: SAX2 base classes and convenience functions -* :mod:`xml.parsers.expat`: the Expat parser binding - - -.. _xml-vulnerabilities: - -XML vulnerabilities -------------------- - -The XML processing modules are not secure against maliciously constructed data. -An attacker can abuse XML features to carry out denial of service attacks, -access local files, generate network connections to other machines, or -circumvent firewalls. - -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. - -========================= ======== ========= ========= ======== ========= -kind sax etree minidom pulldom xmlrpc -========================= ======== ========= ========= ======== ========= -billion laughs **Yes** **Yes** **Yes** **Yes** **Yes** -quadratic blowup **Yes** **Yes** **Yes** **Yes** **Yes** -external entity expansion **Yes** No (1) No (2) **Yes** No (3) -DTD retrieval **Yes** No No **Yes** No -decompression bomb No No No No **Yes** -========================= ======== ========= ========= ======== ========= - -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. - - -billion laughs / exponential entity expansion - The `Billion Laughs`_ attack -- also known as exponential entity expansion -- - uses multiple levels of nested entities. Each entity refers to another entity - several times, and the final entity definition contains a small string. - The exponential expansion results in several gigabytes of text and - consumes lots of memory and CPU time. - -quadratic blowup entity expansion - A quadratic blowup attack is similar to a `Billion Laughs`_ attack; it abuses - entity expansion, too. Instead of nested entities it repeats one large entity - with a couple of thousand chars over and over again. The attack isn't as - efficient as the exponential case but it avoids triggering parser countermeasures - that forbid deeply-nested entities. - -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -DTD retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - -decompression bomb - Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries - that can parse compressed XML streams such as gzipped HTTP streams or - LZMA-compressed - files. For an attacker it can reduce the amount of transmitted data by three - magnitudes or more. - -The documentation for `defusedxml`_ on PyPI has further information about -all known attack vectors with examples and references. - -.. _defused-packages: - -The :mod:`defusedxml` and :mod:`defusedexpat` Packages ------------------------------------------------------- - -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - -`defusedexpat`_ provides a modified libexpat and a patched -:mod:`pyexpat` module that have countermeasures against entity expansion -DoS attacks. The :mod:`defusedexpat` module still allows a sane and configurable amount of entity -expansions. The modifications may be included in some future release of Python, -but will not be included in any bugfix releases of -Python because they break backward compatibility. - - -.. _defusedxml: https://pypi.python.org/pypi/defusedxml/ -.. _defusedexpat: https://pypi.python.org/pypi/defusedexpat/ -.. _Billion Laughs: http://en.wikipedia.org/wiki/Billion_laughs -.. _ZIP bomb: http://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: http://en.wikipedia.org/wiki/Document_Type_Definition diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.sax.handler.rst --- a/Doc/library/xml.sax.handler.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.sax.handler.rst Mon Jan 25 17:05:13 2016 +0100 @@ -237,8 +237,7 @@ Signals the start of an element in non-namespace mode. The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` + string and the *attrs* parameter holds an object of the :class:`Attributes` interface (see :ref:`attributes-objects`) containing the attributes of the element. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not a reliable way to keep a copy of the attributes. @@ -261,8 +260,7 @@ The *name* parameter contains the name of the element type as a ``(uri, localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) + :class:`AttributesNS` interface (see :ref:`attributes-ns-objects`) containing the attributes of the element. If no namespace is associated with the element, the *uri* component of *name* will be ``None``. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not @@ -378,9 +376,8 @@ -------------------- Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser +from the :class:`XMLReader`. If you create an object that implements this +interface, then register the object with your :class:`XMLReader`, the parser will call the methods in your object to report all warnings and errors. There are three levels of errors available: warnings, (possibly) recoverable errors, and unrecoverable errors. All methods take a :exc:`SAXParseException` as the diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.sax.reader.rst --- a/Doc/library/xml.sax.reader.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.sax.reader.rst Mon Jan 25 17:05:13 2016 +0100 @@ -100,58 +100,53 @@ system identifier (a string identifying the input source -- typically a file name or an URL), a file-like object, or an :class:`InputSource` object. When :meth:`parse` returns, the input is completely processed, and the parser object - can be discarded or reset. - - .. versionchanged:: 3.5 - Added support of character streams. + can be discarded or reset. As a limitation, the current implementation only + accepts byte streams; processing of character streams is for further study. .. method:: XMLReader.getContentHandler() - Return the current :class:`~xml.sax.handler.ContentHandler`. + Return the current :class:`ContentHandler`. .. method:: XMLReader.setContentHandler(handler) - Set the current :class:`~xml.sax.handler.ContentHandler`. If no - :class:`~xml.sax.handler.ContentHandler` is set, content events will be - discarded. + Set the current :class:`ContentHandler`. If no :class:`ContentHandler` is set, + content events will be discarded. .. method:: XMLReader.getDTDHandler() - Return the current :class:`~xml.sax.handler.DTDHandler`. + Return the current :class:`DTDHandler`. .. method:: XMLReader.setDTDHandler(handler) - Set the current :class:`~xml.sax.handler.DTDHandler`. If no - :class:`~xml.sax.handler.DTDHandler` is set, DTD + Set the current :class:`DTDHandler`. If no :class:`DTDHandler` is set, DTD events will be discarded. .. method:: XMLReader.getEntityResolver() - Return the current :class:`~xml.sax.handler.EntityResolver`. + Return the current :class:`EntityResolver`. .. method:: XMLReader.setEntityResolver(handler) - Set the current :class:`~xml.sax.handler.EntityResolver`. If no - :class:`~xml.sax.handler.EntityResolver` is set, + Set the current :class:`EntityResolver`. If no :class:`EntityResolver` is set, attempts to resolve an external entity will result in opening the system identifier for the entity, and fail if it is not available. .. method:: XMLReader.getErrorHandler() - Return the current :class:`~xml.sax.handler.ErrorHandler`. + Return the current :class:`ErrorHandler`. .. method:: XMLReader.setErrorHandler(handler) - Set the current error handler. If no :class:`~xml.sax.handler.ErrorHandler` - is set, errors will be raised as exceptions, and warnings will be printed. + Set the current error handler. If no :class:`ErrorHandler` is set, errors will + be raised as exceptions, and warnings will be printed. .. method:: XMLReader.setLocale(locale) @@ -290,7 +285,8 @@ .. method:: InputSource.setByteStream(bytefile) - Set the byte stream (a :term:`binary file`) for this input source. + Set the byte stream (a Python file-like object which does not perform + byte-to-character conversion) for this input source. The SAX parser will ignore this if there is also a character stream specified, but it will use a byte stream in preference to opening a URI connection itself. @@ -309,7 +305,8 @@ .. method:: InputSource.setCharacterStream(charfile) - Set the character stream (a :term:`text file`) for this input source. + Set the character stream for this input source. (The stream must be a Python 1.6 + Unicode-wrapped file-like that performs conversion to strings.) If there is a character stream specified, the SAX parser will ignore any byte stream and will not attempt to open a URI connection to the system identifier. @@ -325,11 +322,9 @@ The :class:`Attributes` Interface --------------------------------- -:class:`Attributes` objects implement a portion of the :term:`mapping protocol -`, including the methods :meth:`~collections.abc.Mapping.copy`, -:meth:`~collections.abc.Mapping.get`, :meth:`~object.__contains__`, -:meth:`~collections.abc.Mapping.items`, :meth:`~collections.abc.Mapping.keys`, -and :meth:`~collections.abc.Mapping.values`. The following methods +:class:`Attributes` objects implement a portion of the mapping protocol, +including the methods :meth:`copy`, :meth:`get`, :meth:`__contains__`, +:meth:`items`, :meth:`keys`, and :meth:`values`. The following methods are also provided: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.sax.rst --- a/Doc/library/xml.sax.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.sax.rst Mon Jan 25 17:05:13 2016 +0100 @@ -13,21 +13,12 @@ SAX exceptions and the convenience functions which will be most used by users of the SAX API. - -.. warning:: - - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - The convenience functions are: .. function:: make_parser(parser_list=[]) - Create and return a SAX :class:`~xml.sax.xmlreader.XMLReader` object. The - first parser found will + Create and return a SAX :class:`XMLReader` object. The first parser found will be used. If *parser_list* is provided, it must be a sequence of strings which name modules that have a function named :func:`create_parser`. Modules listed in *parser_list* will be used before modules in the default list of parsers. @@ -37,9 +28,8 @@ Create a SAX parser and use it to parse a document. The document, passed in as *filename_or_stream*, can be a filename or a file object. The *handler* - parameter needs to be a SAX :class:`~handler.ContentHandler` instance. If - *error_handler* is given, it must be a SAX :class:`~handler.ErrorHandler` - instance; if + parameter needs to be a SAX :class:`ContentHandler` instance. If + *error_handler* is given, it must be a SAX :class:`ErrorHandler` instance; if omitted, :exc:`SAXParseException` will be raised on all errors. There is no return value; all work must be done by the *handler* passed in. @@ -47,11 +37,7 @@ .. function:: parseString(string, handler, error_handler=handler.ErrorHandler()) Similar to :func:`parse`, but parses from a buffer *string* received as a - parameter. *string* must be a :class:`str` instance or a - :term:`bytes-like object`. - - .. versionchanged:: 3.5 - Added support of :class:`str` instances. + parameter. A typical SAX application uses three kinds of objects: readers, handlers and input sources. "Reader" in this context is another term for parser, i.e. some @@ -68,12 +54,10 @@ instantiated by the application itself. Since Python does not have an explicit notion of interface, they are formally introduced as classes, but applications may use implementations which do not inherit from the provided classes. The -:class:`~xml.sax.xmlreader.InputSource`, :class:`~xml.sax.xmlreader.Locator`, -:class:`~xml.sax.xmlreader.Attributes`, :class:`~xml.sax.xmlreader.AttributesNS`, -and :class:`~xml.sax.xmlreader.XMLReader` interfaces are defined in the +:class:`InputSource`, :class:`Locator`, :class:`Attributes`, +:class:`AttributesNS`, and :class:`XMLReader` interfaces are defined in the module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in -:mod:`xml.sax.handler`. For convenience, -:class:`~xml.sax.xmlreader.InputSource` (which is often +:mod:`xml.sax.handler`. For convenience, :class:`InputSource` (which is often instantiated directly) and the handler classes are also available from :mod:`xml.sax`. These interfaces are described below. @@ -86,8 +70,7 @@ Encapsulate an XML error or warning. This class can contain basic error or warning information from either the XML parser or the application: it can be subclassed to provide additional functionality or to add localization. Note - that although the handlers defined in the - :class:`~xml.sax.handler.ErrorHandler` interface + that although the handlers defined in the :class:`ErrorHandler` interface receive instances of this exception, it is not required to actually raise the exception --- it is also useful as a container for information. @@ -100,26 +83,22 @@ .. exception:: SAXParseException(msg, exception, locator) - Subclass of :exc:`SAXException` raised on parse errors. Instances of this - class are passed to the methods of the SAX - :class:`~xml.sax.handler.ErrorHandler` interface to provide information - about the parse error. This class supports the SAX - :class:`~xml.sax.xmlreader.Locator` interface as well as the - :class:`SAXException` interface. + Subclass of :exc:`SAXException` raised on parse errors. Instances of this class + are passed to the methods of the SAX :class:`ErrorHandler` interface to provide + information about the parse error. This class supports the SAX :class:`Locator` + interface as well as the :class:`SAXException` interface. .. exception:: SAXNotRecognizedException(msg, exception=None) - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is + Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is confronted with an unrecognized feature or property. SAX applications and extensions may use this class for similar purposes. .. exception:: SAXNotSupportedException(msg, exception=None) - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is asked to + Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is asked to enable a feature that is not supported, or to set a property to a value that the implementation does not support. SAX applications and extensions may use this class for similar purposes. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xml.sax.utils.rst --- a/Doc/library/xml.sax.utils.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xml.sax.utils.rst Mon Jan 25 17:05:13 2016 +0100 @@ -52,8 +52,7 @@ .. class:: XMLGenerator(out=None, encoding='iso-8859-1', short_empty_elements=False) - This class implements the :class:`~xml.sax.handler.ContentHandler` interface - by writing SAX + This class implements the :class:`ContentHandler` interface by writing SAX events back into an XML document. In other words, using an :class:`XMLGenerator` as the content handler will reproduce the original document being parsed. *out* should be a file-like object which will default to *sys.stdout*. *encoding* is @@ -63,13 +62,12 @@ tags, if set to *True* they are emitted as a single self-closed tag. .. versionadded:: 3.2 - The *short_empty_elements* parameter. + short_empty_elements .. class:: XMLFilterBase(base) - This class is designed to sit between an - :class:`~xml.sax.xmlreader.XMLReader` and the client + This class is designed to sit between an :class:`XMLReader` and the client application's event handlers. By default, it does nothing but pass requests up to the reader and events on to the handlers unmodified, but subclasses can override specific methods to modify the event stream or the configuration @@ -78,10 +76,9 @@ .. function:: prepare_input_source(source, base='') - This function takes an input source and an optional base URL and returns a - fully resolved :class:`~xml.sax.xmlreader.InputSource` object ready for - reading. The input source can be given as a string, a file-like object, or - an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this - function to implement the polymorphic *source* argument to their - :meth:`parse` method. + This function takes an input source and an optional base URL and returns a fully + resolved :class:`InputSource` object ready for reading. The input source can be + given as a string, a file-like object, or an :class:`InputSource` object; + parsers will use this function to implement the polymorphic *source* argument to + their :meth:`parse` method. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xmlrpc.client.rst --- a/Doc/library/xmlrpc.client.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xmlrpc.client.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,20 +21,9 @@ between conformable Python objects and XML on the wire. -.. warning:: - - The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.5 - - For https URIs, :mod:`xmlrpc.client` now performs all the necessary - certificate and hostname checks by default - .. class:: ServerProxy(uri, transport=None, encoding=None, verbose=False, \ allow_none=False, use_datetime=False, \ - use_builtin_types=False, *, context=None) + use_builtin_types=False) .. versionchanged:: 3.3 The *use_builtin_types* flag was added. @@ -63,9 +52,7 @@ portion will be base64-encoded as an HTTP 'Authorization' header, and sent to the remote server as part of the connection process when invoking an XML-RPC method. You only need to use this if the remote server requires a Basic - Authentication user and password. If an HTTPS url is provided, *context* may - be :class:`ssl.SSLContext` and configures the SSL settings of the underlying - HTTPS connection. + Authentication user and password. The returned instance is a proxy object with methods that can be used to invoke corresponding RPC calls on the remote server. If the remote server supports the @@ -78,8 +65,6 @@ (e.g. that can be marshalled through XML), include the following (and except where noted, they are unmarshalled as the same Python type): - .. tabularcolumns:: |l|L| - +---------------------------------+---------------------------------------------+ | Name | Meaning | +=================================+=============================================+ @@ -129,9 +114,6 @@ :class:`Server` is retained as an alias for :class:`ServerProxy` for backwards compatibility. New code should use :class:`ServerProxy`. - .. versionchanged:: 3.5 - Added the *context* argument. - .. seealso:: @@ -200,11 +182,6 @@ no such string is available, an empty string is returned. The documentation string may contain HTML markup. -.. versionchanged:: 3.5 - - Instances of :class:`ServerProxy` support the :term:`context manager` protocol - for closing the underlying transport. - A working example follows. The server code:: @@ -222,9 +199,9 @@ import xmlrpc.client - with xmlrpc.client.ServerProxy("http://localhost:8000/") as proxy: - print("3 is even: %s" % str(proxy.is_even(3))) - print("100 is even: %s" % str(proxy.is_even(100))) + proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") + print("3 is even: %s" % str(proxy.is_even(3))) + print("100 is even: %s" % str(proxy.is_even(100))) .. _datetime-objects: @@ -449,21 +426,21 @@ is a :term:`generator`; iterating over this generator yields the individual results. -A usage example of this class follows. The server code:: +A usage example of this class follows. The server code :: from xmlrpc.server import SimpleXMLRPCServer - def add(x, y): - return x + y + def add(x,y): + return x+y def subtract(x, y): - return x - y + return x-y def multiply(x, y): - return x * y + return x*y def divide(x, y): - return x // y + return x/y # A simple server with simple arithmetic functions server = SimpleXMLRPCServer(("localhost", 8000)) @@ -481,13 +458,13 @@ proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") multicall = xmlrpc.client.MultiCall(proxy) - multicall.add(7, 3) - multicall.subtract(7, 3) - multicall.multiply(7, 3) - multicall.divide(7, 3) + multicall.add(7,3) + multicall.subtract(7,3) + multicall.multiply(7,3) + multicall.divide(7,3) result = multicall() - print("7+3=%d, 7-3=%d, 7*3=%d, 7//3=%d" % tuple(result)) + print("7+3=%d, 7-3=%d, 7*3=%d, 7/3=%d" % tuple(result)) Convenience Functions @@ -532,14 +509,14 @@ from xmlrpc.client import ServerProxy, Error # server = ServerProxy("http://localhost:8000") # local server - with ServerProxy("http://betty.userland.com") as proxy: + server = ServerProxy("http://betty.userland.com") - print(proxy) + print(server) - try: - print(proxy.examples.getStateName(41)) - except Error as v: - print("ERROR", v) + try: + print(server.examples.getStateName(41)) + except Error as v: + print("ERROR", v) To access an XML-RPC server through a proxy, you need to define a custom transport. The following example shows how: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xmlrpc.rst --- a/Doc/library/xmlrpc.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ -:mod:`xmlrpc` --- XMLRPC server and client modules -================================================== - -XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a -transport. With it, a client can call methods with parameters on a remote -server (the server is named by a URI) and get back structured data. - -``xmlrpc`` is a package that collects server and client modules implementing -XML-RPC. The modules are: - -* :mod:`xmlrpc.client` -* :mod:`xmlrpc.server` diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/xmlrpc.server.rst --- a/Doc/library/xmlrpc.server.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/xmlrpc.server.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,13 +16,6 @@ :class:`CGIXMLRPCRequestHandler`. -.. warning:: - - The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - .. class:: SimpleXMLRPCServer(addr, requestHandler=SimpleXMLRPCRequestHandler,\ logRequests=True, allow_none=False, encoding=None,\ bind_and_activate=True, use_builtin_types=False) @@ -184,70 +177,6 @@ # Print list of available methods print(s.system.listMethods()) -The following example included in the :file:`Lib/xmlrpc/server.py` module shows -a server allowing dotted names and registering a multicall function. - -.. warning:: - - Enabling the *allow_dotted_names* option allows intruders to access your - module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this example only within a secure, closed network. - -:: - - import datetime - - class ExampleService: - def getData(self): - return '42' - - class currentTime: - @staticmethod - def getCurrentTime(): - return datetime.datetime.now() - - server = SimpleXMLRPCServer(("localhost", 8000)) - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - server.server_close() - sys.exit(0) - -This ExampleService demo can be invoked from the command line:: - - python -m xmlrpc.server - - -The client that interacts with the above server is included in -`Lib/xmlrpc/client.py`:: - - server = ServerProxy("http://localhost:8000") - - try: - print(server.currentTime.getCurrentTime()) - except Error as v: - print("ERROR", v) - - multi = MultiCall(server) - multi.getData() - multi.pow(2,9) - multi.add(1,2) - try: - for response in multi(): - print(response) - except Error as v: - print("ERROR", v) - -This client which interacts with the demo XMLRPC server can be invoked as:: - - python -m xmlrpc.client - CGIXMLRPCRequestHandler ----------------------- diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/zipapp.rst --- a/Doc/library/zipapp.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,259 +0,0 @@ -:mod:`zipapp` --- Manage executable python zip archives -======================================================= - -.. module:: zipapp - :synopsis: Manage executable python zip archives - - -.. index:: - single: Executable Zip Files - -.. versionadded:: 3.5 - -**Source code:** :source:`Lib/zipapp.py` - --------------- - -This module provides tools to manage the creation of zip files containing -Python code, which can be :ref:`executed directly by the Python interpreter -`. The module provides both a -:ref:`zipapp-command-line-interface` and a :ref:`zipapp-python-api`. - - -Basic Example -------------- - -The following example shows how the :ref:`command-line-interface` -can be used to create an executable archive from a directory containing -Python code. When run, the archive will execute the ``main`` function from -the module ``myapp`` in the archive. - -.. code-block:: sh - - $ python -m zipapp myapp -m "myapp:main" - $ python myapp.pyz - - - -.. _zipapp-command-line-interface: - -Command-Line Interface ----------------------- - -When called as a program from the command line, the following form is used: - -.. code-block:: sh - - $ python -m zipapp source [options] - -If *source* is a directory, this will create an archive from the contents of -*source*. If *source* is a file, it should be an archive, and it will be -copied to the target archive (or the contents of its shebang line will be -displayed if the --info option is specified). - -The following options are understood: - -.. program:: zipapp - -.. cmdoption:: -o , --output= - - Write the output to a file named *output*. If this option is not specified, - the output filename will be the same as the input *source*, with the - extension ``.pyz`` added. If an explicit filename is given, it is used as - is (so a ``.pyz`` extension should be included if required). - - An output filename must be specified if the *source* is an archive (and in - that case, *output* must not be the same as *source*). - -.. cmdoption:: -p , --python= - - Add a ``#!`` line to the archive specifying *interpreter* as the command - to run. Also, on POSIX, make the archive executable. The default is to - write no ``#!`` line, and not make the file executable. - -.. cmdoption:: -m , --main= - - Write a ``__main__.py`` file to the archive that executes *mainfn*. The - *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a - package/module in the archive, and "fn" is a callable in the given module. - The ``__main__.py`` file will execute that callable. - - :option:`--main` cannot be specified when copying an archive. - -.. cmdoption:: --info - - Display the interpreter embedded in the archive, for diagnostic purposes. In - this case, any other options are ignored and SOURCE must be an archive, not a - directory. - -.. cmdoption:: -h, --help - - Print a short usage message and exit. - - -.. _zipapp-python-api: - -Python API ----------- - -The module defines two convenience functions: - - -.. function:: create_archive(source, target=None, interpreter=None, main=None) - - Create an application archive from *source*. The source can be any - of the following: - - * The name of a directory, or a :class:`pathlib.Path` object referring - to a directory, in which case a new application archive will be - created from the content of that directory. - * The name of an existing application archive file, or a :class:`pathlib.Path` - object referring to such a file, in which case the file is copied to - the target (modifying it to reflect the value given for the *interpreter* - argument). The file name should include the ``.pyz`` extension, if required. - * A file object open for reading in bytes mode. The content of the - file should be an application archive, and the file object is - assumed to be positioned at the start of the archive. - - The *target* argument determines where the resulting archive will be - written: - - * If it is the name of a file, or a :class:`pathlb.Path` object, - the archive will be written to that file. - * If it is an open file object, the archive will be written to that - file object, which must be open for writing in bytes mode. - * If the target is omitted (or None), the source must be a directory - and the target will be a file with the same name as the source, with - a ``.pyz`` extension added. - - The *interpreter* argument specifies the name of the Python - interpreter with which the archive will be executed. It is written as - a "shebang" line at the start of the archive. On POSIX, this will be - interpreted by the OS, and on Windows it will be handled by the Python - launcher. Omitting the *interpreter* results in no shebang line being - written. If an interpreter is specified, and the target is a - filename, the executable bit of the target file will be set. - - The *main* argument specifies the name of a callable which will be - used as the main program for the archive. It can only be specified if - the source is a directory, and the source does not already contain a - ``__main__.py`` file. The *main* argument should take the form - "pkg.module:callable" and the archive will be run by importing - "pkg.module" and executing the given callable with no arguments. It - is an error to omit *main* if the source is a directory and does not - contain a ``__main__.py`` file, as otherwise the resulting archive - would not be executable. - - If a file object is specified for *source* or *target*, it is the - caller's responsibility to close it after calling create_archive. - - When copying an existing archive, file objects supplied only need - ``read`` and ``readline``, or ``write`` methods. When creating an - archive from a directory, if the target is a file object it will be - passed to the ``zipfile.ZipFile`` class, and must supply the methods - needed by that class. - -.. function:: get_interpreter(archive) - - Return the interpreter specified in the ``#!`` line at the start of the - archive. If there is no ``#!`` line, return :const:`None`. - The *archive* argument can be a filename or a file-like object open - for reading in bytes mode. It is assumed to be at the start of the archive. - - -.. _zipapp-examples: - -Examples --------- - -Pack up a directory into an archive, and run it. - -.. code-block:: sh - - $ python -m zipapp myapp - $ python myapp.pyz - - -The same can be done using the :func:`create_archive` functon:: - - >>> import zipapp - >>> zipapp.create_archive('myapp.pyz', 'myapp') - -To make the application directly executable on POSIX, specify an interpreter -to use. - -.. code-block:: sh - - $ python -m zipapp myapp -p "/usr/bin/env python" - $ ./myapp.pyz - - -To replace the shebang line on an existing archive, create a modified archive -using the :func:`create_archive` function:: - - >>> import zipapp - >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') - -To update the file in place, do the replacement in memory using a :class:`BytesIO` -object, and then overwrite the source afterwards. Note that there is a risk -when overwriting a file in place that an error will result in the loss of -the original file. This code does not protect against such errors, but -production code should do so. Also, this method will only work if the archive -fits in memory:: - - >>> import zipapp - >>> import io - >>> temp = io.BytesIO() - >>> zipapp.create_archive('myapp.pyz', temp, '/usr/bin/python2') - >>> with open('myapp.pyz', 'wb') as f: - >>> f.write(temp.getvalue()) - -Note that if you specify an interpreter and then distribute your application -archive, you need to ensure that the interpreter used is portable. The Python -launcher for Windows supports most common forms of POSIX ``#!`` line, but there -are other issues to consider: - -* If you use "/usr/bin/env python" (or other forms of the "python" command, - such as "/usr/bin/python"), you need to consider that your users may have - either Python 2 or Python 3 as their default, and write your code to work - under both versions. -* If you use an explicit version, for example "/usr/bin/env python3" your - application will not work for users who do not have that version. (This - may be what you want if you have not made your code Python 2 compatible). -* There is no way to say "python X.Y or later", so be careful of using an - exact version like "/usr/bin/env python3.4" as you will need to change your - shebang line for users of Python 3.5, for example. - -The Python Zip Application Archive Format ------------------------------------------ - -Python has been able to execute zip files which contain a ``__main__.py`` file -since version 2.6. In order to be executed by Python, an application archive -simply has to be a standard zip file containing a ``__main__.py`` file which -will be run as the entry point for the application. As usual for any Python -script, the parent of the script (in this case the zip file) will be placed on -:data:`sys.path` and thus further modules can be imported from the zip file. - -The zip file format allows arbitrary data to be prepended to a zip file. The -zip application format uses this ability to prepend a standard POSIX "shebang" -line to the file (``#!/path/to/interpreter``). - -Formally, the Python zip application format is therefore: - -1. An optional shebang line, containing the characters ``b'#!'`` followed by an - interpreter name, and then a newline (``b'\n'``) character. The interpreter - name can be anything acceptable to the OS "shebang" processing, or the Python - launcher on Windows. The interpreter should be encoded in UTF-8 on Windows, - and in :func:`sys.getfilesystemencoding()` on POSIX. -2. Standard zipfile data, as generated by the :mod:`zipfile` module. The - zipfile content *must* include a file called ``__main__.py`` (which must be - in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The - zipfile data can be compressed or uncompressed. - -If an application archive has a shebang line, it may have the executable bit set -on POSIX systems, to allow it to be executed directly. - -There is no requirement that the tools in this module are used to create -application archives - the module is a convenience, but archives in the above -format created by any means are acceptable to Python. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/zipfile.rst --- a/Doc/library/zipfile.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/zipfile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -18,7 +18,7 @@ This module does not currently handle multi-disk ZIP files. It can handle ZIP files that use the ZIP64 extensions -(that is ZIP files that are more than 4 GiB in size). It supports +(that is ZIP files that are more than 4 GByte in size). It supports decryption of encrypted files in ZIP archives, but it currently cannot create an encrypted file. Decryption is extremely slow as it is implemented in native Python rather than C. @@ -61,7 +61,7 @@ .. class:: ZipInfo(filename='NoName', date_time=(1980,1,1,0,0,0)) Class used to represent information about a member of an archive. Instances - of this class are returned by the :meth:`.getinfo` and :meth:`.infolist` + of this class are returned by the :meth:`getinfo` and :meth:`infolist` methods of :class:`ZipFile` objects. Most users of the :mod:`zipfile` module will not need to create these, but only use those created by this module. *filename* should be the full name of the archive member, and @@ -87,30 +87,7 @@ .. data:: ZIP_DEFLATED The numeric constant for the usual ZIP compression method. This requires the - :mod:`zlib` module. - - -.. data:: ZIP_BZIP2 - - The numeric constant for the BZIP2 compression method. This requires the - :mod:`bz2` module. - - .. versionadded:: 3.3 - -.. data:: ZIP_LZMA - - The numeric constant for the LZMA compression method. This requires the - :mod:`lzma` module. - - .. versionadded:: 3.3 - - .. note:: - - The ZIP file format specification has included support for bzip2 compression - since 2001, and for LZMA compression since 2006. However, some tools - (including older Python releases) do not support these compression - methods, and may either refuse to process the ZIP file altogether, - or fail to extract individual files. + zlib module. No other compression methods are currently supported. .. seealso:: @@ -130,33 +107,30 @@ --------------- -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) +.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False) Open a ZIP file, where *file* can be either a path to a file (a string) or a file-like object. The *mode* parameter should be ``'r'`` to read an existing - file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an - existing file, or ``'x'`` to exclusively create and write a new file. - If *mode* is ``'x'`` and *file* refers to an existing file, - a :exc:`FileExistsError` will be raised. - If *mode* is ``'a'`` and *file* refers to an existing ZIP + file, ``'w'`` to truncate and write a new file, or ``'a'`` to append to an + existing file. If *mode* is ``'a'`` and *file* refers to an existing ZIP file, then additional files are added to it. If *file* does not refer to a ZIP file, then a new ZIP archive is appended to the file. This is meant for adding a ZIP archive to another file (such as :file:`python.exe`). If *mode* is ``a`` and the file does not exist at all, it is created. - If *mode* is ``r`` or ``a``, the file should be seekable. *compression* is the ZIP compression method to use when writing the archive, - and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`RuntimeError` to be raised. If :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module - (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError` + and should be :const:`ZIP_STORED` or :const:`ZIP_DEFLATED`; unrecognized + values will cause :exc:`RuntimeError` to be raised. If :const:`ZIP_DEFLATED` + is specified but the :mod:`zlib` module is not available, :exc:`RuntimeError` is also raised. The default is :const:`ZIP_STORED`. If *allowZip64* is - ``True`` (the default) zipfile will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 2 GiB. If it is false :mod:`zipfile` + ``True`` zipfile will create ZIP files that use the ZIP64 extensions when + the zipfile is larger than 2 GB. If it is false (the default) :mod:`zipfile` will raise an exception when the ZIP file would require ZIP64 extensions. + ZIP64 extensions are disabled by default because the default :program:`zip` + and :program:`unzip` commands on Unix (the InfoZIP utilities) don't support + these extensions. - If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then - :meth:`closed ` without adding any files to the archive, the appropriate + If the file is created with mode ``'a'`` or ``'w'`` and then + :meth:`close`\ d without adding any files to the archive, the appropriate ZIP structures for an empty archive will be written to the file. ZipFile is also a context manager and therefore supports the @@ -169,16 +143,6 @@ .. versionadded:: 3.2 Added the ability to use :class:`ZipFile` as a context manager. - .. versionchanged:: 3.3 - Added support for :mod:`bzip2 ` and :mod:`lzma` compression. - - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - - .. versionchanged:: 3.5 - Added support for writing to unseekable streams. - Added support for the ``'x'`` mode. - .. method:: ZipFile.close() @@ -205,47 +169,38 @@ Return a list of archive members by name. -.. index:: - single: universal newlines; zipfile.ZipFile.open method - .. method:: ZipFile.open(name, mode='r', pwd=None) - Extract a member from the archive as a file-like object (ZipExtFile). *name* - is the name of the file in the archive, or a :class:`ZipInfo` object. The - *mode* parameter, if included, must be one of the following: ``'r'`` (the - default), ``'U'``, or ``'rU'``. Choosing ``'U'`` or ``'rU'`` will enable - :term:`universal newlines` support in the read-only object. *pwd* is the - password used for encrypted files. Calling :meth:`.open` on a closed - ZipFile will raise a :exc:`RuntimeError`. - - :meth:`~ZipFile.open` is also a context manager and therefore supports the - :keyword:`with` statement:: - - with ZipFile('spam.zip') as myzip: - with myzip.open('eggs.txt') as myfile: - print(myfile.read()) + Extract a member from the archive as a file-like object (ZipExtFile). *name* is + the name of the file in the archive, or a :class:`ZipInfo` object. The *mode* + parameter, if included, must be one of the following: ``'r'`` (the default), + ``'U'``, or ``'rU'``. Choosing ``'U'`` or ``'rU'`` will enable universal newline + support in the read-only object. *pwd* is the password used for encrypted files. + Calling :meth:`open` on a closed ZipFile will raise a :exc:`RuntimeError`. .. note:: The file-like object is read-only and provides the following methods: - :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. + :meth:`!read`, :meth:`!readline`, :meth:`!readlines`, :meth:`!__iter__`, + :meth:`!__next__`. .. note:: - Objects returned by :meth:`.open` can operate independently of the - ZipFile. + If the ZipFile was created by passing in a file-like object as the first + argument to the constructor, then the object returned by :meth:`.open` shares the + ZipFile's file pointer. Under these circumstances, the object returned by + :meth:`.open` should not be used after any additional operations are performed + on the ZipFile object. If the ZipFile was created by passing in a string (the + filename) as the first argument to the constructor, then :meth:`.open` will + create a new file object that will be held by the ZipExtFile, allowing it to + operate independently of the ZipFile. .. note:: - The :meth:`.open`, :meth:`read` and :meth:`extract` methods can take a filename + The :meth:`open`, :meth:`read` and :meth:`extract` methods can take a filename or a :class:`ZipInfo` object. You will appreciate this when trying to read a ZIP file that contains members with duplicate names. - .. deprecated-removed:: 3.4 3.6 - The ``'U'`` or ``'rU'`` mode. Use :class:`io.TextIOWrapper` for reading - compressed text files in :term:`universal newlines` mode. .. method:: ZipFile.extract(member, path=None, pwd=None) @@ -255,18 +210,6 @@ to extract to. *member* can be a filename or a :class:`ZipInfo` object. *pwd* is the password used for encrypted files. - Returns the normalized path created (a directory or new file). - - .. note:: - - If a member filename is an absolute path, a drive/UNC sharepoint and - leading (back)slashes will be stripped, e.g.: ``///foo/bar`` becomes - ``foo/bar`` on Unix, and ``C:\foo\bar`` becomes ``foo\bar`` on Windows. - And all ``".."`` components in a member filename will be removed, e.g.: - ``../../foo../../ba..r`` becomes ``foo../ba..r``. On Windows illegal - characters (``:``, ``<``, ``>``, ``|``, ``"``, ``?``, and ``*``) - replaced by underscore (``_``). - .. method:: ZipFile.extractall(path=None, members=None, pwd=None) @@ -280,8 +223,7 @@ Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of *path*, e.g. members that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. This module attempts to prevent that. - See :meth:`extract` note. + dots ``".."``. .. method:: ZipFile.printdir() @@ -300,11 +242,7 @@ file in the archive, or a :class:`ZipInfo` object. The archive must be open for read or append. *pwd* is the password used for encrypted files and, if specified, it will override the default password set with :meth:`setpassword`. Calling - :meth:`read` on a closed ZipFile will raise a :exc:`RuntimeError`. Calling - :meth:`read` on a ZipFile that uses a compression method other than - :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or - :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also - be raised if the corresponding compression module is not available. + :meth:`read` on a closed ZipFile will raise a :exc:`RuntimeError`. .. method:: ZipFile.testzip() @@ -320,8 +258,7 @@ *arcname* (by default, this will be the same as *filename*, but without a drive letter and with leading path separators removed). If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for - the new entry. - The archive must be open with mode ``'w'``, ``'x'`` or ``'a'`` -- calling + the new entry. The archive must be open with mode ``'w'`` or ``'a'`` -- calling :meth:`write` on a ZipFile created with mode ``'r'`` will raise a :exc:`RuntimeError`. Calling :meth:`write` on a closed ZipFile will raise a :exc:`RuntimeError`. @@ -343,16 +280,16 @@ If ``arcname`` (or ``filename``, if ``arcname`` is not given) contains a null byte, the name of the file in the archive will be truncated at the null byte. + .. method:: ZipFile.writestr(zinfo_or_arcname, bytes[, compress_type]) Write the string *bytes* to the archive; *zinfo_or_arcname* is either the file name it will be given in the archive, or a :class:`ZipInfo` instance. If it's an instance, at least the filename, date, and time must be given. If it's a - name, the date and time is set to the current date and time. - The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'`` -- calling - :meth:`writestr` on a ZipFile created with mode ``'r'`` will raise a - :exc:`RuntimeError`. Calling :meth:`writestr` on a closed ZipFile will - raise a :exc:`RuntimeError`. + name, the date and time is set to the current date and time. The archive must be + opened with mode ``'w'`` or ``'a'`` -- calling :meth:`writestr` on a ZipFile + created with mode ``'r'`` will raise a :exc:`RuntimeError`. Calling + :meth:`writestr` on a closed ZipFile will raise a :exc:`RuntimeError`. If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for the new entry, or in the *zinfo_or_arcname* @@ -366,7 +303,7 @@ :class:`ZipInfo` constructor sets this member to :const:`ZIP_STORED`. .. versionchanged:: 3.2 - The *compress_type* argument. + The *compression_type* argument. The following data attributes are also available: @@ -380,10 +317,9 @@ .. attribute:: ZipFile.comment The comment text associated with the ZIP file. If assigning a comment to a - :class:`ZipFile` instance created with mode ``'w'``, ``'x'`` or ``'a'``, - this should be a + :class:`ZipFile` instance created with mode 'a' or 'w', this should be a string no longer than 65535 bytes. Comments longer than this will be - truncated in the written archive when :meth:`close` is called. + truncated in the written archive when :meth:`ZipFile.close` is called. .. _pyzipfile-objects: @@ -394,56 +330,37 @@ The :class:`PyZipFile` constructor takes the same parameters as the :class:`ZipFile` constructor, and one additional parameter, *optimize*. -.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ +.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False, \ optimize=-1) .. versionadded:: 3.2 The *optimize* parameter. - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - Instances have one method in addition to those of :class:`ZipFile` objects: - .. method:: PyZipFile.writepy(pathname, basename='', filterfunc=None) + .. method:: PyZipFile.writepy(pathname, basename='') Search for files :file:`\*.py` and add the corresponding file to the archive. If the *optimize* parameter to :class:`PyZipFile` was not given or ``-1``, - the corresponding file is a :file:`\*.pyc` file, compiling if necessary. + the corresponding file is a :file:`\*.pyo` file if available, else a + :file:`\*.pyc` file, compiling if necessary. If the *optimize* parameter to :class:`PyZipFile` was ``0``, ``1`` or ``2``, only files with that optimization level (see :func:`compile`) are added to the archive, compiling if necessary. - If *pathname* is a file, the filename must end with :file:`.py`, and + If the pathname is a file, the filename must end with :file:`.py`, and just the (corresponding :file:`\*.py[co]`) file is added at the top level - (no path information). If *pathname* is a file that does not end with + (no path information). If the pathname is a file that does not end with :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, and the directory is not a package directory, then all the files :file:`\*.py[co]` are added at the top level. If the directory is a package directory, then all :file:`\*.py[co]` are added under the package name as a file path, and if any subdirectories are package directories, - all of these are added recursively. - - *basename* is intended for internal use only. - - *filterfunc*, if given, must be a function taking a single string - argument. It will be passed each path (including each individual full - file path) before it is added to the archive. If *filterfunc* returns a - false value, the path will not be added, and if it is a directory its - contents will be ignored. For example, if our test files are all either - in ``test`` directories or start with the string ``test_``, we can use a - *filterfunc* to exclude them:: - - >>> zf = PyZipFile('myprog.zip') - >>> def notests(s): - ... fn = os.path.basename(s) - ... return (not (fn == 'test' or fn.startswith('test_'))) - >>> zf.writepy('myprog', filterfunc=notests) - - The :meth:`writepy` method makes archives with file names like + all of these are added recursively. *basename* is intended for internal + use only. The :meth:`writepy` method makes archives with file names like this:: string.pyc # Top level name @@ -452,17 +369,14 @@ test/bogus/__init__.pyc # Subpackage directory test/bogus/myfile.pyc # Submodule test.bogus.myfile - .. versionadded:: 3.4 - The *filterfunc* parameter. - .. _zipinfo-objects: ZipInfo Objects --------------- -Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and -:meth:`.infolist` methods of :class:`ZipFile` objects. Each object stores +Instances of the :class:`ZipInfo` class are returned by the :meth:`getinfo` and +:meth:`infolist` methods of :class:`ZipFile` objects. Each object stores information about a single member of the ZIP archive. Instances have the following attributes: @@ -574,3 +488,4 @@ .. attribute:: ZipInfo.file_size Size of the uncompressed file. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/zipimport.rst --- a/Doc/library/zipimport.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/zipimport.rst Mon Jan 25 17:05:13 2016 +0100 @@ -16,14 +16,14 @@ also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. The ZIP archive can contain a subdirectory structure to support package imports, and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only +subdirectory. For example, the path :file:`/tmp/example.zip/lib/` would only import from the :file:`lib/` subdirectory within the archive. Any files may be present in the ZIP archive, but only files :file:`.py` and -:file:`.pyc` are available for import. ZIP import of dynamic modules +:file:`.py[co]` are available for import. ZIP import of dynamic modules (:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains :file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive +corresponding :file:`.pyc` or :file:`.pyo` file, meaning that if a ZIP archive doesn't contain :file:`.pyc` files, importing may be rather slow. ZIP archives with an archive comment are currently not supported. @@ -111,7 +111,7 @@ .. method:: is_package(fullname) - Return ``True`` if the module specified by *fullname* is a package. Raise + Return True if the module specified by *fullname* is a package. Raise :exc:`ZipImportError` if the module couldn't be found. @@ -147,8 +147,8 @@ Here is an example that imports a module from a ZIP archive - note that the :mod:`zipimport` module is not explicitly used. :: - $ unzip -l example.zip - Archive: example.zip + $ unzip -l /tmp/example.zip + Archive: /tmp/example.zip Length Date Time Name -------- ---- ---- ---- 8467 11-26-02 22:30 jwzthreading.py @@ -157,7 +157,8 @@ $ ./python Python 2.3 (#1, Aug 1 2003, 19:54:32) >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path + >>> sys.path.insert(0, '/tmp/example.zip') # Add .zip file to front of path >>> import jwzthreading >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' + '/tmp/example.zip/jwzthreading.py' + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/library/zlib.rst --- a/Doc/library/zlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/library/zlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -30,62 +30,40 @@ .. function:: adler32(data[, value]) - Computes an Adler-32 checksum of *data*. (An Adler-32 checksum is almost as - reliable as a CRC32 but can be computed much more quickly.) The result - is an unsigned 32-bit integer. If *value* is present, it is used as - the starting value of the checksum; otherwise, a default value of 1 - is used. Passing in *value* allows computing a running checksum over the + Computes a Adler-32 checksum of *data*. (An Adler-32 checksum is almost as + reliable as a CRC32 but can be computed much more quickly.) If *value* is + present, it is used as the starting value of the checksum; otherwise, a fixed + default value is used. This allows computing a running checksum over the concatenation of several inputs. The algorithm is not cryptographically strong, and should not be used for authentication or digital signatures. Since the algorithm is designed for use as a checksum algorithm, it is not suitable for use as a general hash algorithm. - .. versionchanged:: 3.0 - Always returns an unsigned value. - To generate the same numeric value across all Python versions and - platforms, use ``adler32(data) & 0xffffffff``. + Always returns an unsigned 32-bit integer. + +.. note:: + To generate the same numeric value across all Python versions and + platforms use adler32(data) & 0xffffffff. If you are only using + the checksum in packed binary format this is not necessary as the + return value is the correct 32bit binary representation + regardless of sign. .. function:: compress(data[, level]) Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` controlling the level of compression; + *level* is an integer from ``1`` to ``9`` controlling the level of compression; ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``6``. - Raises the :exc:`error` exception if any error occurs. + produces the most. The default value is ``6``. Raises the :exc:`error` + exception if any error occurs. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj([level]) Returns a compression object, to be used for compressing data streams that won't - fit into memory at once. - - *level* is the compression level -- an integer from ``0`` to ``9``. A value - of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``6``. - - *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. - - *wbits* is the base two logarithm of the size of the window buffer. This - should be an integer from ``8`` to ``15``. Higher values give better - compression, but use more memory. - - The *memLevel* argument controls the amount of memory used for the - internal compression state. Valid values range from ``1`` to ``9``. - Higher values use more memory, but are faster and produce smaller output. - - *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. - - *zdict* is a predefined compression dictionary. This is a sequence of bytes - (such as a :class:`bytes` object) containing subsequences that are expected - to occur frequently in the data that is to be compressed. Those subsequences - that are expected to be most common should come at the end of the dictionary. - - .. versionchanged:: 3.3 - Added the *zdict* parameter and keyword argument support. + fit into memory at once. *level* is an integer from ``1`` to ``9`` controlling + the level of compression; ``1`` is fastest and produces the least compression, + ``9`` is slowest and produces the most. The default value is ``6``. .. function:: crc32(data[, value]) @@ -94,19 +72,22 @@ single: Cyclic Redundancy Check single: checksum; Cyclic Redundancy Check - Computes a CRC (Cyclic Redundancy Check) checksum of *data*. The - result is an unsigned 32-bit integer. If *value* is present, it is used - as the starting value of the checksum; otherwise, a default value of 0 - is used. Passing in *value* allows computing a running checksum over the + Computes a CRC (Cyclic Redundancy Check) checksum of *data*. If *value* is + present, it is used as the starting value of the checksum; otherwise, a fixed + default value is used. This allows computing a running checksum over the concatenation of several inputs. The algorithm is not cryptographically strong, and should not be used for authentication or digital signatures. Since the algorithm is designed for use as a checksum algorithm, it is not suitable for use as a general hash algorithm. - .. versionchanged:: 3.0 - Always returns an unsigned value. - To generate the same numeric value across all Python versions and - platforms, use ``crc32(data) & 0xffffffff``. + Always returns an unsigned 32-bit integer. + +.. note:: + To generate the same numeric value across all Python versions and + platforms use crc32(data) & 0xffffffff. If you are only using + the checksum in packed binary format this is not necessary as the + return value is the correct 32bit binary representation + regardless of sign. .. function:: decompress(data[, wbits[, bufsize]]) @@ -133,25 +114,11 @@ to :c:func:`malloc`. The default size is 16384. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj([wbits]) Returns a decompression object, to be used for decompressing data streams that - won't fit into memory at once. - - The *wbits* parameter controls the size of the window buffer. - - The *zdict* parameter specifies a predefined compression dictionary. If - provided, this must be the same dictionary as was used by the compressor that - produced the data that is to be decompressed. - - .. note:: - - If *zdict* is a mutable object (such as a :class:`bytearray`), you must not - modify its contents between the call to :func:`decompressobj` and the first - call to the decompressor's ``decompress()`` method. - - .. versionchanged:: 3.3 - Added the *zdict* parameter. + won't fit into memory at once. The *wbits* parameter controls the size of the + window buffer. Compression objects support the following methods: @@ -190,7 +157,7 @@ .. attribute:: Decompress.unused_data A bytes object which contains any bytes past the end of the compressed data. That is, - this remains ``b""`` until the last byte that contains compression data is + this remains ``""`` until the last byte that contains compression data is available. If the whole bytestring turned out to contain compressed data, this is ``b""``, an empty bytes object. @@ -223,7 +190,7 @@ :meth:`decompress` method. Some of the input data may be preserved in internal buffers for later processing. - If the optional parameter *max_length* is non-zero then the return value will be + If the optional parameter *max_length* is supplied then the return value will be no longer than *max_length*. This may mean that not all of the compressed input can be processed; and unconsumed data will be stored in the attribute :attr:`unconsumed_tail`. This bytestring must be passed to a subsequent call to diff -r 6db40a9955dc -r 0d413f60cc23 Doc/license.rst --- a/Doc/license.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/license.rst Mon Jan 25 17:05:13 2016 +0100 @@ -23,11 +23,11 @@ form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations (now Zope Corporation; see http://www.zope.com/). In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization created +http://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation is a sponsoring member of the PSF. -All Python releases are Open Source (see http://opensource.org/ for the Open +All Python releases are Open Source (see http://www.opensource.org/ for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases. @@ -50,11 +50,75 @@ +----------------+--------------+------------+------------+-----------------+ | 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | +----------------+--------------+------------+------------+-----------------+ +| 2.2 | 2.1.1 | 2001 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ | 2.1.2 | 2.1.1 | 2002 | PSF | yes | +----------------+--------------+------------+------------+-----------------+ | 2.1.3 | 2.1.2 | 2002 | PSF | yes | +----------------+--------------+------------+------------+-----------------+ -| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | +| 2.2.1 | 2.2 | 2002 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.2.2 | 2.2.1 | 2002 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.2.3 | 2.2.2 | 2002-2003 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3 | 2.2.2 | 2002-2003 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3.1 | 2.3 | 2002-2003 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3.2 | 2.3.1 | 2003 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3.3 | 2.3.2 | 2003 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3.4 | 2.3.3 | 2004 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.3.5 | 2.3.4 | 2005 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.4 | 2.3 | 2004 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.4.1 | 2.4 | 2005 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.4.2 | 2.4.1 | 2005 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.4.3 | 2.4.2 | 2006 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.4.4 | 2.4.3 | 2006 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.5 | 2.4 | 2006 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.5.1 | 2.5 | 2007 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.6 | 2.5 | 2008 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.6.1 | 2.6 | 2008 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.6.2 | 2.6.1 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.6.3 | 2.6.2 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 2.6.4 | 2.6.3 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.0 | 2.6 | 2008 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.0.1 | 3.0 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.1 | 3.0.1 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.1.1 | 3.1 | 2009 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.1.2 | 3.1.1 | 2010 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.1.3 | 3.1.2 | 2010 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.1.4 | 3.1.3 | 2011 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.2 | 3.1 | 2011 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.2.1 | 3.2 | 2011 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.2.2 | 3.2.1 | 2011 | PSF | yes | ++----------------+--------------+------------+------------+-----------------+ +| 3.3.0 | 3.2 | 2012 | PSF | yes | +----------------+--------------+------------+------------+-----------------+ .. note:: @@ -73,189 +137,181 @@ ============================================================ -PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------- +.. centered:: PSF LICENSE AGREEMENT FOR PYTHON |release| -.. parsed-literal:: +#. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and + the Individual or Organization ("Licensee") accessing and otherwise using Python + |release| software in source or binary form and its associated documentation. - 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and - the Individual or Organization ("Licensee") accessing and otherwise using Python - |release| software in source or binary form and its associated documentation. +#. Subject to the terms and conditions of this License Agreement, PSF hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + analyze, test, perform and/or display publicly, prepare derivative works, + distribute, and otherwise use Python |release| alone or in any derivative + version, provided, however, that PSF's License Agreement and PSF's notice of + copyright, i.e., "Copyright © 2001-2012 Python Software Foundation; All Rights + Reserved" are retained in Python |release| alone or in any derivative version + prepared by Licensee. - 2. Subject to the terms and conditions of this License Agreement, PSF hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python |release| alone or in any derivative - version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2016 Python Software Foundation; All Rights - Reserved" are retained in Python |release| alone or in any derivative version - prepared by Licensee. +#. In the event Licensee prepares a derivative work that is based on or + incorporates Python |release| or any part thereof, and wants to make the + derivative work available to others as provided herein, then Licensee hereby + agrees to include in any such work a brief summary of the changes made to Python + |release|. - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python |release| or any part thereof, and wants to make the - derivative work available to others as provided herein, then Licensee hereby - agrees to include in any such work a brief summary of the changes made to Python - |release|. +#. PSF is making Python |release| available to Licensee on an "AS IS" basis. + PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF + EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR + WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE + USE OF PYTHON |release| WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - 4. PSF is making Python |release| available to Licensee on an "AS IS" basis. - PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF PYTHON |release| WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. +#. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON |release| + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF + MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON |release|, OR ANY DERIVATIVE + THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON |release| - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON |release|, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. +#. This License Agreement will automatically terminate upon a material breach of + its terms and conditions. - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. +#. Nothing in this License Agreement shall be deemed to create any relationship + of agency, partnership, or joint venture between PSF and Licensee. This License + Agreement does not grant permission to use PSF trademarks or trade name in a + trademark sense to endorse or promote products or services of Licensee, or any + third party. - 7. Nothing in this License Agreement shall be deemed to create any relationship - of agency, partnership, or joint venture between PSF and Licensee. This License - Agreement does not grant permission to use PSF trademarks or trade name in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. +#. By copying, installing or otherwise using Python |release|, Licensee agrees + to be bound by the terms and conditions of this License Agreement. - 8. By copying, installing or otherwise using Python |release|, Licensee agrees - to be bound by the terms and conditions of this License Agreement. +.. centered:: BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 +.. centered:: BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 -.. parsed-literal:: +#. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at + 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization + ("Licensee") accessing and otherwise using this software in source or binary + form and its associated documentation ("the Software"). - 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at - 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization - ("Licensee") accessing and otherwise using this software in source or binary - form and its associated documentation ("the Software"). +#. Subject to the terms and conditions of this BeOpen Python License Agreement, + BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license + to reproduce, analyze, test, perform and/or display publicly, prepare derivative + works, distribute, and otherwise use the Software alone or in any derivative + version, provided, however, that the BeOpen Python License is retained in the + Software, alone or in any derivative version prepared by Licensee. - 2. Subject to the terms and conditions of this BeOpen Python License Agreement, - BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license - to reproduce, analyze, test, perform and/or display publicly, prepare derivative - works, distribute, and otherwise use the Software alone or in any derivative - version, provided, however, that the BeOpen Python License is retained in the - Software, alone or in any derivative version prepared by Licensee. +#. BeOpen is making the Software available to Licensee on an "AS IS" basis. + BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF + EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR + WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE + USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - 3. BeOpen is making the Software available to Licensee on an "AS IS" basis. - BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. +#. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR + ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, + MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF + ADVISED OF THE POSSIBILITY THEREOF. - 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, - MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF - ADVISED OF THE POSSIBILITY THEREOF. +#. This License Agreement will automatically terminate upon a material breach of + its terms and conditions. - 5. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. +#. This License Agreement shall be governed by and interpreted in all respects + by the law of the State of California, excluding conflict of law provisions. + Nothing in this License Agreement shall be deemed to create any relationship of + agency, partnership, or joint venture between BeOpen and Licensee. This License + Agreement does not grant permission to use BeOpen trademarks or trade names in a + trademark sense to endorse or promote products or services of Licensee, or any + third party. As an exception, the "BeOpen Python" logos available at + http://www.pythonlabs.com/logos.html may be used according to the permissions + granted on that web page. - 6. This License Agreement shall be governed by and interpreted in all respects - by the law of the State of California, excluding conflict of law provisions. - Nothing in this License Agreement shall be deemed to create any relationship of - agency, partnership, or joint venture between BeOpen and Licensee. This License - Agreement does not grant permission to use BeOpen trademarks or trade names in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. As an exception, the "BeOpen Python" logos available at - http://www.pythonlabs.com/logos.html may be used according to the permissions - granted on that web page. +#. By copying, installing or otherwise using the software, Licensee agrees to be + bound by the terms and conditions of this License Agreement. - 7. By copying, installing or otherwise using the software, Licensee agrees to be - bound by the terms and conditions of this License Agreement. +.. centered:: CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- +#. This LICENSE AGREEMENT is between the Corporation for National Research + Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 + ("CNRI"), and the Individual or Organization ("Licensee") accessing and + otherwise using Python 1.6.1 software in source or binary form and its + associated documentation. -.. parsed-literal:: +#. Subject to the terms and conditions of this License Agreement, CNRI hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + analyze, test, perform and/or display publicly, prepare derivative works, + distribute, and otherwise use Python 1.6.1 alone or in any derivative version, + provided, however, that CNRI's License Agreement and CNRI's notice of copyright, + i.e., "Copyright © 1995-2001 Corporation for National Research Initiatives; All + Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version + prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, + Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 + is made available subject to the terms and conditions in CNRI's License + Agreement. This Agreement together with Python 1.6.1 may be located on the + Internet using the following unique, persistent identifier (known as a handle): + 1895.22/1013. This Agreement may also be obtained from a proxy server on the + Internet using the following URL: http://hdl.handle.net/1895.22/1013." - 1. This LICENSE AGREEMENT is between the Corporation for National Research - Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 - ("CNRI"), and the Individual or Organization ("Licensee") accessing and - otherwise using Python 1.6.1 software in source or binary form and its - associated documentation. +#. In the event Licensee prepares a derivative work that is based on or + incorporates Python 1.6.1 or any part thereof, and wants to make the derivative + work available to others as provided herein, then Licensee hereby agrees to + include in any such work a brief summary of the changes made to Python 1.6.1. - 2. Subject to the terms and conditions of this License Agreement, CNRI hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python 1.6.1 alone or in any derivative version, - provided, however, that CNRI's License Agreement and CNRI's notice of copyright, - i.e., "Copyright © 1995-2001 Corporation for National Research Initiatives; All - Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version - prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, - Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 - is made available subject to the terms and conditions in CNRI's License - Agreement. This Agreement together with Python 1.6.1 may be located on the - Internet using the following unique, persistent identifier (known as a handle): - 1895.22/1013. This Agreement may also be obtained from a proxy server on the - Internet using the following URL: http://hdl.handle.net/1895.22/1013." +#. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI + MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, + BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY + OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF + PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python 1.6.1 or any part thereof, and wants to make the derivative - work available to others as provided herein, then Licensee hereby agrees to - include in any such work a brief summary of the changes made to Python 1.6.1. +#. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR + ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF + MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE + THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI - MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, - BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY - OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF - PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. +#. This License Agreement will automatically terminate upon a material breach of + its terms and conditions. - 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. +#. This License Agreement shall be governed by the federal intellectual property + law of the United States, including without limitation the federal copyright + law, and, to the extent such U.S. federal law does not apply, by the law of the + Commonwealth of Virginia, excluding Virginia's conflict of law provisions. + Notwithstanding the foregoing, with regard to derivative works based on Python + 1.6.1 that incorporate non-separable material that was previously distributed + under the GNU General Public License (GPL), the law of the Commonwealth of + Virginia shall govern this License Agreement only as to issues arising under or + with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in + this License Agreement shall be deemed to create any relationship of agency, + partnership, or joint venture between CNRI and Licensee. This License Agreement + does not grant permission to use CNRI trademarks or trade name in a trademark + sense to endorse or promote products or services of Licensee, or any third + party. - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. +#. By clicking on the "ACCEPT" button where indicated, or by copying, installing + or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and + conditions of this License Agreement. - 7. This License Agreement shall be governed by the federal intellectual property - law of the United States, including without limitation the federal copyright - law, and, to the extent such U.S. federal law does not apply, by the law of the - Commonwealth of Virginia, excluding Virginia's conflict of law provisions. - Notwithstanding the foregoing, with regard to derivative works based on Python - 1.6.1 that incorporate non-separable material that was previously distributed - under the GNU General Public License (GPL), the law of the Commonwealth of - Virginia shall govern this License Agreement only as to issues arising under or - with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in - this License Agreement shall be deemed to create any relationship of agency, - partnership, or joint venture between CNRI and Licensee. This License Agreement - does not grant permission to use CNRI trademarks or trade name in a trademark - sense to endorse or promote products or services of Licensee, or any third - party. - 8. By clicking on the "ACCEPT" button where indicated, or by copying, installing - or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and - conditions of this License Agreement. +.. centered:: ACCEPT -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- +.. centered:: CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 -.. parsed-literal:: +Copyright © 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The +Netherlands. All rights reserved. - Copyright © 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The - Netherlands. All rights reserved. +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of Stichting Mathematisch Centrum or CWI not be used in advertising or +publicity pertaining to distribution of the software without specific, written +prior permission. - Permission to use, copy, modify, and distribute this software and its - documentation for any purpose and without fee is hereby granted, provided that - the above copyright notice appear in all copies and that both that copyright - notice and this permission notice appear in supporting documentation, and that - the name of Stichting Mathematisch Centrum or CWI not be used in advertising or - publicity pertaining to distribution of the software without specific, written - prior permission. - - STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS - SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO - EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT - OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, - DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS - SOFTWARE. +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT +OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. Licenses and Acknowledgements for Incorporated Software @@ -269,7 +325,7 @@ ---------------- The :mod:`_random` module includes code based on a download from -http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html. The following are +http://www.math.keio.ac.jp/ matumoto/MT2002/emt19937ar.html. The following are the verbatim comments from the original code:: A C-program for MT19937, with initialization improved 2002/1/26. @@ -310,8 +366,8 @@ Any feedback is very welcome. - http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html - email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) + http://www.math.keio.ac.jp/matumoto/emt.html + email: matumoto@math.keio.ac.jp Sockets @@ -598,35 +654,6 @@ SUCH DAMAGE. -SipHash24 ---------- - -The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. The contains the following note:: - - - Copyright (c) 2013 Marek Majkowski - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - - Original location: - https://github.com/majek/csiphash/ - - Solution inspired by code from: - Samuel Neves (supercop/crypto_auth/siphash24/little) - djb (supercop/crypto_auth/siphash24/little2) - Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) - - strtod and dtoa --------------- @@ -662,9 +689,9 @@ The modules :mod:`hashlib`, :mod:`posix`, :mod:`ssl`, :mod:`crypt` use the OpenSSL library for added performance if made available by the -operating system. Additionally, the Windows and Mac OS X installers for -Python may include a copy of the OpenSSL libraries, so we include a copy -of the OpenSSL license here:: +operating system. Additionally, the Windows installers for Python +include a copy of the OpenSSL libraries, so we include a copy of the +OpenSSL license here:: LICENSE ISSUES @@ -882,75 +909,3 @@ Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu - -cfuhash -------- - -The implementation of the hash table used by the :mod:`tracemalloc` is based -on the cfuhash project:: - - Copyright (c) 2005 Don Owens - All rights reserved. - - This code is released under the BSD license: - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the author nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - OF THE POSSIBILITY OF SUCH DAMAGE. - - -libmpdec --------- - -The :mod:`_decimal` Module is built using an included copy of the libmpdec -library unless the build is configured ``--with-system-libmpdec``:: - - Copyright (c) 2008-2016 Stefan Krah. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/make.bat --- a/Doc/make.bat Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/make.bat Mon Jan 25 17:05:13 2016 +0100 @@ -1,130 +1,59 @@ -@echo off -setlocal - -pushd %~dp0 - -set this=%~n0 - -if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build -if "%PYTHON%" EQU "" set PYTHON=py - -if "%1" NEQ "htmlhelp" goto :skiphhcsearch -if exist "%HTMLHELP%" goto :skiphhcsearch - -rem Search for HHC in likely places -set HTMLHELP= -where hhc /q && set HTMLHELP=hhc && goto :skiphhcsearch -where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" ( - echo. - echo.The HTML Help Workshop was not found. Set the HTMLHELP variable - echo.to the path to hhc.exe or download and install it from - echo.http://msdn.microsoft.com/en-us/library/ms669985 - exit /B 1 -) -:skiphhcsearch - -if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v - -if "%BUILDDIR%" EQU "" set BUILDDIR=build - -rem Targets that don't require sphinx-build -if "%1" EQU "" goto help -if "%1" EQU "help" goto help -if "%1" EQU "check" goto check -if "%1" EQU "serve" goto serve -if "%1" == "clean" ( - rmdir /q /s %BUILDDIR% - goto end -) - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - popd - exit /B 1 -) - -rem Targets that do require sphinx-build and have their own label -if "%1" EQU "htmlview" goto htmlview - -rem Everything else -goto build - -:help -echo.usage: %this% BUILDER [filename ...] -echo. -echo.Call %this% with the desired Sphinx builder as the first argument, e.g. -echo.``%this% html`` or ``%this% doctest``. Interesting targets that are -echo.always available include: -echo. -echo. Provided by Sphinx: -echo. html, htmlhelp, latex, text -echo. suspicious, linkcheck, changes, doctest -echo. Provided by this script: -echo. clean, check, serve, htmlview -echo. -echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.txt in this directory or -echo.the documentation for your version of Sphinx for more exhaustive lists -echo.of available targets and descriptions of each. -echo. -echo.This script assumes that the SPHINXBUILD environment variable contains -echo.a legitimate command for calling sphinx-build, or that sphinx-build is -echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can -echo.be passed by setting the SPHINXOPTS environment variable. -goto end - -:build -if NOT "%PAPER%" == "" ( - set SPHINXOPTS=-D latex_paper_size=%PAPER% %SPHINXOPTS% -) -cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%* - -if "%1" EQU "htmlhelp" ( - cmd /C "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp - rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 - if not errorlevel 2 cmd /C exit /b 0 -) - -echo. -if errorlevel 1 ( - echo.Build failed (exit code %ERRORLEVEL%^), check for error messages - echo.above. Any output will be found in %BUILDDIR%\%1 -) else ( - echo.Build succeeded. All output should be in %BUILDDIR%\%1 -) -goto end - -:htmlview -if NOT "%2" EQU "" ( - echo.Can't specify filenames to build with htmlview target, ignoring. -) -cmd /C %this% html - -if EXIST %BUILDDIR%\html\index.html ( - echo.Opening %BUILDDIR%\html\index.html in the default web browser... - start %BUILDDIR%\html\index.html -) - -goto end - -:check -cmd /C %PYTHON% tools\rstlint.py -i tools -goto end - -:serve -cmd /C %PYTHON% ..\Tools\scripts\serve.py %BUILDDIR%\html -goto end - -:end -popd +@@echo off +setlocal + +set SVNROOT=http://svn.python.org/projects +if "%PYTHON%" EQU "" set PYTHON=..\pcbuild\python +if "%HTMLHELP%" EQU "" set HTMLHELP=%ProgramFiles%\HTML Help Workshop\hhc.exe +if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/sphinxext/patchlevel.py`) do set DISTVERSION=%%v + +if "%1" EQU "" goto help +if "%1" EQU "html" goto build +if "%1" EQU "htmlhelp" goto build +if "%1" EQU "latex" goto build +if "%1" EQU "text" goto build +if "%1" EQU "suspicious" goto build +if "%1" EQU "linkcheck" goto build +if "%1" EQU "changes" goto build +if "%1" EQU "checkout" goto checkout +if "%1" EQU "update" goto update + +:help +set this=%~n0 +echo HELP +echo. +echo %this% checkout +echo %this% update +echo %this% html +echo %this% htmlhelp +echo %this% latex +echo %this% text +echo %this% suspicious +echo %this% linkcheck +echo %this% changes +echo. +goto end + +:checkout +svn co %SVNROOT%/external/Sphinx-1.0.7/sphinx tools/sphinx +svn co %SVNROOT%/external/docutils-0.6/docutils tools/docutils +svn co %SVNROOT%/external/Jinja-2.3.1/jinja2 tools/jinja2 +svn co %SVNROOT%/external/Pygments-1.3.1/pygments tools/pygments +goto end + +:update +svn update tools/sphinx +svn update tools/docutils +svn update tools/jinja2 +svn update tools/pygments +goto end + +:build +if not exist build mkdir build +if not exist build\%1 mkdir build\%1 +if not exist build\doctrees mkdir build\doctrees +cmd /C %PYTHON% --version +cmd /C %PYTHON% tools\sphinx-build.py -b%1 -dbuild\doctrees . build\%* +if "%1" EQU "htmlhelp" "%HTMLHELP%" build\htmlhelp\python%DISTVERSION:.=%.hhp +goto end + +:end diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/builtdist.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/builtdist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,302 @@ +.. _packaging-built-dist: + +**************************** +Creating Built Distributions +**************************** + +A "built distribution" is what you're probably used to thinking of either as a +"binary package" or an "installer" (depending on your background). It's not +necessarily binary, though, because it might contain only Python source code +and/or byte-code; and we don't call it a package, because that word is already +spoken for in Python. (And "installer" is a term specific to the world of +mainstream desktop systems.) + +A built distribution is how you make life as easy as possible for installers of +your module distribution: for users of RPM-based Linux systems, it's a binary +RPM; for Windows users, it's an executable installer; for Debian-based Linux +users, it's a Debian package; and so forth. Obviously, no one person will be +able to create built distributions for every platform under the sun, so the +Distutils are designed to enable module developers to concentrate on their +specialty---writing code and creating source distributions---while an +intermediary species called *packagers* springs up to turn source distributions +into built distributions for as many platforms as there are packagers. + +Of course, the module developer could be his own packager; or the packager could +be a volunteer "out there" somewhere who has access to a platform which the +original developer does not; or it could be software periodically grabbing new +source distributions and turning them into built distributions for as many +platforms as the software has access to. Regardless of who they are, a packager +uses the setup script and the :command:`bdist` command family to generate built +distributions. + +As a simple example, if I run the following command in the Distutils source +tree:: + + python setup.py bdist + +then the Distutils builds my module distribution (the Distutils itself in this +case), does a "fake" installation (also in the :file:`build` directory), and +creates the default type of built distribution for my platform. The default +format for built distributions is a "dumb" tar file on Unix, and a simple +executable installer on Windows. (That tar file is considered "dumb" because it +has to be unpacked in a specific location to work.) + +Thus, the above command on a Unix system creates +:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place +installs the Distutils just as though you had downloaded the source distribution +and run ``python setup.py install``. (The "right place" is either the root of +the filesystem or Python's :file:`{prefix}` directory, depending on the options +given to the :command:`bdist_dumb` command; the default is to make dumb +distributions relative to :file:`{prefix}`.) + +Obviously, for pure Python distributions, this isn't any simpler than just +running ``python setup.py install``\ ---but for non-pure distributions, which +include extensions that would need to be compiled, it can mean the difference +between someone being able to use your extensions or not. And creating "smart" +built distributions, such as an executable installer for +Windows, is far more convenient for users even if your distribution doesn't +include any extensions. + +The :command:`bdist` command has a :option:`--formats` option, similar to the +:command:`sdist` command, which you can use to select the types of built +distribution to generate: for example, :: + + python setup.py bdist --format=zip + +would, when run on a Unix system, create :file:`Distutils-1.0.{plat}.zip`\ +---again, this archive would be unpacked from the root directory to install the +Distutils. + +The available formats for built distributions are: + ++-------------+------------------------------+---------+ +| Format | Description | Notes | ++=============+==============================+=========+ +| ``gztar`` | gzipped tar file | (1),(3) | +| | (:file:`.tar.gz`) | | ++-------------+------------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | \(3) | ++-------------+------------------------------+---------+ +| ``zip`` | zip file (:file:`.zip`) | (2),(4) | ++-------------+------------------------------+---------+ +| ``wininst`` | self-extracting ZIP file for | \(4) | +| | Windows | | ++-------------+------------------------------+---------+ +| ``msi`` | Microsoft Installer. | | ++-------------+------------------------------+---------+ + + +Notes: + +(1) + default on Unix + +(2) + default on Windows + +(3) + requires external utilities: :program:`tar` and possibly one of :program:`gzip` + or :program:`bzip2` + +(4) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +You don't have to use the :command:`bdist` command with the :option:`--formats` +option; you can also use the command that directly implements the format you're +interested in. Some of these :command:`bdist` "sub-commands" actually generate +several similar formats; for instance, the :command:`bdist_dumb` command +generates all the "dumb" archive formats (``tar``, ``gztar``, and +``zip``). The :command:`bdist` sub-commands, and the formats generated by +each, are: + ++--------------------------+-----------------------+ +| Command | Formats | ++==========================+=======================+ +| :command:`bdist_dumb` | tar, gztar, zip | ++--------------------------+-----------------------+ +| :command:`bdist_wininst` | wininst | ++--------------------------+-----------------------+ +| :command:`bdist_msi` | msi | ++--------------------------+-----------------------+ + +The following sections give details on the individual :command:`bdist_\*` +commands. + + +.. _packaging-creating-dumb: + +Creating dumb built distributions +================================= + +.. XXX Need to document absolute vs. prefix-relative packages here, but first + I have to implement it! + + +.. _packaging-creating-wininst: + +Creating Windows Installers +=========================== + +Executable installers are the natural format for binary distributions on +Windows. They display a nice graphical user interface, display some information +about the module distribution to be installed taken from the metadata in the +setup script, let the user select a few options, and start or cancel the +installation. + +Since the metadata is taken from the setup script, creating Windows installers +is usually as easy as running:: + + python setup.py bdist_wininst + +or the :command:`bdist` command with the :option:`--formats` option:: + + python setup.py bdist --formats=wininst + +If you have a pure module distribution (only containing pure Python modules and +packages), the resulting installer will be version independent and have a name +like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix +platforms or Mac OS X. + +If you have a non-pure distribution, the extensions can only be created on a +Windows platform, and will be Python version dependent. The installer filename +will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You +have to create a separate installer for every Python version you want to +support. + +The installer will try to compile pure modules into :term:`bytecode` after installation +on the target system in normal and optimizing mode. If you don't want this to +happen for some reason, you can run the :command:`bdist_wininst` command with +the :option:`--no-target-compile` and/or the :option:`--no-target-optimize` +option. + +By default the installer will display the cool "Python Powered" logo when it is +run, but you can also supply your own 152x261 bitmap which must be a Windows +:file:`.bmp` file with the :option:`--bitmap` option. + +The installer will also display a large title on the desktop background window +when it is run, which is constructed from the name of your distribution and the +version number. This can be changed to another text by using the +:option:`--title` option. + +The installer file will be written to the "distribution directory" --- normally +:file:`dist/`, but customizable with the :option:`--dist-dir` option. + +.. _packaging-cross-compile-windows: + +Cross-compiling on Windows +========================== + +Starting with Python 2.6, packaging is capable of cross-compiling between +Windows platforms. In practice, this means that with the correct tools +installed, you can use a 32bit version of Windows to create 64bit extensions +and vice-versa. + +To build for an alternate platform, specify the :option:`--plat-name` option +to the build command. Valid values are currently 'win32', 'win-amd64' and +'win-ia64'. For example, on a 32bit version of Windows, you could execute:: + + python setup.py build --plat-name=win-amd64 + +to build a 64bit version of your extension. The Windows Installers also +support this option, so the command:: + + python setup.py build --plat-name=win-amd64 bdist_wininst + +would create a 64bit installation executable on your 32bit version of Windows. + +To cross-compile, you must download the Python source code and cross-compile +Python itself for the platform you are targetting - it is not possible from a +binary installtion of Python (as the .lib etc file for other platforms are +not included.) In practice, this means the user of a 32 bit operating +system will need to use Visual Studio 2008 to open the +:file:`PCBuild/PCbuild.sln` solution in the Python source tree and build the +"x64" configuration of the 'pythoncore' project before cross-compiling +extensions is possible. + +Note that by default, Visual Studio 2008 does not install 64bit compilers or +tools. You may need to reexecute the Visual Studio setup process and select +these tools (using Control Panel->[Add/Remove] Programs is a convenient way to +check or modify your existing install.) + +.. _packaging-postinstallation-script: + +The Postinstallation script +--------------------------- + +Starting with Python 2.3, a postinstallation script can be specified with the +:option:`--install-script` option. The basename of the script must be +specified, and the script filename must also be listed in the scripts argument +to the setup function. + +This script will be run at installation time on the target system after all the +files have been copied, with ``argv[1]`` set to :option:`-install`, and again at +uninstallation time before the files are removed with ``argv[1]`` set to +:option:`-remove`. + +The installation script runs embedded in the windows installer, every output +(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be +displayed in the GUI after the script has finished. + +Some functions especially useful in this context are available as additional +built-in functions in the installation script. + +.. currentmodule:: bdist_wininst-postinst-script + +.. function:: directory_created(path) + file_created(path) + + These functions should be called when a directory or file is created by the + postinstall script at installation time. It will register *path* with the + uninstaller, so that it will be removed when the distribution is uninstalled. + To be safe, directories are only removed if they are empty. + + +.. function:: get_special_folder_path(csidl_string) + + This function can be used to retrieve special folder locations on Windows like + the Start Menu or the Desktop. It returns the full path to the folder. + *csidl_string* must be one of the following strings:: + + "CSIDL_APPDATA" + + "CSIDL_COMMON_STARTMENU" + "CSIDL_STARTMENU" + + "CSIDL_COMMON_DESKTOPDIRECTORY" + "CSIDL_DESKTOPDIRECTORY" + + "CSIDL_COMMON_STARTUP" + "CSIDL_STARTUP" + + "CSIDL_COMMON_PROGRAMS" + "CSIDL_PROGRAMS" + + "CSIDL_FONTS" + + If the folder cannot be retrieved, :exc:`OSError` is raised. + + Which folders are available depends on the exact Windows version, and probably + also the configuration. For details refer to Microsoft's documentation of the + :c:func:`SHGetSpecialFolderPath` function. + + +.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]]) + + This function creates a shortcut. *target* is the path to the program to be + started by the shortcut. *description* is the description of the shortcut. + *filename* is the title of the shortcut that the user will see. *arguments* + specifies the command-line arguments, if any. *workdir* is the working directory + for the program. *iconpath* is the file containing the icon for the shortcut, + and *iconindex* is the index of the icon in the file *iconpath*. Again, for + details consult the Microsoft documentation for the :class:`IShellLink` + interface. + + +Vista User Access Control (UAC) +=============================== + +Starting with Python 2.6, bdist_wininst supports a :option:`--user-access-control` +option. The default is 'none' (meaning no UAC handling is done), and other +valid values are 'auto' (meaning prompt for UAC elevation if Python was +installed for all users) and 'force' (meaning always prompt for elevation). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/commandhooks.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/commandhooks.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,47 @@ +.. TODO integrate this in commandref and configfile + +.. _packaging-command-hooks: + +============= +Command hooks +============= + +Packaging provides a way of extending its commands by the use of pre- and +post-command hooks. Hooks are Python functions (or any callable object) that +take a command object as argument. They're specified in :ref:`config files +` using their fully qualified names. After a +command is finalized (its options are processed), the pre-command hooks are +executed, then the command itself is run, and finally the post-command hooks are +executed. + +See also global setup hooks in :ref:`setupcfg-spec`. + + +.. _packaging-finding-hooks: + +Finding hooks +============= + +As a hook is configured with a Python dotted name, it must either be defined in +a module installed on the system, or in a module present in the project +directory, where the :file:`setup.cfg` file lives:: + + # file: _setuphooks.py + + def hook(install_cmd): + metadata = install_cmd.dist.metadata + print('Hooked while installing %r %s!' % (metadata['Name'], + metadata['Version'])) + +Then you need to configure it in :file:`setup.cfg`:: + + [install_dist] + pre-hook.a = _setuphooks.hook + +Packaging will add the project directory to :data:`sys.path` and find the +``_setuphooks`` module. + +Hooks defined in different config files (system-wide, user-wide and +project-wide) do not override each other as long as they are specified with +different aliases (additional names after the dot). The alias in the example +above is ``a``. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/commandref.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/commandref.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,374 @@ +.. _packaging-command-reference: + +***************** +Command Reference +***************** + +This reference briefly documents all standard Packaging commands and some of +their options. + +.. FIXME does not work: Use pysetup run --help-commands to list all + standard and extra commands availavble on your system, with their + description. Use pysetup run --help to get help about the options + of one command. + +.. XXX sections from this document should be merged with other docs (e.g. check + and upload with uploading.rst, install_* with install/install.rst, etc.); + there is no value in partially duplicating information. this file could + however serve as an index, i.e. just a list of all commands with links to + every section that describes options or usage + + +Preparing distributions +======================= + +:command:`check` +---------------- + +Perform some tests on the metadata of a distribution. + +For example, it verifies that all required metadata fields are provided in the +:file:`setup.cfg` file. + +.. TODO document reST checks + + +:command:`test` +--------------- + +Run a test suite. + +When doing test-driven development, or running automated builds that need +testing before they are installed for downloading or use, it's often useful to +be able to run a project's unit tests without actually installing the project +anywhere. The :command:`test` command runs project's unit tests without +actually installing it, by temporarily putting the project's source on +:data:`sys.path`, after first running :command:`build_ext -i` to ensure that any +C extensions are built. + +You can use this command in one of two ways: either by specifying a +unittest-compatible test suite for your project (or any callable that returns +it) or by passing a test runner function that will run your tests and display +results in the console. Both options take a Python dotted name in the form +``package.module.callable`` to specify the object to use. + +If none of these options are specified, Packaging will try to perform test +discovery using either unittest (for Python 3.2 and higher) or unittest2 (for +older versions, if installed). + +.. this is a pseudo-command name used to disambiguate the options in indexes and + links +.. program:: packaging test + +.. cmdoption:: --suite=NAME, -s NAME + + Specify the test suite (or module, class, or method) to be run. The default + for this option can be set by in the project's :file:`setup.cfg` file: + + .. code-block:: cfg + + [test] + suite = mypackage.tests.get_all_tests + +.. cmdoption:: --runner=NAME, -r NAME + + Specify the test runner to be called. + + +:command:`config` +----------------- + +Perform distribution configuration. + + +The build step +============== + +This step is mainly useful to compile C/C++ libraries or extension modules. The +build commands can be run manually to check for syntax errors or packaging +issues (for example if the addition of a new source file was forgotten in the +:file:`setup.cfg` file), and is also run automatically by commands which need +it. Packaging checks the mtime of source and built files to avoid re-building +if it's not necessary. + + +:command:`build` +---------------- + +Build all files of a distribution, delegating to the other :command:`build_*` +commands to do the work. + + +:command:`build_clib` +--------------------- + +Build C libraries. + + +:command:`build_ext` +-------------------- + +Build C/C++ extension modules. + + +:command:`build_py` +------------------- + +Build the Python modules (just copy them to the build directory) and +:term:`byte-compile ` them to :file:`.pyc` and/or :file:`.pyo` files. + +The byte compilation is controlled by two sets of options: + +- ``--compile`` and ``--no-compile`` are used to control the creation of + :file:`.pyc` files; the default is ``--no-compile``. + +- ``--optimize N`` (or ``-ON``) is used to control the creation of :file:`.pyo` + files: ``-O1`` turns on basic optimizations, ``-O2`` also discards docstrings, + ``-O0`` does not create :file:`.pyo` files; the default is ``-O0``. + +You can mix and match these options: for example, ``--no-compile --optimize 2`` +will create :file:`.pyo` files but no :file:`.pyc` files. + +.. XXX these option roles do not work + +Calling Python with :option:`-O` or :option:`-B` does not control the creation +of bytecode files, only the options described above do. + + +:command:`build_scripts` +------------------------ +Build the scripts (just copy them to the build directory and adjust their +shebang if they're Python scripts). + + +:command:`clean` +---------------- + +Clean the build tree of the release. + +.. program:: packaging clean + +.. cmdoption:: --all, -a + + Remove build directories for modules, scripts, etc., not only temporary build + by-products. + + +Creating source and built distributions +======================================= + +:command:`sdist` +---------------- + +Build a source distribution for a release. + +It is recommended that you always build and upload a source distribution. Users +of OSes with easy access to compilers and users of advanced packaging tools will +prefer to compile from source rather than using pre-built distributions. For +Windows users, providing a binary installer is also recommended practice. + + +:command:`bdist` +---------------- + +Build a binary distribution for a release. + +This command will call other :command:`bdist_*` commands to create one or more +distributions depending on the options given. The default is to create a +.tar.gz archive on Unix and a zip archive on Windows or OS/2. + +.. program:: packaging bdist + +.. cmdoption:: --formats + + Binary formats to build (comma-separated list). + +.. cmdoption:: --show-formats + + Dump list of available formats. + + +:command:`bdist_dumb` +--------------------- + +Build a "dumb" installer, a simple archive of files that could be unpacked under +``$prefix`` or ``$exec_prefix``. + + +:command:`bdist_wininst` +------------------------ + +Build a Windows installer. + + +:command:`bdist_msi` +-------------------- + +Build a `Microsoft Installer`_ (.msi) file. + +.. _Microsoft Installer: http://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx + +In most cases, the :command:`bdist_msi` installer is a better choice than the +:command:`bdist_wininst` installer, because it provides better support for Win64 +platforms, allows administrators to perform non-interactive installations, and +allows installation through group policies. + + +Publishing distributions +======================== + +:command:`register` +------------------- + +This command registers the current release with the Python Package Index. This +is described in more detail in :PEP:`301`. + +.. TODO explain user and project registration with the web UI + + +:command:`upload` +----------------- + +Upload source and/or binary distributions to PyPI. + +The distributions have to be built on the same command line as the +:command:`upload` command; see :ref:`packaging-package-upload` for more info. + +.. program:: packaging upload + +.. cmdoption:: --sign, -s + + Sign each uploaded file using GPG (GNU Privacy Guard). The ``gpg`` program + must be available for execution on the system ``PATH``. + +.. cmdoption:: --identity=NAME, -i NAME + + Specify the identity or key name for GPG to use when signing. The value of + this option will be passed through the ``--local-user`` option of the + ``gpg`` program. + +.. cmdoption:: --show-response + + Display the full response text from server; this is useful for debugging + PyPI problems. + +.. cmdoption:: --repository=URL, -r URL + + The URL of the repository to upload to. Defaults to + http://pypi.python.org/pypi (i.e., the main PyPI installation). + +.. cmdoption:: --upload-docs + + Also run :command:`upload_docs`. Mainly useful as a default value in + :file:`setup.cfg` (on the command line, it's shorter to just type both + commands). + + +:command:`upload_docs` +---------------------- + +Upload HTML documentation to PyPI. + +PyPI now supports publishing project documentation at a URI of the form +``http://packages.python.org/``. :command:`upload_docs` will create +the necessary zip file out of a documentation directory and will post to the +repository. + +Note that to upload the documentation of a project, the corresponding version +must already be registered with PyPI, using the :command:`register` command --- +just like with :command:`upload`. + +Assuming there is an ``Example`` project with documentation in the subdirectory +:file:`docs`, for example:: + + Example/ + example.py + setup.cfg + docs/ + build/ + html/ + index.html + tips_tricks.html + conf.py + index.txt + tips_tricks.txt + +You can simply specify the directory with the HTML files in your +:file:`setup.cfg` file: + +.. code-block:: cfg + + [upload_docs] + upload-dir = docs/build/html + + +.. program:: packaging upload_docs + +.. cmdoption:: --upload-dir + + The directory to be uploaded to the repository. By default documentation + is searched for in ``docs`` (or ``doc``) directory in project root. + +.. cmdoption:: --show-response + + Display the full response text from server; this is useful for debugging + PyPI problems. + +.. cmdoption:: --repository=URL, -r URL + + The URL of the repository to upload to. Defaults to + http://pypi.python.org/pypi (i.e., the main PyPI installation). + + +The install step +================ + +These commands are used by end-users of a project using :program:`pysetup` or +another compatible installer. Each command will run the corresponding +:command:`build_*` command and then move the built files to their destination on +the target system. + + +:command:`install_dist` +----------------------- + +Install a distribution, delegating to the other :command:`install_*` commands to +do the work. See :ref:`packaging-how-install-works` for complete usage +instructions. + + +:command:`install_data` +----------------------- + +Install data files. + + +:command:`install_distinfo` +--------------------------- + +Install files recording details of the installation as specified in :PEP:`376`. + + +:command:`install_headers` +-------------------------- + +Install C/C++ header files. + + +:command:`install_lib` +---------------------- + +Install all modules (extensions and pure Python). + +.. XXX what about C libraries created with build_clib? + +Similarly to ``build_py``, there are options to control the compilation of +Python code to :term:`bytecode` files (see above). By default, :file:`.pyc` +files will be created (``--compile``) and :file:`.pyo` files will not +(``--optimize 0``). + + +:command:`install_scripts` +-------------------------- + +Install scripts. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/configfile.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/configfile.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,125 @@ +.. _packaging-setup-config: + +************************************ +Writing the Setup Configuration File +************************************ + +Often, it's not possible to write down everything needed to build a distribution +*a priori*: you may need to get some information from the user, or from the +user's system, in order to proceed. As long as that information is fairly +simple---a list of directories to search for C header files or libraries, for +example---then providing a configuration file, :file:`setup.cfg`, for users to +edit is a cheap and easy way to solicit it. Configuration files also let you +provide default values for any command option, which the installer can then +override either on the command line or by editing the config file. + +The setup configuration file is a useful middle-ground between the setup script +---which, ideally, would be opaque to installers [#]_---and the command line to +the setup script, which is outside of your control and entirely up to the +installer. In fact, :file:`setup.cfg` (and any other Distutils configuration +files present on the target system) are processed after the contents of the +setup script, but before the command line. This has several useful +consequences: + +.. If you have more advanced needs, such as determining which extensions to + build based on what capabilities are present on the target system, then you + need the Distutils auto-configuration facility. This started to appear in + Distutils 0.9 but, as of this writing, isn't mature or stable enough yet + for real-world use. + +* installers can override some of what you put in :file:`setup.py` by editing + :file:`setup.cfg` + +* you can provide non-standard defaults for options that are not easily set in + :file:`setup.py` + +* installers can override anything in :file:`setup.cfg` using the command-line + options to :file:`setup.py` + +The basic syntax of the configuration file is simple:: + + [command] + option = value + ... + +where *command* is one of the Distutils commands (e.g. :command:`build_py`, +:command:`install_dist`), and *option* is one of the options that command supports. +Any number of options can be supplied for each command, and any number of +command sections can be included in the file. Blank lines are ignored, as are +comments, which run from a ``'#'`` character until the end of the line. Long +option values can be split across multiple lines simply by indenting the +continuation lines. + +You can find out the list of options supported by a particular command with the +universal :option:`--help` option, e.g. :: + + > python setup.py --help build_ext + [...] + Options for 'build_ext' command: + --build-lib (-b) directory for compiled extension modules + --build-temp (-t) directory for temporary files (build by-products) + --inplace (-i) ignore build-lib and put compiled extensions into the + source directory alongside your pure Python modules + --include-dirs (-I) list of directories to search for header files + --define (-D) C preprocessor macros to define + --undef (-U) C preprocessor macros to undefine + --swig-opts list of SWIG command-line options + [...] + +.. XXX do we want to support ``setup.py --help metadata``? + +Note that an option spelled :option:`--foo-bar` on the command line is spelled +:option:`foo_bar` in configuration files. + +For example, say you want your extensions to be built "in-place"---that is, you +have an extension :mod:`pkg.ext`, and you want the compiled extension file +(:file:`ext.so` on Unix, say) to be put in the same source directory as your +pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the +:option:`--inplace` option on the command line to ensure this:: + + python setup.py build_ext --inplace + +But this requires that you always specify the :command:`build_ext` command +explicitly, and remember to provide :option:`--inplace`. An easier way is to +"set and forget" this option, by encoding it in :file:`setup.cfg`, the +configuration file for this distribution:: + + [build_ext] + inplace = 1 + +This will affect all builds of this module distribution, whether or not you +explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in +your source distribution, it will also affect end-user builds---which is +probably a bad idea for this option, since always building extensions in-place +would break installation of the module distribution. In certain peculiar cases, +though, modules are built right in their installation directory, so this is +conceivably a useful ability. (Distributing extensions that expect to be built +in their installation directory is almost always a bad idea, though.) + +Another example: certain commands take options that vary from project to +project but not depending on the installation system, for example, +:command:`test` needs to know where your test suite is located and what test +runner to use; likewise, :command:`upload_docs` can find HTML documentation in +a :file:`doc` or :file:`docs` directory, but needs an option to find files in +:file:`docs/build/html`. Instead of having to type out these options each +time you want to run the command, you can put them in the project's +:file:`setup.cfg`:: + + [test] + suite = packaging.tests + + [upload_docs] + upload-dir = docs/build/html + + +.. seealso:: + + :ref:`packaging-config-syntax` in "Installing Python Projects" + More information on the configuration files is available in the manual for + system administrators. + + +.. rubric:: Footnotes + +.. [#] This ideal probably won't be achieved until auto-configuration is fully + supported by the Distutils. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/examples.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/examples.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,334 @@ +.. _packaging-examples: + +******** +Examples +******** + +This chapter provides a number of basic examples to help get started with +Packaging. + + +.. _packaging-pure-mod: + +Pure Python distribution (by module) +==================================== + +If you're just distributing a couple of modules, especially if they don't live +in a particular package, you can specify them individually using the +:option:`py_modules` option in the setup script. + +In the simplest case, you'll have two files to worry about: a setup script and +the single module you're distributing, :file:`foo.py` in this example:: + + / + setup.py + foo.py + +(In all diagrams in this section, ** will refer to the distribution root +directory.) A minimal setup script to describe this situation would be:: + + from packaging.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo']) + +Note that the name of the distribution is specified independently with the +:option:`name` option, and there's no rule that says it has to be the same as +the name of the sole module in the distribution (although that's probably a good +convention to follow). However, the distribution name is used to generate +filenames, so you should stick to letters, digits, underscores, and hyphens. + +Since :option:`py_modules` is a list, you can of course specify multiple +modules, e.g. if you're distributing modules :mod:`foo` and :mod:`bar`, your +setup might look like this:: + + / + setup.py + foo.py + bar.py + +and the setup script might be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + py_modules=['foo', 'bar']) + +You can put module source files into another directory, but if you have enough +modules to do that, it's probably easier to specify modules by package rather +than listing them individually. + + +.. _packaging-pure-pkg: + +Pure Python distribution (by package) +===================================== + +If you have more than a couple of modules to distribute, especially if they are +in multiple packages, it's probably easier to specify whole packages rather than +individual modules. This works even if your modules are not in a package; you +can just tell the Distutils to process modules from the root package, and that +works the same as any other package (except that you don't have to have an +:file:`__init__.py` file). + +The setup script from the last example could also be written as :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['']) + +(The empty string stands for the root package.) + +If those two files are moved into a subdirectory, but remain in the root +package, e.g.:: + + / + setup.py + src/ + foo.py + bar.py + +then you would still specify the root package, but you have to tell the +Distutils where source files in the root package live:: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'': 'src'}, + packages=['']) + +More typically, though, you will want to distribute multiple modules in the same +package (or in sub-packages). For example, if the :mod:`foo` and :mod:`bar` +modules belong in package :mod:`foobar`, one way to lay out your source tree is + +:: + + / + setup.py + foobar/ + __init__.py + foo.py + bar.py + +This is in fact the default layout expected by the Distutils, and the one that +requires the least work to describe in your setup script:: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar']) + +If you want to put modules in directories not named for their package, then you +need to use the :option:`package_dir` option again. For example, if the +:file:`src` directory holds modules in the :mod:`foobar` package:: + + / + setup.py + src/ + __init__.py + foo.py + bar.py + +an appropriate setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': 'src'}, + packages=['foobar']) + +Or, you might put modules from your main package right in the distribution +root:: + + / + setup.py + __init__.py + foo.py + bar.py + +in which case your setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': ''}, + packages=['foobar']) + +(The empty string also stands for the current directory.) + +If you have sub-packages, they must be explicitly listed in :option:`packages`, +but any entries in :option:`package_dir` automatically extend to sub-packages. +(In other words, the Distutils does *not* scan your source tree, trying to +figure out which directories correspond to Python packages by looking for +:file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: + + / + setup.py + foobar/ + __init__.py + foo.py + bar.py + subfoo/ + __init__.py + blah.py + +then the corresponding setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar', 'foobar.subfoo']) + +(Again, the empty string in :option:`package_dir` stands for the current +directory.) + + +.. _packaging-single-ext: + +Single extension module +======================= + +Extension modules are specified using the :option:`ext_modules` option. +:option:`package_dir` has no effect on where extension source files are found; +it only affects the source for pure Python modules. The simplest case, a +single extension module in a single C source file, is:: + + / + setup.py + foo.c + +If the :mod:`foo` extension belongs in the root package, the setup script for +this could be :: + + from packaging.core import setup, Extension + setup(name='foobar', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])]) + +If the extension actually belongs in a package, say :mod:`foopkg`, then + +With exactly the same source tree layout, this extension can be put in the +:mod:`foopkg` package simply by changing the name of the extension:: + + from packaging.core import setup, Extension + setup(name='foobar', + version='1.0', + packages=['foopkg'], + ext_modules=[Extension('foopkg.foo', ['foo.c'])]) + + +Checking metadata +================= + +The ``check`` command allows you to verify if your project's metadata +meets the minimum requirements to build a distribution. + +To run it, just call it using your :file:`setup.py` script. If something is +missing, ``check`` will display a warning. + +Let's take an example with a simple script:: + + from packaging.core import setup + + setup(name='foobar') + +.. TODO configure logging StreamHandler to match this output + +Running the ``check`` command will display some warnings:: + + $ python setup.py check + running check + warning: check: missing required metadata: version, home_page + warning: check: missing metadata: either (author and author_email) or + (maintainer and maintainer_email) must be supplied + + +If you use the reStructuredText syntax in the ``long_description`` field and +`Docutils `_ is installed you can check if +the syntax is fine with the ``check`` command, using the ``restructuredtext`` +option. + +For example, if the :file:`setup.py` script is changed like this:: + + from packaging.core import setup + + desc = """\ + Welcome to foobar! + =============== + + This is the description of the ``foobar`` project. + """ + + setup(name='foobar', + version='1.0', + author=u'Tarek Ziadé', + author_email='tarek@ziade.org', + summary='Foobar utilities' + description=desc, + home_page='http://example.com') + +Where the long description is broken, ``check`` will be able to detect it +by using the :mod:`docutils` parser:: + + $ python setup.py check --restructuredtext + running check + warning: check: Title underline too short. (line 2) + warning: check: Could not finish the parsing. + + +.. _packaging-reading-metadata: + +Reading the metadata +==================== + +The :func:`packaging.core.setup` function provides a command-line interface +that allows you to query the metadata fields of a project through the +:file:`setup.py` script of a given project:: + + $ python setup.py --name + foobar + +This call reads the ``name`` metadata by running the +:func:`packaging.core.setup` function. When a source or binary +distribution is created with Distutils, the metadata fields are written +in a static file called :file:`PKG-INFO`. When a Distutils-based project is +installed in Python, the :file:`PKG-INFO` file is copied alongside the modules +and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, +where ``NAME`` is the name of the project, ``VERSION`` its version as defined +in the Metadata, and ``pyX.X`` the major and minor version of Python like +``2.7`` or ``3.2``. + +You can read back this static file, by using the +:class:`packaging.dist.Metadata` class and its +:func:`read_pkg_file` method:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata() + >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) + >>> metadata.name + 'distribute' + >>> metadata.version + '0.6.8' + >>> metadata.description + 'Easily download, build, install, upgrade, and uninstall Python packages' + +Notice that the class can also be instantiated with a metadata file path to +loads its values:: + + >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' + >>> Metadata(pkg_info_path).name + 'distribute' + + +.. XXX These comments have been here for at least ten years. Write the + sections or delete the comments (we can maybe ask Greg Ward about + the planned contents). (Unindent to make them section titles) + + .. multiple-ext:: + + Multiple extension modules + ========================== + + Putting it all together + ======================= diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/extending.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/extending.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,95 @@ +.. _extending-packaging: + +******************* +Extending Distutils +******************* + +Distutils can be extended in various ways. Most extensions take the form of new +commands or replacements for existing commands. New commands may be written to +support new types of platform-specific packaging, for example, while +replacements for existing commands may be made to modify details of how the +command operates on a package. + +Most extensions of the packaging are made within :file:`setup.py` scripts that +want to modify existing commands; many simply add a few file extensions that +should be copied into packages in addition to :file:`.py` files as a +convenience. + +Most packaging command implementations are subclasses of the +:class:`packaging.cmd.Command` class. New commands may directly inherit from +:class:`Command`, while replacements often derive from :class:`Command` +indirectly, directly subclassing the command they are replacing. Commands are +required to derive from :class:`Command`. + +.. .. _extend-existing: + Extending existing commands + =========================== + + +.. .. _new-commands: + Writing new commands + ==================== + + +Integrating new commands +======================== + +There are different ways to integrate new command implementations into +packaging. The most difficult is to lobby for the inclusion of the new features +in packaging itself, and wait for (and require) a version of Python that +provides that support. This is really hard for many reasons. + +The most common, and possibly the most reasonable for most needs, is to include +the new implementations with your :file:`setup.py` script, and cause the +:func:`packaging.core.setup` function use them:: + + from packaging.core import setup + from packaging.command.build_py import build_py as _build_py + + class build_py(_build_py): + """Specialized Python source builder.""" + + # implement whatever needs to be different... + + setup(..., cmdclass={'build_py': build_py}) + +This approach is most valuable if the new implementations must be used to use a +particular package, as everyone interested in the package will need to have the +new command implementation. + +Beginning with Python 2.4, a third option is available, intended to allow new +commands to be added which can support existing :file:`setup.py` scripts without +requiring modifications to the Python installation. This is expected to allow +third-party extensions to provide support for additional packaging systems, but +the commands can be used for anything packaging commands can be used for. A new +configuration option, :option:`command_packages` (command-line option +:option:`--command-packages`), can be used to specify additional packages to be +searched for modules implementing commands. Like all packaging options, this +can be specified on the command line or in a configuration file. This option +can only be set in the ``[global]`` section of a configuration file, or before +any commands on the command line. If set in a configuration file, it can be +overridden from the command line; setting it to an empty string on the command +line causes the default to be used. This should never be set in a configuration +file provided with a package. + +This new option can be used to add any number of packages to the list of +packages searched for command implementations; multiple package names should be +separated by commas. When not specified, the search is only performed in the +:mod:`packaging.command` package. When :file:`setup.py` is run with the option +:option:`--command-packages` :option:`distcmds,buildcmds`, however, the packages +:mod:`packaging.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched +in that order. New commands are expected to be implemented in modules of the +same name as the command by classes sharing the same name. Given the example +command-line option above, the command :command:`bdist_openpkg` could be +implemented by the class :class:`distcmds.bdist_openpkg.bdist_openpkg` or +:class:`buildcmds.bdist_openpkg.bdist_openpkg`. + + +Adding new distribution types +============================= + +Commands that create distributions (files in the :file:`dist/` directory) need +to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that +:command:`upload` can upload it to PyPI. The *filename* in the pair contains no +path information, only the name of the file itself. In dry-run mode, pairs +should still be added to represent what would have been created. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/index.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,45 @@ +.. _packaging-index: + +############################## + Distributing Python Projects +############################## + +:Authors: The Fellowship of the Packaging +:Email: distutils-sig@python.org +:Release: |version| +:Date: |today| + +This document describes Packaging for Python authors, describing how to use the +module to make Python applications, packages or modules easily available to a +wider audience with very little overhead for build/release/install mechanics. + +.. toctree:: + :maxdepth: 2 + :numbered: + + tutorial + setupcfg + introduction + setupscript + configfile + sourcedist + builtdist + packageindex + uploading + examples + extending + commandhooks + commandref + + +.. seealso:: + + :ref:`packaging-install-index` + A user-centered manual which includes information on adding projects + into an existing Python installation. You do not need to be a Python + programmer to read this manual. + + :mod:`packaging` + A library reference for developers of packaging tools wanting to use + standalone building blocks like :mod:`~packaging.version` or + :mod:`~packaging.metadata`, or extend Packaging itself. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/introduction.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/introduction.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,193 @@ +.. _packaging-intro: + +***************************** +An Introduction to Packaging +***************************** + +This document covers using Packaging to distribute your Python modules, +concentrating on the role of developer/distributor. If you're looking for +information on installing Python modules you should refer to the +:ref:`packaging-install-index` chapter. + +Throughout this documentation, the terms "Distutils", "the Distutils" and +"Packaging" will be used interchangeably. + +.. _packaging-concepts: + +Concepts & Terminology +====================== + +Using Distutils is quite simple both for module developers and for +users/administrators installing third-party modules. As a developer, your +responsibilities (apart from writing solid, well-documented and well-tested +code, of course!) are: + +* writing a setup script (:file:`setup.py` by convention) + +* (optional) writing a setup configuration file + +* creating a source distribution + +* (optional) creating one or more "built" (binary) distributions of your + project + +All of these tasks are covered in this document. + +Not all module developers have access to multiple platforms, so one cannot +expect them to create buildt distributions for every platform. To remedy +this, it is hoped that intermediaries called *packagers* will arise to address +this need. Packagers take source distributions released by module developers, +build them on one or more platforms and release the resulting built +distributions. Thus, users on a greater range of platforms will be able to +install the most popular Python modules in the most natural way for their +platform without having to run a setup script or compile a single line of code. + + +.. _packaging-simple-example: + +A Simple Example +================ + +A setup script is usually quite simple, although since it's written in Python +there are no arbitrary limits to what you can do with it, though you should be +careful about putting expensive operations in your setup script. +Unlike, say, Autoconf-style configure scripts the setup script may be run +multiple times in the course of building and installing a module +distribution. + +If all you want to do is distribute a module called :mod:`foo`, contained in a +file :file:`foo.py`, then your setup script can be as simple as:: + + from packaging.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo']) + +Some observations: + +* most information that you supply to the Distutils is supplied as keyword + arguments to the :func:`setup` function + +* those keyword arguments fall into two categories: package metadata (name, + version number, etc.) and information about what's in the package (a list + of pure Python modules in this case) + +* modules are specified by module name, not filename (the same will hold true + for packages and extensions) + +* it's recommended that you supply a little more metadata than we have in the + example. In particular your name, email address and a URL for the + project if appropriate (see section :ref:`packaging-setup-script` for an example) + +To create a source distribution for this module you would create a setup +script, :file:`setup.py`, containing the above code and run:: + + python setup.py sdist + +which will create an archive file (e.g., tarball on Unix, ZIP file on Windows) +containing your setup script :file:`setup.py`, and your module :file:`foo.py`. +The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and +will unpack into a directory :file:`foo-1.0`. + +If an end-user wishes to install your :mod:`foo` module all he has to do is +download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and from the +:file:`foo-1.0` directory run :: + + python setup.py install + +which will copy :file:`foo.py` to the appropriate directory for +third-party modules in their Python installation. + +This simple example demonstrates some fundamental concepts of Distutils. +First, both developers and installers have the same basic user interface, i.e. +the setup script. The difference is which Distutils *commands* they use: the +:command:`sdist` command is almost exclusively for module developers, while +:command:`install` is more often used by installers (although some developers +will want to install their own code occasionally). + +If you want to make things really easy for your users, you can create more +than one built distributions for them. For instance, if you are running on a +Windows machine and want to make things easy for other Windows users, you can +create an executable installer (the most appropriate type of built distribution +for this platform) with the :command:`bdist_wininst` command. For example:: + + python setup.py bdist_wininst + +will create an executable installer, :file:`foo-1.0.win32.exe`, in the current +directory. You can find out what distribution formats are available at any time +by running :: + + python setup.py bdist --help-formats + + +.. _packaging-python-terms: + +General Python terminology +========================== + +If you're reading this document, you probably have a good idea of what Python +modules, extensions and so forth are. Nevertheless, just to be sure that +everyone is on the same page, here's a quick overview of Python terms: + +module + The basic unit of code reusability in Python: a block of code imported by + some other code. Three types of modules are important to us here: pure + Python modules, extension modules and packages. + +pure Python module + A module written in Python and contained in a single :file:`.py` file (and + possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes + referred to as a "pure module." + +extension module + A module written in the low-level language of the Python implementation: C/C++ + for Python, Java for Jython. Typically contained in a single dynamically + loaded pre-compiled file, e.g. a shared object (:file:`.so`) file for Python + extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python + extensions on Windows, or a Java class file for Jython extensions. Note that + currently Distutils only handles C/C++ extensions for Python. + +package + A module that contains other modules, typically contained in a directory of + the filesystem and distinguished from other directories by the presence of a + file :file:`__init__.py`. + +root package + The root of the hierarchy of packages. (This isn't really a package, + since it doesn't have an :file:`__init__.py` file. But... we have to + call it something, right?) The vast majority of the standard library is + in the root package, as are many small standalone third-party modules that + don't belong to a larger module collection. Unlike regular packages, + modules in the root package can be found in many directories: in fact, + every directory listed in ``sys.path`` contributes modules to the root + package. + + +.. _packaging-term: + +Distutils-specific terminology +============================== + +The following terms apply more specifically to the domain of distributing Python +modules using Distutils: + +module distribution + A collection of Python modules distributed together as a single downloadable + resource and meant to be installed all as one. Examples of some well-known + module distributions are NumPy, SciPy, PIL (the Python Imaging + Library) or mxBase. (Module distributions would be called a *package*, + except that term is already taken in the Python context: a single module + distribution may contain zero, one, or many Python packages.) + +pure module distribution + A module distribution that contains only pure Python modules and packages. + Sometimes referred to as a "pure distribution." + +non-pure module distribution + A module distribution that contains at least one extension module. Sometimes + referred to as a "non-pure distribution." + +distribution root + The top-level directory of your source tree (or source distribution). The + directory where :file:`setup.py` exists. Generally :file:`setup.py` will + be run from this directory. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/packageindex.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/packageindex.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,104 @@ +.. _packaging-package-index: + +********************************** +Registering with the Package Index +********************************** + +The Python Package Index (PyPI) holds metadata describing distributions +packaged with packaging. The packaging command :command:`register` is used to +submit your distribution's metadata to the index. It is invoked as follows:: + + python setup.py register + +Distutils will respond with the following prompt:: + + running register + We need to know who you are, so please choose either: + 1. use your existing login, + 2. register as a new user, + 3. have the server generate a new password for you (and email it to you), or + 4. quit + Your selection [default 1]: + +Note: if your username and password are saved locally, you will not see this +menu. + +If you have not registered with PyPI, then you will need to do so now. You +should choose option 2, and enter your details as required. Soon after +submitting your details, you will receive an email which will be used to confirm +your registration. + +Once you are registered, you may choose option 1 from the menu. You will be +prompted for your PyPI username and password, and :command:`register` will then +submit your metadata to the index. + +You may submit any number of versions of your distribution to the index. If you +alter the metadata for a particular version, you may submit it again and the +index will be updated. + +PyPI holds a record for each (name, version) combination submitted. The first +user to submit information for a given name is designated the Owner of that +name. They may submit changes through the :command:`register` command or through +the web interface. They may also designate other users as Owners or Maintainers. +Maintainers may edit the package information, but not designate other Owners or +Maintainers. + +By default PyPI will list all versions of a given package. To hide certain +versions, the Hidden property should be set to yes. This must be edited through +the web interface. + + +.. _packaging-pypirc: + +The .pypirc file +================ + +The format of the :file:`.pypirc` file is as follows:: + + [packaging] + index-servers = + pypi + + [pypi] + repository: + username: + password: + +The *packaging* section defines a *index-servers* variable that lists the +name of all sections describing a repository. + +Each section describing a repository defines three variables: + +- *repository*, that defines the url of the PyPI server. Defaults to + ``http://www.python.org/pypi``. +- *username*, which is the registered username on the PyPI server. +- *password*, that will be used to authenticate. If omitted the user + will be prompt to type it when needed. + +If you want to define another server a new section can be created and +listed in the *index-servers* variable:: + + [packaging] + index-servers = + pypi + other + + [pypi] + repository: + username: + password: + + [other] + repository: http://example.com/pypi + username: + password: + +:command:`register` can then be called with the -r option to point the +repository to work with:: + + python setup.py register -r http://example.com/pypi + +For convenience, the name of the section that describes the repository +may also be used:: + + python setup.py register -r other diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/setupcfg.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/setupcfg.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,890 @@ +.. highlightlang:: cfg + +.. _setupcfg-spec: + +******************************************* +Specification of the :file:`setup.cfg` file +******************************************* + +:version: 0.9 + +This document describes the :file:`setup.cfg`, an ini-style configuration file +used by Packaging to replace the :file:`setup.py` file used by Distutils. +This specification is language-agnostic, and will therefore repeat some +information that's already documented for Python in the +:class:`configparser.RawConfigParser` documentation. + +.. contents:: + :depth: 3 + :local: + + +.. _setupcfg-syntax: + +Syntax +====== + +The ini-style format used in the configuration file is a simple collection of +sections that group sets of key-value fields separated by ``=`` or ``:`` and +optional whitespace. Lines starting with ``#`` or ``;`` are comments and will +be ignored. Empty lines are also ignored. Example:: + + [section1] + # comment + name = value + name2 = "other value" + + [section2] + foo = bar + + +Parsing values +--------------- + +Here are a set of rules to parse values: + +- If a value is quoted with ``"`` chars, it's a string. If a quote character is + present in the quoted value, it can be escaped as ``\"`` or left as-is. + +- If the value is ``true``, ``t``, ``yes``, ``y`` (case-insensitive) or ``1``, + it's converted to the language equivalent of a ``True`` value; if it's + ``false``, ``f``, ``no``, ``n`` (case-insensitive) or ``0``, it's converted to + the equivalent of ``False``. + +- A value can contain multiple lines. When read, lines are converted into a + sequence of values. Each line after the first must start with a least one + space or tab character; this leading indentation will be stripped. + +- All other values are considered strings. + +Examples:: + + [section] + foo = one + two + three + + bar = false + baz = 1.3 + boo = "ok" + beee = "wqdqw pojpj w\"ddq" + + +Extending files +--------------- + +A configuration file can be extended (i.e. included) by other files. For this, +a ``DEFAULT`` section must contain an ``extends`` key whose value points to one +or more files which will be merged into the current files by adding new sections +and fields. If a file loaded by ``extends`` contains sections or keys that +already exist in the original file, they will not override the previous values. + +Contents of :file:`one.cfg`:: + + [section1] + name = value + + [section2] + foo = foo from one.cfg + +Contents of :file:`two.cfg`:: + + [DEFAULT] + extends = one.cfg + + [section2] + foo = foo from two.cfg + baz = baz from two.cfg + +The result of parsing :file:`two.cfg` is equivalent to this file:: + + [section1] + name = value + + [section2] + foo = foo from one.cfg + baz = baz from two.cfg + +Example use of multi-line notation to include more than one file:: + + [DEFAULT] + extends = one.cfg + two.cfg + +When several files are provided, they are processed sequentially, following the +precedence rules explained above. This means that the list of files should go +from most specialized to most common. + +**Tools will need to provide a way to produce a merged version of the +file**. This will be useful to let users publish a single file. + + +.. _setupcfg-sections: + +Description of sections and fields +================================== + +Each section contains a description of its options. + +- Options that are marked *multi* can have multiple values, one value per + line. +- Options that are marked *optional* can be omitted. +- Options that are marked *environ* can use environment markers, as described + in :PEP:`345`. + + +The sections are: + +global + Global options not related to one command. + +metadata + Name, version and other information defined by :PEP:`345`. + +files + Modules, scripts, data, documentation and other files to include in the + distribution. + +extension sections + Options used to build extension modules. + +command sections + Options given for specific commands, identical to those that can be given + on the command line. + + +.. _setupcfg-section-global: + +Global options +-------------- + +Contains global options for Packaging. This section is shared with Distutils. + + +commands + Defined Packaging command. A command is defined by its fully + qualified name. *optional*, *multi* + + Examples:: + + [global] + commands = + package.setup.CustomSdistCommand + package.setup.BdistDeb + +compilers + Defined Packaging compiler. A compiler is defined by its fully + qualified name. *optional*, *multi* + + Example:: + + [global] + compilers = + hotcompiler.SmartCCompiler + +setup_hooks + Defines a list of callables to be called right after the :file:`setup.cfg` + file is read, before any other processing. Each value is a Python dotted + name to an object, which has to be defined in a module present in the project + directory alonside :file:`setup.cfg` or on Python's :data:`sys.path` (see + :ref:`packaging-finding-hooks`). The callables are executed in the + order they're found in the file; if one of them cannot be found, tools should + not stop, but for example produce a warning and continue with the next line. + Each callable receives the configuration as a dictionary (keys are + :file:`setup.cfg` sections, values are dictionaries of fields) and can make + any change to it. *optional*, *multi* + + Example:: + + [global] + setup_hooks = _setuphooks.customize_config + + + +.. _setupcfg-section-metadata: + +Metadata +-------- + +The metadata section contains the metadata for the project as described in +:PEP:`345`. Field names are case-insensitive. + +Fields: + +name + Name of the project. + +version + Version of the project. Must comply with :PEP:`386`. + +platform + Platform specification describing an operating system + supported by the distribution which is not listed in the "Operating System" + Trove classifiers (:PEP:`301`). *optional*, *multi* + +supported-platform + Binary distributions containing a PKG-INFO file will + use the Supported-Platform field in their metadata to specify the OS and + CPU for which the binary distribution was compiled. The semantics of + the Supported-Platform field are free form. *optional*, *multi* + +summary + A one-line summary of what the distribution does. + (Used to be called *description* in Distutils1.) + +description + A longer description. (Used to be called *long_description* + in Distutils1.) A file can be provided in the *description-file* field. + *optional* + +keywords + A list of additional keywords to be used to assist searching + for the distribution in a larger catalog. Comma or space-separated. + *optional* + +home-page + The URL for the distribution's home page. + +download-url + The URL from which this version of the distribution + can be downloaded. *optional* + +author + Author's name. *optional* + +author-email + Author's e-mail. *optional* + +maintainer + Maintainer's name. *optional* + +maintainer-email + Maintainer's e-mail. *optional* + +license + A text indicating the term of uses, when a trove classifier does + not match. *optional*. + +classifiers + Classification for the distribution, as described in PEP 301. + *optional*, *multi*, *environ* + +requires-dist + name of another packaging project required as a dependency. + The format is *name (version)* where version is an optional + version declaration, as described in PEP 345. *optional*, *multi*, *environ* + +provides-dist + name of another packaging project contained within this + distribution. Same format than *requires-dist*. *optional*, *multi*, + *environ* + +obsoletes-dist + name of another packaging project this version obsoletes. + Same format than *requires-dist*. *optional*, *multi*, *environ* + +requires-python + Specifies the Python version the distribution requires. The value is a + comma-separated list of version predicates, as described in PEP 345. + *optional*, *environ* + +requires-externals + a dependency in the system. This field is free-form, + and just a hint for downstream maintainers. *optional*, *multi*, + *environ* + +project-url + A label, followed by a browsable URL for the project. + "label, url". The label is limited to 32 signs. *optional*, *multi* + +One extra field not present in PEP 345 is supported: + +description-file + Path to a text file that will be used to fill the ``description`` field. + Multiple values are accepted; they must be separated by whitespace. + ``description-file`` and ``description`` are mutually exclusive. *optional* + + + +Example:: + + [metadata] + name = pypi2rpm + version = 0.1 + author = Tarek Ziadé + author-email = tarek@ziade.org + summary = Script that transforms an sdist archive into a RPM package + description-file = README + home-page = http://bitbucket.org/tarek/pypi2rpm/wiki/Home + project-url: + Repository, http://bitbucket.org/tarek/pypi2rpm/ + RSS feed, https://bitbucket.org/tarek/pypi2rpm/rss + classifier = + Development Status :: 3 - Alpha + License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1) + +You should not give any explicit value for metadata-version: it will be guessed +from the fields present in the file. + + +.. _setupcfg-section-files: + +Files +----- + +This section describes the files included in the project. + +packages_root + the root directory containing all packages and modules + (default: current directory, i.e. the project's top-level + directory where :file:`setup.cfg` lives). *optional* + +packages + a list of packages the project includes *optional*, *multi* + +modules + a list of packages the project includes *optional*, *multi* + +scripts + a list of scripts the project includes *optional*, *multi* + +extra_files + a list of patterns for additional files to include in source distributions + (see :ref:`packaging-manifest`) *optional*, *multi* + +Example:: + + [files] + packages_root = src + packages = + pypi2rpm + pypi2rpm.command + + scripts = + pypi2rpm/pypi2rpm.py + + extra_files = + setup.py + README + + +.. Note:: + The :file:`setup.cfg` configuration file is included by default. Contrary to + Distutils, :file:`README` (or :file:`README.txt`) and :file:`setup.py` are + not included by default. + + +Resources +^^^^^^^^^ + +This section describes the files used by the project which must not be installed +in the same place that python modules or libraries, they are called +**resources**. They are for example documentation files, script files, +databases, etc... + +For declaring resources, you must use this notation:: + + source = destination + +Data-files are declared in the **resources** field in the **file** section, for +example:: + + [files] + resources = + source1 = destination1 + source2 = destination2 + +The **source** part of the declaration are relative paths of resources files +(using unix path separator **/**). For example, if you've this source tree:: + + foo/ + doc/ + doc.man + scripts/ + foo.sh + +Your setup.cfg will look like:: + + [files] + resources = + doc/doc.man = destination_doc + scripts/foo.sh = destination_scripts + +The final paths where files will be placed are composed by : **source** + +**destination**. In the previous example, **doc/doc.man** will be placed in +**destination_doc/doc/doc.man** and **scripts/foo.sh** will be placed in +**destination_scripts/scripts/foo.sh**. (If you want more control on the final +path, take a look at :ref:`setupcfg-resources-base-prefix`). + +The **destination** part of resources declaration are paths with categories. +Indeed, it's generally a bad idea to give absolute path as it will be cross +incompatible. So, you must use resources categories in your **destination** +declaration. Categories will be replaced by their real path at the installation +time. Using categories is all benefit, your declaration will be simpler, cross +platform and it will allow packager to place resources files where they want +without breaking your code. + +Categories can be specified by using this syntax:: + + {category} + +Default categories are: + +* config +* appdata +* appdata.arch +* appdata.persistent +* appdata.disposable +* help +* icon +* scripts +* doc +* info +* man + +A special category also exists **{distribution.name}** that will be replaced by +the name of the distribution, but as most of the defaults categories use them, +so it's not necessary to add **{distribution.name}** into your destination. + +If you use categories in your declarations, and you are encouraged to do, final +path will be:: + + source + destination_expanded + +.. _example_final_path: + +For example, if you have this setup.cfg:: + + [metadata] + name = foo + + [files] + resources = + doc/doc.man = {doc} + +And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final +path will be:: + + {datadir}/doc/foo/doc/doc.man + +Where {datafir} category will be platform-dependent. + + +More control on source part +""""""""""""""""""""""""""" + +Glob syntax +''''''''''' + +When you declare source file, you can use a glob-like syntax to match multiples file, for example:: + + scripts/* = {script} + +Will match all the files in the scripts directory and placed them in the script category. + +Glob tokens are: + + * ``*``: match all files. + * ``?``: match any character. + * ``**``: match any level of tree recursion (even 0). + * ``{}``: will match any part separated by comma (example: ``{sh,bat}``). + +.. TODO Add examples + +Order of declaration +'''''''''''''''''''' + +The order of declaration is important if one file match multiple rules. The last +rules matched by file is used, this is useful if you have this source tree:: + + foo/ + doc/ + index.rst + setup.rst + documentation.txt + doc.tex + README + +And you want all the files in the doc directory to be placed in {doc} category, +but README must be placed in {help} category, instead of listing all the files +one by one, you can declare them in this way:: + + [files] + resources = + doc/* = {doc} + doc/README = {help} + +Exclude +''''''' + +You can exclude some files of resources declaration by giving no destination, it +can be useful if you have a non-resources file in the same directory of +resources files:: + + foo/ + doc/ + RELEASES + doc.tex + documentation.txt + docu.rst + +Your **files** section will be:: + + [files] + resources = + doc/* = {doc} + doc/RELEASES = + +More control on destination part +"""""""""""""""""""""""""""""""" + +.. _setupcfg-resources-base-prefix: + +Defining a base prefix +'''''''''''''''''''''' + +When you define your resources, you can have more control of how the final path +is computed. + +By default, the final path is:: + + destination + source + +This can generate long paths, for example (example_final_path_):: + + {datadir}/doc/foo/doc/doc.man + +When you declare your source, you can use whitespace to split the source in +**prefix** **suffix**. So, for example, if you have this source:: + + docs/ doc.man + +The **prefix** is "docs/" and the **suffix** is "doc.html". + +.. note:: + + Separator can be placed after a path separator or replace it. So these two + sources are equivalent:: + + docs/ doc.man + docs doc.man + +.. note:: + + Glob syntax is working the same way with standard source and split source. + So these rules:: + + docs/* + docs/ * + docs * + + Will match all the files in the docs directory. + +When you use split source, the final path is computed this way:: + + destination + prefix + +So for example, if you have this setup.cfg:: + + [metadata] + name = foo + + [files] + resources = + doc/ doc.man = {doc} + +And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final +path will be:: + + {datadir}/doc/foo/doc.man + + +Overwriting paths for categories +"""""""""""""""""""""""""""""""" + +This part is intended for system administrators or downstream OS packagers. + +The real paths of categories are registered in the *sysconfig.cfg* file +installed in your python installation. This file uses an ini format too. +The content of the file is organized into several sections: + +* globals: Standard categories's paths. +* posix_prefix: Standard paths for categories and installation paths for posix + system. +* other ones XXX + +Standard categories paths are platform independent, they generally refers to +other categories, which are platform dependent. :mod:`sysconfig` will choose +these category from sections matching os.name. For example:: + + doc = {datadir}/doc/{distribution.name} + +It refers to datadir category, which can be different between platforms. In +posix system, it may be:: + + datadir = /usr/share + +So the final path will be:: + + doc = /usr/share/doc/{distribution.name} + +The platform-dependent categories are: + +* confdir +* datadir +* libdir +* base + + +Defining extra categories +""""""""""""""""""""""""" + +.. TODO + + +Examples +"""""""" + +These examples are incremental but work unitarily. + +Resources in root dir +''''''''''''''''''''' + +Source tree:: + + babar-1.0/ + README + babar.sh + launch.sh + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + *.sh = {scripts} + +So babar.sh and launch.sh will be placed in {scripts} directory. + +Now let's move all the scripts into a scripts directory. + +Resources in sub-directory +'''''''''''''''''''''''''' + +Source tree:: + + babar-1.1/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.sh = {scripts} + +It's important to use the separator after scripts/ to install all the shell +scripts into {scripts} instead of {scripts}/scripts. + +Now let's add some docs. + +Resources in multiple sub-directories +''''''''''''''''''''''''''''''''''''' + +Source tree:: + + babar-1.2/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + docs/ + api + man + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.sh = {scripts} + doc/ * = {doc} + doc/ man = {man} + +You want to place all the file in the docs script into {doc} category, instead +of man, which must be placed into {man} category, we will use the order of +declaration of globs to choose the destination, the last glob that match the +file is used. + +Now let's add some scripts for windows users. + +Complete example +'''''''''''''''' + +Source tree:: + + babar-1.3/ + README + doc/ + api + man + scripts/ + babar.sh + launch.sh + babar.bat + launch.bat + LAUNCH + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.{sh,bat} = {scripts} + doc/ * = {doc} + doc/ man = {man} + +We use brace expansion syntax to place all the shell and batch scripts into +{scripts} category. + + +.. _setupcfg-section-extensions: + +Extension modules sections +-------------------------- + +If a project includes extension modules written in C or C++, each one of them +needs to have its options defined in a dedicated section. Here's an example:: + + [files] + packages = coconut + + [extension: coconut._fastcoconut] + language = cxx + sources = cxx_src/cononut_utils.cxx + cxx_src/python_module.cxx + include_dirs = /usr/include/gecode + /usr/include/blitz + extra_compile_args = + -fPIC -O2 + -DGECODE_VERSION=$(./gecode_version) -- sys.platform != 'win32' + /DGECODE_VERSION=win32 -- sys.platform == 'win32' + +The section name must start with ``extension:``; the right-hand part is used as +the full name (including a parent package, if any) of the extension. Whitespace +around the extension name is allowed. If the extension module is not standalone +(e.g. ``_bisect``) but part of a package (e.g. ``thing._speedups``), the parent +package must be listed in the ``packages`` field. +Valid fields and their values are listed in the documentation of the +:class:`packaging.compiler.extension.Extension` class; values documented as +Python lists translate to multi-line values in the configuration file. In +addition, multi-line values accept environment markers on each line, after a +``--``. + + +.. _setupcfg-section-commands: + +Commands sections +----------------- + +To pass options to commands without having to type them on the command line +for each invocation, you can write them in the :file:`setup.cfg` file, in a +section named after the command. Example:: + + [sdist] + # special function to add custom files + manifest-builders = package.setup.list_extra_files + + [build] + use-2to3 = True + + [build_ext] + inplace = on + + [check] + strict = on + all = on + +Option values given in the configuration file can be overriden on the command +line. See :ref:`packaging-setup-config` for more information. + +These sections are also used to define :ref:`command hooks +`. + + +.. _setupcfg-extensibility: + +Extensibility +============= + +Every section can have fields that are not part of this specification. They are +called **extensions**. + +An extension field starts with ``X-``. Example:: + + [metadata] + name = Distribute + X-Debian-Name = python-distribute + + +.. _setupcfg-changes: + +Changes in the specification +============================ + +The versioning scheme for this specification is **MAJOR.MINOR**. Changes in the +specification will cause the version number to be updated. + +Changes to the minor number reflect backwards-compatible changes: + +- New fields and sections (optional or mandatory) can be added. +- Optional fields can be removed. + +The major number will be incremented for backwards-incompatible changes: + +- Mandatory fields or sections are removed. +- Fields change their meaning. + +As a consequence, a tool written to consume 1.5 has these properties: + +- Can read 1.1, 1.2 and all versions < 1.5, since the tool knows what + optional fields weren't there. + + .. XXX clarify + +- Can also read 1.6 and other 1.x versions: The tool will just ignore fields it + doesn't know about, even if they are mandatory in the new version. If + optional fields were removed, the tool will just consider them absent. + +- Cannot read 2.x and should refuse to interpret such files. + +A tool written to produce 1.x should have these properties: + +- Writes all mandatory fields. +- May write optional fields. + + +.. _setupcfg-acks: + +Acknowledgments +=============== + +This specification includes work and feedback from these people: + +- Tarek Ziadé +- Julien Jehannet +- Boris Feld +- Éric Araujo + +(If your name is missing, please :ref:`let us know `.) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/setupscript.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/setupscript.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,693 @@ +.. _packaging-setup-script: + +************************ +Writing the Setup Script +************************ + +The setup script is the center of all activity in building, distributing, and +installing modules using Distutils. The main purpose of the setup script is +to describe your module distribution to Distutils, so that the various +commands that operate on your modules do the right thing. As we saw in section +:ref:`packaging-simple-example`, the setup script consists mainly of a +call to :func:`setup` where the most information is supplied as +keyword arguments to :func:`setup`. + +Here's a slightly more involved example, which we'll follow for the next couple +of sections: a setup script that could be used for Packaging itself:: + + #!/usr/bin/env python + + from packaging.core import setup, find_packages + + setup(name='Packaging', + version='1.0', + summary='Python Distribution Utilities', + keywords=['packaging', 'packaging'], + author=u'Tarek Ziadé', + author_email='tarek@ziade.org', + home_page='http://bitbucket.org/tarek/packaging/wiki/Home', + license='PSF', + packages=find_packages()) + + +There are only two differences between this and the trivial one-file +distribution presented in section :ref:`packaging-simple-example`: more +metadata and the specification of pure Python modules by package rather than +by module. This is important since Ristutils consist of a couple of dozen +modules split into (so far) two packages; an explicit list of every module +would be tedious to generate and difficult to maintain. For more information +on the additional metadata, see section :ref:`packaging-metadata`. + +Note that any pathnames (files or directories) supplied in the setup script +should be written using the Unix convention, i.e. slash-separated. The +Distutils will take care of converting this platform-neutral representation into +whatever is appropriate on your current platform before actually using the +pathname. This makes your setup script portable across operating systems, which +of course is one of the major goals of the Distutils. In this spirit, all +pathnames in this document are slash-separated. + +This, of course, only applies to pathnames given to Distutils functions. If +you, for example, use standard Python functions such as :func:`glob.glob` or +:func:`os.listdir` to specify files, you should be careful to write portable +code instead of hardcoding path separators:: + + glob.glob(os.path.join('mydir', 'subdir', '*.html')) + os.listdir(os.path.join('mydir', 'subdir')) + + +.. _packaging-listing-packages: + +Listing whole packages +====================== + +The :option:`packages` option tells the Distutils to process (build, distribute, +install, etc.) all pure Python modules found in each package mentioned in the +:option:`packages` list. In order to do this, of course, there has to be a +correspondence between package names and directories in the filesystem. The +default correspondence is the most obvious one, i.e. package :mod:`packaging` is +found in the directory :file:`packaging` relative to the distribution root. +Thus, when you say ``packages = ['foo']`` in your setup script, you are +promising that the Distutils will find a file :file:`foo/__init__.py` (which +might be spelled differently on your system, but you get the idea) relative to +the directory where your setup script lives. If you break this promise, the +Distutils will issue a warning but still process the broken package anyway. + +If you use a different convention to lay out your source directory, that's no +problem: you just have to supply the :option:`package_dir` option to tell the +Distutils about your convention. For example, say you keep all Python source +under :file:`lib`, so that modules in the "root package" (i.e., not in any +package at all) are in :file:`lib`, modules in the :mod:`foo` package are in +:file:`lib/foo`, and so forth. Then you would put :: + + package_dir = {'': 'lib'} + +in your setup script. The keys to this dictionary are package names, and an +empty package name stands for the root package. The values are directory names +relative to your distribution root. In this case, when you say ``packages = +['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists. + +Another possible convention is to put the :mod:`foo` package right in +:file:`lib`, the :mod:`foo.bar` package in :file:`lib/bar`, etc. This would be +written in the setup script as :: + + package_dir = {'foo': 'lib'} + +A ``package: dir`` entry in the :option:`package_dir` dictionary implicitly +applies to all packages below *package*, so the :mod:`foo.bar` case is +automatically handled here. In this example, having ``packages = ['foo', +'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and +:file:`lib/bar/__init__.py`. (Keep in mind that although :option:`package_dir` +applies recursively, you must explicitly list all packages in +:option:`packages`: the Distutils will *not* recursively scan your source tree +looking for any directory with an :file:`__init__.py` file.) + + +.. _packaging-listing-modules: + +Listing individual modules +========================== + +For a small module distribution, you might prefer to list all modules rather +than listing packages---especially the case of a single module that goes in the +"root package" (i.e., no package at all). This simplest case was shown in +section :ref:`packaging-simple-example`; here is a slightly more involved +example:: + + py_modules = ['mod1', 'pkg.mod2'] + +This describes two modules, one of them in the "root" package, the other in the +:mod:`pkg` package. Again, the default package/directory layout implies that +these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and +that :file:`pkg/__init__.py` exists as well. And again, you can override the +package/directory correspondence using the :option:`package_dir` option. + + +.. _packaging-describing-extensions: + +Describing extension modules +============================ + +Just as writing Python extension modules is a bit more complicated than writing +pure Python modules, describing them to the Distutils is a bit more complicated. +Unlike pure modules, it's not enough just to list modules or packages and expect +the Distutils to go out and find the right files; you have to specify the +extension name, source file(s), and any compile/link requirements (include +directories, libraries to link with, etc.). + +.. XXX read over this section + +All of this is done through another keyword argument to :func:`setup`, the +:option:`ext_modules` option. :option:`ext_modules` is just a list of +:class:`Extension` instances, each of which describes a single extension module. +Suppose your distribution includes a single extension, called :mod:`foo` and +implemented by :file:`foo.c`. If no additional instructions to the +compiler/linker are needed, describing this extension is quite simple:: + + Extension('foo', ['foo.c']) + +The :class:`Extension` class can be imported from :mod:`packaging.core` along +with :func:`setup`. Thus, the setup script for a module distribution that +contains only this one extension and nothing else might be:: + + from packaging.core import setup, Extension + setup(name='foo', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])]) + +The :class:`Extension` class (actually, the underlying extension-building +machinery implemented by the :command:`build_ext` command) supports a great deal +of flexibility in describing Python extensions, which is explained in the +following sections. + + +Extension names and packages +---------------------------- + +The first argument to the :class:`Extension` constructor is always the name of +the extension, including any package names. For example, :: + + Extension('foo', ['src/foo1.c', 'src/foo2.c']) + +describes an extension that lives in the root package, while :: + + Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c']) + +describes the same extension in the :mod:`pkg` package. The source files and +resulting object code are identical in both cases; the only difference is where +in the filesystem (and therefore where in Python's namespace hierarchy) the +resulting extension lives. + +If your distribution contains only one or more extension modules in a package, +you need to create a :file:`{package}/__init__.py` file anyway, otherwise Python +won't be able to import anything. + +If you have a number of extensions all in the same package (or all under the +same base package), use the :option:`ext_package` keyword argument to +:func:`setup`. For example, :: + + setup(..., + ext_package='pkg', + ext_modules=[Extension('foo', ['foo.c']), + Extension('subpkg.bar', ['bar.c'])]) + +will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to +:mod:`pkg.subpkg.bar`. + + +Extension source files +---------------------- + +The second argument to the :class:`Extension` constructor is a list of source +files. Since the Distutils currently only support C, C++, and Objective-C +extensions, these are normally C/C++/Objective-C source files. (Be sure to use +appropriate extensions to distinguish C++\ source files: :file:`.cc` and +:file:`.cpp` seem to be recognized by both Unix and Windows compilers.) + +However, you can also include SWIG interface (:file:`.i`) files in the list; the +:command:`build_ext` command knows how to deal with SWIG extensions: it will run +SWIG on the interface file and compile the resulting C/C++ file into your +extension. + +.. XXX SWIG support is rough around the edges and largely untested! + +This warning notwithstanding, options to SWIG can be currently passed like +this:: + + setup(..., + ext_modules=[Extension('_foo', ['foo.i'], + swig_opts=['-modern', '-I../include'])], + py_modules=['foo']) + +Or on the command line like this:: + + > python setup.py build_ext --swig-opts="-modern -I../include" + +On some platforms, you can include non-source files that are processed by the +compiler and included in your extension. Currently, this just means Windows +message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for +Visual C++. These will be compiled to binary resource (:file:`.res`) files and +linked into the executable. + + +Preprocessor options +-------------------- + +Three optional arguments to :class:`Extension` will help if you need to specify +include directories to search or preprocessor macros to define/undefine: +``include_dirs``, ``define_macros``, and ``undef_macros``. + +For example, if your extension requires header files in the :file:`include` +directory under your distribution root, use the ``include_dirs`` option:: + + Extension('foo', ['foo.c'], include_dirs=['include']) + +You can specify absolute directories there; if you know that your extension will +only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get +away with :: + + Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11']) + +You should avoid this sort of non-portable usage if you plan to distribute your +code: it's probably better to write C code like :: + + #include + +If you need to include header files from some other Python extension, you can +take advantage of the fact that header files are installed in a consistent way +by the Distutils :command:`install_header` command. For example, the Numerical +Python header files are installed (on a standard Unix installation) to +:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ +according to your platform and Python installation.) Since the Python include +directory---\ :file:`/usr/local/include/python1.5` in this case---is always +included in the search path when building Python extensions, the best approach +is to write C code like :: + + #include + +.. TODO check if it's d2.sysconfig or the new sysconfig module now + +If you must put the :file:`Numerical` include directory right into your header +search path, though, you can find that directory using the Distutils +:mod:`packaging.sysconfig` module:: + + from packaging.sysconfig import get_python_inc + incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical') + setup(..., + Extension(..., include_dirs=[incdir])) + +Even though this is quite portable---it will work on any Python installation, +regardless of platform---it's probably easier to just write your C code in the +sensible way. + +You can define and undefine preprocessor macros with the ``define_macros`` and +``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)`` +tuples, where ``name`` is the name of the macro to define (a string) and +``value`` is its value: either a string or ``None``. (Defining a macro ``FOO`` +to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with +most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is +just a list of macros to undefine. + +For example:: + + Extension(..., + define_macros=[('NDEBUG', '1'), + ('HAVE_STRFTIME', None)], + undef_macros=['HAVE_FOO', 'HAVE_BAR']) + +is the equivalent of having this at the top of every C source file:: + + #define NDEBUG 1 + #define HAVE_STRFTIME + #undef HAVE_FOO + #undef HAVE_BAR + + +Library options +--------------- + +You can also specify the libraries to link against when building your extension, +and the directories to search for those libraries. The ``libraries`` option is +a list of libraries to link against, ``library_dirs`` is a list of directories +to search for libraries at link-time, and ``runtime_library_dirs`` is a list of +directories to search for shared (dynamically loaded) libraries at run-time. + +For example, if you need to link against libraries known to be in the standard +library search path on target systems :: + + Extension(..., + libraries=['gdbm', 'readline']) + +If you need to link with libraries in a non-standard location, you'll have to +include the location in ``library_dirs``:: + + Extension(..., + library_dirs=['/usr/X11R6/lib'], + libraries=['X11', 'Xt']) + +(Again, this sort of non-portable construct should be avoided if you intend to +distribute your code.) + +.. XXX Should mention clib libraries here or somewhere else! + + +Other options +------------- + +There are still some other options which can be used to handle special cases. + +The :option:`optional` option is a boolean; if it is true, +a build failure in the extension will not abort the build process, but +instead simply not install the failing extension. + +The :option:`extra_objects` option is a list of object files to be passed to the +linker. These files must not have extensions, as the default extension for the +compiler is used. + +:option:`extra_compile_args` and :option:`extra_link_args` can be used to +specify additional command-line options for the respective compiler and linker +command lines. + +:option:`export_symbols` is only useful on Windows. It can contain a list of +symbols (functions or variables) to be exported. This option is not needed when +building compiled extensions: Distutils will automatically add ``initmodule`` +to the list of exported symbols. + +The :option:`depends` option is a list of files that the extension depends on +(for example header files). The build command will call the compiler on the +sources to rebuild extension if any on this files has been modified since the +previous build. + +Relationships between Distributions and Packages +================================================ + +.. FIXME rewrite to update to PEP 345 (but without dist/release confusion) + +A distribution may relate to packages in three specific ways: + +#. It can require packages or modules. + +#. It can provide packages or modules. + +#. It can obsolete packages or modules. + +These relationships can be specified using keyword arguments to the +:func:`packaging.core.setup` function. + +Dependencies on other Python modules and packages can be specified by supplying +the *requires* keyword argument to :func:`setup`. The value must be a list of +strings. Each string specifies a package that is required, and optionally what +versions are sufficient. + +To specify that any version of a module or package is required, the string +should consist entirely of the module or package name. Examples include +``'mymodule'`` and ``'xml.parsers.expat'``. + +If specific versions are required, a sequence of qualifiers can be supplied in +parentheses. Each qualifier may consist of a comparison operator and a version +number. The accepted comparison operators are:: + + < > == + <= >= != + +These can be combined by using multiple qualifiers separated by commas (and +optional whitespace). In this case, all of the qualifiers must be matched; a +logical AND is used to combine the evaluations. + +Let's look at a bunch of examples: + ++-------------------------+----------------------------------------------+ +| Requires Expression | Explanation | ++=========================+==============================================+ +| ``==1.0`` | Only version ``1.0`` is compatible | ++-------------------------+----------------------------------------------+ +| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` | +| | is compatible, except ``1.5.1`` | ++-------------------------+----------------------------------------------+ + +Now that we can specify dependencies, we also need to be able to specify what we +provide that other distributions can require. This is done using the *provides* +keyword argument to :func:`setup`. The value for this keyword is a list of +strings, each of which names a Python module or package, and optionally +identifies the version. If the version is not specified, it is assumed to match +that of the distribution. + +Some examples: + ++---------------------+----------------------------------------------+ +| Provides Expression | Explanation | ++=====================+==============================================+ +| ``mypkg`` | Provide ``mypkg``, using the distribution | +| | version | ++---------------------+----------------------------------------------+ +| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of | +| | the distribution version | ++---------------------+----------------------------------------------+ + +A package can declare that it obsoletes other packages using the *obsoletes* +keyword argument. The value for this is similar to that of the *requires* +keyword: a list of strings giving module or package specifiers. Each specifier +consists of a module or package name optionally followed by one or more version +qualifiers. Version qualifiers are given in parentheses after the module or +package name. + +The versions identified by the qualifiers are those that are obsoleted by the +distribution being described. If no qualifiers are given, all versions of the +named module or package are understood to be obsoleted. + +.. _packaging-installing-scripts: + +Installing Scripts +================== + +So far we have been dealing with pure and non-pure Python modules, which are +usually not run by themselves but imported by scripts. + +Scripts are files containing Python source code, intended to be started from the +command line. Scripts don't require Distutils to do anything very complicated. +The only clever feature is that if the first line of the script starts with +``#!`` and contains the word "python", the Distutils will adjust the first line +to refer to the current interpreter location. By default, it is replaced with +the current interpreter location. The :option:`--executable` (or :option:`-e`) +option will allow the interpreter path to be explicitly overridden. + +The :option:`scripts` option simply is a list of files to be handled in this +way. From the PyXML setup script:: + + setup(..., + scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val']) + +All the scripts will also be added to the ``MANIFEST`` file if no template is +provided. See :ref:`packaging-manifest`. + +.. _packaging-installing-package-data: + +Installing Package Data +======================= + +Often, additional files need to be installed into a package. These files are +often data that's closely related to the package's implementation, or text files +containing documentation that might be of interest to programmers using the +package. These files are called :dfn:`package data`. + +Package data can be added to packages using the ``package_data`` keyword +argument to the :func:`setup` function. The value must be a mapping from +package name to a list of relative path names that should be copied into the +package. The paths are interpreted as relative to the directory containing the +package (information from the ``package_dir`` mapping is used if appropriate); +that is, the files are expected to be part of the package in the source +directories. They may contain glob patterns as well. + +The path names may contain directory portions; any necessary directories will be +created in the installation. + +For example, if a package should contain a subdirectory with several data files, +the files can be arranged like this in the source tree:: + + setup.py + src/ + mypkg/ + __init__.py + module.py + data/ + tables.dat + spoons.dat + forks.dat + +The corresponding call to :func:`setup` might be:: + + setup(..., + packages=['mypkg'], + package_dir={'mypkg': 'src/mypkg'}, + package_data={'mypkg': ['data/*.dat']}) + + +All the files that match ``package_data`` will be added to the ``MANIFEST`` +file if no template is provided. See :ref:`packaging-manifest`. + + +.. _packaging-additional-files: + +Installing Additional Files +=========================== + +The :option:`data_files` option can be used to specify additional files needed +by the module distribution: configuration files, message catalogs, data files, +anything which doesn't fit in the previous categories. + +:option:`data_files` specifies a sequence of (*directory*, *files*) pairs in the +following way:: + + setup(..., + data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), + ('config', ['cfg/data.cfg']), + ('/etc/init.d', ['init-script'])]) + +Note that you can specify the directory names where the data files will be +installed, but you cannot rename the data files themselves. + +Each (*directory*, *files*) pair in the sequence specifies the installation +directory and the files to install there. If *directory* is a relative path, it +is interpreted relative to the installation prefix (Python's ``sys.prefix`` for +pure-Python packages, ``sys.exec_prefix`` for packages that contain extension +modules). Each file name in *files* is interpreted relative to the +:file:`setup.py` script at the top of the package source distribution. No +directory information from *files* is used to determine the final location of +the installed file; only the name of the file is used. + +You can specify the :option:`data_files` options as a simple sequence of files +without specifying a target directory, but this is not recommended, and the +:command:`install_dist` command will print a warning in this case. To install data +files directly in the target directory, an empty string should be given as the +directory. + +All the files that match ``data_files`` will be added to the ``MANIFEST`` file +if no template is provided. See :ref:`packaging-manifest`. + + + +.. _packaging-metadata: + +Metadata reference +================== + +The setup script may include additional metadata beyond the name and version. +This table describes required and additional information: + +.. TODO synchronize with setupcfg; link to it (but don't remove it, it's a + useful summary) + ++----------------------+---------------------------+-----------------+--------+ +| Meta-Data | Description | Value | Notes | ++======================+===========================+=================+========+ +| ``name`` | name of the project | short string | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``version`` | version of this release | short string | (1)(2) | ++----------------------+---------------------------+-----------------+--------+ +| ``author`` | project author's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``author_email`` | email address of the | email address | \(3) | +| | project author | | | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer`` | project maintainer's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer_email`` | email address of the | email address | \(3) | +| | project maintainer | | | ++----------------------+---------------------------+-----------------+--------+ +| ``home_page`` | home page for the project | URL | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``summary`` | short description of the | short string | | +| | project | | | ++----------------------+---------------------------+-----------------+--------+ +| ``description`` | longer description of the | long string | \(5) | +| | project | | | ++----------------------+---------------------------+-----------------+--------+ +| ``download_url`` | location where the | URL | | +| | project may be downloaded | | | ++----------------------+---------------------------+-----------------+--------+ +| ``classifiers`` | a list of classifiers | list of strings | \(4) | ++----------------------+---------------------------+-----------------+--------+ +| ``platforms`` | a list of platforms | list of strings | | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the release | short string | \(6) | ++----------------------+---------------------------+-----------------+--------+ + +Notes: + +(1) + These fields are required. + +(2) + It is recommended that versions take the form *major.minor[.patch[.sub]]*. + +(3) + Either the author or the maintainer must be identified. + +(4) + The list of classifiers is available from the `PyPI website + `_. See also :mod:`packaging.create`. + +(5) + The ``description`` field is used by PyPI when you are registering a + release, to build its PyPI page. + +(6) + The ``license`` field is a text indicating the license covering the + distribution where the license is not a selection from the "License" Trove + classifiers. See the ``Classifier`` field. Notice that + there's a ``licence`` distribution option which is deprecated but still + acts as an alias for ``license``. + +'short string' + A single line of text, not more than 200 characters. + +'long string' + Multiple lines of plain text in reStructuredText format (see + http://docutils.sf.net/). + +'list of strings' + See below. + +In Python 2.x, "string value" means a unicode object. If a byte string (str or +bytes) is given, it has to be valid ASCII. + +.. TODO move this section to the version document, keep a summary, add a link + +Encoding the version information is an art in itself. Python projects generally +adhere to the version format *major.minor[.patch][sub]*. The major number is 0 +for initial, experimental releases of software. It is incremented for releases +that represent major milestones in a project. The minor number is incremented +when important new features are added to the project. The patch number +increments when bug-fix releases are made. Additional trailing version +information is sometimes used to indicate sub-releases. These are +"a1,a2,...,aN" (for alpha releases, where functionality and API may change), +"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN" +(for final pre-release release testing). Some examples: + +0.1.0 + the first, experimental release of a project + +1.0.1a2 + the second alpha release of the first patch version of 1.0 + +:option:`classifiers` are specified in a Python list:: + + setup(..., + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Python Software Foundation License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Topic :: Communications :: Email', + 'Topic :: Office/Business', + 'Topic :: Software Development :: Bug Tracking', + ]) + + +Debugging the setup script +========================== + +Sometimes things go wrong, and the setup script doesn't do what the developer +wants. + +Distutils catches any exceptions when running the setup script, and print a +simple error message before the script is terminated. The motivation for this +behaviour is to not confuse administrators who don't know much about Python and +are trying to install a project. If they get a big long traceback from deep +inside the guts of Distutils, they may think the project or the Python +installation is broken because they don't read all the way down to the bottom +and see that it's a permission problem. + +.. FIXME DISTUTILS_DEBUG is dead, document logging/warnings here + +On the other hand, this doesn't help the developer to find the cause of the +failure. For this purpose, the DISTUTILS_DEBUG environment variable can be set +to anything except an empty string, and Packaging will now print detailed +information about what it is doing, and prints the full traceback in case an +exception occurs. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/sourcedist.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/sourcedist.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,266 @@ +.. _packaging-source-dist: + +****************************** +Creating a Source Distribution +****************************** + +As shown in section :ref:`packaging-simple-example`, you use the :command:`sdist` command +to create a source distribution. In the simplest case, :: + + python setup.py sdist + +(assuming you haven't specified any :command:`sdist` options in the setup script +or config file), :command:`sdist` creates the archive of the default format for +the current platform. The default format is a gzip'ed tar file +(:file:`.tar.gz`) on Unix, and ZIP file on Windows. + +You can specify as many formats as you like using the :option:`--formats` +option, for example:: + + python setup.py sdist --formats=gztar,zip + +to create a gzipped tarball and a zip file. The available formats are: + ++-----------+-------------------------+---------+ +| Format | Description | Notes | ++===========+=========================+=========+ +| ``zip`` | zip file (:file:`.zip`) | (1),(3) | ++-----------+-------------------------+---------+ +| ``gztar`` | gzip'ed tar file | \(2) | +| | (:file:`.tar.gz`) | | ++-----------+-------------------------+---------+ +| ``bztar`` | bzip2'ed tar file | | +| | (:file:`.tar.bz2`) | | ++-----------+-------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | | ++-----------+-------------------------+---------+ + +Notes: + +(1) + default on Windows + +(2) + default on Unix + +(3) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +When using any ``tar`` format (``gztar``, ``bztar`` or +``tar``) under Unix, you can specify the ``owner`` and ``group`` names +that will be set for each member of the archive. + +For example, if you want all files of the archive to be owned by root:: + + python setup.py sdist --owner=root --group=root + + +.. _packaging-manifest: + +Specifying the files to distribute +================================== + +If you don't supply an explicit list of files (or instructions on how to +generate one), the :command:`sdist` command puts a minimal default set into the +source distribution: + +* all Python source files implied by the :option:`py_modules` and + :option:`packages` options + +* all C source files mentioned in the :option:`ext_modules` or + :option:`libraries` options + +* scripts identified by the :option:`scripts` option + See :ref:`packaging-installing-scripts`. + +* anything that looks like a test script: :file:`test/test\*.py` (currently, the + Packaging don't do anything with test scripts except include them in source + distributions, but in the future there will be a standard for testing Python + module distributions) + +* the configuration file :file:`setup.cfg` + +* all files that matches the ``package_data`` metadata. + See :ref:`packaging-installing-package-data`. + +* all files that matches the ``data_files`` metadata. + See :ref:`packaging-additional-files`. + +Contrary to Distutils, :file:`README` (or :file:`README.txt`) and +:file:`setup.py` are not included by default. + +Sometimes this is enough, but usually you will want to specify additional files +to distribute. The typical way to do this is to write a *manifest template*, +called :file:`MANIFEST.in` by default. The manifest template is just a list of +instructions for how to generate your manifest file, :file:`MANIFEST`, which is +the exact list of files to include in your source distribution. The +:command:`sdist` command processes this template and generates a manifest based +on its instructions and what it finds in the filesystem. + +If you prefer to roll your own manifest file, the format is simple: one filename +per line, regular files (or symlinks to them) only. If you do supply your own +:file:`MANIFEST`, you must specify everything: the default set of files +described above does not apply in this case. + +:file:`MANIFEST` files start with a comment indicating they are generated. +Files without this comment are not overwritten or removed. + +See :ref:`packaging-manifest-template` section for a syntax reference. + + +.. _packaging-manifest-options: + +Manifest-related options +======================== + +The normal course of operations for the :command:`sdist` command is as follows: + +* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` + and create the manifest + +* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest + with just the default file set + +* if either :file:`MANIFEST.in` or the setup script (:file:`setup.py`) are more + recent than :file:`MANIFEST`, recreate :file:`MANIFEST` by reading + :file:`MANIFEST.in` + +* use the list of files now in :file:`MANIFEST` (either just generated or read + in) to create the source distribution archive(s) + +There are a couple of options that modify this behaviour. First, use the +:option:`--no-defaults` and :option:`--no-prune` to disable the standard +"include" and "exclude" sets. + +Second, you might just want to (re)generate the manifest, but not create a +source distribution:: + + python setup.py sdist --manifest-only + +:option:`-o` is a shortcut for :option:`--manifest-only`. + + +.. _packaging-manifest-template: + +The MANIFEST.in template +======================== + +A :file:`MANIFEST.in` file can be added in a project to define the list of +files to include in the distribution built by the :command:`sdist` command. + +When :command:`sdist` is run, it will look for the :file:`MANIFEST.in` file +and interpret it to generate the :file:`MANIFEST` file that contains the +list of files that will be included in the package. + +This mechanism can be used when the default list of files is not enough. +(See :ref:`packaging-manifest`). + +Principle +--------- + +The manifest template has one command per line, where each command specifies a +set of files to include or exclude from the source distribution. For an +example, let's look at the Packaging' own manifest template:: + + include *.txt + recursive-include examples *.txt *.py + prune examples/sample?/build + +The meanings should be fairly clear: include all files in the distribution root +matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory +matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching +:file:`examples/sample?/build`. All of this is done *after* the standard +include set, so you can exclude files from the standard set with explicit +instructions in the manifest template. (Or, you can use the +:option:`--no-defaults` option to disable the standard set entirely.) + +The order of commands in the manifest template matters: initially, we have the +list of default files as described above, and each command in the template adds +to or removes from that list of files. Once we have fully processed the +manifest template, we remove files that should not be included in the source +distribution: + +* all files in the Packaging "build" tree (default :file:`build/`) + +* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`, + :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs` + +Now we have our complete list of files, which is written to the manifest for +future reference, and then used to build the source distribution archive(s). + +You can disable the default set of included files with the +:option:`--no-defaults` option, and you can disable the standard exclude set +with :option:`--no-prune`. + +Following the Packaging' own manifest template, let's trace how the +:command:`sdist` command builds the list of files to include in the Packaging +source distribution: + +#. include all Python source files in the :file:`packaging` and + :file:`packaging/command` subdirectories (because packages corresponding to + those two directories were mentioned in the :option:`packages` option in the + setup script---see section :ref:`packaging-setup-script`) + +#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard + files) + +#. include :file:`test/test\*.py` (standard files) + +#. include :file:`\*.txt` in the distribution root (this will find + :file:`README.txt` a second time, but such redundancies are weeded out later) + +#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree + under :file:`examples`, + +#. exclude all files in the sub-trees starting at directories matching + :file:`examples/sample?/build`\ ---this may exclude files included by the + previous two steps, so it's important that the ``prune`` command in the manifest + template comes after the ``recursive-include`` command + +#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`, + :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs` + directories + +Just like in the setup script, file and directory names in the manifest template +should always be slash-separated; the Packaging will take care of converting +them to the standard representation on your platform. That way, the manifest +template is portable across operating systems. + +Commands +-------- + +The manifest template commands are: + ++-------------------------------------------+-----------------------------------------------+ +| Command | Description | ++===========================================+===============================================+ +| :command:`include pat1 pat2 ...` | include all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`prune dir` | exclude all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ +| :command:`graft dir` | include all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ + +The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of +regular filename characters, ``?`` matches any single regular filename +character, and ``[range]`` matches any of the characters in *range* (e.g., +``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename +character" is platform-specific: on Unix it is anything except slash; on Windows +anything except backslash or colon. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/tutorial.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/tutorial.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,112 @@ +================== +Packaging tutorial +================== + +Welcome to the Packaging tutorial! We will learn how to use Packaging +to package your project. + +.. TODO merge with introduction.rst + + +Getting started +--------------- + +Packaging works with the *setup.cfg* file. It contains all the metadata for +your project, as defined in PEP 345, but also declare what your project +contains. + +Let's say you have a project called *CLVault* containing one package called +*clvault*, and a few scripts inside. You can use the *pysetup* script to create +a *setup.cfg* file for the project. The script will ask you a few questions:: + + $ mkdir CLVault + $ cd CLVault + $ pysetup create + Project name [CLVault]: + Current version number: 0.1 + Package description: + >Command-line utility to store and retrieve passwords + Author name: Tarek Ziade + Author e-mail address: tarek@ziade.org + Project Home Page: http://bitbucket.org/tarek/clvault + Do you want to add a package ? (y/n): y + Package name: clvault + Do you want to add a package ? (y/n): n + Do you want to set Trove classifiers? (y/n): y + Please select the project status: + + 1 - Planning + 2 - Pre-Alpha + 3 - Alpha + 4 - Beta + 5 - Production/Stable + 6 - Mature + 7 - Inactive + + Status: 3 + What license do you use: GPL + Matching licenses: + + 1) License :: OSI Approved :: GNU General Public License (GPL) + 2) License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) + + Type the number of the license you wish to use or ? to try again:: 1 + Do you want to set other trove identifiers (y/n) [n]: n + Wrote "setup.cfg". + + +A setup.cfg file is created, containing the metadata of your project and the +list of the packages it contains:: + + $ cat setup.cfg + [metadata] + name = CLVault + version = 0.1 + author = Tarek Ziade + author_email = tarek@ziade.org + description = Command-line utility to store and retrieve passwords + home_page = http://bitbucket.org/tarek/clvault + + classifier = Development Status :: 3 - Alpha + License :: OSI Approved :: GNU General Public License (GPL) + + [files] + packages = clvault + + +Our project will depend on the *keyring* project. Let's add it in the +[metadata] section:: + + [metadata] + ... + requires_dist = + keyring + + +Running commands +---------------- + +You can run useful commands on your project once the setup.cfg file is ready: + +- sdist: creates a source distribution +- register: register your project to PyPI +- upload: upload the distribution to PyPI +- install_dist: install it + +All commands are run using the run script:: + + $ pysetup run install_dist + $ pysetup run sdist + $ pysetup run upload + +If you want to push a source distribution of your project to PyPI, do:: + + $ pysetup run sdist register upload + + +Installing the project +---------------------- + +The project can be installed by manually running the packaging install command:: + + $ pysetup run install_dist diff -r 6db40a9955dc -r 0d413f60cc23 Doc/packaging/uploading.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/packaging/uploading.rst Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,80 @@ +.. _packaging-package-upload: + +*************************************** +Uploading Packages to the Package Index +*************************************** + +The Python Package Index (PyPI) not only stores the package info, but also the +package data if the author of the package wishes to. The packaging command +:command:`upload` pushes the distribution files to PyPI. + +The command is invoked immediately after building one or more distribution +files. For example, the command :: + + python setup.py sdist bdist_wininst upload + +will cause the source distribution and the Windows installer to be uploaded to +PyPI. Note that these will be uploaded even if they are built using an earlier +invocation of :file:`setup.py`, but that only distributions named on the command +line for the invocation including the :command:`upload` command are uploaded. + +The :command:`upload` command uses the username, password, and repository URL +from the :file:`$HOME/.pypirc` file (see section :ref:`packaging-pypirc` for more on this +file). If a :command:`register` command was previously called in the same +command, and if the password was entered in the prompt, :command:`upload` will +reuse the entered password. This is useful if you do not want to store a clear +text password in the :file:`$HOME/.pypirc` file. + +You can specify another PyPI server with the :option:`--repository=*url*` +option:: + + python setup.py sdist bdist_wininst upload -r http://example.com/pypi + +See section :ref:`packaging-pypirc` for more on defining several servers. + +You can use the :option:`--sign` option to tell :command:`upload` to sign each +uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must +be available for execution on the system :envvar:`PATH`. You can also specify +which key to use for signing using the :option:`--identity=*name*` option. + +Other :command:`upload` options include :option:`--repository=` or +:option:`--repository=
` where *url* is the url of the server and +*section* the name of the section in :file:`$HOME/.pypirc`, and +:option:`--show-response` (which displays the full response text from the PyPI +server for help in debugging upload problems). + +PyPI package display +==================== + +The ``description`` field plays a special role at PyPI. It is used by +the server to display a home page for the registered package. + +If you use the `reStructuredText `_ +syntax for this field, PyPI will parse it and display an HTML output for +the package home page. + +The ``description`` field can be filled from a text file located in the +project:: + + from packaging.core import setup + + fp = open('README.txt') + try: + description = fp.read() + finally: + fp.close() + + setup(name='Packaging', + description=description) + +In that case, :file:`README.txt` is a regular reStructuredText text file located +in the root of the package besides :file:`setup.py`. + +To prevent registering broken reStructuredText content, you can use the +:program:`rst2html` program that is provided by the :mod:`docutils` package +and check the ``description`` from the command line:: + + $ python setup.py --description | rst2html.py > output.html + +:mod:`docutils` will display a warning if there's something wrong with your +syntax. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/compound_stmts.rst --- a/Doc/reference/compound_stmts.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/compound_stmts.rst Mon Jan 25 17:05:13 2016 +0100 @@ -22,14 +22,14 @@ single: clause single: suite -A compound statement consists of one or more 'clauses.' A clause consists of a +Compound statements consist of one or more 'clauses.' A clause consists of a header and a 'suite.' The clause headers of a particular compound statement are all at the same indentation level. Each clause header begins with a uniquely identifying keyword and ends with a colon. A suite is a group of statements controlled by a clause. A suite can be one or more semicolon-separated simple statements on the same line as the header, following the header's colon, or it can be one or more indented statements on subsequent lines. Only the latter -form of a suite can contain nested compound statements; the following is illegal, +form of suite can contain nested compound statements; the following is illegal, mostly because it wouldn't be clear to which :keyword:`if` clause a following :keyword:`else` clause would belong:: @@ -51,9 +51,6 @@ : | `with_stmt` : | `funcdef` : | `classdef` - : | `async_with_stmt` - : | `async_for_stmt` - : | `async_funcdef` suite: `stmt_list` NEWLINE | NEWLINE INDENT `statement`+ DEDENT statement: `stmt_list` NEWLINE | `compound_stmt` stmt_list: `simple_stmt` (";" `simple_stmt`)* [";"] @@ -159,8 +156,8 @@ The expression list is evaluated once; it should yield an iterable object. An iterator is created for the result of the ``expression_list``. The suite is -then executed once for each item provided by the iterator, in the order returned -by the iterator. Each item in turn is assigned to the target list using the +then executed once for each item provided by the iterator, in the order of +ascending indices. Each item in turn is assigned to the target list using the standard rules for assignments (see :ref:`assignment`), and then the suite is executed. When the items are exhausted (which is immediately when the sequence is empty or an iterator raises a :exc:`StopIteration` exception), the suite in @@ -173,25 +170,17 @@ A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`else` clause's suite. A :keyword:`continue` statement executed in the first suite skips the rest of the suite and continues -with the next item, or with the :keyword:`else` clause if there is no next +with the next item, or with the :keyword:`else` clause if there was no next item. -The for-loop makes assignments to the variables(s) in the target list. -This overwrites all previous assignments to those variables including -those made in the suite of the for-loop:: - - for i in range(10): - print(i) - i = 5 # this will not affect the for-loop - # because i will be overwritten with the next - # index in the range - +The suite may assign to the variable(s) in the target list; this does not affect +the next item assigned to it. .. index:: builtin: range Names in the target list are not deleted when the loop is finished, but if the -sequence is empty, they will not have been assigned to at all by the loop. Hint: +sequence is empty, it will not have been assigned to at all by the loop. Hint: the built-in function :func:`range` returns an iterator of integers suitable to emulate the effect of Pascal's ``for i := a to b do``; e.g., ``list(range(3))`` returns the list ``[0, 1, 2]``. @@ -237,7 +226,7 @@ .. productionlist:: try_stmt: try1_stmt | try2_stmt try1_stmt: "try" ":" `suite` - : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ + : ("except" [`expression` ["as" `target`]] ":" `suite`)+ : ["else" ":" `suite`] : ["finally" ":" `suite`] try2_stmt: "try" ":" `suite` @@ -295,7 +284,7 @@ object: traceback Before an except clause's suite is executed, details about the exception are -stored in the :mod:`sys` module and can be accessed via :func:`sys.exc_info`. +stored in the :mod:`sys` module and can be access via :func:`sys.exc_info`. :func:`sys.exc_info` returns a 3-tuple consisting of the exception class, the exception instance and a traceback object (see section :ref:`types`) identifying the point in the program where the exception occurred. :func:`sys.exc_info` @@ -318,23 +307,12 @@ :keyword:`try` clause is executed, including any :keyword:`except` and :keyword:`else` clauses. If an exception occurs in any of the clauses and is not handled, the exception is temporarily saved. The :keyword:`finally` clause -is executed. If there is a saved exception it is re-raised at the end of the -:keyword:`finally` clause. If the :keyword:`finally` clause raises another -exception, the saved exception is set as the context of the new exception. -If the :keyword:`finally` clause executes a :keyword:`return` or :keyword:`break` -statement, the saved exception is discarded:: - - >>> def f(): - ... try: - ... 1/0 - ... finally: - ... return 42 - ... - >>> f() - 42 - -The exception information is not available to the program during execution of -the :keyword:`finally` clause. +is executed. If there is a saved exception, it is re-raised at the end of the +:keyword:`finally` clause. If the :keyword:`finally` clause raises another +exception or executes a :keyword:`return` or :keyword:`break` statement, the +saved exception is set as the context of the new exception. The exception +information is not available to the program during execution of the +:keyword:`finally` clause. .. index:: statement: return @@ -348,20 +326,6 @@ reason is a problem with the current implementation --- this restriction may be lifted in the future). -The return value of a function is determined by the last :keyword:`return` -statement executed. Since the :keyword:`finally` clause always executes, a -:keyword:`return` statement executed in the :keyword:`finally` clause will -always be the last one executed:: - - >>> def foo(): - ... try: - ... return 'try' - ... finally: - ... return 'finally' - ... - >>> foo() - 'finally' - Additional information on exceptions can be found in section :ref:`exceptions`, and information on using the :keyword:`raise` statement to generate exceptions may be found in section :ref:`raise`. @@ -373,9 +337,7 @@ The :keyword:`with` statement ============================= -.. index:: - statement: with - single: as; with statement +.. index:: statement: with The :keyword:`with` statement is used to wrap the execution of a block with methods defined by a context manager (see section :ref:`context-managers`). @@ -444,9 +406,6 @@ statement. -.. index:: - single: parameter; function definition - .. _function: .. _def: @@ -471,10 +430,11 @@ decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`parameter_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] - : | "**" `parameter` [","] + parameter_list: (`defparameter` ",")* + : ( "*" [`parameter`] ("," `defparameter`)* + : [, "**" `parameter`] + : | "**" `parameter` + : | `defparameter` [","] ) parameter: `identifier` [":" `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` @@ -508,27 +468,23 @@ def func(): pass func = f1(arg)(f2(func)) -.. index:: - triple: default; parameter; value - single: argument; function definition +.. index:: triple: default; parameter; value -When one or more :term:`parameters ` have the form *parameter* ``=`` -*expression*, the function is said to have "default parameter values." For a -parameter with a default value, the corresponding :term:`argument` may be -omitted from a call, in which +When one or more parameters have the form *parameter* ``=`` *expression*, the +function is said to have "default parameter values." For a parameter with a +default value, the corresponding argument may be omitted from a call, in which case the parameter's default value is substituted. If a parameter has a default value, all following parameters up until the "``*``" must also have a default value --- this is a syntactic restriction that is not expressed by the grammar. -**Default parameter values are evaluated from left to right when the function -definition is executed.** This means that the expression is evaluated once, when -the function is defined, and that the same "pre-computed" value is used for each -call. This is especially important to understand when a default parameter is a -mutable object, such as a list or a dictionary: if the function modifies the -object (e.g. by appending an item to a list), the default value is in effect -modified. This is generally not what was intended. A way around this is to use -``None`` as the default, and explicitly test for it in the body of the function, -e.g.:: +**Default parameter values are evaluated when the function definition is +executed.** This means that the expression is evaluated once, when the function +is defined, and that the same "pre-computed" value is used for each call. This +is especially important to understand when a default parameter is a mutable +object, such as a list or a dictionary: if the function modifies the object +(e.g. by appending an item to a list), the default value is in effect modified. +This is generally not what was intended. A way around this is to use ``None`` +as the default, and explicitly test for it in the body of the function, e.g.:: def whats_on_the_telly(penguin=None): if penguin is None: @@ -563,17 +519,17 @@ by the parameters' names in the :attr:`__annotations__` attribute of the function object. -.. index:: pair: lambda; expression +.. index:: pair: lambda; form It is also possible to create anonymous functions (functions not bound to a -name), for immediate use in expressions. This uses lambda expressions, described in -section :ref:`lambda`. Note that the lambda expression is merely a shorthand for a +name), for immediate use in expressions. This uses lambda forms, described in +section :ref:`lambda`. Note that the lambda form is merely a shorthand for a simplified function definition; a function defined in a ":keyword:`def`" statement can be passed around or assigned to another name just like a function -defined by a lambda expression. The ":keyword:`def`" form is actually more powerful +defined by a lambda form. The ":keyword:`def`" form is actually more powerful since it allows the execution of multiple statements and annotations. -**Programmer's note:** Functions are first-class objects. A "``def``" statement +**Programmer's note:** Functions are first-class objects. A "``def``" form executed inside a function definition defines a local function that can be returned or passed around. Free variables used in the nested function can access the local variables of the function containing the def. See section @@ -663,130 +619,6 @@ :pep:`3129` - Class Decorators -Coroutines -========== - -.. versionadded:: 3.5 - -.. index:: statement: async def -.. _`async def`: - -Coroutine function definition ------------------------------ - -.. productionlist:: - async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" ["->" `expression`] ":" `suite` - -.. index:: - keyword: async - keyword: await - -Execution of Python coroutines can be suspended and resumed at many points -(see :term:`coroutine`). In the body of a coroutine, any ``await`` and -``async`` identifiers become reserved keywords; :keyword:`await` expressions, -:keyword:`async for` and :keyword:`async with` can only be used in -coroutine bodies. - -Functions defined with ``async def`` syntax are always coroutine functions, -even if they do not contain ``await`` or ``async`` keywords. - -It is a :exc:`SyntaxError` to use :keyword:`yield` expressions in -``async def`` coroutines. - -An example of a coroutine function:: - - async def func(param1, param2): - do_stuff() - await some_coroutine() - - -.. index:: statement: async for -.. _`async for`: - -The :keyword:`async for` statement ----------------------------------- - -.. productionlist:: - async_for_stmt: "async" `for_stmt` - -An :term:`asynchronous iterable` is able to call asynchronous code in its -*iter* implementation, and :term:`asynchronous iterator` can call asynchronous -code in its *next* method. - -The ``async for`` statement allows convenient iteration over asynchronous -iterators. - -The following code:: - - async for TARGET in ITER: - BLOCK - else: - BLOCK2 - -Is semantically equivalent to:: - - iter = (ITER) - iter = await type(iter).__aiter__(iter) - running = True - while running: - try: - TARGET = await type(iter).__anext__(iter) - except StopAsyncIteration: - running = False - else: - BLOCK - else: - BLOCK2 - -See also :meth:`__aiter__` and :meth:`__anext__` for details. - -It is a :exc:`SyntaxError` to use ``async for`` statement outside of an -:keyword:`async def` function. - - -.. index:: statement: async with -.. _`async with`: - -The :keyword:`async with` statement ------------------------------------ - -.. productionlist:: - async_with_stmt: "async" `with_stmt` - -An :term:`asynchronous context manager` is a :term:`context manager` that is -able to suspend execution in its *enter* and *exit* methods. - -The following code:: - - async with EXPR as VAR: - BLOCK - -Is semantically equivalent to:: - - mgr = (EXPR) - aexit = type(mgr).__aexit__ - aenter = type(mgr).__aenter__(mgr) - exc = True - - VAR = await aenter - try: - BLOCK - except: - if not await aexit(mgr, *sys.exc_info()): - raise - else: - await aexit(mgr, None, None, None) - -See also :meth:`__aenter__` and :meth:`__aexit__` for details. - -It is a :exc:`SyntaxError` to use ``async with`` statement outside of an -:keyword:`async def` function. - -.. seealso:: - - :pep:`492` - Coroutines with async and await syntax - - .. rubric:: Footnotes .. [#] The exception is propagated to the invocation stack unless diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/datamodel.rst --- a/Doc/reference/datamodel.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/datamodel.rst Mon Jan 25 17:05:13 2016 +0100 @@ -35,19 +35,12 @@ Every object has an identity, a type and a value. An object's *identity* never changes once it has been created; you may think of it as the object's address in memory. The ':keyword:`is`' operator compares the identity of two objects; the -:func:`id` function returns an integer representing its identity. - -.. impl-detail:: - - For CPython, ``id(x)`` is the memory address where ``x`` is stored. - +:func:`id` function returns an integer representing its identity (currently +implemented as its address). An object's :dfn:`type` is also unchangeable. [#]_ An object's type determines the operations that the object supports (e.g., "does it have a length?") and also defines the possible values for objects of that type. The :func:`type` function returns an object's type (which is an object -itself). Like its identity, an object's :dfn:`type` is also unchangeable. -[#]_ - -The *value* of some objects can change. Objects whose value can +itself). The *value* of some objects can change. Objects whose value can change are said to be *mutable*; objects whose value is unchangeable once they are created are called *immutable*. (The value of an immutable container object that contains a reference to a mutable object can change when the latter's value @@ -77,7 +70,7 @@ module for information on controlling the collection of cyclic garbage. Other implementations act differently and CPython may change. Do not depend on immediate finalization of objects when they become - unreachable (so you should always close files explicitly). + unreachable (ex: always close files). Note that the use of the implementation's tracing or debugging facilities may keep objects alive that would normally be collectable. Also note that catching @@ -154,16 +147,11 @@ This type has a single value. There is a single object with this value. This object is accessed through the built-in name ``NotImplemented``. Numeric methods - and rich comparison methods should return this value if they do not implement the + and rich comparison methods may return this value if they do not implement the operation for the operands provided. (The interpreter will then try the reflected operation, or some other fallback, depending on the operator.) Its truth value is true. - See - :ref:`implementing-the-arithmetic-operations` - for more details. - - Ellipsis .. index:: object: Ellipsis @@ -206,7 +194,7 @@ single: True These represent the truth values False and True. The two objects representing - the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a + the values False and True are the only Boolean objects. The Boolean type is a subtype of the integer type, and Boolean values behave like the values 0 and 1, respectively, in almost all contexts, the exception being that when converted to a string, the strings ``"False"`` or ``"True"`` are returned, respectively. @@ -227,7 +215,7 @@ at the mercy of the underlying machine architecture (and C or Java implementation) for the accepted range and handling of overflow. Python does not support single-precision floating point numbers; the savings in processor and - memory usage that are usually the reason for using these are dwarfed by the + memory usage that are usually the reason for using these is dwarfed by the overhead of using objects in Python, so there is no reason to complicate the language with two kinds of floating point numbers. @@ -279,28 +267,25 @@ The following types are immutable sequences: - .. index:: - single: string; immutable sequences - Strings .. index:: builtin: chr builtin: ord + builtin: str single: character single: integer single: Unicode - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:type:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` - converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. + A string is a sequence of values that represent Unicode codepoints. + All the codepoints in range ``U+0000 - U+10FFFF`` can be represented + in a string. Python doesn't have a :c:type:`chr` type, and + every character in the string is represented as a string object + with length ``1``. The built-in function :func:`ord` converts a + character to its codepoint (as an integer); :func:`chr` converts + an integer in range ``0 - 10FFFF`` to the corresponding character. :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given text encoding, and - :meth:`bytes.decode` can be used to achieve the opposite. + :class:`bytes` using the given encoding, and :meth:`bytes.decode` can + be used to achieve the opposite. Tuples .. index:: @@ -320,15 +305,17 @@ A bytes object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in function :func:`bytes` can be used to + (like ``b'abc'`` and the built-in function :func:`bytes` can be used to construct bytes objects. Also, bytes objects can be decoded to strings - via the :meth:`~bytes.decode` method. + via the :meth:`decode` method. Mutable sequences .. index:: object: mutable sequence object: mutable pair: assignment; statement + single: delete + statement: del single: subscription single: slicing @@ -382,7 +369,7 @@ These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as - :meth:`~set.add`. + :meth:`add`. Frozen sets .. index:: object: frozenset @@ -452,15 +439,12 @@ Special attributes: - .. tabularcolumns:: |l|L|l| - +-------------------------+-------------------------------+-----------+ | Attribute | Meaning | | +=========================+===============================+===========+ | :attr:`__doc__` | The function's documentation | Writable | | | string, or ``None`` if | | - | | unavailable; not inherited by | | - | | subclasses | | + | | unavailable | | +-------------------------+-------------------------------+-----------+ | :attr:`__name__` | The function's name | Writable | +-------------------------+-------------------------------+-----------+ @@ -500,7 +484,7 @@ | :attr:`__annotations__` | A dict containing annotations | Writable | | | of parameters. The keys of | | | | the dict are the parameter | | - | | names, and ``'return'`` for | | + | | names, or ``'return'`` for | | | | the return annotation, if | | | | provided. | | +-------------------------+-------------------------------+-----------+ @@ -609,23 +593,13 @@ A function or method which uses the :keyword:`yield` statement (see section :ref:`yield`) is called a :dfn:`generator function`. Such a function, when called, always returns an iterator object which can be used to execute the - body of the function: calling the iterator's :meth:`iterator.__next__` - method will cause the function to execute until it provides a value - using the :keyword:`yield` statement. When the function executes a + body of the function: calling the iterator's :meth:`__next__` method will + cause the function to execute until it provides a value using the + :keyword:`yield` statement. When the function executes a :keyword:`return` statement or falls off the end, a :exc:`StopIteration` exception is raised and the iterator will have reached the end of the set of values to be returned. - Coroutine functions - .. index:: - single: coroutine; function - - A function or method which is defined using :keyword:`async def` is called - a :dfn:`coroutine function`. Such a function, when called, returns a - :term:`coroutine` object. It may contain :keyword:`await` expressions, - as well as :keyword:`async with` and :keyword:`async for` statements. See - also the :ref:`coroutine-objects` section. - Built-in functions .. index:: object: built-in function @@ -670,20 +644,17 @@ statement: import object: module - Modules are a basic organizational unit of Python code, and are created by - the :ref:`import system ` as invoked either by the - :keyword:`import` statement (see :keyword:`import`), or by calling - functions such as :func:`importlib.import_module` and built-in - :func:`__import__`. A module object has a namespace implemented by a - dictionary object (this is the dictionary referenced by the ``__globals__`` - attribute of functions defined in the module). Attribute references are - translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to - ``m.__dict__["x"]``. A module object does not contain the code object used - to initialize the module (since it isn't needed once the initialization is - done). + Modules are imported by the :keyword:`import` statement (see section + :ref:`import`). A module object has a + namespace implemented by a dictionary object (this is the dictionary referenced + by the __globals__ attribute of functions defined in the module). Attribute + references are translated to lookups in this dictionary, e.g., ``m.x`` is + equivalent to ``m.__dict__["x"]``. A module object does not contain the code + object used to initialize the module (since it isn't needed once the + initialization is done). - Attribute assignment updates the module's namespace dictionary, e.g., - ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. + Attribute assignment updates the module's namespace dictionary, e.g., ``m.x = + 1`` is equivalent to ``m.__dict__["x"] = 1``. .. index:: single: __dict__ (module attribute) @@ -705,12 +676,11 @@ Predefined (writable) attributes: :attr:`__name__` is the module's name; :attr:`__doc__` is the module's documentation string, or ``None`` if - unavailable; :attr:`__file__` is the pathname of the file from which the - module was loaded, if it was loaded from a file. The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter; for extension modules - loaded dynamically from a shared library, it is the pathname of the shared - library file. + unavailable; :attr:`__file__` is the pathname of the file from which the module + was loaded, if it was loaded from a file. The :attr:`__file__` attribute is not + present for C modules that are statically linked into the interpreter; for + extension modules loaded dynamically from a shared library, it is the pathname + of the shared library file. Custom classes Custom class types are typically created by class definitions (see section @@ -724,7 +694,7 @@ where there are multiple inheritance paths leading back to a common ancestor. Additional details on the C3 MRO used by Python can be found in the documentation accompanying the 2.3 release at - https://www.python.org/download/releases/2.3/mro/. + http://www.python.org/download/releases/2.3/mro/. .. XXX: Could we add that MRO doc as an appendix to the language ref? @@ -763,10 +733,10 @@ Special attributes: :attr:`__name__` is the class name; :attr:`__module__` is the module name in which the class was defined; :attr:`__dict__` is the - dictionary containing the class's namespace; :attr:`~class.__bases__` is a - tuple (possibly empty or a singleton) containing the base classes, in the - order of their occurrence in the base class list; :attr:`__doc__` is the - class's documentation string, or None if undefined. + dictionary containing the class's namespace; :attr:`__bases__` is a tuple + (possibly empty or a singleton) containing the base classes, in the order of + their occurrence in the base class list; :attr:`__doc__` is the class's + documentation string, or None if undefined. Class instances .. index:: @@ -808,8 +778,8 @@ single: __dict__ (instance attribute) single: __class__ (instance attribute) - Special attributes: :attr:`~object.__dict__` is the attribute dictionary; - :attr:`~instance.__class__` is the instance's class. + Special attributes: :attr:`__dict__` is the attribute dictionary; + :attr:`__class__` is the instance's class. I/O objects (also known as file objects) .. index:: @@ -827,9 +797,9 @@ A :term:`file object` represents an open file. Various shortcuts are available to create file objects: the :func:`open` built-in function, and - also :func:`os.popen`, :func:`os.fdopen`, and the - :meth:`~socket.socket.makefile` method of socket objects (and perhaps by - other functions or methods provided by extension modules). + also :func:`os.popen`, :func:`os.fdopen`, and the :meth:`makefile` method + of socket objects (and perhaps by other functions or methods provided + by extension modules). The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are initialized to file objects corresponding to the interpreter's standard @@ -949,20 +919,6 @@ frame). A debugger can implement a Jump command (aka Set Next Statement) by writing to f_lineno. - Frame objects support one method: - - .. method:: frame.clear() - - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator - is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). - - :exc:`RuntimeError` is raised if the frame is currently executing. - - .. versionadded:: 3.4 - Traceback objects .. index:: object: traceback @@ -1012,9 +968,9 @@ single: stop (slice object attribute) single: step (slice object attribute) - Special read-only attributes: :attr:`~slice.start` is the lower bound; - :attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step - value; each is ``None`` if omitted. These attributes can have any type. + Special read-only attributes: :attr:`start` is the lower bound; :attr:`stop` is + the upper bound; :attr:`step` is the step value; each is ``None`` if omitted. + These attributes can have any type. Slice objects support one method: @@ -1068,8 +1024,7 @@ the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval of individual elements, but extracting a slice may not make sense. (One example -of this is the :class:`~xml.dom.NodeList` interface in the W3C's Document -Object Model.) +of this is the :class:`NodeList` interface in the W3C's Document Object Model.) .. _customization: @@ -1110,17 +1065,13 @@ .. index:: pair: class; constructor - Called after the instance has been created (by :meth:`__new__`), but before - it is returned to the caller. The arguments are those passed to the - class constructor expression. If a base class has an :meth:`__init__` - method, the derived class's :meth:`__init__` method, if any, must explicitly - call it to ensure proper initialization of the base class part of the - instance; for example: ``BaseClass.__init__(self, [args...])``. - - Because :meth:`__new__` and :meth:`__init__` work together in constructing - objects (:meth:`__new__` to create it, and :meth:`__init__` to customise it), - no non-``None`` value may be returned by :meth:`__init__`; doing so will - cause a :exc:`TypeError` to be raised at runtime. + Called when the instance is created. The arguments are those passed to the + class constructor expression. If a base class has an :meth:`__init__` method, + the derived class's :meth:`__init__` method, if any, must explicitly call it to + ensure proper initialization of the base class part of the instance; for + example: ``BaseClass.__init__(self, [args...])``. As a special constraint on + constructors, no value may be returned; doing so will cause a :exc:`TypeError` + to be raised at runtime. .. method:: object.__del__(self) @@ -1152,14 +1103,14 @@ reference to the object on the stack frame that raised an unhandled exception in interactive mode (the traceback stored in ``sys.last_traceback`` keeps the stack frame alive). The first situation - can only be remedied by explicitly breaking the cycles; the second can be - resolved by freeing the reference to the traceback object when it is no - longer useful, and the third can be resolved by storing ``None`` in - ``sys.last_traceback``. - Circular references which are garbage are detected and cleaned up when - the cyclic garbage collector is enabled (it's on by default). Refer to the - documentation for the :mod:`gc` module for more information about this - topic. + can only be remedied by explicitly breaking the cycles; the latter two + situations can be resolved by storing ``None`` in ``sys.last_traceback``. + Circular references which are garbage are detected when the option cycle + detector is enabled (it's on by default), but can only be cleaned up if + there are no Python- level :meth:`__del__` methods involved. Refer to the + documentation for the :mod:`gc` module for more information about how + :meth:`__del__` methods are handled by the cycle detector, particularly + the description of the ``garbage`` value. .. warning:: @@ -1178,12 +1129,11 @@ modules are still available at the time when the :meth:`__del__` method is called. - .. index:: - single: repr() (built-in function); __repr__() (object method) - .. method:: object.__repr__(self) + .. index:: builtin: repr + Called by the :func:`repr` built-in function to compute the "official" string representation of an object. If at all possible, this should look like a valid Python expression that could be used to recreate an object with the @@ -1196,25 +1146,18 @@ This is typically used for debugging, so it is important that the representation is information-rich and unambiguous. - .. index:: - single: string; __str__() (object method) - single: format() (built-in function); __str__() (object method) - single: print() (built-in function); __str__() (object method) - .. method:: object.__str__(self) - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely - printable string representation of an object. The return value must be a - :ref:`string ` object. + .. index:: + builtin: str + builtin: print - This method differs from :meth:`object.__repr__` in that there is no - expectation that :meth:`__str__` return a valid Python expression: a more - convenient or concise representation can be used. - - The default implementation defined by the built-in type :class:`object` - calls :meth:`object.__repr__`. + Called by the :func:`str` built-in function and by the :func:`print` function + to compute the "informal" string representation of an object. This differs + from :meth:`__repr__` in that it does not have to be a valid Python + expression: a more convenient or concise representation may be used instead. + The return value must be a string object. .. XXX what about subclasses of string? @@ -1226,16 +1169,16 @@ Called by :func:`bytes` to compute a byte-string representation of an object. This should return a ``bytes`` object. - .. index:: - single: string; __format__() (object method) - pair: string; conversion - builtin: print - .. method:: object.__format__(self, format_spec) + .. index:: + pair: string; conversion + builtin: str + builtin: print + Called by the :func:`format` built-in function (and by extension, the - :meth:`str.format` method of class :class:`str`) to produce a "formatted" + :meth:`format` method of class :class:`str`) to produce a "formatted" string representation of an object. The ``format_spec`` argument is a string that contains a description of the formatting options desired. The interpretation of the ``format_spec`` argument is up to the type @@ -1247,10 +1190,6 @@ The return value must be a string object. - .. versionchanged:: 3.4 - The __format__ method of ``object`` itself raises a :exc:`TypeError` - if passed any non-empty string. - .. _richcmpfuncs: .. method:: object.__lt__(self, other) @@ -1276,14 +1215,10 @@ context (e.g., in the condition of an ``if`` statement), Python will call :func:`bool` on the value to determine if the result is true or false. - By default, :meth:`__ne__` delegates to :meth:`__eq__` and - inverts the result unless it is ``NotImplemented``. There are no other - implied relationships among the comparison operators, for example, - the truth of ``(x.__hash__``. - - If a class that does not override :meth:`__eq__` wishes to suppress hash - support, it should include ``__hash__ = None`` in the class definition. - A class which defines its own :meth:`__hash__` that explicitly raises - a :exc:`TypeError` would be incorrectly identified as hashable by - an ``isinstance(obj, collections.Hashable)`` call. + explicitly by setting ``__hash__ = .__hash__``. Otherwise the + inheritance of :meth:`__hash__` will be blocked, just as if :attr:`__hash__` + had been explicitly set to :const:`None`. .. note:: - By default, the :meth:`__hash__` values of str, bytes and datetime + Note by default the :meth:`__hash__` values of str, bytes and datetime objects are "salted" with an unpredictable random value. Although they remain constant within an individual Python process, they are not predictable between repeated invocations of Python. @@ -1366,9 +1290,9 @@ dict insertion, O(n^2) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. - Changing hash values affects the iteration order of dicts, sets and - other mappings. Python has never made guarantees about this ordering - (and it typically varies between 32-bit and 64-bit builds). + Changing hash values affects the order in which keys are retrieved from a + dict. Note Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds). See also :envvar:`PYTHONHASHSEED`. @@ -1492,14 +1416,6 @@ Called to delete the attribute on an instance *instance* of the owner class. -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). - - .. _descriptor-invocation: Invoking Descriptors @@ -1581,9 +1497,9 @@ .. data:: object.__slots__ This class variable can be assigned a string, iterable, or sequence of - strings with variable names used by instances. *__slots__* reserves space - for the declared variables and prevents the automatic creation of *__dict__* - and *__weakref__* for each instance. + strings with variable names used by instances. If defined in a + class, *__slots__* reserves space for the declared variables and prevents the + automatic creation of *__dict__* and *__weakref__* for each instance. Notes on using *__slots__* @@ -1620,7 +1536,7 @@ program undefined. In the future, a check may be added to prevent this. * Nonempty *__slots__* does not work for classes derived from "variable-length" - built-in types such as :class:`int`, :class:`bytes` and :class:`tuple`. + built-in types such as :class:`int`, :class:`str` and :class:`tuple`. * Any non-string iterable may be assigned to *__slots__*. Mappings may also be used; however, in the future, special meaning may be assigned to the values @@ -1634,122 +1550,58 @@ Customizing class creation -------------------------- -By default, classes are constructed using :func:`type`. The class body is -executed in a new namespace and the class name is bound locally to the -result of ``type(name, bases, namespace)``. +By default, classes are constructed using :func:`type`. A class definition is +read into a separate namespace and the value of class name is bound to the +result of ``type(name, bases, dict)``. -The class creation process can be customised by passing the ``metaclass`` -keyword argument in the class definition line, or by inheriting from an -existing class that included such an argument. In the following example, -both ``MyClass`` and ``MySubclass`` are instances of ``Meta``:: +When the class definition is read, if a callable ``metaclass`` keyword argument +is passed after the bases in the class definition, the callable given will be +called instead of :func:`type`. If other keyword arguments are passed, they +will also be passed to the metaclass. This allows classes or functions to be +written which monitor or alter the class creation process: - class Meta(type): - pass +* Modifying the class dictionary prior to the class being created. - class MyClass(metaclass=Meta): - pass +* Returning an instance of another class -- essentially performing the role of a + factory function. - class MySubclass(MyClass): - pass +These steps will have to be performed in the metaclass's :meth:`__new__` method +-- :meth:`type.__new__` can then be called from this method to create a class +with different properties. This example adds a new element to the class +dictionary before creating the class:: -Any other keyword arguments that are specified in the class definition are -passed through to all metaclass operations described below. + class metacls(type): + def __new__(mcs, name, bases, dict): + dict['foo'] = 'metacls was here' + return type.__new__(mcs, name, bases, dict) -When a class definition is executed, the following steps occur: +You can of course also override other class methods (or add new methods); for +example defining a custom :meth:`__call__` method in the metaclass allows custom +behavior when the class is called, e.g. not always creating a new instance. -* the appropriate metaclass is determined -* the class namespace is prepared -* the class body is executed -* the class object is created +If the metaclass has a :meth:`__prepare__` attribute (usually implemented as a +class or static method), it is called before the class body is evaluated with +the name of the class and a tuple of its bases for arguments. It should return +an object that supports the mapping interface that will be used to store the +namespace of the class. The default is a plain dictionary. This could be used, +for example, to keep track of the order that class attributes are declared in by +returning an ordered dictionary. -Determining the appropriate metaclass -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The appropriate metaclass is determined by the following precedence rules: -The appropriate metaclass for a class definition is determined as follows: +* If the ``metaclass`` keyword argument is passed with the bases, it is used. -* if no bases and no explicit metaclass are given, then :func:`type` is used -* if an explicit metaclass is given and it is *not* an instance of - :func:`type`, then it is used directly as the metaclass -* if an instance of :func:`type` is given as the explicit metaclass, or - bases are defined, then the most derived metaclass is used +* Otherwise, if there is at least one base class, its metaclass is used. -The most derived metaclass is selected from the explicitly specified -metaclass (if any) and the metaclasses (i.e. ``type(cls)``) of all specified -base classes. The most derived metaclass is one which is a subtype of *all* -of these candidate metaclasses. If none of the candidate metaclasses meets -that criterion, then the class definition will fail with ``TypeError``. - - -.. _prepare: - -Preparing the class namespace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Once the appropriate metaclass has been identified, then the class namespace -is prepared. If the metaclass has a ``__prepare__`` attribute, it is called -as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the -additional keyword arguments, if any, come from the class definition). - -If the metaclass has no ``__prepare__`` attribute, then the class namespace -is initialised as an empty :func:`dict` instance. - -.. seealso:: - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - - -Executing the class body -^^^^^^^^^^^^^^^^^^^^^^^^ - -The class body is executed (approximately) as -``exec(body, globals(), namespace)``. The key difference from a normal -call to :func:`exec` is that lexical scoping allows the class body (including -any methods) to reference names from the current and outer scopes when the -class definition occurs inside a function. - -However, even when the class definition occurs inside the function, methods -defined inside the class still cannot see names defined at the class scope. -Class variables must be accessed through the first parameter of instance or -class methods, and cannot be accessed at all from static methods. - - -Creating the class object -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Once the class namespace has been populated by executing the class body, -the class object is created by calling -``metaclass(name, bases, namespace, **kwds)`` (the additional keywords -passed here are the same as those passed to ``__prepare__``). - -This class object is the one that will be referenced by the zero-argument -form of :func:`super`. ``__class__`` is an implicit closure reference -created by the compiler if any methods in a class body refer to either -``__class__`` or ``super``. This allows the zero argument form of -:func:`super` to correctly identify the class being defined based on -lexical scoping, while the class or instance that was used to make the -current call is identified based on the first argument passed to the method. - -After the class object is created, it is passed to the class decorators -included in the class definition (if any) and the resulting object is bound -in the local namespace as the defined class. - -.. seealso:: - - :pep:`3135` - New super - Describes the implicit ``__class__`` closure reference - - -Metaclass example -^^^^^^^^^^^^^^^^^ +* Otherwise, the default metaclass (:class:`type`) is used. The potential uses for metaclasses are boundless. Some ideas that have been -explored include logging, interface checking, automatic delegation, automatic +explored including logging, interface checking, automatic delegation, automatic property creation, proxies, frameworks, and automatic resource locking/synchronization. Here is an example of a metaclass that uses an :class:`collections.OrderedDict` -to remember the order that class variables are defined:: +to remember the order that class members were defined:: class OrderedClass(type): @@ -1757,9 +1609,9 @@ def __prepare__(metacls, name, bases, **kwds): return collections.OrderedDict() - def __new__(cls, name, bases, namespace, **kwds): - result = type.__new__(cls, name, bases, dict(namespace)) - result.members = tuple(namespace) + def __new__(cls, name, bases, classdict): + result = type.__new__(cls, name, bases, dict(classdict)) + result.members = tuple(classdict) return result class A(metaclass=OrderedClass): @@ -1815,10 +1667,10 @@ :pep:`3119` - Introducing Abstract Base Classes Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality - in the context of adding Abstract Base Classes (see the :mod:`abc` - module) to the language. + :func:`issubclass` behavior through :meth:`__instancecheck__` and + :meth:`__subclasscheck__`, with motivation for this functionality in the + context of adding Abstract Base Classes (see the :mod:`abc` module) to the + language. .. _callable-types: @@ -1848,10 +1700,9 @@ N`` where *N* is the length of the sequence, or slice objects, which define a range of items. It is also recommended that mappings provide the methods :meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and +:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`copy`, and :meth:`update` behaving similar to those for Python's standard dictionary -objects. The :mod:`collections` module provides a -:class:`~collections.abc.MutableMapping` +objects. The :mod:`collections` module provides a :class:`MutableMapping` abstract base class to help create those methods from a base set of :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`. Mutable sequences should provide methods :meth:`append`, :meth:`count`, @@ -1881,15 +1732,6 @@ considered to be false in a Boolean context. -.. method:: object.__length_hint__(self) - - Called to implement :func:`operator.length_hint`. Should return an estimated - length for the object (which may be greater or less than the actual length). - The length must be an integer ``>=`` 0. This method is purely an - optimization and is never required for correctness. - - .. versionadded:: 3.4 - .. note:: Slicing is done exclusively with the following three methods. A call like :: @@ -1922,12 +1764,6 @@ indexes to allow proper detection of the end of the sequence. -.. method:: object.__missing__(self, key) - - Called by :class:`dict`\ .\ :meth:`__getitem__` to implement ``self[key]`` for dict subclasses - when key is not in the dictionary. - - .. method:: object.__setitem__(self, key, value) Called to implement assignment to ``self[key]``. Same note as for @@ -1950,7 +1786,8 @@ This method is called when an iterator is required for a container. This method should return a new iterator object that can iterate over all the objects in the - container. For mappings, it should iterate over the keys of the container. + container. For mappings, it should iterate over the keys of the container, and + should also be made available as the method :meth:`keys`. Iterator objects also need to implement this method; they are required to return themselves. For more information on iterator objects, see :ref:`typeiter`. @@ -2000,7 +1837,6 @@ .. method:: object.__add__(self, other) object.__sub__(self, other) object.__mul__(self, other) - object.__matmul__(self, other) object.__truediv__(self, other) object.__floordiv__(self, other) object.__mod__(self, other) @@ -2017,16 +1853,15 @@ builtin: pow builtin: pow - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For instance, to - evaluate the expression ``x + y``, where *x* is an instance of a class that - has an :meth:`__add__` method, ``x.__add__(y)`` is called. The - :meth:`__divmod__` method should be the equivalent to using - :meth:`__floordiv__` and :meth:`__mod__`; it should not be related to - :meth:`__truediv__`. Note that :meth:`__pow__` should be defined to accept - an optional third argument if the ternary version of the built-in :func:`pow` - function is to be supported. + These methods are called to implement the binary arithmetic operations (``+``, + ``-``, ``*``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``, ``<<``, + ``>>``, ``&``, ``^``, ``|``). For instance, to evaluate the expression + ``x + y``, where *x* is an instance of a class that has an :meth:`__add__` + method, ``x.__add__(y)`` is called. The :meth:`__divmod__` method should be the + equivalent to using :meth:`__floordiv__` and :meth:`__mod__`; it should not be + related to :meth:`__truediv__`. Note that :meth:`__pow__` should be defined + to accept an optional third argument if the ternary version of the built-in + :func:`pow` function is to be supported. If one of those methods does not support the operation with the supplied arguments, it should return ``NotImplemented``. @@ -2035,7 +1870,6 @@ .. method:: object.__radd__(self, other) object.__rsub__(self, other) object.__rmul__(self, other) - object.__rmatmul__(self, other) object.__rtruediv__(self, other) object.__rfloordiv__(self, other) object.__rmod__(self, other) @@ -2051,14 +1885,14 @@ builtin: divmod builtin: pow - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected - (swapped) operands. These functions are only called if the left operand does - not support the corresponding operation and the operands are of different - types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is - an instance of a class that has an :meth:`__rsub__` method, ``y.__rsub__(x)`` - is called if ``x.__sub__(y)`` returns *NotImplemented*. + These methods are called to implement the binary arithmetic operations (``+``, + ``-``, ``*``, ``/``, ``//``, ``%``, :func:`divmod`, :func:`pow`, ``**``, + ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected (swapped) operands. + These functions are only called if the left operand does not support the + corresponding operation and the operands are of different types. [#]_ For + instance, to evaluate the expression ``x - y``, where *y* is an instance of + a class that has an :meth:`__rsub__` method, ``y.__rsub__(x)`` is called if + ``x.__sub__(y)`` returns *NotImplemented*. .. index:: builtin: pow @@ -2076,7 +1910,6 @@ .. method:: object.__iadd__(self, other) object.__isub__(self, other) object.__imul__(self, other) - object.__imatmul__(self, other) object.__itruediv__(self, other) object.__ifloordiv__(self, other) object.__imod__(self, other) @@ -2088,17 +1921,15 @@ object.__ior__(self, other) These methods are called to implement the augmented arithmetic assignments - (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, - ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the - operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the - augmented assignment falls back to the normal methods. For instance, if *x* - is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and - ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In - certain situations, augmented assignment can result in unexpected errors (see - :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact - part of the data model. + (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, ``>>=``, + ``&=``, ``^=``, ``|=``). These methods should attempt to do the operation + in-place (modifying *self*) and return the result (which could be, but does + not have to be, *self*). If a specific method is not defined, the augmented + assignment falls back to the normal methods. For instance, to execute the + statement ``x += y``, where *x* is an instance of a class that has an + :meth:`__iadd__` method, ``x.__iadd__(y)`` is called. If *x* is an instance + of a class that does not define a :meth:`__iadd__` method, ``x.__add__(y)`` + and ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. .. method:: object.__neg__(self) @@ -2130,17 +1961,9 @@ .. method:: object.__index__(self) - Called to implement :func:`operator.index`, and whenever Python needs to - losslessly convert the numeric object to an integer object (such as in - slicing, or in the built-in :func:`bin`, :func:`hex` and :func:`oct` - functions). Presence of this method indicates that the numeric object is - an integer type. Must return an integer. - - .. note:: - - In order to have a coherent integer type class, when :meth:`__index__` is - defined :meth:`__int__` should also be defined, and both should return - the same value. + Called to implement :func:`operator.index`. Also called whenever Python needs + an integer object (such as in slicing, or in the built-in :func:`bin`, + :func:`hex` and :func:`oct` functions). Must return an integer. .. _context-managers: @@ -2240,9 +2063,9 @@ :meth:`__getattribute__` method even of the object's metaclass:: >>> class Meta(type): - ... def __getattribute__(*args): - ... print("Metaclass getattribute invoked") - ... return type.__getattribute__(*args) + ... def __getattribute__(*args): + ... print("Metaclass getattribute invoked") + ... return type.__getattribute__(*args) ... >>> class C(object, metaclass=Meta): ... def __len__(self): @@ -2268,155 +2091,6 @@ object itself in order to be consistently invoked by the interpreter). -.. index:: - single: coroutine - -Coroutines -========== - - -Awaitable Objects ------------------ - -An :term:`awaitable` object generally implements an :meth:`__await__` method. -:term:`Coroutine` objects returned from :keyword:`async def` functions -are awaitable. - -.. note:: - - The :term:`generator iterator` objects returned from generators - decorated with :func:`types.coroutine` or :func:`asyncio.coroutine` - are also awaitable, but they do not implement :meth:`__await__`. - -.. method:: object.__await__(self) - - Must return an :term:`iterator`. Should be used to implement - :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements - this method to be compatible with the :keyword:`await` expression. - -.. versionadded:: 3.5 - -.. seealso:: :pep:`492` for additional information about awaitable objects. - - -.. _coroutine-objects: - -Coroutine Objects ------------------ - -:term:`Coroutine` objects are :term:`awaitable` objects. -A coroutine's execution can be controlled by calling :meth:`__await__` and -iterating over the result. When the coroutine has finished executing and -returns, the iterator raises :exc:`StopIteration`, and the exception's -:attr:`~StopIteration.value` attribute holds the return value. If the -coroutine raises an exception, it is propagated by the iterator. Coroutines -should not directly raise unhandled :exc:`StopIteration` exceptions. - -Coroutines also have the methods listed below, which are analogous to -those of generators (see :ref:`generator-methods`). However, unlike -generators, coroutines do not directly support iteration. - -.. method:: coroutine.send(value) - - Starts or resumes execution of the coroutine. If *value* is ``None``, - this is equivalent to advancing the iterator returned by - :meth:`__await__`. If *value* is not ``None``, this method delegates - to the :meth:`~generator.send` method of the iterator that caused - the coroutine to suspend. The result (return value, - :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. - -.. method:: coroutine.throw(type[, value[, traceback]]) - - Raises the specified exception in the coroutine. This method delegates - to the :meth:`~generator.throw` method of the iterator that caused - the coroutine to suspend, if it has such a method. Otherwise, - the exception is raised at the suspension point. The result - (return value, :exc:`StopIteration`, or other exception) is the same as - when iterating over the :meth:`__await__` return value, described - above. If the exception is not caught in the coroutine, it propagates - back to the caller. - -.. method:: coroutine.close() - - Causes the coroutine to clean itself up and exit. If the coroutine - is suspended, this method first delegates to the :meth:`~generator.close` - method of the iterator that caused the coroutine to suspend, if it - has such a method. Then it raises :exc:`GeneratorExit` at the - suspension point, causing the coroutine to immediately clean itself up. - Finally, the coroutine is marked as having finished executing, even if - it was never started. - - Coroutine objects are automatically closed using the above process when - they are about to be destroyed. - - -Asynchronous Iterators ----------------------- - -An *asynchronous iterable* is able to call asynchronous code in its -``__aiter__`` implementation, and an *asynchronous iterator* can call -asynchronous code in its ``__anext__`` method. - -Asynchronous iterators can be used in an :keyword:`async for` statement. - -.. method:: object.__aiter__(self) - - Must return an *awaitable* resulting in an *asynchronous iterator* object. - -.. method:: object.__anext__(self) - - Must return an *awaitable* resulting in a next value of the iterator. Should - raise a :exc:`StopAsyncIteration` error when the iteration is over. - -An example of an asynchronous iterable object:: - - class Reader: - async def readline(self): - ... - - async def __aiter__(self): - return self - - async def __anext__(self): - val = await self.readline() - if val == b'': - raise StopAsyncIteration - return val - -.. versionadded:: 3.5 - - -Asynchronous Context Managers ------------------------------ - -An *asynchronous context manager* is a *context manager* that is able to -suspend execution in its ``__aenter__`` and ``__aexit__`` methods. - -Asynchronous context managers can be used in an :keyword:`async with` statement. - -.. method:: object.__aenter__(self) - - This method is semantically similar to the :meth:`__enter__`, with only - difference that it must return an *awaitable*. - -.. method:: object.__aexit__(self, exc_type, exc_value, traceback) - - This method is semantically similar to the :meth:`__exit__`, with only - difference that it must return an *awaitable*. - -An example of an asynchronous context manager class:: - - class AsyncContextManager: - async def __aenter__(self): - await log('entering context') - - async def __aexit__(self, exc_type, exc, tb): - await log('exiting context') - -.. versionadded:: 3.5 - - .. rubric:: Footnotes .. [#] It *is* possible in some cases to change an object's type, under certain diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/executionmodel.rst --- a/Doc/reference/executionmodel.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/executionmodel.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,26 +5,37 @@ Execution model *************** +.. index:: single: execution model + + +.. _naming: + +Naming and binding +================== + .. index:: - single: execution model pair: code; block + single: namespace + single: scope -.. _prog_structure: +.. index:: + single: name + pair: binding; name -Structure of a program -====================== +:dfn:`Names` refer to objects. Names are introduced by name binding operations. +Each occurrence of a name in the program text refers to the :dfn:`binding` of +that name established in the innermost function block containing the use. .. index:: block -A Python program is constructed from code blocks. A :dfn:`block` is a piece of Python program text that is executed as a unit. The following are blocks: a module, a function body, and a class definition. Each command typed interactively is a block. A script file (a file given as -standard input to the interpreter or specified as a command line argument to the -interpreter) is a code block. A script command (a command specified on the -interpreter command line with the '**-c**' option) is a code block. The string -argument passed to the built-in functions :func:`eval` and :func:`exec` is a -code block. +standard input to the interpreter or specified on the interpreter command line +the first argument) is a code block. A script command (a command specified on +the interpreter command line with the '**-c**' option) is a code block. The +string argument passed to the built-in functions :func:`eval` and :func:`exec` +is a code block. .. index:: pair: execution; frame @@ -32,25 +43,43 @@ administrative information (used for debugging) and determines where and how execution continues after the code block's execution has completed. -.. _naming: +.. index:: scope -Naming and binding -================== +A :dfn:`scope` defines the visibility of a name within a block. If a local +variable is defined in a block, its scope includes that block. If the +definition occurs in a function block, the scope extends to any blocks contained +within the defining one, unless a contained block introduces a different binding +for the name. The scope of names defined in a class block is limited to the +class block; it does not extend to the code blocks of methods -- this includes +comprehensions and generator expressions since they are implemented using a +function scope. This means that the following will fail:: + + class A: + a = 42 + b = list(a + i for i in range(10)) + +.. index:: single: environment + +When a name is used in a code block, it is resolved using the nearest enclosing +scope. The set of all such scopes visible to a code block is called the block's +:dfn:`environment`. + +.. index:: pair: free; variable + +If a name is bound in a block, it is a local variable of that block, unless +declared as :keyword:`nonlocal`. If a name is bound at the module level, it is +a global variable. (The variables of the module code block are local and +global.) If a variable is used in a code block but not defined there, it is a +:dfn:`free variable`. .. index:: - single: namespace - single: scope + single: NameError (built-in exception) + single: UnboundLocalError -.. _bind_names: - -Binding of names ----------------- - -.. index:: - single: name - pair: binding; name - -:dfn:`Names` refer to objects. Names are introduced by name binding operations. +When a name is not found at all, a :exc:`NameError` exception is raised. If the +name refers to a local variable that has not been bound, a +:exc:`UnboundLocalError` exception is raised. :exc:`UnboundLocalError` is a +subclass of :exc:`NameError`. .. index:: statement: from @@ -70,46 +99,6 @@ Each assignment or import statement occurs within a block defined by a class or function definition or at the module level (the top-level code block). -.. index:: pair: free; variable - -If a name is bound in a block, it is a local variable of that block, unless -declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at -the module level, it is a global variable. (The variables of the module code -block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. - -Each occurrence of a name in the program text refers to the :dfn:`binding` of -that name established by the following name resolution rules. - -.. _resolve_names: - -Resolution of names -------------------- - -.. index:: scope - -A :dfn:`scope` defines the visibility of a name within a block. If a local -variable is defined in a block, its scope includes that block. If the -definition occurs in a function block, the scope extends to any blocks contained -within the defining one, unless a contained block introduces a different binding -for the name. - -.. index:: single: environment - -When a name is used in a code block, it is resolved using the nearest enclosing -scope. The set of all such scopes visible to a code block is called the block's -:dfn:`environment`. - -.. index:: - single: NameError (built-in exception) - single: UnboundLocalError - -When a name is not found at all, a :exc:`NameError` exception is raised. -If the current scope is a function scope, and the name refers to a local -variable that has not yet been bound to a value at the point where the name is -used, an :exc:`UnboundLocalError` exception is raised. -:exc:`UnboundLocalError` is a subclass of :exc:`NameError`. - If a name binding operation occurs anywhere within a code block, all uses of the name within the block are treated as references to the current block. This can lead to errors when a name is used within a block before it is bound. This rule @@ -122,45 +111,10 @@ namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the name is not found there, the -builtins namespace is searched. The :keyword:`global` statement must precede -all uses of the name. +global namespace is searched first. If the name is not found there, the builtins +namespace is searched. The global statement must precede all uses of the name. -The :keyword:`global` statement has the same scope as a name binding operation -in the same block. If the nearest enclosing scope for a free variable contains -a global statement, the free variable is treated as a global. - -.. XXX say more about "nonlocal" semantics here - -The :keyword:`nonlocal` statement causes corresponding names to refer -to previously bound variables in the nearest enclosing function scope. -:exc:`SyntaxError` is raised at compile time if the given name does not -exist in any enclosing function scope. - -.. index:: module: __main__ - -The namespace for a module is automatically created the first time a module is -imported. The main module for a script is always called :mod:`__main__`. - -Class definition blocks and arguments to :func:`exec` and :func:`eval` are -special in the context of name resolution. -A class definition is an executable statement that may use and define names. -These references follow the normal rules for name resolution with an exception -that unbound local variables are looked up in the global namespace. -The namespace of the class definition becomes the attribute dictionary of -the class. The scope of names defined in a class block is limited to the -class block; it does not extend to the code blocks of methods -- this includes -comprehensions and generator expressions since they are implemented using a -function scope. This means that the following will fail:: - - class A: - a = 42 - b = list(a + i for i in range(10)) - -.. _restrict_exec: - -Builtins and restricted execution ---------------------------------- +.. XXX document "nonlocal" semantics here .. index:: pair: restricted; execution @@ -180,26 +134,36 @@ :keyword:`import` the :mod:`builtins` module and modify its attributes appropriately. +.. index:: module: __main__ + +The namespace for a module is automatically created the first time a module is +imported. The main module for a script is always called :mod:`__main__`. + +The :keyword:`global` statement has the same scope as a name binding operation +in the same block. If the nearest enclosing scope for a free variable contains +a global statement, the free variable is treated as a global. + +A class definition is an executable statement that may use and define names. +These references follow the normal rules for name resolution. The namespace of +the class definition becomes the attribute dictionary of the class. Names +defined at the class scope are not visible in methods. + + .. _dynamic-features: Interaction with dynamic features --------------------------------- -Name resolution of free variables occurs at runtime, not at compile time. -This means that the following code will print 42:: - - i = 10 - def f(): - print(i) - i = 42 - f() - There are several cases where Python statements are illegal when used in conjunction with nested scopes that contain free variables. If a variable is referenced in an enclosing scope, it is illegal to delete the name. An error will be reported at compile time. +If the wild card form of import --- ``import *`` --- is used in a function and +the function contains or is a nested block with free variables, the compiler +will raise a :exc:`SyntaxError`. + .. XXX from * also invalid with relative imports (at least currently) The :func:`eval` and :func:`exec` functions do not have access to the full @@ -270,3 +234,4 @@ .. [#] This limitation occurs because the code that is executed by these operations is not available at the time the module is compiled. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/expressions.rst --- a/Doc/reference/expressions.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/expressions.rst Mon Jan 25 17:05:13 2016 +0100 @@ -29,7 +29,7 @@ When a description of an arithmetic operator below uses the phrase "the numeric arguments are converted to a common type," this means that the operator -implementation for built-in types works as follows: +implementation for built-in types works that way: * If either argument is a complex number, the other is converted to complex; @@ -38,9 +38,8 @@ * otherwise, both must be integers and no conversion is necessary. -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. +Some additional rules apply for certain operators (e.g., a string left argument +to the '%' operator). Extensions must define their own conversion behavior. .. _atoms: @@ -85,13 +84,14 @@ definition begins with two or more underscore characters and does not end in two or more underscores, it is considered a :dfn:`private name` of that class. Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. +them. The transformation inserts the class name in front of the name, with +leading underscores removed, and a single underscore inserted in front of the +class name. For example, the identifier ``__spam`` occurring in a class named +``Ham`` will be transformed to ``_Ham__spam``. This transformation is +independent of the syntactical context in which the identifier is used. If the +transformed name is extremely long (longer than 255 characters), implementation +defined truncation may happen. If the class name consists only of underscores, +no transformation is done. .. _atom-literals: @@ -184,7 +184,7 @@ each time the innermost block is reached. Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" into the enclosing scope. +to in the target list don't "leak" in the enclosing scope. .. _lists: @@ -294,16 +294,16 @@ brackets or curly braces. Variables used in the generator expression are evaluated lazily when the -:meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the leftmost :keyword:`for` clause is -immediately evaluated, so that an error produced by it can be seen before any -other possible error in the code that handles the generator expression. -Subsequent :keyword:`for` clauses cannot be evaluated immediately since they -may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in bar(x))``. +:meth:`__next__` method is called for generator object (in the same fashion as +normal generators). However, the leftmost :keyword:`for` clause is immediately +evaluated, so that an error produced by it can be seen before any other possible +error in the code that handles the generator expression. Subsequent +:keyword:`for` clauses cannot be evaluated immediately since they may depend on +the previous :keyword:`for` loop. For example: ``(x*y for x in range(10) for y +in bar(x))``. The parentheses can be omitted on calls with only one argument. See section -:ref:`calls` for details. +:ref:`calls` for the detail. .. _yieldexpr: @@ -320,49 +320,48 @@ yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] -The yield expression is only used when defining a :term:`generator` function and -thus can only be used in the body of a function definition. Using a yield -expression in a function's body causes that function to be a generator. +The :keyword:`yield` expression is only used when defining a generator function, +and can only be used in the body of a function definition. Using a +:keyword:`yield` expression in a function definition is sufficient to cause that +definition to create a generator function instead of a normal function. When a generator function is called, it returns an iterator known as a -generator. That generator then controls the execution of the generator function. +generator. That generator then controls the execution of a generator function. The execution starts when one of the generator's methods is called. At that -time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`expression_list` to the generator's -caller. By suspended, we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, the internal -evaluation stack, and the state of any exception handling. When the execution -is resumed by calling one of the -generator's methods, the function can proceed exactly as if the yield expression -were just another external call. The value of the yield expression after -resuming depends on the method which resumed the execution. If -:meth:`~generator.__next__` is used (typically via either a :keyword:`for` or -the :func:`next` builtin) then the result is :const:`None`. Otherwise, if -:meth:`~generator.send` is used, then the result will be the value passed in to -that method. +time, the execution proceeds to the first :keyword:`yield` expression, where it +is suspended again, returning the value of :token:`expression_list` to +generator's caller. By suspended we mean that all local state is retained, +including the current bindings of local variables, the instruction pointer, and +the internal evaluation stack. When the execution is resumed by calling one of +the generator's methods, the function can proceed exactly as if the +:keyword:`yield` expression was just another external call. The value of the +:keyword:`yield` expression after resuming depends on the method which resumed +the execution. If :meth:`__next__` is used (typically via either a +:keyword:`for` or the :func:`next` builtin) then the result is :const:`None`, +otherwise, if :meth:`send` is used, then the result will be the value passed +in to that method. .. index:: single: coroutine All of this makes generator functions quite similar to coroutines; they yield multiple times, they have more than one entry point and their execution can be suspended. The only difference is that a generator function cannot control -where the execution should continue after it yields; the control is always +where should the execution continue after it yields; the control is always transferred to the generator's caller. -Yield expressions are allowed anywhere in a :keyword:`try` construct. If the -generator is not resumed before it is -finalized (by reaching a zero reference count or by being garbage collected), -the generator-iterator's :meth:`~generator.close` method will be called, -allowing any pending :keyword:`finally` clauses to execute. +:keyword:`yield` expressions are allowed in the :keyword:`try` clause of a +:keyword:`try` ... :keyword:`finally` construct. If the generator is not +resumed before it is finalized (by reaching a zero reference count or by being +garbage collected), the generator-iterator's :meth:`close` method will be +called, allowing any pending :keyword:`finally` clauses to execute. When ``yield from `` is used, it treats the supplied expression as a subiterator. All values produced by that subiterator are passed directly to the caller of the current generator's methods. Any values passed in with -:meth:`~generator.send` and any exceptions passed in with -:meth:`~generator.throw` are passed to the underlying iterator if it has the -appropriate methods. If this is not the case, then :meth:`~generator.send` -will raise :exc:`AttributeError` or :exc:`TypeError`, while -:meth:`~generator.throw` will just raise the passed in exception immediately. +:meth:`send` and any exceptions passed in with :meth:`throw` are passed to +the underlying iterator if it has the appropriate methods. If this is not the +case, then :meth:`send` will raise :exc:`AttributeError` or :exc:`TypeError`, +while :meth:`throw` will just raise the passed in exception immediately. When the underlying iterator is complete, the :attr:`~StopIteration.value` attribute of the raised :exc:`StopIteration` instance becomes the value of @@ -371,35 +370,15 @@ (by returning a value from the sub-generator). .. versionchanged:: 3.3 - Added ``yield from `` to delegate control flow to a subiterator. + Added ``yield from `` to delegate control flow to a subiterator -The parentheses may be omitted when the yield expression is the sole expression -on the right hand side of an assignment statement. - -.. seealso:: - - :pep:`0255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. - - :pep:`0342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. - - :pep:`0380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to sub-generators easy. +The parentheses can be omitted when the :keyword:`yield` expression is the +sole expression on the right hand side of an assignment statement. .. index:: object: generator -.. _generator-methods: -Generator-iterator methods -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the methods of a generator iterator. They can -be used to control the execution of a generator function. - -Note that calling any of the generator methods below when the generator -is already executing raises a :exc:`ValueError` exception. +The following generator's methods can be used to control the execution of a +generator function: .. index:: exception: StopIteration @@ -407,12 +386,12 @@ .. method:: generator.__next__() Starts the execution of a generator function or resumes it at the last - executed yield expression. When a generator function is resumed with a - :meth:`~generator.__next__` method, the current yield expression always - evaluates to :const:`None`. The execution then continues to the next yield - expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`__next__`'s caller. If the - generator exits without yielding another value, a :exc:`StopIteration` + executed :keyword:`yield` expression. When a generator function is resumed + with a :meth:`__next__` method, the current :keyword:`yield` expression + always evaluates to :const:`None`. The execution then continues to the next + :keyword:`yield` expression, where the generator is suspended again, and the + value of the :token:`expression_list` is returned to :meth:`next`'s caller. + If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. This method is normally called implicitly, e.g. by a :keyword:`for` loop, or @@ -422,17 +401,17 @@ .. method:: generator.send(value) Resumes the execution and "sends" a value into the generator function. The - *value* argument becomes the result of the current yield expression. The - :meth:`send` method returns the next value yielded by the generator, or - raises :exc:`StopIteration` if the generator exits without yielding another - value. When :meth:`send` is called to start the generator, it must be called - with :const:`None` as the argument, because there is no yield expression that - could receive the value. + ``value`` argument becomes the result of the current :keyword:`yield` + expression. The :meth:`send` method returns the next value yielded by the + generator, or raises :exc:`StopIteration` if the generator exits without + yielding another value. When :meth:`send` is called to start the generator, + it must be called with :const:`None` as the argument, because there is no + :keyword:`yield` expression that could receive the value. .. method:: generator.throw(type[, value[, traceback]]) - Raises an exception of type ``type`` at the point where the generator was paused, + Raises an exception of type ``type`` at the point where generator was paused, and returns the next value yielded by the generator function. If the generator exits without yielding another value, a :exc:`StopIteration` exception is raised. If the generator function does not catch the passed-in exception, or @@ -444,17 +423,12 @@ .. method:: generator.close() Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. - -.. index:: single: yield; examples - -Examples -^^^^^^^^ + paused. If the generator function then raises :exc:`StopIteration` (by + exiting normally, or due to already being closed) or :exc:`GeneratorExit` (by + not catching the exception), close returns to its caller. If the generator + yields a value, a :exc:`RuntimeError` is raised. If the generator raises any + other exception, it is propagated to the caller. :meth:`close` does nothing + if the generator has already exited due to an exception or normal exit. Here is a simple example that demonstrates the behavior of generators and generator functions:: @@ -483,8 +457,19 @@ >>> generator.close() Don't forget to clean up when 'close()' is called. -For examples using ``yield from``, see :ref:`pep-380` in "What's New in -Python." + +.. seealso:: + + :pep:`0255` - Simple Generators + The proposal for adding generators and the :keyword:`yield` statement to Python. + + :pep:`0342` - Coroutines via Enhanced Generators + The proposal to enhance the API and syntax of generators, making them + usable as simple coroutines. + + :pep:`0380` - Syntax for Delegating to a Subgenerator + The proposal to introduce the :token:`yield_from` syntax, making delegation + to sub-generators easy. .. _primaries: @@ -520,11 +505,11 @@ The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier. This production can be customized by -overriding the :meth:`__getattr__` method. If this attribute is not available, -the exception :exc:`AttributeError` is raised. Otherwise, the type and value of -the object produced is determined by the object. Multiple evaluations of the -same attribute reference may yield different objects. +attribute whose name is the identifier (which can be customized by overriding +the :meth:`__getattr__` method). If this attribute is not available, the +exception :exc:`AttributeError` is raised. Otherwise, the type and value of the +object produced is determined by the object. Multiple evaluations of the same +attribute reference may yield different objects. .. _subscriptions: @@ -549,9 +534,9 @@ .. productionlist:: subscription: `primary` "[" `expression_list` "]" -The primary must evaluate to an object that supports subscription (lists or -dictionaries for example). User-defined objects can support subscription by -defining a :meth:`__getitem__` method. +The primary must evaluate to an object that supports subscription, e.g. a list +or dictionary. User-defined objects can support subscription by defining a +:meth:`__getitem__` method. For built-in objects, there are two types of objects that support subscription: @@ -621,31 +606,30 @@ single: stop (slice object attribute) single: step (slice object attribute) -The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`__getitem__` method as +The semantics for a slicing are as follows. The primary must evaluate to a +mapping object, and it is indexed (using the same :meth:`__getitem__` method as normal subscription) with a key that is constructed from the slice list, as follows. If the slice list contains at least one comma, the key is a tuple containing the conversion of the slice items; otherwise, the conversion of the lone slice item is the key. The conversion of a slice item that is an expression is that expression. The conversion of a proper slice is a slice -object (see section :ref:`types`) whose :attr:`~slice.start`, -:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the -expressions given as lower bound, upper bound and stride, respectively, -substituting ``None`` for missing expressions. +object (see section :ref:`types`) whose :attr:`start`, :attr:`stop` and +:attr:`step` attributes are the values of the expressions given as lower bound, +upper bound and stride, respectively, substituting ``None`` for missing +expressions. -.. index:: - object: callable - single: call - single: argument; call semantics - .. _calls: Calls ----- -A call calls a callable object (e.g., a :term:`function`) with a possibly empty -series of :term:`arguments `: +.. index:: single: call + +.. index:: object: callable + +A call calls a callable object (e.g., a function) with a possibly empty series +of arguments: .. productionlist:: call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" @@ -660,17 +644,14 @@ keyword_arguments: `keyword_item` ("," `keyword_item`)* keyword_item: `identifier` "=" `expression` -An optional trailing comma may be present after the positional and keyword arguments -but does not affect the semantics. - -.. index:: - single: parameter; call semantics +A trailing comma may be present after the positional and keyword arguments but +does not affect the semantics. The primary must evaluate to a callable object (user-defined functions, built-in functions, methods of built-in objects, class objects, methods of class instances, and all objects having a :meth:`__call__` method are callable). All argument expressions are evaluated before the call is attempted. Please refer -to section :ref:`function` for the syntax of formal :term:`parameter` lists. +to section :ref:`function` for the syntax of formal parameter lists. .. XXX update with kwonly args PEP @@ -812,20 +793,6 @@ if that method was called. -.. _await: - -Await expression -================ - -Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. -Can only be used inside a :term:`coroutine function`. - -.. productionlist:: - await: ["await"] `primary` - -.. versionadded:: 3.5 - - .. _power: The power operator @@ -835,7 +802,7 @@ less tightly than unary operators on its right. The syntax is: .. productionlist:: - power: `await` ["**" `u_expr`] + power: `primary` ["**" `u_expr`] Thus, in an unparenthesized sequence of power and unary operators, the operators are evaluated from right to left (this does not constrain the evaluation order @@ -906,9 +873,8 @@ operators and one for additive operators: .. productionlist:: - m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | - : `m_expr` "//" `u_expr`| `m_expr` "/" `u_expr` | - : `m_expr` "%" `u_expr` + m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` + : | `m_expr` "%" `u_expr` a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr` .. index:: single: multiplication @@ -919,20 +885,13 @@ common type and then multiplied together. In the latter case, sequence repetition is performed; a negative repetition factor yields an empty sequence. -.. index:: single: matrix multiplication - -The ``@`` (at) operator is intended to be used for matrix multiplication. No -builtin Python types implement this operator. - -.. versionadded:: 3.5 - .. index:: exception: ZeroDivisionError single: division The ``/`` (division) and ``//`` (floor division) operators yield the quotient of their arguments. The numeric arguments are first converted to a common type. -Division of integers yields a float, while floor division of integers results in an +Integer division yields a float, while floor division of integers results in an integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. @@ -965,9 +924,9 @@ .. index:: single: addition The ``+`` (addition) operator yields the sum of its arguments. The arguments -must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common type and then added together. -In the latter case, the sequences are concatenated. +must either both be numbers or both sequences of the same type. In the former +case, the numbers are converted to a common type and then added together. In +the latter case, the sequences are concatenated. .. index:: single: subtraction @@ -992,8 +951,8 @@ .. index:: exception: ValueError -A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left -shift by *n* bits is defined as multiplication with ``pow(2,n)``. +A right shift by *n* bits is defined as division by ``pow(2,n)``. A left shift +by *n* bits is defined as multiplication with ``pow(2,n)``. .. note:: @@ -1036,6 +995,10 @@ .. _comparisons: +.. _is: +.. _is not: +.. _in: +.. _not in: Comparisons =========== @@ -1071,187 +1034,76 @@ *c*, so that, e.g., ``x < y > z`` is perfectly legal (though perhaps not pretty). -Value comparisons ------------------ +The operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare the +values of two objects. The objects need not have the same type. If both are +numbers, they are converted to a common type. Otherwise, the ``==`` and ``!=`` +operators *always* consider objects of different types to be unequal, while the +``<``, ``>``, ``>=`` and ``<=`` operators raise a :exc:`TypeError` when +comparing objects of different types that do not implement these operators for +the given pair of types. You can control comparison behavior of objects of +non-built-in types by defining rich comparison methods like :meth:`__gt__`, +described in section :ref:`customization`. -The operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare the -values of two objects. The objects do not need to have the same type. +Comparison of objects of the same type depends on the type: -Chapter :ref:`objects` states that objects have a value (in addition to type -and identity). The value of an object is a rather abstract notion in Python: -For example, there is no canonical access method for an object's value. Also, -there is no requirement that the value of an object should be constructed in a -particular way, e.g. comprised of all its data attributes. Comparison operators -implement a particular notion of what the value of an object is. One can think -of them as defining the value of an object indirectly, by means of their -comparison implementation. +* Numbers are compared arithmetically. -Because all types are (direct or indirect) subtypes of :class:`object`, they -inherit the default comparison behavior from :class:`object`. Types can -customize their comparison behavior by implementing -:dfn:`rich comparison methods` like :meth:`__lt__`, described in -:ref:`customization`. - -The default behavior for equality comparison (``==`` and ``!=``) is based on -the identity of the objects. Hence, equality comparison of instances with the -same identity results in equality, and equality comparison of instances with -different identities results in inequality. A motivation for this default -behavior is the desire that all objects should be reflexive (i.e. ``x is y`` -implies ``x == y``). - -A default order comparison (``<``, ``>``, ``<=``, and ``>=``) is not provided; -an attempt raises :exc:`TypeError`. A motivation for this default behavior is -the lack of a similar invariant as for equality. - -The behavior of the default equality comparison, that instances with different -identities are always unequal, may be in contrast to what types will need that -have a sensible definition of object value and value-based equality. Such -types will need to customize their comparison behavior, and in fact, a number -of built-in types have done that. - -The following list describes the comparison behavior of the most important -built-in types. - -* Numbers of built-in numeric types (:ref:`typesnumeric`) and of the standard - library types :class:`fractions.Fraction` and :class:`decimal.Decimal` can be - compared within and across their types, with the restriction that complex - numbers do not support order comparison. Within the limits of the types - involved, they compare mathematically (algorithmically) correct without loss - of precision. - - The not-a-number values :const:`float('NaN')` and :const:`Decimal('NaN')` - are special. They are identical to themselves (``x is x`` is true) but - are not equal to themselves (``x == x`` is false). Additionally, - comparing any number to a not-a-number value +* The values :const:`float('NaN')` and :const:`Decimal('NaN')` are special. + The are identical to themselves, ``x is x`` but are not equal to themselves, + ``x != x``. Additionally, comparing any value to a not-a-number value will return ``False``. For example, both ``3 < float('NaN')`` and ``float('NaN') < 3`` will return ``False``. -* Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be - compared within and across their types. They compare lexicographically using - the numeric values of their elements. +* Bytes objects are compared lexicographically using the numeric values of their + elements. -* Strings (instances of :class:`str`) compare lexicographically using the - numerical Unicode code points (the result of the built-in function - :func:`ord`) of their characters. [#]_ +* Strings are compared lexicographically using the numeric equivalents (the + result of the built-in function :func:`ord`) of their characters. [#]_ String + and bytes object can't be compared! - Strings and binary sequences cannot be directly compared. +* Tuples and lists are compared lexicographically using comparison of + corresponding elements. This means that to compare equal, each element must + compare equal and the two sequences must be of the same type and have the same + length. -* Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can - be compared only within each of their types, with the restriction that ranges - do not support order comparison. Equality comparison across these types - results in unequality, and ordering comparison across these types raises - :exc:`TypeError`. + If not equal, the sequences are ordered the same as their first differing + elements. For example, ``[1,2,x] <= [1,2,y]`` has the same value as + ``x <= y``. If the corresponding element does not exist, the shorter + sequence is ordered first (for example, ``[1,2] < [1,2,3]``). - Sequences compare lexicographically using comparison of corresponding - elements, whereby reflexivity of the elements is enforced. +* Mappings (dictionaries) compare equal if and only if they have the same + ``(key, value)`` pairs. Order comparisons ``('<', '<=', '>=', '>')`` + raise :exc:`TypeError`. - In enforcing reflexivity of elements, the comparison of collections assumes - that for a collection element ``x``, ``x == x`` is always true. Based on - that assumption, element identity is compared first, and element comparison - is performed only for distinct elements. This approach yields the same - result as a strict element comparison would, if the compared elements are - reflexive. For non-reflexive elements, the result is different than for - strict element comparison, and may be surprising: The non-reflexive - not-a-number values for example result in the following comparison behavior - when used in a list:: +* Sets and frozensets define comparison operators to mean subset and superset + tests. Those relations do not define total orderings (the two sets ``{1,2}`` + and {2,3} are not equal, nor subsets of one another, nor supersets of one + another). Accordingly, sets are not appropriate arguments for functions + which depend on total ordering. For example, :func:`min`, :func:`max`, and + :func:`sorted` produce undefined results given a list of sets as inputs. - >>> nan = float('NaN') - >>> nan is nan - True - >>> nan == nan - False <-- the defined non-reflexive behavior of NaN - >>> [nan] == [nan] - True <-- list enforces reflexivity and tests identity first +* Most other objects of built-in types compare unequal unless they are the same + object; the choice whether one object is considered smaller or larger than + another one is made arbitrarily but consistently within one execution of a + program. - Lexicographical comparison between built-in collections works as follows: +Comparison of objects of the differing types depends on whether either +of the types provide explicit support for the comparison. Most numeric types +can be compared with one another, but comparisons of :class:`float` and +:class:`Decimal` are not supported to avoid the inevitable confusion arising +from representation issues such as ``float('1.1')`` being inexactly represented +and therefore not exactly equal to ``Decimal('1.1')`` which is. When +cross-type comparison is not supported, the comparison method returns +``NotImplemented``. This can create the illusion of non-transitivity between +supported cross-type comparisons and unsupported comparisons. For example, +``Decimal(2) == 2`` and ``2 == float(2)`` but ``Decimal(2) != float(2)``. - - For two collections to compare equal, they must be of the same type, have - the same length, and each pair of corresponding elements must compare - equal (for example, ``[1,2] == (1,2)`` is false because the type is not the - same). - - - Collections that support order comparison are ordered the same as their - first unequal elements (for example, ``[1,2,x] <= [1,2,y]`` has the same - value as ``x <= y``). If a corresponding element does not exist, the - shorter collection is ordered first (for example, ``[1,2] < [1,2,3]`` is - true). - -* Mappings (instances of :class:`dict`) compare equal if and only if they have - equal `(key, value)` pairs. Equality comparison of the keys and elements - enforces reflexivity. - - Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. - -* Sets (instances of :class:`set` or :class:`frozenset`) can be compared within - and across their types. - - They define order - comparison operators to mean subset and superset tests. Those relations do - not define total orderings (for example, the two sets ``{1,2}`` and ``{2,3}`` - are not equal, nor subsets of one another, nor supersets of one - another). Accordingly, sets are not appropriate arguments for functions - which depend on total ordering (for example, :func:`min`, :func:`max`, and - :func:`sorted` produce undefined results given a list of sets as inputs). - - Comparison of sets enforces reflexivity of its elements. - -* Most other built-in types have no comparison methods implemented, so they - inherit the default comparison behavior. - -User-defined classes that customize their comparison behavior should follow -some consistency rules, if possible: - -* Equality comparison should be reflexive. - In other words, identical objects should compare equal: - - ``x is y`` implies ``x == y`` - -* Comparison should be symmetric. - In other words, the following expressions should have the same result: - - ``x == y`` and ``y == x`` - - ``x != y`` and ``y != x`` - - ``x < y`` and ``y > x`` - - ``x <= y`` and ``y >= x`` - -* Comparison should be transitive. - The following (non-exhaustive) examples illustrate that: - - ``x > y and y > z`` implies ``x > z`` - - ``x < y and y <= z`` implies ``x < z`` - -* Inverse comparison should result in the boolean negation. - In other words, the following expressions should have the same result: - - ``x == y`` and ``not x != y`` - - ``x < y`` and ``not x >= y`` (for total ordering) - - ``x > y`` and ``not x <= y`` (for total ordering) - - The last two expressions apply to totally ordered collections (e.g. to - sequences, but not to sets or mappings). See also the - :func:`~functools.total_ordering` decorator. - -Python does not enforce these consistency rules. In fact, the not-a-number -values are an example for not following these rules. - - -.. _in: -.. _not in: .. _membership-test-details: -Membership test operations --------------------------- - The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in s`` evaluates to true if *x* is a member of *s*, and false otherwise. ``x not in s`` returns the negation of ``x in s``. All built-in sequences and set types -support this as well as dictionary, for which :keyword:`in` tests whether the +support this as well as dictionary, for which :keyword:`in` tests whether a the dictionary has a given key. For container types such as list, tuple, set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent to ``any(x is e or x == e for e in y)``. @@ -1289,13 +1141,6 @@ operator: is not pair: identity; test - -.. _is: -.. _is not: - -Identity comparisons --------------------- - The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x is y`` is true if and only if *x* and *y* are the same object. ``x is not y`` yields the inverse truth value. [#]_ @@ -1344,9 +1189,9 @@ they return to ``False`` and ``True``, but rather return the last evaluated argument. This is sometimes useful, e.g., if ``s`` is a string that should be replaced by a default value if it is empty, the expression ``s or 'foo'`` yields -the desired value. Because :keyword:`not` has to create a new value, it -returns a boolean value regardless of the type of its argument -(for example, ``not 'foo'`` produces ``False`` rather than ``''``.) +the desired value. Because :keyword:`not` has to invent a value anyway, it does +not bother to return a value of the same type as its argument, so e.g., ``not +'foo'`` yields ``False``, not ``''``.) Conditional expressions @@ -1358,14 +1203,14 @@ .. productionlist:: conditional_expression: `or_test` ["if" `or_test` "else" `expression`] - expression: `conditional_expression` | `lambda_expr` - expression_nocond: `or_test` | `lambda_expr_nocond` + expression: `conditional_expression` | `lambda_form` + expression_nocond: `or_test` | `lambda_form_nocond` Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. -The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. -If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is +The expression ``x if C else y`` first evaluates the condition, *C* (*not* *x*); +if *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is evaluated and its value is returned. See :pep:`308` for more details about conditional expressions. @@ -1383,19 +1228,19 @@ pair: anonymous; function .. productionlist:: - lambda_expr: "lambda" [`parameter_list`]: `expression` - lambda_expr_nocond: "lambda" [`parameter_list`]: `expression_nocond` + lambda_form: "lambda" [`parameter_list`]: `expression` + lambda_form_nocond: "lambda" [`parameter_list`]: `expression_nocond` -Lambda expressions (sometimes called lambda forms) are used to create anonymous -functions. The expression ``lambda arguments: expression`` yields a function -object. The unnamed object behaves like a function object defined with :: +Lambda forms (lambda expressions) have the same syntactic position as +expressions. They are a shorthand to create anonymous functions; the expression +``lambda arguments: expression`` yields a function object. The unnamed object +behaves like a function object defined with :: def (arguments): return expression See section :ref:`function` for the syntax of parameter lists. Note that -functions created with lambda expressions cannot contain statements or -annotations. +functions created with lambda forms cannot contain statements or annotations. .. _exprlists: @@ -1446,20 +1291,18 @@ .. _operator-summary: -Operator precedence -=================== +Summary +======= .. index:: pair: operator; precedence -The following table summarizes the operator precedence in Python, from lowest +The following table summarizes the operator precedences in Python, from lowest precedence (least binding) to highest precedence (most binding). Operators in the same box have the same precedence. Unless the syntax is explicitly given, operators are binary. Operators in the same box group left to right (except for -exponentiation, which groups from right to left). - -Note that comparisons, membership tests, and identity tests, all have the same -precedence and have a left-to-right chaining feature as described in the -:ref:`comparisons` section. +comparisons, including tests, which all have the same precedence and chain from +left to right --- see section :ref:`comparisons` --- and exponentiation, which +groups from right to left). +-----------------------------------------------+-------------------------------------+ @@ -1473,10 +1316,10 @@ +-----------------------------------------------+-------------------------------------+ | :keyword:`and` | Boolean AND | +-----------------------------------------------+-------------------------------------+ -| :keyword:`not` ``x`` | Boolean NOT | +| :keyword:`not` *x* | Boolean NOT | +-----------------------------------------------+-------------------------------------+ -| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | -| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | +| :keyword:`in`, :keyword:`not` :keyword:`in`, | Comparisons, including membership | +| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests, | | ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +-----------------------------------------------+-------------------------------------+ | ``|`` | Bitwise OR | @@ -1489,22 +1332,19 @@ +-----------------------------------------------+-------------------------------------+ | ``+``, ``-`` | Addition and subtraction | +-----------------------------------------------+-------------------------------------+ -| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication division, | -| | remainder [#]_ | +| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder | +| | [#]_ | +-----------------------------------------------+-------------------------------------+ | ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +-----------------------------------------------+-------------------------------------+ | ``**`` | Exponentiation [#]_ | +-----------------------------------------------+-------------------------------------+ -| ``await`` ``x`` | Await expression | -+-----------------------------------------------+-------------------------------------+ | ``x[index]``, ``x[index:index]``, | Subscription, slicing, | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ | ``(expressions...)``, | Binding or tuple display, | | ``[expressions...]``, | list display, | -| ``{key: value...}``, | dictionary display, | +| ``{key:datum...}``, | dictionary display, | | ``{expressions...}`` | set display | +-----------------------------------------------+-------------------------------------+ @@ -1525,24 +1365,12 @@ cases, Python returns the latter result, in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very close to ``x``. -.. [#] The Unicode standard distinguishes between :dfn:`code points` - (e.g. U+0041) and :dfn:`abstract characters` (e.g. "LATIN CAPITAL LETTER A"). - While most abstract characters in Unicode are only represented using one - code point, there is a number of abstract characters that can in addition be - represented using a sequence of more than one code point. For example, the - abstract character "LATIN CAPITAL LETTER C WITH CEDILLA" can be represented - as a single :dfn:`precomposed character` at code position U+00C7, or as a - sequence of a :dfn:`base character` at code position U+0043 (LATIN CAPITAL - LETTER C), followed by a :dfn:`combining character` at code position U+0327 - (COMBINING CEDILLA). - - The comparison operators on strings compare at the level of Unicode code - points. This may be counter-intuitive to humans. For example, - ``"\u00C7" == "\u0043\u0327"`` is ``False``, even though both strings - represent the same abstract character "LATIN CAPITAL LETTER C WITH CEDILLA". - - To compare strings at the level of abstract characters (that is, in a way - intuitive to humans), use :func:`unicodedata.normalize`. +.. [#] While comparisons between strings make sense at the byte level, they may + be counter-intuitive to users. For example, the strings ``"\u00C7"`` and + ``"\u0327\u0043"`` compare differently, even though they both represent the + same unicode character (LATIN CAPITAL LETTER C WITH CEDILLA). To compare + strings in a human recognizable way, compare using + :func:`unicodedata.normalize`. .. [#] Due to automatic garbage-collection, free lists, and the dynamic nature of descriptors, you may notice seemingly unusual behaviour in certain uses of diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/import.rst --- a/Doc/reference/import.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1004 +0,0 @@ - -.. _importsystem: - -***************** -The import system -***************** - -.. index:: single: import machinery - -Python code in one :term:`module` gains access to the code in another module -by the process of :term:`importing` it. The :keyword:`import` statement is -the most common way of invoking the import machinery, but it is not the only -way. Functions such as :func:`importlib.import_module` and built-in -:func:`__import__` can also be used to invoke the import machinery. - -The :keyword:`import` statement combines two operations; it searches for the -named module, then it binds the results of that search to a name in the local -scope. The search operation of the :keyword:`import` statement is defined as -a call to the :func:`__import__` function, with the appropriate arguments. -The return value of :func:`__import__` is used to perform the name -binding operation of the :keyword:`import` statement. See the -:keyword:`import` statement for the exact details of that name binding -operation. - -A direct call to :func:`__import__` performs only the module search and, if -found, the module creation operation. While certain side-effects may occur, -such as the importing of parent packages, and the updating of various caches -(including :data:`sys.modules`), only the :keyword:`import` statement performs -a name binding operation. - -When calling :func:`__import__` as part of an import statement, the -standard builtin :func:`__import__` is called. Other mechanisms for -invoking the import system (such as :func:`importlib.import_module`) may -choose to subvert :func:`__import__` and use its own solution to -implement import semantics. - -When a module is first imported, Python searches for the module and if found, -it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, an :exc:`ImportError` is raised. Python implements various -strategies to search for the named module when the import machinery is -invoked. These strategies can be modified and extended by using various hooks -described in the sections below. - -.. versionchanged:: 3.3 - The import system has been updated to fully implement the second phase - of :pep:`302`. There is no longer any implicit import machinery - the full - import system is exposed through :data:`sys.meta_path`. In addition, - native namespace package support has been implemented (see :pep:`420`). - - -:mod:`importlib` -================ - -The :mod:`importlib` module provides a rich API for interacting with the -import system. For example :func:`importlib.import_module` provides a -recommended, simpler API than built-in :func:`__import__` for invoking the -import machinery. Refer to the :mod:`importlib` library documentation for -additional detail. - - - -Packages -======== - -.. index:: - single: package - -Python has only one type of module object, and all modules are of this type, -regardless of whether the module is implemented in Python, C, or something -else. To help organize modules and provide a naming hierarchy, Python has a -concept of :term:`packages `. - -You can think of packages as the directories on a file system and modules as -files within directories, but don't take this analogy too literally since -packages and modules need not originate from the file system. For the -purposes of this documentation, we'll use this convenient analogy of -directories and files. Like file system directories, packages are organized -hierarchically, and packages may themselves contain subpackages, as well as -regular modules. - -It's important to keep in mind that all packages are modules, but not all -modules are packages. Or put another way, packages are just a special kind of -module. Specifically, any module that contains a ``__path__`` attribute is -considered a package. - -All modules have a name. Subpackage names are separated from their parent -package name by dots, akin to Python's standard attribute access syntax. Thus -you might have a module called :mod:`sys` and a package called :mod:`email`, -which in turn has a subpackage called :mod:`email.mime` and a module within -that subpackage called :mod:`email.mime.text`. - - -Regular packages ----------------- - -.. index:: - pair: package; regular - -Python defines two types of packages, :term:`regular packages ` and :term:`namespace packages `. Regular -packages are traditional packages as they existed in Python 3.2 and earlier. -A regular package is typically implemented as a directory containing an -``__init__.py`` file. When a regular package is imported, this -``__init__.py`` file is implicitly executed, and the objects it defines are -bound to names in the package's namespace. The ``__init__.py`` file can -contain the same Python code that any other module can contain, and Python -will add some additional attributes to the module when it is imported. - -For example, the following file system layout defines a top level ``parent`` -package with three subpackages:: - - parent/ - __init__.py - one/ - __init__.py - two/ - __init__.py - three/ - __init__.py - -Importing ``parent.one`` will implicitly execute ``parent/__init__.py`` and -``parent/one/__init__.py``. Subsequent imports of ``parent.two`` or -``parent.three`` will execute ``parent/two/__init__.py`` and -``parent/three/__init__.py`` respectively. - - -Namespace packages ------------------- - -.. index:: - pair:: package; namespace - pair:: package; portion - -A namespace package is a composite of various :term:`portions `, -where each portion contributes a subpackage to the parent package. Portions -may reside in different locations on the file system. Portions may also be -found in zip files, on the network, or anywhere else that Python searches -during import. Namespace packages may or may not correspond directly to -objects on the file system; they may be virtual modules that have no concrete -representation. - -Namespace packages do not use an ordinary list for their ``__path__`` -attribute. They instead use a custom iterable type which will automatically -perform a new search for package portions on the next import attempt within -that package if the path of their parent package (or :data:`sys.path` for a -top level package) changes. - -With namespace packages, there is no ``parent/__init__.py`` file. In fact, -there may be multiple ``parent`` directories found during import search, where -each one is provided by a different portion. Thus ``parent/one`` may not be -physically located next to ``parent/two``. In this case, Python will create a -namespace package for the top-level ``parent`` package whenever it or one of -its subpackages is imported. - -See also :pep:`420` for the namespace package specification. - - -Searching -========= - -To begin the search, Python needs the :term:`fully qualified ` -name of the module (or package, but for the purposes of this discussion, the -difference is immaterial) being imported. This name may come from various -arguments to the :keyword:`import` statement, or from the parameters to the -:func:`importlib.import_module` or :func:`__import__` functions. - -This name will be used in various phases of the import search, and it may be -the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python -first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, an :exc:`ImportError` is raised. - - -The module cache ----------------- - -.. index:: - single: sys.modules - -The first place checked during import search is :data:`sys.modules`. This -mapping serves as a cache of all modules that have been previously imported, -including the intermediate paths. So if ``foo.bar.baz`` was previously -imported, :data:`sys.modules` will contain entries for ``foo``, ``foo.bar``, -and ``foo.bar.baz``. Each key will have as its value the corresponding module -object. - -During import, the module name is looked up in :data:`sys.modules` and if -present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then an -:exc:`ImportError` is raised. If the module name is missing, Python will -continue searching for the module. - -:data:`sys.modules` is writable. Deleting a key may not destroy the -associated module (as other modules may hold references to it), -but it will invalidate the cache entry for the named module, causing -Python to search anew for the named module upon its next -import. The key can also be assigned to ``None``, forcing the next import -of the module to result in an :exc:`ImportError`. - -Beware though, as if you keep a reference to the module object, -invalidate its cache entry in :data:`sys.modules`, and then re-import the -named module, the two module objects will *not* be the same. By contrast, -:func:`importlib.reload` will reuse the *same* module object, and simply -reinitialise the module contents by rerunning the module's code. - - -Finders and loaders -------------------- - -.. index:: - single: finder - single: loader - single: module spec - -If the named module is not found in :data:`sys.modules`, then Python's import -protocol is invoked to find and load the module. This protocol consists of -two conceptual objects, :term:`finders ` and :term:`loaders `. -A finder's job is to determine whether it can find the named module using -whatever strategy it knows about. Objects that implement both of these -interfaces are referred to as :term:`importers ` - they return -themselves when they find that they can load the requested module. - -Python includes a number of default finders and importers. The first one -knows how to locate built-in modules, and the second knows how to locate -frozen modules. A third default finder searches an :term:`import path` -for modules. The :term:`import path` is a list of locations that may -name file system paths or zip files. It can also be extended to search -for any locatable resource, such as those identified by URLs. - -The import machinery is extensible, so new finders can be added to extend the -range and scope of module searching. - -Finders do not actually load modules. If they can find the named module, they -return a :dfn:`module spec`, an encapsulation of the module's import-related -information, which the import machinery then uses when loading the module. - -The following sections describe the protocol for finders and loaders in more -detail, including how you can create and register new ones to extend the -import machinery. - -.. versionchanged:: 3.4 - In previous versions of Python, finders returned :term:`loaders ` - directly, whereas now they return module specs which *contain* loaders. - Loaders are still used during import but have fewer responsibilities. - -Import hooks ------------- - -.. index:: - single: import hooks - single: meta hooks - single: path hooks - pair: hooks; import - pair: hooks; meta - pair: hooks; path - -The import machinery is designed to be extensible; the primary mechanism for -this are the *import hooks*. There are two types of import hooks: *meta -hooks* and *import path hooks*. - -Meta hooks are called at the start of import processing, before any other -import processing has occurred, other than :data:`sys.modules` cache look up. -This allows meta hooks to override :data:`sys.path` processing, frozen -modules, or even built-in modules. Meta hooks are registered by adding new -finder objects to :data:`sys.meta_path`, as described below. - -Import path hooks are called as part of :data:`sys.path` (or -``package.__path__``) processing, at the point where their associated path -item is encountered. Import path hooks are registered by adding new callables -to :data:`sys.path_hooks` as described below. - - -The meta path -------------- - -.. index:: - single: sys.meta_path - pair: finder; find_spec - -When the named module is not found in :data:`sys.modules`, Python next -searches :data:`sys.meta_path`, which contains a list of meta path finder -objects. These finders are queried in order to see if they know how to handle -the named module. Meta path finders must implement a method called -:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: -a name, an import path, and (optionally) a target module. The meta path -finder can use any strategy it wants to determine whether it can handle -the named module or not. - -If the meta path finder knows how to handle the named module, it returns a -spec object. If it cannot handle the named module, it returns ``None``. If -:data:`sys.meta_path` processing reaches the end of its list without returning -a spec, then an :exc:`ImportError` is raised. Any other exceptions raised -are simply propagated up, aborting the import process. - -The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path -finders is called with two or three arguments. The first is the fully -qualified name of the module being imported, for example ``foo.bar.baz``. -The second argument is the path entries to use for the module search. For -top-level modules, the second argument is ``None``, but for submodules or -subpackages, the second argument is the value of the parent package's -``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, an :exc:`ImportError` is raised. The third argument is an -existing module object that will be the target of loading later. The -import system passes in a target module only during reload. - -The meta path may be traversed multiple times for a single import request. -For example, assuming none of the modules involved has already been cached, -importing ``foo.bar.baz`` will first perform a top level import, calling -``mpf.find_spec("foo", None, None)`` on each meta path finder (``mpf``). After -``foo`` has been imported, ``foo.bar`` will be imported by traversing the -meta path a second time, calling -``mpf.find_spec("foo.bar", foo.__path__, None)``. Once ``foo.bar`` has been -imported, the final traversal will call -``mpf.find_spec("foo.bar.baz", foo.bar.__path__, None)``. - -Some meta path finders only support top level imports. These importers will -always return ``None`` when anything other than ``None`` is passed as the -second argument. - -Python's default :data:`sys.meta_path` has three meta path finders, one that -knows how to import built-in modules, one that knows how to import frozen -modules, and one that knows how to import modules from an :term:`import path` -(i.e. the :term:`path based finder`). - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path - finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which - is now deprecated. While it will continue to work without change, the - import machinery will try it only if the finder does not implement - ``find_spec()``. - - -Loading -======= - -If and when a module spec is found, the import machinery will use it (and -the loader it contains) when loading the module. Here is an approximation -of what happens during the loading portion of import:: - - module = None - if spec.loader is not None and hasattr(spec.loader, 'create_module'): - # It is assumed 'exec_module' will also be defined on the loader. - module = spec.loader.create_module(spec) - if module is None: - module = ModuleType(spec.name) - # The import-related module attributes get set here: - _init_module_attrs(spec, module) - - if spec.loader is None: - if spec.submodule_search_locations is not None: - # namespace package - sys.modules[spec.name] = module - else: - # unsupported - raise ImportError - elif not hasattr(spec.loader, 'exec_module'): - module = spec.loader.load_module(spec.name) - # Set __loader__ and __package__ if missing. - else: - sys.modules[spec.name] = module - try: - spec.loader.exec_module(module) - except BaseException: - try: - del sys.modules[spec.name] - except KeyError: - pass - raise - return sys.modules[spec.name] - -Note the following details: - - * If there is an existing module object with the given name in - :data:`sys.modules`, import will have already returned it. - - * The module will exist in :data:`sys.modules` before the loader - executes the module code. This is crucial because the module code may - (directly or indirectly) import itself; adding it to :data:`sys.modules` - beforehand prevents unbounded recursion in the worst case and multiple - loading in the best. - - * If loading fails, the failing module -- and only the failing module -- - gets removed from :data:`sys.modules`. Any module already in the - :data:`sys.modules` cache, and any module that was successfully loaded - as a side-effect, must remain in the cache. This contrasts with - reloading where even the failing module is left in :data:`sys.modules`. - - * After the module is created but before execution, the import machinery - sets the import-related module attributes ("_init_module_attrs" in - the pseudo-code example above), as summarized in a - :ref:`later section `. - - * Module execution is the key moment of loading in which the module's - namespace gets populated. Execution is entirely delegated to the - loader, which gets to decide what gets populated and how. - - * The module created during loading and passed to exec_module() may - not be the one returned at the end of import [#fnlo]_. - -.. versionchanged:: 3.4 - The import system has taken over the boilerplate responsibilities of - loaders. These were previously performed by the - :meth:`importlib.abc.Loader.load_module` method. - -Loaders -------- - -Module loaders provide the critical function of loading: module execution. -The import machinery calls the :meth:`importlib.abc.Loader.exec_module` -method with a single argument, the module object to execute. Any value -returned from :meth:`~importlib.abc.Loader.exec_module` is ignored. - -Loaders must satisfy the following requirements: - - * If the module is a Python module (as opposed to a built-in module or a - dynamically loaded extension), the loader should execute the module's code - in the module's global name space (``module.__dict__``). - - * If the loader cannot execute the module, it should raise an - :exc:`ImportError`, although any other exception raised during - :meth:`~importlib.abc.Loader.exec_module` will be propagated. - -In many cases, the finder and loader can be the same object; in such cases the -:meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a -spec with the loader set to ``self``. - -Module loaders may opt in to creating the module object during loading -by implementing a :meth:`~importlib.abc.Loader.create_module` method. -It takes one argument, the module spec, and returns the new module object -to use during loading. ``create_module()`` does not need to set any attributes -on the module object. If the method returns ``None``, the -import machinery will create the new module itself. - -.. versionadded:: 3.4 - The create_module() method of loaders. - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.Loader.load_module` method was replaced by - :meth:`~importlib.abc.Loader.exec_module` and the import - machinery assumed all the boilerplate responsibilities of loading. - - For compatibility with existing loaders, the import machinery will use - the ``load_module()`` method of loaders if it exists and the loader does - not also implement ``exec_module()``. However, ``load_module()`` has been - deprecated and loaders should implement ``exec_module()`` instead. - - The ``load_module()`` method must implement all the boilerplate loading - functionality described above in addition to executing the module. All - the same constraints apply, with some additional clarification: - - * If there is an existing module object with the given name in - :data:`sys.modules`, the loader must use that existing module. - (Otherwise, :func:`importlib.reload` will not work correctly.) If the - named module does not exist in :data:`sys.modules`, the loader - must create a new module object and add it to :data:`sys.modules`. - - * The module *must* exist in :data:`sys.modules` before the loader - executes the module code, to prevent unbounded recursion or multiple - loading. - - * If loading fails, the loader must remove any modules it has inserted - into :data:`sys.modules`, but it must remove **only** the failing - module(s), and only if the loader itself has loaded the module(s) - explicitly. - -.. versionchanged:: 3.5 - A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. Starting in Python 3.6 it will be an error to not - define ``create_module()`` on a loader attached to a ModuleSpec. - -Submodules ----------- - -When a submodule is loaded using any mechanism (e.g. ``importlib`` APIs, the -``import`` or ``import-from`` statements, or built-in ``__import__()``) a -binding is placed in the parent module's namespace to the submodule object. -For example, if package ``spam`` has a submodule ``foo``, after importing -``spam.foo``, ``spam`` will have an attribute ``foo`` which is bound to the -submodule. Let's say you have the following directory structure:: - - spam/ - __init__.py - foo.py - bar.py - -and ``spam/__init__.py`` has the following lines in it:: - - from .foo import Foo - from .bar import Bar - -then executing the following puts a name binding to ``foo`` and ``bar`` in the -``spam`` module:: - - >>> import spam - >>> spam.foo - - >>> spam.bar - - -Given Python's familiar name binding rules this might seem surprising, but -it's actually a fundamental feature of the import system. The invariant -holding is that if you have ``sys.modules['spam']`` and -``sys.modules['spam.foo']`` (as you would after the above import), the latter -must appear as the ``foo`` attribute of the former. - -Module spec ------------ - -The import machinery uses a variety of information about each module -during import, especially before loading. Most of the information is -common to all modules. The purpose of a module's spec is to encapsulate -this import-related information on a per-module basis. - -Using a spec during import allows state to be transferred between import -system components, e.g. between the finder that creates the module spec -and the loader that executes it. Most importantly, it allows the -import machinery to perform the boilerplate operations of loading, -whereas without a module spec the loader had that responsibility. - -See :class:`~importlib.machinery.ModuleSpec` for more specifics on what -information a module's spec may hold. - -.. versionadded:: 3.4 - -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully-qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - -.. attribute:: __package__ - - The module's ``__package__`` attribute must be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. It is - expected to have the same value as ``__spec__.parent``. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__package__`` is not defined, ``__spec__.parent`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional. If set, this attribute's value must be a - string. The import system may opt to leave ``__file__`` unset if it - has no semantic meaning (e.g. a module loaded from a database). - - If ``__file__`` is set, it may also be appropriate to set the - ``__cached__`` attribute which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - It is also appropriate to set ``__cached__`` when ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of ``__file__`` and/or ``__cached__``. So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - -.. _package-path-rules: - -module.__path__ ---------------- - -By definition, if a module has an ``__path__`` attribute, it is a package, -regardless of its value. - -A package's ``__path__`` attribute is used during imports of its subpackages. -Within the import machinery, it functions much the same as :data:`sys.path`, -i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. - -``__path__`` must be an iterable of strings, but it may be empty. -The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. - -A package's ``__init__.py`` file may set or alter the package's ``__path__`` -attribute, and this was typically the way namespace packages were implemented -prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` -correctly for the namespace package. - -Module reprs ------------- - -By default, all modules have a usable repr, however depending on the -attributes set above, and in the module's spec, you can more explicitly -control the repr of module objects. - -If the module has a spec (``__spec__``), the import machinery will try -to generate a repr from it. If that fails or there is no spec, the import -system will craft a default repr using whatever information is available -on the module. It will try to use the ``module.__name__``, -``module.__file__``, and ``module.__loader__`` as input into the repr, -with defaults for whatever information is missing. - -Here are the exact rules used: - - * If the module has a ``__spec__`` attribute, the information in the spec - is used to generate the repr. The "name", "loader", "origin", and - "has_location" attributes are consulted. - - * If the module has a ``__file__`` attribute, this is used as part of the - module's repr. - - * If the module has no ``__file__`` but does have a ``__loader__`` that is not - ``None``, then the loader's repr is used as part of the module's repr. - - * Otherwise, just use the module's ``__name__`` in the repr. - -.. versionchanged:: 3.4 - Use of :meth:`loader.module_repr() ` - has been deprecated and the module spec is now used by the import - machinery to generate a module repr. - - For backward compatibility with Python 3.3, the module repr will be - generated by calling the loader's - :meth:`~importlib.abc.Loader.module_repr` method, if defined, before - trying either approach described above. However, the method is deprecated. - - -The Path Based Finder -===================== - -.. index:: - single: path based finder - -As mentioned previously, Python comes with several default meta path finders. -One of these, called the :term:`path based finder` -(:class:`~importlib.machinery.PathFinder`), searches an :term:`import path`, -which contains a list of :term:`path entries `. Each path -entry names a location to search for modules. - -The path based finder itself doesn't know how to import anything. Instead, it -traverses the individual path entries, associating each of them with a -path entry finder that knows how to handle that particular kind of path. - -The default set of path entry finders implement all the semantics for finding -modules on the file system, handling special file types such as Python source -code (``.py`` files), Python byte code (``.pyc`` files) and -shared libraries (e.g. ``.so`` files). When supported by the :mod:`zipimport` -module in the standard library, the default path entry finders also handle -loading all of these file types (other than shared libraries) from zipfiles. - -Path entries need not be limited to file system locations. They can refer to -URLs, database queries, or any other location that can be specified as a -string. - -The path based finder provides additional hooks and protocols so that you -can extend and customize the types of searchable path entries. For example, -if you wanted to support path entries as network URLs, you could write a hook -that implements HTTP semantics to find modules on the web. This hook (a -callable) would return a :term:`path entry finder` supporting the protocol -described below, which was then used to get a loader for the module from the -web. - -A word of warning: this section and the previous both use the term *finder*, -distinguishing between them by using the terms :term:`meta path finder` and -:term:`path entry finder`. These two types of finders are very similar, -support similar protocols, and function in similar ways during the import -process, but it's important to keep in mind that they are subtly different. -In particular, meta path finders operate at the beginning of the import -process, as keyed off the :data:`sys.meta_path` traversal. - -By contrast, path entry finders are in a sense an implementation detail -of the path based finder, and in fact, if the path based finder were to be -removed from :data:`sys.meta_path`, none of the path entry finder semantics -would be invoked. - - -Path entry finders ------------------- - -.. index:: - single: sys.path - single: sys.path_hooks - single: sys.path_importer_cache - single: PYTHONPATH - -The :term:`path based finder` is responsible for finding and loading -Python modules and packages whose location is specified with a string -:term:`path entry`. Most path entries name locations in the file system, -but they need not be limited to this. - -As a meta path finder, the :term:`path based finder` implements the -:meth:`~importlib.abc.MetaPathFinder.find_spec` protocol previously -described, however it exposes additional hooks that can be used to -customize how modules are found and loaded from the :term:`import path`. - -Three variables are used by the :term:`path based finder`, :data:`sys.path`, -:data:`sys.path_hooks` and :data:`sys.path_importer_cache`. The ``__path__`` -attributes on package objects are also used. These provide additional ways -that the import machinery can be customized. - -:data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` -environment variable and various other installation- and -implementation-specific defaults. Entries in :data:`sys.path` can name -directories on the file system, zip files, and potentially other "locations" -(see the :mod:`site` module) that should be searched for modules, such as -URLs, or database queries. Only strings and bytes should be present on -:data:`sys.path`; all other data types are ignored. The encoding of bytes -entries is determined by the individual :term:`path entry finders `. - -The :term:`path based finder` is a :term:`meta path finder`, so the import -machinery begins the :term:`import path` search by calling the path -based finder's :meth:`~importlib.machinery.PathFinder.find_spec` method as -described previously. When the ``path`` argument to -:meth:`~importlib.machinery.PathFinder.find_spec` is given, it will be a -list of string paths to traverse - typically a package's ``__path__`` -attribute for an import within that package. If the ``path`` argument is -``None``, this indicates a top level import and :data:`sys.path` is used. - -The path based finder iterates over every entry in the search path, and -for each of these, looks for an appropriate :term:`path entry finder` -(:class:`~importlib.abc.PathEntryFinder`) for the -path entry. Because this can be an expensive operation (e.g. there may be -`stat()` call overheads for this search), the path based finder maintains -a cache mapping path entries to path entry finders. This cache is maintained -in :data:`sys.path_importer_cache` (despite the name, this cache actually -stores finder objects rather than being limited to :term:`importer` objects). -In this way, the expensive search for a particular :term:`path entry` -location's :term:`path entry finder` need only be done once. User code is -free to remove cache entries from :data:`sys.path_importer_cache` forcing -the path based finder to perform the path entry search again [#fnpic]_. - -If the path entry is not present in the cache, the path based finder iterates -over every callable in :data:`sys.path_hooks`. Each of the :term:`path entry -hooks ` in this list is called with a single argument, the -path entry to be searched. This callable may either return a :term:`path -entry finder` that can handle the path entry, or it may raise -:exc:`ImportError`. An :exc:`ImportError` is used by the path based finder to -signal that the hook cannot find a :term:`path entry finder`. -for that :term:`path entry`. The -exception is ignored and :term:`import path` iteration continues. The hook -should expect either a string or bytes object; the encoding of bytes objects -is up to the hook (e.g. it may be a file system encoding, UTF-8, or something -else), and if the hook cannot decode the argument, it should raise -:exc:`ImportError`. - -If :data:`sys.path_hooks` iteration ends with no :term:`path entry finder` -being returned, then the path based finder's -:meth:`~importlib.machinery.PathFinder.find_spec` method will store ``None`` -in :data:`sys.path_importer_cache` (to indicate that there is no finder for -this path entry) and return ``None``, indicating that this -:term:`meta path finder` could not find the module. - -If a :term:`path entry finder` *is* returned by one of the :term:`path entry -hook` callables on :data:`sys.path_hooks`, then the following protocol is used -to ask the finder for a module spec, which is then used when loading the -module. - -The current working directory -- denoted by an empty string -- is handled -slightly differently from other entries on :data:`sys.path`. First, if the -current working directory is found to not exist, no value is stored in -:data:`sys.path_importer_cache`. Second, the value for the current working -directory is looked up fresh for each module lookup. Third, the path used for -:data:`sys.path_importer_cache` and returned by -:meth:`importlib.machinery.PathFinder.find_spec` will be the actual current -working directory and not the empty string. - -Path entry finder protocol --------------------------- - -In order to support imports of modules and initialized packages and also to -contribute portions to namespace packages, path entry finders must implement -the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. - -:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the -fully qualified name of the module being imported, and the (optional) target -module. ``find_spec()`` returns a fully populated spec for the module. -This spec will always have "loader" set (with one exception). - -To indicate to the import machinery that the spec represents a namespace -:term:`portion`. the path entry finder sets "loader" on the spec to -``None`` and "submodule_search_locations" to a list containing the -portion. - -.. versionchanged:: 3.4 - :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced - :meth:`~importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which - are now deprecated, but will be used if ``find_spec()`` is not defined. - - Older path entry finders may implement one of these two deprecated methods - instead of ``find_spec()``. The methods are still respected for the - sake of backward compatibility. Howevever, if ``find_spec()`` is - implemented on the path entry finder, the legacy methods are ignored. - - :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the - fully qualified name of the module being imported. ``find_loader()`` - returns a 2-tuple where the first item is the loader and the second item - is a namespace :term:`portion`. When the first item (i.e. the loader) is - ``None``, this means that while the path entry finder does not have a - loader for the named module, it knows that the path entry contributes to - a namespace portion for the named module. This will almost always be the - case where Python is asked to import a namespace package that has no - physical presence on the file system. When a path entry finder returns - ``None`` for the loader, the second item of the 2-tuple return value must - be a sequence, although it can be empty. - - If ``find_loader()`` returns a non-``None`` loader value, the portion is - ignored and the loader is returned from the path based finder, terminating - the search through the path entries. - - For backwards compatibility with other implementations of the import - protocol, many path entry finders also support the same, - traditional ``find_module()`` method that meta path finders support. - However path entry finder ``find_module()`` methods are never called - with a ``path`` argument (they are expected to record the appropriate - path information from the initial call to the path hook). - - The ``find_module()`` method on path entry finders is deprecated, - as it does not allow the path entry finder to contribute portions to - namespace packages. If both ``find_loader()`` and ``find_module()`` - exist on a path entry finder, the import system will always call - ``find_loader()`` in preference to ``find_module()``. - - -Replacing the standard import system -==================================== - -The most reliable mechanism for replacing the entire import system is to -delete the default contents of :data:`sys.meta_path`, replacing them -entirely with a custom meta path hook. - -If it is acceptable to only alter the behaviour of import statements -without affecting other APIs that access the import system, then replacing -the builtin :func:`__import__` function may be sufficient. This technique -may also be employed at the module level to only alter the behaviour of -import statements within that module. - -To selectively prevent import of some modules from a hook early on the -meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ImportError` directly from -:meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning -``None``. The latter indicates that the meta path search should continue, -while raising an exception terminates it immediately. - - -Special considerations for __main__ -=================================== - -The :mod:`__main__` module is a special case relative to Python's import -system. As noted :ref:`elsewhere `, the ``__main__`` module -is directly initialized at interpreter startup, much like :mod:`sys` and -:mod:`builtins`. However, unlike those two, it doesn't strictly -qualify as a built-in module. This is because the manner in which -``__main__`` is initialized depends on the flags and other options with -which the interpreter is invoked. - -.. _main_spec: - -__main__.__spec__ ------------------ - -Depending on how :mod:`__main__` is initialized, ``__main__.__spec__`` -gets set appropriately or to ``None``. - -When Python is started with the :option:`-m` option, ``__spec__`` is set -to the module spec of the corresponding module or package. ``__spec__`` is -also populated when the ``__main__`` module is loaded as part of executing a -directory, zipfile or other :data:`sys.path` entry. - -In :ref:`the remaining cases ` -``__main__.__spec__`` is set to ``None``, as the code used to populate the -:mod:`__main__` does not correspond directly with an importable module: - -- interactive prompt -- -c switch -- running from stdin -- running directly from a source or bytecode file - -Note that ``__main__.__spec__`` is always ``None`` in the last case, -*even if* the file could technically be imported directly as a module -instead. Use the :option:`-m` switch if valid module metadata is desired -in :mod:`__main__`. - -Note also that even when ``__main__`` corresponds with an importable module -and ``__main__.__spec__`` is set accordingly, they're still considered -*distinct* modules. This is due to the fact that blocks guarded by -``if __name__ == "__main__":`` checks only execute when the module is used -to populate the ``__main__`` namespace, and not during normal import. - - -Open issues -=========== - -XXX It would be really nice to have a diagram. - -XXX * (import_machinery.rst) how about a section devoted just to the -attributes of modules and packages, perhaps expanding upon or supplanting the -related entries in the data model reference page? - -XXX runpy, pkgutil, et al in the library manual should all get "See Also" -links at the top pointing to the new import system section. - -XXX Add more explanation regarding the different ways in which -``__main__`` is initialized? - -XXX Add more info on ``__main__`` quirks/pitfalls (i.e. copy from -:pep:`395`). - - -References -========== - -The import machinery has evolved considerably since Python's early days. The -original `specification for packages -`_ is still available to read, -although some details have changed since the writing of that document. - -The original specification for :data:`sys.meta_path` was :pep:`302`, with -subsequent extension in :pep:`420`. - -:pep:`420` introduced :term:`namespace packages ` for -Python 3.3. :pep:`420` also introduced the :meth:`find_loader` protocol as an -alternative to :meth:`find_module`. - -:pep:`366` describes the addition of the ``__package__`` attribute for -explicit relative imports in main modules. - -:pep:`328` introduced absolute and explicit relative imports and initially -proposed ``__name__`` for semantics :pep:`366` would eventually specify for -``__package__``. - -:pep:`338` defines executing modules as scripts. - -:pep:`451` adds the encapsulation of per-module import state in spec -objects. It also off-loads most of the boilerplate responsibilities of -loaders back onto the import machinery. These changes allow the -deprecation of several APIs in the import system and also addition of new -methods to finders and loaders. - -.. rubric:: Footnotes - -.. [#fnmo] See :class:`types.ModuleType`. - -.. [#fnlo] The importlib implementation avoids using the return value - directly. Instead, it gets the module object by looking the module name up - in :data:`sys.modules`. The indirect effect of this is that an imported - module may replace itself in :data:`sys.modules`. This is - implementation-specific behavior that is not guaranteed to work in other - Python implementations. - -.. [#fnpic] In legacy code, it is possible to find instances of - :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It - is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/index.rst --- a/Doc/reference/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,9 @@ The Python Language Reference ################################# +:Release: |version| +:Date: |today| + This reference manual describes the syntax and "core semantics" of the language. It is terse, but attempts to be exact and complete. The semantics of non-essential built-in object types and of the built-in functions and modules @@ -21,7 +24,6 @@ lexical_analysis.rst datamodel.rst executionmodel.rst - import.rst expressions.rst simple_stmts.rst compound_stmts.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/introduction.rst --- a/Doc/reference/introduction.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/introduction.rst Mon Jan 25 17:05:13 2016 +0100 @@ -66,7 +66,7 @@ An alternate Python for .NET. Unlike Python.NET, this is a complete Python implementation that generates IL, and compiles Python code directly to .NET assemblies. It was created by Jim Hugunin, the original creator of Jython. For - more information, see `the IronPython website `_. + more information, see `the IronPython website `_. PyPy An implementation of Python written completely in Python. It supports several diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/lexical_analysis.rst --- a/Doc/reference/lexical_analysis.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/lexical_analysis.rst Mon Jan 25 17:05:13 2016 +0100 @@ -76,14 +76,12 @@ Encoding declarations --------------------- -.. index:: source character set, encoding declarations (source file) +.. index:: source character set, encodings If a comment in the first or second line of the Python script matches the regular expression ``coding[=:]\s*([-\w.]+)``, this comment is processed as an encoding declaration; the first group of this expression names the encoding of -the source code file. The encoding declaration must appear on a line of its -own. If it is the second line, the first line must also be a comment-only line. -The recommended forms of an encoding expression are :: +the source code file. The recommended forms of this expression are :: # -*- coding: -*- @@ -100,7 +98,7 @@ If an encoding is declared, the encoding name must be recognized by Python. The encoding is used for all lexical analysis, including string literals, comments -and identifiers. +and identifiers. The encoding declaration must appear on a line of its own. .. XXX there should be a list of supported encodings. @@ -312,9 +310,7 @@ * *Mc* - spacing combining marks * *Nd* - decimal numbers * *Pc* - connector punctuations -* *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards - compatibility +* *Other_ID_Start* - explicit list of characters in `PropList.txt `_ to support backwards compatibility * *Other_ID_Continue* - likewise All identifiers are converted into the normal form NFKC while parsing; comparison @@ -405,7 +401,7 @@ .. productionlist:: stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" + stringprefix: "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR" shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' shortstringitem: `shortstringchar` | `stringescapeseq` @@ -445,34 +441,32 @@ may only contain ASCII characters; bytes with a numeric value of 128 or greater must be expressed with escapes. -As of Python 3.3 it is possible again to prefix string literals with a +As of Python 3.3 it is possible again to prefix unicode strings with a ``u`` prefix to simplify maintenance of dual 2.x and 3.x codebases. Both string and bytes literals may optionally be prefixed with a letter ``'r'`` or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` -escapes in raw strings are not treated specially. Given that Python 2.x's raw -unicode literals behave differently than Python 3.x's the ``'ur'`` syntax -is not supported. +escapes in raw strings are not treated specially. -.. versionadded:: 3.3 - The ``'rb'`` prefix of raw bytes literals has been added as a synonym - of ``'br'``. + .. versionadded:: 3.3 + The ``'rb'`` prefix of raw bytes literals has been added as a synonym + of ``'br'``. -.. versionadded:: 3.3 - Support for the unicode legacy literal (``u'value'``) was reintroduced - to simplify the maintenance of dual Python 2.x and 3.x codebases. - See :pep:`414` for more information. + .. versionadded:: 3.3 + Support for the unicode legacy literal (``u'value'``) and other + versions were reintroduced to simplify the maintenance of dual + Python 2.x and 3.x codebases. See :pep:`414` for more information. -In triple-quoted literals, unescaped newlines and quotes are allowed (and are -retained), except that three unescaped quotes in a row terminate the literal. (A -"quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) +In triple-quoted strings, unescaped newlines and quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the string. (A +"quote" is the character used to open the string, i.e. either ``'`` or ``"``.) .. index:: physical line, escape sequence, Standard C, C -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and -bytes literals are interpreted according to rules similar to those used by -Standard C. The recognized escape sequences are: +Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in strings are +interpreted according to rules similar to those used by Standard C. The +recognized escape sequences are: +-----------------+---------------------------------+-------+ | Escape Sequence | Meaning | Notes | @@ -542,27 +536,29 @@ this escape sequence. Exactly four hex digits are required. (6) - Any Unicode character can be encoded this way. Exactly eight hex digits + Any Unicode character can be encoded this way, but characters outside the Basic + Multilingual Plane (BMP) will be encoded using a surrogate pair if Python is + compiled to use 16-bit code units (the default). Exactly eight hex digits are required. .. index:: unrecognized escape sequence Unlike Standard C, all unrecognized escape sequences are left in the string -unchanged, i.e., *the backslash is left in the result*. (This behavior is +unchanged, i.e., *the backslash is left in the string*. (This behavior is useful when debugging: if an escape sequence is mistyped, the resulting output is more easily recognized as broken.) It is also important to note that the escape sequences only recognized in string literals fall into the category of unrecognized escapes for bytes literals. -Even in a raw literal, quotes can be escaped with a backslash, but the -backslash remains in the result; for example, ``r"\""`` is a valid string +Even in a raw string, string quotes can be escaped with a backslash, but the +backslash remains in the string; for example, ``r"\""`` is a valid string literal consisting of two characters: a backslash and a double quote; ``r"\"`` is not a valid string literal (even a raw string cannot end in an odd number of -backslashes). Specifically, *a raw literal cannot end in a single backslash* +backslashes). Specifically, *a raw string cannot end in a single backslash* (since the backslash would escape the following quote character). Note also that a single backslash followed by a newline is interpreted as those two -characters as part of the literal, *not* as a line continuation. +characters as part of the string, *not* as a line continuation. .. _string-catenation: @@ -634,7 +630,8 @@ Some examples of integer literals:: 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef + 3 79228162514264337593543950336 0o377 0x100000000 + 79228162514264337593543950336 0xdeadbeef .. _floating: @@ -692,7 +689,7 @@ The following tokens are operators:: - + - * ** / // % @ + + - * ** / // % << >> & | ^ ~ < > <= >= == != @@ -707,8 +704,8 @@ The following tokens serve as delimiters in the grammar:: ( ) [ ] { } - , : . ; @ = -> - += -= *= /= //= %= @= + , : . ; @ = + += -= *= /= //= %= &= |= ^= >>= <<= **= The period can also occur in floating-point and imaginary literals. A sequence @@ -729,4 +726,4 @@ .. rubric:: Footnotes -.. [#] http://www.unicode.org/Public/8.0.0/ucd/NameAliases.txt +.. [#] http://www.unicode.org/Public/6.1.0/ucd/NameAliases.txt diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/simple_stmts.rst --- a/Doc/reference/simple_stmts.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/simple_stmts.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,7 +7,7 @@ .. index:: pair: simple; statement -A simple statement is comprised within a single logical line. Several simple +Simple statements are comprised within a single logical line. Several simple statements may occur on a single line separated by semicolons. The syntax for simple statements is: @@ -70,7 +70,6 @@ ===================== .. index:: - single: =; assignment statement pair: assignment; statement pair: binding; name pair: rebinding; name @@ -91,8 +90,8 @@ : | `slicing` : | "*" `target` -(See section :ref:`primaries` for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) +(See section :ref:`primaries` for the syntax definitions for the last three +symbols.) An assignment statement evaluates the expression list (remember that this can be a single expression or a comma-separated list, the latter yielding a tuple) and @@ -228,7 +227,7 @@ inclusive. Finally, the sequence object is asked to replace the slice with the items of the assigned sequence. The length of the slice may be different from the length of the assigned sequence, thus changing the length of the - target sequence, if the target sequence allows it. + target sequence, if the object allows it. .. impl-detail:: @@ -236,15 +235,14 @@ as for expressions, and invalid syntax is rejected during the code generation phase, causing less detailed error messages. -Although the definition of assignment implies that overlaps between the -left-hand side and the right-hand side are 'simultanenous' (for example ``a, b = -b, a`` swaps two variables), overlaps *within* the collection of assigned-to -variables occur left-to-right, sometimes resulting in confusion. For instance, -the following program prints ``[0, 2]``:: +WARNING: Although the definition of assignment implies that overlaps between the +left-hand side and the right-hand side are 'safe' (for example ``a, b = b, a`` +swaps two variables), overlaps *within* the collection of assigned-to variables +are not safe! For instance, the following program prints ``[0, 2]``:: x = [0, 1] i = 0 - i, x[i] = 1, 2 # i is updated, then x[i] is updated + i, x[i] = 1, 2 print(x) @@ -262,18 +260,6 @@ .. index:: pair: augmented; assignment single: statement; assignment, augmented - single: +=; augmented assignment - single: -=; augmented assignment - single: *=; augmented assignment - single: /=; augmented assignment - single: %=; augmented assignment - single: &=; augmented assignment - single: ^=; augmented assignment - single: |=; augmented assignment - single: **=; augmented assignment - single: //=; augmented assignment - single: >>=; augmented assignment - single: <<=; augmented assignment Augmented assignment is the combination, in a single statement, of a binary operation and an assignment statement: @@ -281,10 +267,10 @@ .. productionlist:: augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) augtarget: `identifier` | `attributeref` | `subscription` | `slicing` - augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" + augop: "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**=" : | ">>=" | "<<=" | "&=" | "^=" | "|=" -(See section :ref:`primaries` for the syntax definitions of the last three +(See section :ref:`primaries` for the syntax definitions for the last three symbols.) An augmented assignment evaluates the target (which, unlike normal assignment @@ -298,11 +284,6 @@ is performed *in-place*, meaning that rather than creating a new object and assigning that to the target, the old object is modified instead. -Unlike normal assignments, augmented assignments evaluate the left-hand side -*before* evaluating the right-hand side. For example, ``a[i] += f(x)`` first -looks-up ``a[i]``, then it evaluates ``f(x)`` and performs the addition, and -lastly, it writes the result back to ``a[i]``. - With the exception of assigning to tuples and multiple targets in a single statement, the assignment done by augmented assignment statements is handled the same way as normal assignments. Similarly, with the exception of the possible @@ -412,6 +393,7 @@ the sliced object). .. versionchanged:: 3.2 + Previously it was illegal to delete a name from the local namespace if it occurs as a free variable in a nested block. @@ -464,26 +446,53 @@ .. productionlist:: yield_stmt: `yield_expression` -A :keyword:`yield` statement is semantically equivalent to a :ref:`yield -expression `. The yield statement can be used to omit the parentheses -that would otherwise be required in the equivalent yield expression -statement. For example, the yield statements :: +The :keyword:`yield` statement is only used when defining a generator function, +and is only used in the body of the generator function. Using a :keyword:`yield` +statement in a function definition is sufficient to cause that definition to +create a generator function instead of a normal function. - yield - yield from +When a generator function is called, it returns an iterator known as a generator +iterator, or more commonly, a generator. The body of the generator function is +executed by calling the :func:`next` function on the generator repeatedly until +it raises an exception. -are equivalent to the yield expression statements :: +When a :keyword:`yield` statement is executed, the state of the generator is +frozen and the value of :token:`expression_list` is returned to :meth:`next`'s +caller. By "frozen" we mean that all local state is retained, including the +current bindings of local variables, the instruction pointer, and the internal +evaluation stack: enough information is saved so that the next time :func:`next` +is invoked, the function can proceed exactly as if the :keyword:`yield` +statement were just another external call. - (yield ) - (yield from ) +The :keyword:`yield` statement is allowed in the :keyword:`try` clause of a +:keyword:`try` ... :keyword:`finally` construct. If the generator is not +resumed before it is finalized (by reaching a zero reference count or by being +garbage collected), the generator-iterator's :meth:`close` method will be +called, allowing any pending :keyword:`finally` clauses to execute. -Yield expressions and statements are only used when defining a :term:`generator` -function, and are only used in the body of the generator function. Using yield -in a function definition is sufficient to cause that definition to create a -generator function instead of a normal function. +When ``yield from `` is used, it treats the supplied expression as +a subiterator, producing values from it until the underlying iterator is +exhausted. -For full details of :keyword:`yield` semantics, refer to the -:ref:`yieldexpr` section. + .. versionchanged:: 3.3 + Added ``yield from `` to delegate control flow to a subiterator + +For full details of :keyword:`yield` semantics, refer to the :ref:`yieldexpr` +section. + +.. seealso:: + + :pep:`0255` - Simple Generators + The proposal for adding generators and the :keyword:`yield` statement to Python. + + :pep:`0342` - Coroutines via Enhanced Generators + The proposal to enhance the API and syntax of generators, making them + usable as simple coroutines. + + :pep:`0380` - Syntax for Delegating to a Subgenerator + The proposal to introduce the :token:`yield_from` syntax, making delegation + to sub-generators easy. + .. _raise: @@ -548,8 +557,8 @@ RuntimeError: Something bad happened A similar mechanism works implicitly if an exception is raised inside an -exception handler or a :keyword:`finally` clause: the previous exception is then -attached as the new exception's :attr:`__context__` attribute:: +exception handler: the previous exception is then attached as the new +exception's :attr:`__context__` attribute:: >>> try: ... print(1 / 0) @@ -652,88 +661,166 @@ relative_module: "."* `module` | "."+ name: `identifier` -The basic import statement (no :keyword:`from` clause) is executed in two -steps: +Import statements are executed in two steps: (1) find a module, and initialize +it if necessary; (2) define a name or names in the local namespace (of the scope +where the :keyword:`import` statement occurs). The statement comes in two +forms differing on whether it uses the :keyword:`from` keyword. The first form +(without :keyword:`from`) repeats these steps for each identifier in the list. +The form with :keyword:`from` performs step (1) once, and then performs step +(2) repeatedly. For a reference implementation of step (1), see the +:mod:`importlib` module. -#. find a module, loading and initializing it if necessary -#. define a name or names in the local namespace for the scope where - the :keyword:`import` statement occurs. +.. index:: + single: package -When the statement contains multiple clauses (separated by -commas) the two steps are carried out separately for each clause, just -as though the clauses had been separated out into individiual import -statements. +To understand how step (1) occurs, one must first understand how Python handles +hierarchical naming of modules. To help organize modules and provide a +hierarchy in naming, Python has a concept of packages. A package can contain +other packages and modules while modules cannot contain other modules or +packages. From a file system perspective, packages are directories and modules +are files. The original `specification for packages +`_ is still available to read, +although minor details have changed since the writing of that document. -The details of the first step, finding and loading modules are described in -greater detail in the section on the :ref:`import system `, -which also describes the various types of packages and modules that can -be imported, as well as all the hooks that can be used to customize -the import system. Note that failures in this step may indicate either -that the module could not be located, *or* that an error occurred while -initializing the module, which includes execution of the module's code. +.. index:: + single: sys.modules -If the requested module is retrieved successfully, it will be made -available in the local namespace in one of three ways: +Once the name of the module is known (unless otherwise specified, the term +"module" will refer to both packages and modules), searching +for the module or package can begin. The first place checked is +:data:`sys.modules`, the cache of all modules that have been imported +previously. If the module is found there then it is used in step (2) of import +unless ``None`` is found in :data:`sys.modules`, in which case +:exc:`ImportError` is raised. -.. index:: single: as; import statement +.. index:: + single: sys.meta_path + single: finder + pair: finder; find_module + single: __path__ -* If the module name is followed by :keyword:`as`, then the name - following :keyword:`as` is bound directly to the imported module. -* If no other name is specified, and the module being imported is a top - level module, the module's name is bound in the local namespace as a - reference to the imported module -* If the module being imported is *not* a top level module, then the name - of the top level package that contains the module is bound in the local - namespace as a reference to the top level package. The imported module - must be accessed using its full qualified name rather than directly +If the module is not found in the cache, then :data:`sys.meta_path` is searched +(the specification for :data:`sys.meta_path` can be found in :pep:`302`). +The object is a list of :term:`finder` objects which are queried in order as to +whether they know how to load the module by calling their :meth:`find_module` +method with the name of the module. If the module happens to be contained +within a package (as denoted by the existence of a dot in the name), then a +second argument to :meth:`find_module` is given as the value of the +:attr:`__path__` attribute from the parent package (everything up to the last +dot in the name of the module being imported). If a finder can find the module +it returns a :term:`loader` (discussed later) or returns ``None``. +.. index:: + single: sys.path_hooks + single: sys.path_importer_cache + single: sys.path + +If none of the finders on :data:`sys.meta_path` are able to find the module +then some implicitly defined finders are queried. Implementations of Python +vary in what implicit meta path finders are defined. The one they all do +define, though, is one that handles :data:`sys.path_hooks`, +:data:`sys.path_importer_cache`, and :data:`sys.path`. + +The implicit finder searches for the requested module in the "paths" specified +in one of two places ("paths" do not have to be file system paths). If the +module being imported is supposed to be contained within a package then the +second argument passed to :meth:`find_module`, :attr:`__path__` on the parent +package, is used as the source of paths. If the module is not contained in a +package then :data:`sys.path` is used as the source of paths. + +Once the source of paths is chosen it is iterated over to find a finder that +can handle that path. The dict at :data:`sys.path_importer_cache` caches +finders for paths and is checked for a finder. If the path does not have a +finder cached then :data:`sys.path_hooks` is searched by calling each object in +the list with a single argument of the path, returning a finder or raises +:exc:`ImportError`. If a finder is returned then it is cached in +:data:`sys.path_importer_cache` and then used for that path entry. If no finder +can be found but the path exists then a value of ``None`` is +stored in :data:`sys.path_importer_cache` to signify that an implicit, +file-based finder that handles modules stored as individual files should be +used for that path. If the path does not exist then a finder which always +returns ``None`` is placed in the cache for the path. + +.. index:: + single: loader + pair: loader; load_module + exception: ImportError + +If no finder can find the module then :exc:`ImportError` is raised. Otherwise +some finder returned a loader whose :meth:`load_module` method is called with +the name of the module to load (see :pep:`302` for the original definition of +loaders). A loader has several responsibilities to perform on a module it +loads. First, if the module already exists in :data:`sys.modules` (a +possibility if the loader is called outside of the import machinery) then it +is to use that module for initialization and not a new module. But if the +module does not exist in :data:`sys.modules` then it is to be added to that +dict before initialization begins. If an error occurs during loading of the +module and it was added to :data:`sys.modules` it is to be removed from the +dict. If an error occurs but the module was already in :data:`sys.modules` it +is left in the dict. + +.. index:: + single: __name__ + single: __file__ + single: __path__ + single: __package__ + single: __loader__ + +The loader must set several attributes on the module. :data:`__name__` is to be +set to the name of the module. :data:`__file__` is to be the "path" to the file +unless the module is built-in (and thus listed in +:data:`sys.builtin_module_names`) in which case the attribute is not set. +If what is being imported is a package then :data:`__path__` is to be set to a +list of paths to be searched when looking for modules and packages contained +within the package being imported. :data:`__package__` is optional but should +be set to the name of package that contains the module or package (the empty +string is used for module not contained in a package). :data:`__loader__` is +also optional but should be set to the loader object that is loading the +module. + +.. index:: + exception: ImportError + +If an error occurs during loading then the loader raises :exc:`ImportError` if +some other exception is not already being propagated. Otherwise the loader +returns the module that was loaded and initialized. + +When step (1) finishes without raising an exception, step (2) can begin. + +The first form of :keyword:`import` statement binds the module name in the local +namespace to the module object, and then goes on to import the next identifier, +if any. If the module name is followed by :keyword:`as`, the name following +:keyword:`as` is used as the local name for the module. .. index:: pair: name; binding - keyword: from exception: ImportError -The :keyword:`from` form uses a slightly more complex process: - -#. find the module specified in the :keyword:`from` clause, loading and - initializing it if necessary; -#. for each of the identifiers specified in the :keyword:`import` clauses: - - #. check if the imported module has an attribute by that name - #. if not, attempt to import a submodule with that name and then - check the imported module again for that attribute - #. if the attribute is not found, :exc:`ImportError` is raised. - #. otherwise, a reference to that value is stored in the local namespace, - using the name in the :keyword:`as` clause if it is present, - otherwise using the attribute name - -Examples:: - - import foo # foo imported and bound locally - import foo.bar.baz # foo.bar.baz imported, foo bound locally - import foo.bar.baz as fbb # foo.bar.baz imported and bound as fbb - from foo.bar import baz # foo.bar.baz imported and bound as baz - from foo import attr # foo imported and foo.attr bound as attr - -If the list of identifiers is replaced by a star (``'*'``), all public -names defined in the module are bound in the local namespace for the scope -where the :keyword:`import` statement occurs. +The :keyword:`from` form does not bind the module name: it goes through the list +of identifiers, looks each one of them up in the module found in step (1), and +binds the name in the local namespace to the object thus found. As with the +first form of :keyword:`import`, an alternate local name can be supplied by +specifying ":keyword:`as` localname". If a name is not found, +:exc:`ImportError` is raised. If the list of identifiers is replaced by a star +(``'*'``), all public names defined in the module are bound in the local +namespace of the :keyword:`import` statement. .. index:: single: __all__ (optional module attribute) The *public names* defined by a module are determined by checking the module's -namespace for a variable named ``__all__``; if defined, it must be a sequence -of strings which are names defined or imported by that module. The names -given in ``__all__`` are all considered public and are required to exist. If -``__all__`` is not defined, the set of public names includes all names found -in the module's namespace which do not begin with an underscore character -(``'_'``). ``__all__`` should contain the entire public API. It is intended -to avoid accidentally exporting items that are not part of the API (such as -library modules which were imported and used within the module). +namespace for a variable named ``__all__``; if defined, it must be a sequence of +strings which are names defined or imported by that module. The names given in +``__all__`` are all considered public and are required to exist. If ``__all__`` +is not defined, the set of public names includes all names found in the module's +namespace which do not begin with an underscore character (``'_'``). +``__all__`` should contain the entire public API. It is intended to avoid +accidentally exporting items that are not part of the API (such as library +modules which were imported and used within the module). -The wild card form of import --- ``from module import *`` --- is only allowed at -the module level. Attempting to use it in class or function definitions will -raise a :exc:`SyntaxError`. +The :keyword:`from` form with ``*`` may only occur in a module scope. The wild +card form of import --- ``import *`` --- is only allowed at the module level. +Attempting to use it in class or function definitions will raise a +:exc:`SyntaxError`. .. index:: single: relative; import @@ -752,7 +839,7 @@ The specification for relative imports is contained within :pep:`328`. :func:`importlib.import_module` is provided to support applications that -determine dynamically the modules to be loaded. +determine which modules need to be loaded dynamically. .. _future: @@ -764,12 +851,10 @@ A :dfn:`future statement` is a directive to the compiler that a particular module should be compiled using syntax or semantics that will be available in a -specified future release of Python where the feature becomes standard. - -The future statement is intended to ease migration to future versions of Python -that introduce incompatible changes to the language. It allows use of the new -features on a per-module basis before the release in which the feature becomes -standard. +specified future release of Python. The future statement is intended to ease +migration to future versions of Python that introduce incompatible changes to +the language. It allows use of the new features on a per-module basis before +the release in which the feature becomes standard. .. productionlist:: * future_statement: "from" "__future__" "import" feature ["as" name] @@ -864,7 +949,7 @@ .. impl-detail:: - The current implementation does not enforce the two restrictions, but + The current implementation does not enforce the latter two restrictions, but programs should not abuse this freedom, as future implementations may enforce them or silently change the meaning of the program. @@ -897,16 +982,16 @@ : | "nonlocal" identifier augop expression_list The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. +previously bound variables in the nearest enclosing scope. This is important +because the default behavior for binding is to search the local namespace +first. The statement allows encapsulated code to rebind variables outside of +the local scope besides the global (module) scope. .. XXX not implemented The :keyword:`nonlocal` statement may prepend an assignment or augmented assignment, but not an expression. -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a +Names listed in a :keyword:`nonlocal` statement, unlike to those listed in a :keyword:`global` statement, must refer to pre-existing bindings in an enclosing scope (the scope in which a new binding should be created cannot be determined unambiguously). diff -r 6db40a9955dc -r 0d413f60cc23 Doc/reference/toplevel_components.rst --- a/Doc/reference/toplevel_components.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/reference/toplevel_components.rst Mon Jan 25 17:05:13 2016 +0100 @@ -97,10 +97,20 @@ ================ .. index:: single: input + .. index:: builtin: eval -:func:`eval` is used for expression input. It ignores leading whitespace. The +There are two forms of expression input. Both ignore leading whitespace. The string argument to :func:`eval` must have the following form: .. productionlist:: eval_input: `expression_list` NEWLINE* + +.. index:: + object: file + single: input; raw + single: readline() (file method) + +Note: to read 'raw' input line without interpretation, you can use the +:meth:`readline` method of file objects, including ``sys.stdin``. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/extensions/c_annotations.py --- a/Doc/tools/extensions/c_annotations.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,121 +0,0 @@ -# -*- coding: utf-8 -*- -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: - - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. - - * stable API annotations - - Usage: Set the `refcount_file` config value to the path to the reference - count data file. - - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" - -from os import path -from docutils import nodes -from docutils.parsers.rst import directives - -from sphinx import addnodes -from sphinx.domains.c import CObject - - -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations(dict): - @classmethod - def fromfile(cls, filename): - d = cls() - fp = open(filename, 'r') - try: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = d[function] - except KeyError: - entry = d[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - finally: - fp.close() - return d - - def add_annotations(self, app, doctree): - for node in doctree.traverse(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if par['stableabi']: - node.insert(0, nodes.emphasis(' Part of the stable ABI.', - ' Part of the stable ABI.', - classes=['stableabi'])) - if par['objtype'] != 'function': - continue - if not par[0].has_key('names') or not par[0]['names']: - continue - name = par[0]['names'][0] - if name.startswith("c."): - name = name[2:] - entry = self.get(name) - if not entry: - continue - elif entry.result_type not in ("PyObject*", "PyVarObject*"): - continue - if entry.result_refs is None: - rc = 'Return value: Always NULL.' - elif entry.result_refs: - rc = 'Return value: New reference.' - else: - rc = 'Return value: Borrowed reference.' - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) - - -def init_annotations(app): - refcounts = Annotations.fromfile( - path.join(app.srcdir, app.config.refcount_file)) - app.connect('doctree-read', refcounts.add_annotations) - - -def setup(app): - app.add_config_value('refcount_file', '', True) - app.connect('builder-inited', init_annotations) - - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, - } - old_handle_signature = CObject.handle_signature - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/extensions/patchlevel.py --- a/Doc/tools/extensions/patchlevel.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ - - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. - - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" - -import os -import re -import sys - -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') - - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - - d = {} - f = open(patchlevel_h) - try: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value - finally: - f.close() - - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) - - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release - - -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) - return version, release - - -def get_version_info(): - try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print >>sys.stderr, 'Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release - return version, release - -if __name__ == '__main__': - print(get_header_version_info('.')[1]) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/extensions/pyspecific.py --- a/Doc/tools/extensions/pyspecific.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,386 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pyspecific.py - ~~~~~~~~~~~~~ - - Sphinx extension with Python doc-specific markup. - - :copyright: 2008-2014 by Georg Brandl. - :license: Python license. -""" - -import re -import codecs -from os import path -from time import asctime -from pprint import pformat -from docutils.io import StringOutput -from docutils.utils import new_document - -from docutils import nodes, utils - -from sphinx import addnodes -from sphinx.builders import Builder -from sphinx.util.nodes import split_explicit_title -from sphinx.util.compat import Directive -from sphinx.writers.html import HTMLTranslator -from sphinx.writers.text import TextWriter -from sphinx.writers.latex import LaTeXTranslator -from sphinx.domains.python import PyModulelevel, PyClassmember - -# Support for checking for suspicious markup - -import suspicious - - -ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://hg.python.org/cpython/file/default/%s' - -# monkey-patch reST parser to disable alphabetic and roman enumerated lists -from docutils.parsers.rst.states import Body -Body.enum.converters['loweralpha'] = \ - Body.enum.converters['upperalpha'] = \ - Body.enum.converters['lowerroman'] = \ - Body.enum.converters['upperroman'] = lambda x: None - -# monkey-patch HTML and LaTeX translators to keep doctest blocks in the -# doctest docs themselves -orig_visit_literal_block = HTMLTranslator.visit_literal_block -orig_depart_literal_block = LaTeXTranslator.depart_literal_block - - -def new_visit_literal_block(self, node): - meta = self.builder.env.metadata[self.builder.current_docname] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_visit_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -def new_depart_literal_block(self, node): - meta = self.builder.env.metadata[self.curfilestack[-1]] - old_trim_doctest_flags = self.highlighter.trim_doctest_flags - if 'keepdoctest' in meta: - self.highlighter.trim_doctest_flags = False - try: - orig_depart_literal_block(self, node) - finally: - self.highlighter.trim_doctest_flags = old_trim_doctest_flags - - -HTMLTranslator.visit_literal_block = new_visit_literal_block -LaTeXTranslator.depart_literal_block = new_depart_literal_block - - -# Support for marking up and linking to bugs.python.org issues - -def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) - text = 'issue ' + issue - refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) - return [refnode], [] - - -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - -# Support for marking up implementation details - -class ImplementationDetail(Directive): - - has_content = True - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True - - def run(self): - pnode = nodes.compound(classes=['impl-detail']) - content = self.content - add_text = nodes.strong('CPython implementation detail:', - 'CPython implementation detail:') - if self.arguments: - n, m = self.state.inline_text(self.arguments[0], self.lineno) - pnode.append(nodes.paragraph('', '', *(n + m))) - self.state.nested_parse(content, self.content_offset, pnode) - if pnode.children and isinstance(pnode[0], nodes.paragraph): - pnode[0].insert(0, add_text) - pnode[0].insert(1, nodes.Text(' ')) - else: - pnode.insert(0, nodes.paragraph('', '', add_text)) - return [pnode] - - -# Support for documenting decorators - -class PyDecoratorMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyDecoratorMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_addname('@', '@')) - return ret - - def needs_arglist(self): - return False - - -class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): - def run(self): - # a decorator function is a function after all - self.name = 'py:function' - return PyModulelevel.run(self) - - -class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -class PyCoroutineMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyCoroutineMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine ')) - return ret - - -class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): - def run(self): - self.name = 'py:function' - return PyModulelevel.run(self) - - -class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -class PyAbstractMethod(PyClassmember): - - def handle_signature(self, sig, signode): - ret = super(PyAbstractMethod, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('abstractmethod ', - 'abstractmethod ')) - return ret - - def run(self): - self.name = 'py:method' - return PyClassmember.run(self) - - -# Support for documenting version of removal in deprecations - -class DeprecatedRemoved(Directive): - has_content = True - required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - - _label = 'Deprecated since version %s, will be removed in version %s' - - def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - text = self._label % version - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes) - node.append(para) - else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified'])) - node.append(para) - env = self.state.document.settings.env - env.note_versionchange('deprecated', version[0], node, self.lineno) - return [node] + messages - - -# Support for including Misc/NEWS - -issue_re = re.compile('([Ii])ssue #([0-9]+)') -whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") - - -class MiscNews(Directive): - has_content = False - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = False - option_spec = {} - - def run(self): - fname = self.arguments[0] - source = self.state_machine.input_lines.source( - self.lineno - self.state_machine.input_offset - 1) - source_dir = path.dirname(path.abspath(source)) - fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) - try: - fp = codecs.open(fpath, encoding='utf-8') - try: - content = fp.read() - finally: - fp.close() - except Exception: - text = 'The NEWS file is not available.' - node = nodes.strong(text, text) - return [node] - content = issue_re.sub(r'`\1ssue #\2 `__', - content) - content = whatsnew_re.sub(r'\1', content) - # remove first 3 lines as they are the main heading - lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] - self.state_machine.insert_input(lines, fname) - return [] - - -# Support for building "topic help" for pydoc - -pydoc_topic_labels = [ - 'assert', 'assignment', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'binary', - 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', - 'bltin-null-object', 'bltin-type-objects', 'booleans', - 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', - 'context-managers', 'continue', 'conversions', 'customization', 'debugger', - 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', - 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', - 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', - 'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types', - 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', - 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', - 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', - 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', - 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield' -] - - -class PydocTopicsBuilder(Builder): - name = 'pydoc-topics' - - def init(self): - self.topics = {} - - def get_outdated_docs(self): - return 'all pydoc topics' - - def get_target_uri(self, docname, typ=None): - return '' # no URIs - - def write(self, *ignored): - writer = TextWriter(self) - for label in self.status_iterator(pydoc_topic_labels, - 'building topics... ', - length=len(pydoc_topic_labels)): - if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) - continue - docname, labelid, sectname = self.env.domaindata['std']['labels'][label] - doctree = self.env.get_and_resolve_doctree(docname, self) - document = new_document('
') - document.append(doctree.ids[labelid]) - destination = StringOutput(encoding='utf-8') - writer.write(document, destination) - self.topics[label] = writer.output - - def finish(self): - f = open(path.join(self.outdir, 'topics.py'), 'wb') - try: - f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) - f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) - f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) - finally: - f.close() - - -# Support for documenting Opcodes - -opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?') - - -def parse_opcode_signature(env, sig, signode): - """Transform an opcode signature into RST nodes.""" - m = opcode_sig_re.match(sig) - if m is None: - raise ValueError - opname, arglist = m.groups() - signode += addnodes.desc_name(opname, opname) - if arglist is not None: - paramlist = addnodes.desc_parameterlist() - signode += paramlist - paramlist += addnodes.desc_parameter(arglist, arglist) - return opname.strip() - - -# Support for documenting pdb commands - -pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)') - -# later... -# pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+ | # identifiers -# [.,:]+ | # punctuation -# [\[\]()] | # parens -# \s+ # whitespace -# ''', re.X) - - -def parse_pdb_command(env, sig, signode): - """Transform a pdb command signature into RST nodes.""" - m = pdbcmd_sig_re.match(sig) - if m is None: - raise ValueError - name, args = m.groups() - fullname = name.replace('(', '').replace(')', '') - signode += addnodes.desc_name(name, name) - if args: - signode += addnodes.desc_addname(' '+args, ' '+args) - return fullname - - -def setup(app): - app.add_role('issue', issue_role) - app.add_role('source', source_role) - app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('deprecated-removed', DeprecatedRemoved) - app.add_builder(PydocTopicsBuilder) - app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) - app.add_description_unit('opcode', 'opcode', '%s (opcode)', - parse_opcode_signature) - app.add_description_unit('pdbcommand', 'pdbcmd', '%s (pdb command)', - parse_pdb_command) - app.add_description_unit('2to3fixer', '2to3fixer', '%s (2to3 fixer)') - app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) - app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) - app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) - app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) - app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) - app.add_directive('miscnews', MiscNews) - return {'version': '1.0', 'parallel_read_safe': True} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/extensions/suspicious.py --- a/Doc/tools/extensions/suspicious.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,274 +0,0 @@ -""" -Try to detect suspicious constructs, resembling markup -that has leaked into the final output. - -Suspicious lines are reported in a comma-separated-file, -``suspicious.csv``, located in the output directory. - -The file is utf-8 encoded, and each line contains four fields: - - * document name (normalized) - * line number in the source document - * problematic text - * complete line showing the problematic text in context - -It is common to find many false positives. To avoid reporting them -again and again, they may be added to the ``ignored.csv`` file -(located in the configuration directory). The file has the same -format as ``suspicious.csv`` with a few differences: - - - each line defines a rule; if the rule matches, the issue - is ignored. - - line number may be empty (that is, nothing between the - commas: ",,"). In this case, line numbers are ignored (the - rule matches anywhere in the file). - - the last field does not have to be a complete line; some - surrounding text (never more than a line) is enough for - context. - -Rules are processed sequentially. A rule matches when: - - * document names are the same - * problematic texts are the same - * line numbers are close to each other (5 lines up or down) - * the rule text is completely contained into the source line - -The simplest way to create the ignored.csv file is by copying -undesired entries from suspicious.csv (possibly trimming the last -field.) - -Copyright 2009 Gabriel A. Genellina - -""" - -import os -import re -import csv -import sys - -from docutils import nodes -from sphinx.builders import Builder - -detect_all = re.compile(r''' - ::(?=[^=])| # two :: (but NOT ::=) - :[a-zA-Z][a-zA-Z0-9]+| # :foo - `| # ` (seldom used by itself) - (?= (3, 0) - - -class Rule: - def __init__(self, docname, lineno, issue, line): - """A rule for ignoring issues""" - self.docname = docname # document to which this rule applies - self.lineno = lineno # line number in the original source; - # this rule matches only near that. - # None -> don't care - self.issue = issue # the markup fragment that triggered this rule - self.line = line # text of the container element (single line only) - self.used = False - - def __repr__(self): - return '{0.docname},,{0.issue},{0.line}'.format(self) - - - -class dialect(csv.excel): - """Our dialect: uses only linefeed as newline.""" - lineterminator = '\n' - - -class CheckSuspiciousMarkupBuilder(Builder): - """ - Checks for possibly invalid markup that may leak into the output. - """ - name = 'suspicious' - - def init(self): - # create output file - self.log_file_name = os.path.join(self.outdir, 'suspicious.csv') - open(self.log_file_name, 'w').close() - # load database of previously ignored issues - self.load_rules(os.path.join(os.path.dirname(__file__), '..', - 'susp-ignored.csv')) - - def get_outdated_docs(self): - return self.env.found_docs - - def get_target_uri(self, docname, typ=None): - return '' - - def prepare_writing(self, docnames): - pass - - def write_doc(self, docname, doctree): - # set when any issue is encountered in this document - self.any_issue = False - self.docname = docname - visitor = SuspiciousVisitor(doctree, self) - doctree.walk(visitor) - - def finish(self): - unused_rules = [rule for rule in self.rules if not rule.used] - if unused_rules: - self.warn('Found %s/%s unused rules:' % - (len(unused_rules), len(self.rules))) - for rule in unused_rules: - self.info(repr(rule)) - return - - def check_issue(self, line, lineno, issue): - if not self.is_ignored(line, lineno, issue): - self.report_issue(line, lineno, issue) - - def is_ignored(self, line, lineno, issue): - """Determine whether this issue should be ignored.""" - docname = self.docname - for rule in self.rules: - if rule.docname != docname: continue - if rule.issue != issue: continue - # Both lines must match *exactly*. This is rather strict, - # and probably should be improved. - # Doing fuzzy matches with levenshtein distance could work, - # but that means bringing other libraries... - # Ok, relax that requirement: just check if the rule fragment - # is contained in the document line - if rule.line not in line: continue - # Check both line numbers. If they're "near" - # this rule matches. (lineno=None means "don't care") - if (rule.lineno is not None) and \ - abs(rule.lineno - lineno) > 5: continue - # if it came this far, the rule matched - rule.used = True - return True - return False - - def report_issue(self, text, lineno, issue): - if not self.any_issue: self.info() - self.any_issue = True - self.write_log_entry(lineno, issue, text) - if py3: - self.warn('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) - else: - self.warn('[%s:%d] "%s" found in "%-.120s"' % ( - self.docname.encode(sys.getdefaultencoding(),'replace'), - lineno, - issue.encode(sys.getdefaultencoding(),'replace'), - text.strip().encode(sys.getdefaultencoding(),'replace'))) - self.app.statuscode = 1 - - def write_log_entry(self, lineno, issue, text): - if py3: - f = open(self.log_file_name, 'a') - writer = csv.writer(f, dialect) - writer.writerow([self.docname, lineno, issue, text.strip()]) - f.close() - else: - f = open(self.log_file_name, 'ab') - writer = csv.writer(f, dialect) - writer.writerow([self.docname.encode('utf-8'), - lineno, - issue.encode('utf-8'), - text.strip().encode('utf-8')]) - f.close() - - def load_rules(self, filename): - """Load database of previously ignored issues. - - A csv file, with exactly the same format as suspicious.csv - Fields: document name (normalized), line number, issue, surrounding text - """ - self.info("loading ignore rules... ", nonl=1) - self.rules = rules = [] - try: - if py3: - f = open(filename, 'r') - else: - f = open(filename, 'rb') - except IOError: - return - for i, row in enumerate(csv.reader(f)): - if len(row) != 4: - raise ValueError( - "wrong format in %s, line %d: %s" % (filename, i+1, row)) - docname, lineno, issue, text = row - if lineno: - lineno = int(lineno) - else: - lineno = None - if not py3: - docname = docname.decode('utf-8') - issue = issue.decode('utf-8') - text = text.decode('utf-8') - rule = Rule(docname, lineno, issue, text) - rules.append(rule) - f.close() - self.info('done, %d rules loaded' % len(self.rules)) - - -def get_lineno(node): - """Obtain line number information for a node.""" - lineno = None - while lineno is None and node: - node = node.parent - lineno = node.line - return lineno - - -def extract_line(text, index): - """text may be a multiline string; extract - only the line containing the given character index. - - >>> extract_line("abc\ndefgh\ni", 6) - >>> 'defgh' - >>> for i in (0, 2, 3, 4, 10): - ... print extract_line("abc\ndefgh\ni", i) - abc - abc - abc - defgh - defgh - i - """ - p = text.rfind('\n', 0, index) + 1 - q = text.find('\n', index) - if q < 0: - q = len(text) - return text[p:q] - - -class SuspiciousVisitor(nodes.GenericNodeVisitor): - - lastlineno = 0 - - def __init__(self, document, builder): - nodes.GenericNodeVisitor.__init__(self, document) - self.builder = builder - - def default_visit(self, node): - if isinstance(node, (nodes.Text, nodes.image)): # direct text containers - text = node.astext() - # lineno seems to go backwards sometimes (?) - self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno) - seen = set() # don't report the same issue more than only once per line - for match in detect_all(text): - issue = match.group() - line = extract_line(text, match.start()) - if (issue, line) not in seen: - self.builder.check_issue(line, lineno, issue) - seen.add((issue, line)) - - unknown_visit = default_visit - - def visit_document(self, node): - self.lastlineno = 0 - - def visit_comment(self, node): - # ignore comments -- too much false positives. - # (although doing this could miss some errors; - # there were two sections "commented-out" by mistake - # in the Python docs that would not be catched) - raise nodes.SkipNode diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/pydoctheme/static/pydoctheme.css --- a/Doc/tools/pydoctheme/static/pydoctheme.css Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,178 +0,0 @@ -@import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - border-bottom: 1px solid #ccc; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -div.sphinxsidebar input { - font-family: 'Lucida Grande',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #0072aa; -} - -div.body a:visited { - color: #6363bb; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, code, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt, div.body code { - border-radius: 3px; -} - -div.body tt.descname, div.body code.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt, div.body code.xref, div.body a code { - font-weight: normal; -} - -.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -.refcount { - color: #060; -} - -.stableabi { - color: #229; -} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/pydoctheme/theme.conf --- a/Doc/tools/pydoctheme/theme.conf Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', Arial, sans-serif -headfont = 'Lucida Grande', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/roman.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/roman.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,80 @@ +"""Convert to and from Roman numerals""" + +__author__ = "Mark Pilgrim (f8dy@diveintopython.org)" +__version__ = "1.4" +__date__ = "8 August 2001" +__copyright__ = """Copyright (c) 2001 Mark Pilgrim + +This program is part of "Dive Into Python", a free Python tutorial for +experienced programmers. Visit http://diveintopython.org/ for the +latest version. + +This program is free software; you can redistribute it and/or modify +it under the terms of the Python 2.1.1 license, available at +http://www.python.org/2.1.1/license.html +""" + +import re + +#Define exceptions +class RomanError(Exception): pass +class OutOfRangeError(RomanError): pass +class NotIntegerError(RomanError): pass +class InvalidRomanNumeralError(RomanError): pass + +#Define digit mapping +romanNumeralMap = (('M', 1000), + ('CM', 900), + ('D', 500), + ('CD', 400), + ('C', 100), + ('XC', 90), + ('L', 50), + ('XL', 40), + ('X', 10), + ('IX', 9), + ('V', 5), + ('IV', 4), + ('I', 1)) + +def toRoman(n): + """convert integer to Roman numeral""" + if not (0 < n < 5000): + raise OutOfRangeError("number out of range (must be 1..4999)") + if int(n) != n: + raise NotIntegerError("decimals can not be converted") + + result = "" + for numeral, integer in romanNumeralMap: + while n >= integer: + result += numeral + n -= integer + return result + +#Define pattern to detect valid Roman numerals +romanNumeralPattern = re.compile(""" + ^ # beginning of string + M{0,4} # thousands - 0 to 4 M's + (CM|CD|D?C{0,3}) # hundreds - 900 (CM), 400 (CD), 0-300 (0 to 3 C's), + # or 500-800 (D, followed by 0 to 3 C's) + (XC|XL|L?X{0,3}) # tens - 90 (XC), 40 (XL), 0-30 (0 to 3 X's), + # or 50-80 (L, followed by 0 to 3 X's) + (IX|IV|V?I{0,3}) # ones - 9 (IX), 4 (IV), 0-3 (0 to 3 I's), + # or 5-8 (V, followed by 0 to 3 I's) + $ # end of string + """ ,re.VERBOSE) + +def fromRoman(s): + """convert Roman numeral to integer""" + if not s: + raise InvalidRomanNumeralError('Input can not be blank') + if not romanNumeralPattern.search(s): + raise InvalidRomanNumeralError('Invalid Roman numeral: %s' % s) + + result = 0 + index = 0 + for numeral, integer in romanNumeralMap: + while s[index:index+len(numeral)] == numeral: + result += integer + index += len(numeral) + return result diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/rstlint.py --- a/Doc/tools/rstlint.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tools/rstlint.py Mon Jan 25 17:05:13 2016 +0100 @@ -15,6 +15,7 @@ import re import sys import getopt +import subprocess from os.path import join, splitext, abspath, exists from collections import defaultdict @@ -27,16 +28,14 @@ 'parsed-literal', 'pull-quote', 'raw', 'replace', 'restructuredtext-test-directive', 'role', 'rubric', 'sectnum', 'sidebar', 'table', 'target-notes', 'tip', 'title', 'topic', 'unicode', 'warning', - # Sphinx and Python docs custom ones + # Sphinx custom ones 'acks', 'attribute', 'autoattribute', 'autoclass', 'autodata', 'autoexception', 'autofunction', 'automethod', 'automodule', 'centered', 'cfunction', 'class', 'classmethod', 'cmacro', 'cmdoption', 'cmember', 'code-block', 'confval', 'cssclass', 'ctype', 'currentmodule', 'cvar', - 'data', 'decorator', 'decoratormethod', 'deprecated-removed', - 'deprecated(?!-removed)', 'describe', 'directive', 'doctest', 'envvar', - 'event', 'exception', 'function', 'glossary', 'highlight', 'highlightlang', - 'impl-detail', 'index', 'literalinclude', 'method', 'miscnews', 'module', - 'moduleauthor', 'opcode', 'pdbcommand', 'productionlist', + 'data', 'deprecated', 'describe', 'directive', 'doctest', 'envvar', 'event', + 'exception', 'function', 'glossary', 'highlight', 'highlightlang', 'index', + 'literalinclude', 'method', 'module', 'moduleauthor', 'productionlist', 'program', 'role', 'sectionauthor', 'seealso', 'sourcecode', 'staticmethod', 'tabularcolumns', 'testcode', 'testoutput', 'testsetup', 'toctree', 'todo', 'todolist', 'versionadded', 'versionchanged' @@ -45,14 +44,13 @@ all_directives = '(' + '|'.join(directives) + ')' seems_directive_re = re.compile(r'\.\. %s([^a-z:]|:(?!:))' % all_directives) default_role_re = re.compile(r'(^| )`\w([^`]*?\w)?`($| )') -leaked_markup_re = re.compile(r'[a-z]::\s|`|\.\.\s*\w+:') +leaked_markup_re = re.compile(r'[a-z]::[^=]|:[a-z]+:|`|\.\.\s*\w+:') checkers = {} checker_props = {'severity': 1, 'falsepositives': False} - def checker(*suffixes, **kwds): """Decorator to register a function as a checker.""" def deco(func): @@ -173,6 +171,10 @@ count = defaultdict(int) for root, dirs, files in os.walk(path): + # ignore subdirs controlled by svn + if '.svn' in dirs: + dirs.remove('.svn') + # ignore subdirs in ignore list if abspath(root) in ignore: del dirs[:] @@ -196,7 +198,7 @@ print('Checking %s...' % fn) try: - with open(fn, 'r', encoding='utf-8') as f: + with open(fn, 'r') as f: lines = list(f) except (IOError, OSError) as err: print('%s: cannot open: %s' % (fn, err)) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinx-build.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinx-build.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +""" + Sphinx - Python documentation toolchain + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: 2007-2010 by Georg Brandl. + :license: Python license. +""" + +import sys +import warnings + +# Get rid of UserWarnings reported by pkg_resources. +warnings.filterwarnings('ignore', category=UserWarning, module='jinja2') + +if __name__ == '__main__': + + if sys.version_info[:3] < (2, 4, 0): + sys.stderr.write("""\ +Error: Sphinx needs to be executed with Python 2.4 or newer (not 3.0 though). +(If you run this from the Makefile, you can set the PYTHON variable +to the path of an alternative interpreter executable, e.g., +``make html PYTHON=python2.5``). +""") + sys.exit(1) + + from sphinx import main + sys.exit(main(sys.argv)) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/download.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/download.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,65 @@ +{% extends "layout.html" %} +{% set title = 'Download' %} +{% if daily is defined %} + {% set dlbase = pathto('archives', 1) %} +{% else %} + {% set dlbase = 'http://docs.python.org/ftp/python/doc/' + release %} +{% endif %} + +{% block body %} +

Download Python {{ release }} Documentation

+ +{% if last_updated %}

Last updated on: {{ last_updated }}.

{% endif %} + +

To download an archive containing all the documents for this version of +Python in one of various formats, follow one of links in this table. The numbers +in the table are the size of the download files in megabytes.

+ + + + + + + + + + + + + + + + + + + + + + + +
FormatPacked as .zipPacked as .tar.bz2
PDF (US-Letter paper size)Download (ca. 8 MB)Download (ca. 8 MB)
PDF (A4 paper size)Download (ca. 8 MB)Download (ca. 8 MB)
HTMLDownload (ca. 6 MB)Download (ca. 4 MB)
Plain TextDownload (ca. 2 MB)Download (ca. 1.5 MB)
EPUBDownload (ca. 3.5 MB)Download (ca. 3.5 MB)
+ +

These archives contain all the content in the documentation.

+ +

HTML Help (.chm) files are made available in the "Windows" section +on the Python +download page.

+ + +

Unpacking

+ +

Unix users should download the .tar.bz2 archives; these are bzipped tar +archives and can be handled in the usual way using tar and the bzip2 +program. The InfoZIP unzip program can be +used to handle the ZIP archives if desired. The .tar.bz2 archives provide the +best compression and fastest download times.

+ +

Windows users can use the ZIP archives since those are customary on that +platform. These are created on Unix using the InfoZIP zip program.

+ + +

Problems

+ +

If you have comments or suggestions for the Python documentation, please send +email to docs@python.org.

+{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/indexcontent.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/indexcontent.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,59 @@ +{% extends "defindex.html" %} +{% block tables %} +

Parts of the documentation:

+ + +
+

What's new in Python {{ version }}?
+ or all "What's new" documents since 2.0

+

Tutorial
+ start here

+

Library Reference
+ keep this under your pillow

+

Language Reference
+ describes syntax and language elements

+

Python Setup and Usage
+ how to use Python on different platforms

+

Python HOWTOs
+ in-depth documents on specific topics

+ 		+

Extending and Embedding
+ tutorial for C/C++ programmers

+

Python/C API
+ reference for C/C++ programmers

+

Installing Python Projects
+ finding and installing modules and applications

+

Distributing Python Projects
+ packaging and distributing modules and applications

+

FAQs
+ frequently asked questions (with answers!)

+
+ +

Indices and tables:

+ + +
+

Global Module Index
+ quick access to all modules

+

General Index
+ all functions, classes, terms

+

Glossary
+ the most important terms explained

+ 		+

Search page
+ search this documentation

+

Complete Table of Contents
+ lists all sections and subsections

+
+ +

Meta information:

+ + +
+

Reporting bugs

+

About the documentation

+ 		+

History and License of Python

+

Copyright

+
+{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/indexsidebar.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/indexsidebar.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,22 @@ +

Download

+

Download these documents

+

Docs for other versions

+ + +

Other resources

+ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/layout.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/layout.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,35 @@ +{% extends "!layout.html" %} +{% block rootrellink %} +
    • +
  • Python{{ reldelim1 }}
    • +
  • {{ shorttitle }}{{ reldelim1 }}
    • +{% endblock %} +{% block extrahead %} + + {% if not embedded %}{% endif %} +{{ super() }} +{% endblock %} +{% block footer %} +
    + © Copyright {{ copyright|e }}. +
    + The Python Software Foundation is a non-profit corporation. + Please donate. +
    + Last updated on {{ last_updated|e }}. + Found a bug? +
    + Created using Sphinx {{ sphinx_version|e }}. +
    +{% endblock %} +{% block sidebarsourcelink %} +{%- if show_source and has_source and sourcename %} +

    {{ _('This Page') }}

    + +{%- endif %} +{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/opensearch.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/opensearch.xml Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,4 @@ +{% extends "!opensearch.xml" %} +{% block extra -%} +http://www.python.org/images/favicon16x16.ico +{%- endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/patchlevel.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/patchlevel.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" + patchlevel.py + ~~~~~~~~~~~~~ + + Extract version info from Include/patchlevel.h. + Adapted from Doc/tools/getversioninfo. + + :copyright: 2007-2008 by Georg Brandl. + :license: Python license. +""" + +import os +import re +import sys + +def get_header_version_info(srcdir): + patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') + + # This won't pick out all #defines, but it will pick up the ones we + # care about. + rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') + + d = {} + f = open(patchlevel_h) + try: + for line in f: + m = rx.match(line) + if m is not None: + name, value = m.group(1, 2) + d[name] = value + finally: + f.close() + + release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) + micro = int(d['PY_MICRO_VERSION']) + if micro != 0: + release += '.' + str(micro) + + level = d['PY_RELEASE_LEVEL'] + suffixes = { + 'PY_RELEASE_LEVEL_ALPHA': 'a', + 'PY_RELEASE_LEVEL_BETA': 'b', + 'PY_RELEASE_LEVEL_GAMMA': 'rc', + } + if level != 'PY_RELEASE_LEVEL_FINAL': + release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) + return version, release + + +def get_sys_version_info(): + major, minor, micro, level, serial = sys.version_info + release = version = '%s.%s' % (major, minor) + if micro: + release += '.%s' % micro + if level != 'final': + release += '%s%s' % (level[0], serial) + return version, release + + +def get_version_info(): + try: + return get_header_version_info('.') + except (IOError, OSError): + version, release = get_sys_version_info() + print >>sys.stderr, 'Can\'t get version info from Include/patchlevel.h, ' \ + 'using version of this interpreter (%s).' % release + return version, release + +if __name__ == '__main__': + print(get_header_version_info('.')[1]) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/pydoctheme/static/pydoctheme.css --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/pydoctheme/static/pydoctheme.css Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,170 @@ +@import url("default.css"); + +body { + background-color: white; + margin-left: 1em; + margin-right: 1em; +} + +div.related { + margin-bottom: 1.2em; + padding: 0.5em 0; + border-top: 1px solid #ccc; + margin-top: 0.5em; +} + +div.related a:hover { + color: #0095C4; +} + +div.related:first-child { + border-top: 0; + border-bottom: 1px solid #ccc; +} + +div.sphinxsidebar { + background-color: #eeeeee; + border-radius: 5px; + line-height: 130%; + font-size: smaller; +} + +div.sphinxsidebar h3, div.sphinxsidebar h4 { + margin-top: 1.5em; +} + +div.sphinxsidebarwrapper > h3:first-child { + margin-top: 0.2em; +} + +div.sphinxsidebarwrapper > ul > li > ul > li { + margin-bottom: 0.4em; +} + +div.sphinxsidebar a:hover { + color: #0095C4; +} + +div.sphinxsidebar input { + font-family: 'Lucida Grande',Arial,sans-serif; + border: 1px solid #999999; + font-size: smaller; + border-radius: 3px; +} + +div.sphinxsidebar input[type=text] { + max-width: 150px; +} + +div.body { + padding: 0 0 0 1.2em; +} + +div.body p { + line-height: 140%; +} + +div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { + margin: 0; + border: 0; + padding: 0.3em 0; +} + +div.body hr { + border: 0; + background-color: #ccc; + height: 1px; +} + +div.body pre { + border-radius: 3px; + border: 1px solid #ac9; +} + +div.body div.admonition, div.body div.impl-detail { + border-radius: 3px; +} + +div.body div.impl-detail > p { + margin: 0; +} + +div.body div.seealso { + border: 1px solid #dddd66; +} + +div.body a { + color: #00608f; +} + +div.body a:visited { + color: #30306f; +} + +div.body a:hover { + color: #00B0E4; +} + +tt, pre { + font-family: monospace, sans-serif; + font-size: 96.5%; +} + +div.body tt { + border-radius: 3px; +} + +div.body tt.descname { + font-size: 120%; +} + +div.body tt.xref, div.body a tt { + font-weight: normal; +} + +p.deprecated { + border-radius: 3px; +} + +table.docutils { + border: 1px solid #ddd; + min-width: 20%; + border-radius: 3px; + margin-top: 10px; + margin-bottom: 10px; +} + +table.docutils td, table.docutils th { + border: 1px solid #ddd !important; + border-radius: 3px; +} + +table p, table li { + text-align: left !important; +} + +table.docutils th { + background-color: #eee; + padding: 0.3em 0.5em; +} + +table.docutils td { + background-color: white; + padding: 0.3em 0.5em; +} + +table.footnote, table.footnote td { + border: 0 !important; +} + +div.footer { + line-height: 150%; + margin-top: -2em; + text-align: right; + width: auto; + margin-right: 10px; +} + +div.footer a:hover { + color: #0095C4; +} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/pydoctheme/theme.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/pydoctheme/theme.conf Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,23 @@ +[theme] +inherit = default +stylesheet = pydoctheme.css +pygments_style = sphinx + +[options] +bodyfont = 'Lucida Grande', Arial, sans-serif +headfont = 'Lucida Grande', Arial, sans-serif +footerbgcolor = white +footertextcolor = #555555 +relbarbgcolor = white +relbartextcolor = #666666 +relbarlinkcolor = #444444 +sidebarbgcolor = white +sidebartextcolor = #444444 +sidebarlinkcolor = #444444 +bgcolor = white +textcolor = #222222 +linkcolor = #0090c0 +visitedlinkcolor = #00608f +headtextcolor = #1a1a1a +headbgcolor = white +headlinkcolor = #aaaaaa diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/pyspecific.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/pyspecific.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +""" + pyspecific.py + ~~~~~~~~~~~~~ + + Sphinx extension with Python doc-specific markup. + + :copyright: 2008, 2009, 2010, 2011, 2012 by Georg Brandl. + :license: Python license. +""" + +ISSUE_URI = 'http://bugs.python.org/issue%s' +SOURCE_URI = 'http://hg.python.org/cpython/file/default/%s' + +from docutils import nodes, utils +from sphinx.util.nodes import split_explicit_title + +# monkey-patch reST parser to disable alphabetic and roman enumerated lists +from docutils.parsers.rst.states import Body +Body.enum.converters['loweralpha'] = \ + Body.enum.converters['upperalpha'] = \ + Body.enum.converters['lowerroman'] = \ + Body.enum.converters['upperroman'] = lambda x: None + +# monkey-patch HTML translator to give versionmodified paragraphs a class +def new_visit_versionmodified(self, node): + self.body.append(self.starttag(node, 'p', CLASS=node['type'])) + text = versionlabels[node['type']] % node['version'] + if len(node): + text += ':' + else: + text += '.' + self.body.append('%s ' % text) + +from sphinx.writers.html import HTMLTranslator +from sphinx.locale import versionlabels +HTMLTranslator.visit_versionmodified = new_visit_versionmodified + + +# Support for marking up and linking to bugs.python.org issues + +def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): + issue = utils.unescape(text) + text = 'issue ' + issue + refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) + return [refnode], [] + + +# Support for linking to Python source files easily + +def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): + has_t, title, target = split_explicit_title(text) + title = utils.unescape(title) + target = utils.unescape(target) + refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) + return [refnode], [] + + +# Support for marking up implementation details + +from sphinx.util.compat import Directive + +class ImplementationDetail(Directive): + + has_content = True + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + + def run(self): + pnode = nodes.compound(classes=['impl-detail']) + content = self.content + add_text = nodes.strong('CPython implementation detail:', + 'CPython implementation detail:') + if self.arguments: + n, m = self.state.inline_text(self.arguments[0], self.lineno) + pnode.append(nodes.paragraph('', '', *(n + m))) + self.state.nested_parse(content, self.content_offset, pnode) + if pnode.children and isinstance(pnode[0], nodes.paragraph): + pnode[0].insert(0, add_text) + pnode[0].insert(1, nodes.Text(' ')) + else: + pnode.insert(0, nodes.paragraph('', '', add_text)) + return [pnode] + + +# Support for documenting decorators + +from sphinx import addnodes +from sphinx.domains.python import PyModulelevel, PyClassmember + +class PyDecoratorMixin(object): + def handle_signature(self, sig, signode): + ret = super(PyDecoratorMixin, self).handle_signature(sig, signode) + signode.insert(0, addnodes.desc_addname('@', '@')) + return ret + + def needs_arglist(self): + return False + +class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): + def run(self): + # a decorator function is a function after all + self.name = 'py:function' + return PyModulelevel.run(self) + +class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): + def run(self): + self.name = 'py:method' + return PyClassmember.run(self) + + +# Support for documenting version of removal in deprecations + +from sphinx.locale import versionlabels +from sphinx.util.compat import Directive + +versionlabels['deprecated-removed'] = \ + 'Deprecated since version %s, will be removed in version %s' + +class DeprecatedRemoved(Directive): + has_content = True + required_arguments = 2 + optional_arguments = 1 + final_argument_whitespace = True + option_spec = {} + + def run(self): + node = addnodes.versionmodified() + node.document = self.state.document + node['type'] = 'deprecated-removed' + version = (self.arguments[0], self.arguments[1]) + node['version'] = version + if len(self.arguments) == 3: + inodes, messages = self.state.inline_text(self.arguments[2], + self.lineno+1) + node.extend(inodes) + if self.content: + self.state.nested_parse(self.content, self.content_offset, node) + ret = [node] + messages + else: + ret = [node] + env = self.state.document.settings.env + env.note_versionchange('deprecated', version[0], node, self.lineno) + return ret + + +# Support for building "topic help" for pydoc + +pydoc_topic_labels = [ + 'assert', 'assignment', 'atom-identifiers', 'atom-literals', + 'attribute-access', 'attribute-references', 'augassign', 'binary', + 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', + 'bltin-null-object', 'bltin-type-objects', 'booleans', + 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', + 'context-managers', 'continue', 'conversions', 'customization', 'debugger', + 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', + 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', + 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', + 'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types', + 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', + 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', + 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', + 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', + 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield' +] + +from os import path +from time import asctime +from pprint import pformat +from docutils.io import StringOutput +from docutils.utils import new_document + +from sphinx.builders import Builder +from sphinx.writers.text import TextWriter + + +class PydocTopicsBuilder(Builder): + name = 'pydoc-topics' + + def init(self): + self.topics = {} + + def get_outdated_docs(self): + return 'all pydoc topics' + + def get_target_uri(self, docname, typ=None): + return '' # no URIs + + def write(self, *ignored): + writer = TextWriter(self) + for label in self.status_iterator(pydoc_topic_labels, + 'building topics... ', + length=len(pydoc_topic_labels)): + if label not in self.env.domaindata['std']['labels']: + self.warn('label %r not in documentation' % label) + continue + docname, labelid, sectname = self.env.domaindata['std']['labels'][label] + doctree = self.env.get_and_resolve_doctree(docname, self) + document = new_document('
    ') + document.append(doctree.ids[labelid]) + destination = StringOutput(encoding='utf-8') + writer.write(document, destination) + self.topics[label] = writer.output.encode('utf-8') + + def finish(self): + f = open(path.join(self.outdir, 'topics.py'), 'w') + try: + f.write('# -*- coding: utf-8 -*-\n') + f.write('# Autogenerated by Sphinx on %s\n' % asctime()) + f.write('topics = ' + pformat(self.topics) + '\n') + finally: + f.close() + + +# Support for checking for suspicious markup + +import suspicious + + +# Support for documenting Opcodes + +import re + +opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?') + +def parse_opcode_signature(env, sig, signode): + """Transform an opcode signature into RST nodes.""" + m = opcode_sig_re.match(sig) + if m is None: + raise ValueError + opname, arglist = m.groups() + signode += addnodes.desc_name(opname, opname) + if arglist is not None: + paramlist = addnodes.desc_parameterlist() + signode += paramlist + paramlist += addnodes.desc_parameter(arglist, arglist) + return opname.strip() + + +# Support for documenting pdb commands + +pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)') + +# later... +#pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+ | # identifiers +# [.,:]+ | # punctuation +# [\[\]()] | # parens +# \s+ # whitespace +# ''', re.X) + +def parse_pdb_command(env, sig, signode): + """Transform a pdb command signature into RST nodes.""" + m = pdbcmd_sig_re.match(sig) + if m is None: + raise ValueError + name, args = m.groups() + fullname = name.replace('(', '').replace(')', '') + signode += addnodes.desc_name(name, name) + if args: + signode += addnodes.desc_addname(' '+args, ' '+args) + return fullname + + +def setup(app): + app.add_role('issue', issue_role) + app.add_role('source', source_role) + app.add_directive('impl-detail', ImplementationDetail) + app.add_directive('deprecated-removed', DeprecatedRemoved) + app.add_builder(PydocTopicsBuilder) + app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) + app.add_description_unit('opcode', 'opcode', '%s (opcode)', + parse_opcode_signature) + app.add_description_unit('pdbcommand', 'pdbcmd', '%s (pdb command)', + parse_pdb_command) + app.add_description_unit('2to3fixer', '2to3fixer', '%s (2to3 fixer)') + app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) + app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/static/basic.css --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/static/basic.css Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,438 @@ +/** + * Sphinx stylesheet -- basic theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +img { + border: 0; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + border: 0 solid #dce; + border-collapse: collapse; +} + +table.docutils td, table.docutils th { + padding: 2px 5px 2px 5px; + border-left: 0; + background-color: #eef; +} + +table.docutils td p.last, table.docutils th p.last { + margin-bottom: 0; +} + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +table.docutils th { + border-top: 1px solid #cac; + background-color: #ede; +} + +th { + text-align: left; + padding-right: 5px; +} + +th.head { + text-align: center; +} + +/* -- other body styles ----------------------------------------------------- */ + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +p.deprecated, p.deprecated-removed { + background-color: #ffe4e4; + border: 1px solid #f66; + padding: 7px +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.impl-detail { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; + border: 1px solid #ccc; +} + +.impl-detail .compound-first { + margin-top: 0; +} + +.impl-detail .compound-last { + margin-bottom: 0; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/static/copybutton.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/static/copybutton.js Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,57 @@ +$(document).ready(function() { + /* Add a [>>>] button on the top-right corner of code samples to hide + * the >>> and ... prompts and the output and thus make the code + * copyable. */ + var div = $('.highlight-python .highlight,' + + '.highlight-python3 .highlight') + var pre = div.find('pre'); + + // get the styles from the current theme + pre.parent().parent().css('position', 'relative'); + var hide_text = 'Hide the prompts and output'; + var show_text = 'Show the prompts and output'; + var border_width = pre.css('border-top-width'); + var border_style = pre.css('border-top-style'); + var border_color = pre.css('border-top-color'); + var button_styles = { + 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', + 'border-color': border_color, 'border-style': border_style, + 'border-width': border_width, 'color': border_color, 'text-size': '75%', + 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', + 'border-radius': '0 3px 0 0' + } + + // create and add the button to all the code blocks that contain >>> + div.each(function(index) { + var jthis = $(this); + if (jthis.find('.gp').length > 0) { + var button = $('>>>'); + button.css(button_styles) + button.attr('title', hide_text); + jthis.prepend(button); + } + // tracebacks (.gt) contain bare text elements that need to be + // wrapped in a span to work with .nextUntil() (see later) + jthis.find('pre:has(.gt)').contents().filter(function() { + return ((this.nodeType == 3) && (this.data.trim().length > 0)); + }).wrap(''); + }); + + // define the behavior of the button when it's clicked + $('.copybutton').toggle( + function() { + var button = $(this); + button.parent().find('.go, .gp, .gt').hide(); + button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); + button.css('text-decoration', 'line-through'); + button.attr('title', show_text); + }, + function() { + var button = $(this); + button.parent().find('.go, .gp, .gt').show(); + button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); + button.css('text-decoration', 'none'); + button.attr('title', hide_text); + }); +}); + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/static/py.png Binary file Doc/tools/sphinxext/static/py.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/static/sidebar.js --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/static/sidebar.js Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,155 @@ +/* + * sidebar.js + * ~~~~~~~~~~ + * + * This script makes the Sphinx sidebar collapsible. + * + * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in + * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to + * collapse and expand the sidebar. + * + * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the + * width of the sidebar and the margin-left of the document are decreased. + * When the sidebar is expanded the opposite happens. This script saves a + * per-browser/per-session cookie used to remember the position of the sidebar + * among the pages. Once the browser is closed the cookie is deleted and the + * position reset to the default (expanded). + * + * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +$(function() { + // global elements used by the functions. + // the 'sidebarbutton' element is defined as global after its + // creation, in the add_sidebar_button function + var bodywrapper = $('.bodywrapper'); + var sidebar = $('.sphinxsidebar'); + var sidebarwrapper = $('.sphinxsidebarwrapper'); + + // original margin-left of the bodywrapper and width of the sidebar + // with the sidebar expanded + var bw_margin_expanded = bodywrapper.css('margin-left'); + var ssb_width_expanded = sidebar.width(); + + // margin-left of the bodywrapper and width of the sidebar + // with the sidebar collapsed + var bw_margin_collapsed = '.8em'; + var ssb_width_collapsed = '.8em'; + + // colors used by the current theme + var dark_color = '#AAAAAA'; + var light_color = '#CCCCCC'; + + function sidebar_is_collapsed() { + return sidebarwrapper.is(':not(:visible)'); + } + + function toggle_sidebar() { + if (sidebar_is_collapsed()) + expand_sidebar(); + else + collapse_sidebar(); + } + + function collapse_sidebar() { + sidebarwrapper.hide(); + sidebar.css('width', ssb_width_collapsed); + bodywrapper.css('margin-left', bw_margin_collapsed); + sidebarbutton.css({ + 'margin-left': '0', + 'height': bodywrapper.height(), + 'border-radius': '5px' + }); + sidebarbutton.find('span').text('»'); + sidebarbutton.attr('title', _('Expand sidebar')); + document.cookie = 'sidebar=collapsed'; + } + + function expand_sidebar() { + bodywrapper.css('margin-left', bw_margin_expanded); + sidebar.css('width', ssb_width_expanded); + sidebarwrapper.show(); + sidebarbutton.css({ + 'margin-left': ssb_width_expanded-12, + 'height': bodywrapper.height(), + 'border-radius': '0 5px 5px 0' + }); + sidebarbutton.find('span').text('«'); + sidebarbutton.attr('title', _('Collapse sidebar')); + //sidebarwrapper.css({'padding-top': + // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); + document.cookie = 'sidebar=expanded'; + } + + function add_sidebar_button() { + sidebarwrapper.css({ + 'float': 'left', + 'margin-right': '0', + 'width': ssb_width_expanded - 28 + }); + // create the button + sidebar.append( + '
    «
    ' + ); + var sidebarbutton = $('#sidebarbutton'); + // find the height of the viewport to center the '<<' in the page + var viewport_height; + if (window.innerHeight) + viewport_height = window.innerHeight; + else + viewport_height = $(window).height(); + var sidebar_offset = sidebar.offset().top; + var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); + sidebarbutton.find('span').css({ + 'display': 'block', + 'position': 'fixed', + 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 + }); + + sidebarbutton.click(toggle_sidebar); + sidebarbutton.attr('title', _('Collapse sidebar')); + sidebarbutton.css({ + 'border-radius': '0 5px 5px 0', + 'color': '#444444', + 'background-color': '#CCCCCC', + 'font-size': '1.2em', + 'cursor': 'pointer', + 'height': sidebar_height, + 'padding-top': '1px', + 'padding-left': '1px', + 'margin-left': ssb_width_expanded - 12 + }); + + sidebarbutton.hover( + function () { + $(this).css('background-color', dark_color); + }, + function () { + $(this).css('background-color', light_color); + } + ); + } + + function set_position_from_cookie() { + if (!document.cookie) + return; + var items = document.cookie.split(';'); + for(var k=0; k=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" +faq/programming,,::,for x in sequence[::-1]: +faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," +faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," +faq/windows,229,:EOF,@setlocal enableextensions & python -x %~f0 %* & goto :EOF +faq/windows,393,:REG,.py :REG_SZ: c:\\python.exe -u %s %s +howto/cporting,,:add,"if (!PyArg_ParseTuple(args, ""ii:add_ints"", &one, &two))" +howto/cporting,,:encode,"if (!PyArg_ParseTuple(args, ""O:encode_object"", &myobj))" +howto/cporting,,:say,"if (!PyArg_ParseTuple(args, ""U:say_hello"", &name))" +howto/curses,,:black,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:blue,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:cyan,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:green,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:magenta,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:red,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/curses,,:white,"7:white." +howto/curses,,:yellow,"They are: 0:black, 1:red, 2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and" +howto/logging,,:And,"WARNING:And this, too" +howto/logging,,:And,"WARNING:root:And this, too" +howto/logging,,:Doing,INFO:root:Doing something +howto/logging,,:Finished,INFO:root:Finished +howto/logging,,:logger,severity:logger name:message +howto/logging,,:Look,WARNING:root:Look before you leap! +howto/logging,,:message,severity:logger name:message +howto/logging,,:root,DEBUG:root:This message should go to the log file +howto/logging,,:root,INFO:root:Doing something +howto/logging,,:root,INFO:root:Finished +howto/logging,,:root,INFO:root:So should this +howto/logging,,:root,INFO:root:Started +howto/logging,,:root,"WARNING:root:And this, too" +howto/logging,,:root,WARNING:root:Look before you leap! +howto/logging,,:root,WARNING:root:Watch out! +howto/logging,,:So,INFO:root:So should this +howto/logging,,:So,INFO:So should this +howto/logging,,:Started,INFO:root:Started +howto/logging,,:This,DEBUG:root:This message should go to the log file +howto/logging,,:This,DEBUG:This message should appear on the console +howto/logging,,:Watch,WARNING:root:Watch out! +howto/pyporting,75,::,# make sure to use :: Python *and* :: Python :: 3 so +howto/pyporting,75,::,"'Programming Language :: Python'," +howto/pyporting,75,::,'Programming Language :: Python :: 3' +howto/regex,,::, +howto/regex,,:foo,(?:foo) +howto/urllib2,,:example,"for example ""joe@password:example.com""" +howto/webservers,,.. image:,.. image:: http.png +library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," +library/bisect,32,:hi,all(val >= x for val in a[i:hi]) +library/bisect,42,:hi,all(val > x for val in a[i:hi]) +library/configparser,,:home,my_dir: ${Common:home_dir}/twosheds +library/configparser,,:option,${section:option} +library/configparser,,:path,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} +library/configparser,,:Python,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} +library/configparser,,`,# Set the optional `raw` argument of get() to True if you wish to disable +library/configparser,,:system,path: ${Common:system_dir}/Library/Frameworks/ +library/configparser,,`,# The optional `fallback` argument can be used to provide a fallback value +library/configparser,,`,# The optional `vars` argument is a dict with members that will take +library/datetime,,:MM, +library/datetime,,:SS, +library/decimal,,:optional,"trailneg:optional trailing minus indicator" +library/difflib,,:ahi,a[alo:ahi] +library/difflib,,:bhi,b[blo:bhi] +library/difflib,,:i1, +library/difflib,,:i2, +library/difflib,,:j2, +library/dis,,:TOS, +library/dis,,`,TOS = `TOS` +library/doctest,,`,``factorial`` from the ``example`` module: +library/doctest,,`,The ``example`` module +library/doctest,,`,Using ``factorial`` +library/functions,,:step,a[start:stop:step] +library/functions,,:stop,"a[start:stop, i]" +library/functions,,:stop,a[start:stop:step] +library/hotshot,,:lineno,"ncalls tottime percall cumtime percall filename:lineno(function)" +library/http.client,52,:port,host:port +library/httplib,,:port,host:port +library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS" +library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS" +library/itertools,,:step,elements from seq[start:stop:step] +library/itertools,,:stop,elements from seq[start:stop:step] +library/linecache,,:sys,"sys:x:3:3:sys:/dev:/bin/sh" +library/logging,,:And, +library/logging,,:Doing,INFO:root:Doing something +library/logging,,:Finished,INFO:root:Finished +library/logging,,:logger,severity:logger name:message +library/logging,,:Look,WARNING:root:Look before you leap! +library/logging,,:message,severity:logger name:message +library/logging,,:package1, +library/logging,,:package2, +library/logging,,:port,host:port +library/logging,,:root, +library/logging,,:So,INFO:root:So should this +library/logging,,:So,INFO:So should this +library/logging,,:Started,INFO:root:Started +library/logging,,:This, +library/logging,,:Watch,WARNING:root:Watch out! +library/logging.handlers,,:port,host:port +library/mmap,,:i2,obj[i1:i2] +library/multiprocessing,,`,# Add more tasks using `put()` +library/multiprocessing,,`,# A test file for the `multiprocessing` package +library/multiprocessing,,`,# A test of `multiprocessing.Pool` class +library/multiprocessing,,`,# `BaseManager`. +library/multiprocessing,,`,`Cluster` is a subclass of `SyncManager` so it allows creation of +library/multiprocessing,,`,# create server for a `HostManager` object +library/multiprocessing,,`,# Depends on `multiprocessing` package -- tested with `processing-0.60` +library/multiprocessing,,`,`hostname` gives the name of the host. If hostname is not +library/multiprocessing,,`,# in the original order then consider using `Pool.map()` or +library/multiprocessing,,`,">>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]`" +library/multiprocessing,,`,">>> l._callmethod('__getslice__', (2, 7)) # equiv to `l[2:7]`" +library/multiprocessing,,`,# Not sure if we should synchronize access to `socket.accept()` method by +library/multiprocessing,,`,# object. (We import `multiprocessing.reduction` to enable this pickling.) +library/multiprocessing,,`,# `Pool.imap()` (which will save on the amount of code needed anyway). +library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)" +library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy +library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy +library/multiprocessing,,`,# register the generator function baz; use `GeneratorProxy` to make proxies +library/multiprocessing,,`,`slots` is used to specify the number of slots for processes on +library/nntplib,,:bytes,:bytes +library/nntplib,,:bytes,"['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']" +library/nntplib,,:lines,:lines +library/nntplib,,:lines,"['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject']" +library/optparse,,:len,"del parser.rargs[:len(value)]" +library/os.path,,:foo,c:foo +library/parser,,`,"""Make a function that raises an argument to the exponent `exp`.""" +library/pdb,,:lineno,filename:lineno +library/pdb,,:lineno,[filename:lineno | bpnumber [bpnumber ...]] +library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")" +library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS""" +library/pprint,209,::,"'classifiers': ['Development Status :: 4 - Beta'," +library/pprint,209,::,"'Intended Audience :: Developers'," +library/pprint,209,::,"'License :: OSI Approved :: MIT License'," +library/pprint,209,::,"'Natural Language :: English'," +library/pprint,209,::,"'Operating System :: OS Independent'," +library/pprint,209,::,"'Programming Language :: Python'," +library/pprint,209,::,"'Programming Language :: Python :: 2'," +library/pprint,209,::,"'Programming Language :: Python :: 2.6'," +library/pprint,209,::,"'Programming Language :: Python :: 2.7'," +library/pprint,209,::,"'Topic :: Software Development :: Libraries'," +library/pprint,209,::,"'Topic :: Software Development :: Libraries :: Python Modules']," +library/profile,,:lineno,filename:lineno(function) +library/profile,,:lineno,ncalls tottime percall cumtime percall filename:lineno(function) +library/profile,,:lineno,"(sort by filename:lineno)," +library/pyexpat,,:elem1, +library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">" +library/repr,,`,"return `obj`" +library/smtplib,,:port,"as well as a regular host:port server." +library/smtplib,,:port,method must support that as well as a regular host:port +library/socket,,::,"(10, 1, 6, '', ('2001:888:2000:d::a2', 80, 0, 0))]" +library/socket,,::,'5aef:2b::8' +library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" +library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) +library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" +library/sqlite3,,:age,"select name_last, age from people where name_last=:who and age=:age" +library/sqlite3,,:memory, +library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" +library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" +library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." +library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" +library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState +library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com +library/ssl,,:Some,"Locality Name (eg, city) []:Some City" +library/ssl,,:US,Country Name (2 letter code) [AU]:US +library/stdtypes,,::,>>> a[::-1].tolist() +library/stdtypes,,::,>>> a[::2].tolist() +library/stdtypes,,:end,s[start:end] +library/stdtypes,,::,>>> hash(v[::-2]) == hash(b'abcefg'[::-2]) +library/stdtypes,,:len,s[len(s):len(s)] +library/stdtypes,,::,>>> y = m[::2] +library/string,,:end,s[start:end] +library/subprocess,,`,"output=`dmesg | grep hda`" +library/subprocess,,`,"output=`mycmd myarg`" +library/tarfile,,:bz2, +library/tarfile,,:compression,filemode[:compression] +library/tarfile,,:gz, +library/tarfile,,:xz,'a:xz' +library/tarfile,,:xz,'r:xz' +library/tarfile,,:xz,'w:xz' +library/time,,:mm, +library/time,,:ss, +library/turtle,,::,Example:: +library/urllib2,,:password,"""joe:password@python.org""" +library/urllib,,:port,:port +library/urllib.request,,:close,Connection:close +library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" +library/urllib.request,,:password,"""joe:password@python.org""" +library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 +library/xmlrpc.client,,:pass,http://user:pass@host:port/path +library/xmlrpc.client,,:pass,user:pass +library/xmlrpc.client,,:port,http://user:pass@host:port/path +license,,`,"``Software''), to deal in the Software without restriction, including" +license,,`,"THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND," +license,,`,* THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND +license,,`,THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +license,,`,* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY +license,,`,THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND +license,,:zooko,mailto:zooko@zooko.com +packaging/examples,,`,This is the description of the ``foobar`` project. +packaging/setupcfg,,::,Development Status :: 3 - Alpha +packaging/setupcfg,,::,License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1) +packaging/setupscript,,::,"'Development Status :: 4 - Beta'," +packaging/setupscript,,::,"'Environment :: Console'," +packaging/setupscript,,::,"'Environment :: Web Environment'," +packaging/setupscript,,::,"'Intended Audience :: Developers'," +packaging/setupscript,,::,"'Intended Audience :: End Users/Desktop'," +packaging/setupscript,,::,"'Intended Audience :: System Administrators'," +packaging/setupscript,,::,"'License :: OSI Approved :: Python Software Foundation License'," +packaging/setupscript,,::,"'Operating System :: MacOS :: MacOS X'," +packaging/setupscript,,::,"'Operating System :: Microsoft :: Windows'," +packaging/setupscript,,::,"'Operating System :: POSIX'," +packaging/setupscript,,::,"'Programming Language :: Python'," +packaging/setupscript,,::,"'Topic :: Communications :: Email'," +packaging/setupscript,,::,"'Topic :: Office/Business'," +packaging/setupscript,,::,"'Topic :: Software Development :: Bug Tracking'," +packaging/tutorial,,::,1) License :: OSI Approved :: GNU General Public License (GPL) +packaging/tutorial,,::,2) License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +packaging/tutorial,,::,classifier = Development Status :: 3 - Alpha +packaging/tutorial,,::,License :: OSI Approved :: GNU General Public License (GPL) +packaging/tutorial,,::,Type the number of the license you wish to use or ? to try again:: 1 +reference/datamodel,,:max, +reference/datamodel,,:step,a[i:j:step] +reference/expressions,,:datum,{key:datum...} +reference/expressions,,`,`expressions...` +reference/expressions,,:index,x[index:index] +reference/grammar,,:output,#diagram:output +reference/grammar,,:rules,#diagram:rules +reference/grammar,,`,'`' testlist1 '`' +reference/grammar,,:token,#diagram:token +reference/lexical_analysis,,`,", : . ` = ;" +reference/lexical_analysis,,`,$ ? ` +reference/lexical_analysis,,:fileencoding,# vim:fileencoding= +tutorial/datastructures,,:value,It is also possible to delete a key:value +tutorial/datastructures,,:value,key:value pairs within the braces adds initial key:value pairs +tutorial/stdlib2,,:config,"logging.warning('Warning:config file %s not found', 'server.conf')" +tutorial/stdlib2,,:config,WARNING:root:Warning:config file server.conf not found +tutorial/stdlib2,,:Critical,CRITICAL:root:Critical error -- shutting down +tutorial/stdlib2,,:Error,ERROR:root:Error occurred +tutorial/stdlib2,,:root,CRITICAL:root:Critical error -- shutting down +tutorial/stdlib2,,:root,ERROR:root:Error occurred +tutorial/stdlib2,,:root,WARNING:root:Warning:config file server.conf not found +tutorial/stdlib2,,:start,extra = data[start:start+extra_size] +tutorial/stdlib2,,:start,"fields = struct.unpack('>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo') +whatsnew/2.7,735,:Sunday,'2009:4:Sunday' +whatsnew/2.7,862,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0" +whatsnew/2.7,862,::,"export PYTHONWARNINGS=all,error:::Cookie:0" +whatsnew/3.2,,:affe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," +whatsnew/3.2,,:affe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') +whatsnew/3.2,,:beef,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," +whatsnew/3.2,,:beef,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') +whatsnew/3.2,,:cafe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," +whatsnew/3.2,,:cafe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') +whatsnew/3.2,,:deaf,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," +whatsnew/3.2,,:deaf,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') +whatsnew/3.2,,:directory,... ${buildout:directory}/downloads/dist +whatsnew/3.2,,:directory,${buildout:directory}/downloads/dist +whatsnew/3.2,,::,"$ export PYTHONWARNINGS='ignore::RuntimeWarning::,once::UnicodeWarning::'" +whatsnew/3.2,,:feed,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," +whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') +whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" +whatsnew/3.2,,:location,... zope9-location = ${zope9:location} +whatsnew/3.2,,:location,zope9-location = ${zope9:location} +whatsnew/3.2,,:prefix,... zope-conf = ${custom:prefix}/etc/zope.conf +whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/sphinxext/suspicious.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/tools/sphinxext/suspicious.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,263 @@ +""" +Try to detect suspicious constructs, resembling markup +that has leaked into the final output. + +Suspicious lines are reported in a comma-separated-file, +``suspicious.csv``, located in the output directory. + +The file is utf-8 encoded, and each line contains four fields: + + * document name (normalized) + * line number in the source document + * problematic text + * complete line showing the problematic text in context + +It is common to find many false positives. To avoid reporting them +again and again, they may be added to the ``ignored.csv`` file +(located in the configuration directory). The file has the same +format as ``suspicious.csv`` with a few differences: + + - each line defines a rule; if the rule matches, the issue + is ignored. + - line number may be empty (that is, nothing between the + commas: ",,"). In this case, line numbers are ignored (the + rule matches anywhere in the file). + - the last field does not have to be a complete line; some + surrounding text (never more than a line) is enough for + context. + +Rules are processed sequentially. A rule matches when: + + * document names are the same + * problematic texts are the same + * line numbers are close to each other (5 lines up or down) + * the rule text is completely contained into the source line + +The simplest way to create the ignored.csv file is by copying +undesired entries from suspicious.csv (possibly trimming the last +field.) + +Copyright 2009 Gabriel A. Genellina + +""" + +import os +import re +import csv +import sys + +from docutils import nodes +from sphinx.builders import Builder + +detect_all = re.compile(r''' + ::(?=[^=])| # two :: (but NOT ::=) + :[a-zA-Z][a-zA-Z0-9]+| # :foo + `| # ` (seldom used by itself) + (?= (3, 0) + + +class Rule: + def __init__(self, docname, lineno, issue, line): + """A rule for ignoring issues""" + self.docname = docname # document to which this rule applies + self.lineno = lineno # line number in the original source; + # this rule matches only near that. + # None -> don't care + self.issue = issue # the markup fragment that triggered this rule + self.line = line # text of the container element (single line only) + + + +class dialect(csv.excel): + """Our dialect: uses only linefeed as newline.""" + lineterminator = '\n' + + +class CheckSuspiciousMarkupBuilder(Builder): + """ + Checks for possibly invalid markup that may leak into the output. + """ + name = 'suspicious' + + def init(self): + # create output file + self.log_file_name = os.path.join(self.outdir, 'suspicious.csv') + open(self.log_file_name, 'w').close() + # load database of previously ignored issues + self.load_rules(os.path.join(os.path.dirname(__file__), + 'susp-ignored.csv')) + + def get_outdated_docs(self): + return self.env.found_docs + + def get_target_uri(self, docname, typ=None): + return '' + + def prepare_writing(self, docnames): + pass + + def write_doc(self, docname, doctree): + # set when any issue is encountered in this document + self.any_issue = False + self.docname = docname + visitor = SuspiciousVisitor(doctree, self) + doctree.walk(visitor) + + def finish(self): + return + + def check_issue(self, line, lineno, issue): + if not self.is_ignored(line, lineno, issue): + self.report_issue(line, lineno, issue) + + def is_ignored(self, line, lineno, issue): + """Determine whether this issue should be ignored.""" + docname = self.docname + for rule in self.rules: + if rule.docname != docname: continue + if rule.issue != issue: continue + # Both lines must match *exactly*. This is rather strict, + # and probably should be improved. + # Doing fuzzy matches with levenshtein distance could work, + # but that means bringing other libraries... + # Ok, relax that requirement: just check if the rule fragment + # is contained in the document line + if rule.line not in line: continue + # Check both line numbers. If they're "near" + # this rule matches. (lineno=None means "don't care") + if (rule.lineno is not None) and \ + abs(rule.lineno - lineno) > 5: continue + # if it came this far, the rule matched + return True + return False + + def report_issue(self, text, lineno, issue): + if not self.any_issue: self.info() + self.any_issue = True + self.write_log_entry(lineno, issue, text) + if py3: + self.warn('[%s:%d] "%s" found in "%-.120s"' % + (self.docname, lineno, issue, text)) + else: + self.warn('[%s:%d] "%s" found in "%-.120s"' % ( + self.docname.encode(sys.getdefaultencoding(),'replace'), + lineno, + issue.encode(sys.getdefaultencoding(),'replace'), + text.strip().encode(sys.getdefaultencoding(),'replace'))) + self.app.statuscode = 1 + + def write_log_entry(self, lineno, issue, text): + if py3: + f = open(self.log_file_name, 'a') + writer = csv.writer(f, dialect) + writer.writerow([self.docname, lineno, issue, text.strip()]) + f.close() + else: + f = open(self.log_file_name, 'ab') + writer = csv.writer(f, dialect) + writer.writerow([self.docname.encode('utf-8'), + lineno, + issue.encode('utf-8'), + text.strip().encode('utf-8')]) + f.close() + + def load_rules(self, filename): + """Load database of previously ignored issues. + + A csv file, with exactly the same format as suspicious.csv + Fields: document name (normalized), line number, issue, surrounding text + """ + self.info("loading ignore rules... ", nonl=1) + self.rules = rules = [] + try: + if py3: + f = open(filename, 'r') + else: + f = open(filename, 'rb') + except IOError: + return + for i, row in enumerate(csv.reader(f)): + if len(row) != 4: + raise ValueError( + "wrong format in %s, line %d: %s" % (filename, i+1, row)) + docname, lineno, issue, text = row + if lineno: + lineno = int(lineno) + else: + lineno = None + if not py3: + docname = docname.decode('utf-8') + issue = issue.decode('utf-8') + text = text.decode('utf-8') + rule = Rule(docname, lineno, issue, text) + rules.append(rule) + f.close() + self.info('done, %d rules loaded' % len(self.rules)) + + +def get_lineno(node): + """Obtain line number information for a node.""" + lineno = None + while lineno is None and node: + node = node.parent + lineno = node.line + return lineno + + +def extract_line(text, index): + """text may be a multiline string; extract + only the line containing the given character index. + + >>> extract_line("abc\ndefgh\ni", 6) + >>> 'defgh' + >>> for i in (0, 2, 3, 4, 10): + ... print extract_line("abc\ndefgh\ni", i) + abc + abc + abc + defgh + defgh + i + """ + p = text.rfind('\n', 0, index) + 1 + q = text.find('\n', index) + if q < 0: + q = len(text) + return text[p:q] + + +class SuspiciousVisitor(nodes.GenericNodeVisitor): + + lastlineno = 0 + + def __init__(self, document, builder): + nodes.GenericNodeVisitor.__init__(self, document) + self.builder = builder + + def default_visit(self, node): + if isinstance(node, (nodes.Text, nodes.image)): # direct text containers + text = node.astext() + # lineno seems to go backwards sometimes (?) + self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno) + seen = set() # don't report the same issue more than only once per line + for match in detect_all(text): + issue = match.group() + line = extract_line(text, match.start()) + if (issue, line) not in seen: + self.builder.check_issue(line, lineno, issue) + seen.add((issue, line)) + + unknown_visit = default_visit + + def visit_document(self, node): + self.lastlineno = 0 + + def visit_comment(self, node): + # ignore comments -- too much false positives. + # (although doing this could miss some errors; + # there were two sections "commented-out" by mistake + # in the Python docs that would not be catched) + raise nodes.SkipNode diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/static/copybutton.js --- a/Doc/tools/static/copybutton.js Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,57 +0,0 @@ -$(document).ready(function() { - /* Add a [>>>] button on the top-right corner of code samples to hide - * the >>> and ... prompts and the output and thus make the code - * copyable. */ - var div = $('.highlight-python .highlight,' + - '.highlight-python3 .highlight') - var pre = div.find('pre'); - - // get the styles from the current theme - pre.parent().parent().css('position', 'relative'); - var hide_text = 'Hide the prompts and output'; - var show_text = 'Show the prompts and output'; - var border_width = pre.css('border-top-width'); - var border_style = pre.css('border-top-style'); - var border_color = pre.css('border-top-color'); - var button_styles = { - 'cursor':'pointer', 'position': 'absolute', 'top': '0', 'right': '0', - 'border-color': border_color, 'border-style': border_style, - 'border-width': border_width, 'color': border_color, 'text-size': '75%', - 'font-family': 'monospace', 'padding-left': '0.2em', 'padding-right': '0.2em', - 'border-radius': '0 3px 0 0' - } - - // create and add the button to all the code blocks that contain >>> - div.each(function(index) { - var jthis = $(this); - if (jthis.find('.gp').length > 0) { - var button = $('>>>'); - button.css(button_styles) - button.attr('title', hide_text); - jthis.prepend(button); - } - // tracebacks (.gt) contain bare text elements that need to be - // wrapped in a span to work with .nextUntil() (see later) - jthis.find('pre:has(.gt)').contents().filter(function() { - return ((this.nodeType == 3) && (this.data.trim().length > 0)); - }).wrap(''); - }); - - // define the behavior of the button when it's clicked - $('.copybutton').toggle( - function() { - var button = $(this); - button.parent().find('.go, .gp, .gt').hide(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden'); - button.css('text-decoration', 'line-through'); - button.attr('title', show_text); - }, - function() { - var button = $(this); - button.parent().find('.go, .gp, .gt').show(); - button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible'); - button.css('text-decoration', 'none'); - button.attr('title', hide_text); - }); -}); - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/static/py.png Binary file Doc/tools/static/py.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/static/sidebar.js --- a/Doc/tools/static/sidebar.js Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,193 +0,0 @@ -/* - * sidebar.js - * ~~~~~~~~~~ - * - * This script makes the Sphinx sidebar collapsible and implements intelligent - * scrolling. - * - * .sphinxsidebar contains .sphinxsidebarwrapper. This script adds in - * .sphixsidebar, after .sphinxsidebarwrapper, the #sidebarbutton used to - * collapse and expand the sidebar. - * - * When the sidebar is collapsed the .sphinxsidebarwrapper is hidden and the - * width of the sidebar and the margin-left of the document are decreased. - * When the sidebar is expanded the opposite happens. This script saves a - * per-browser/per-session cookie used to remember the position of the sidebar - * among the pages. Once the browser is closed the cookie is deleted and the - * position reset to the default (expanded). - * - * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - * :license: BSD, see LICENSE for details. - * - */ - -$(function() { - // global elements used by the functions. - // the 'sidebarbutton' element is defined as global after its - // creation, in the add_sidebar_button function - var jwindow = $(window); - var jdocument = $(document); - var bodywrapper = $('.bodywrapper'); - var sidebar = $('.sphinxsidebar'); - var sidebarwrapper = $('.sphinxsidebarwrapper'); - - // original margin-left of the bodywrapper and width of the sidebar - // with the sidebar expanded - var bw_margin_expanded = bodywrapper.css('margin-left'); - var ssb_width_expanded = sidebar.width(); - - // margin-left of the bodywrapper and width of the sidebar - // with the sidebar collapsed - var bw_margin_collapsed = '.8em'; - var ssb_width_collapsed = '.8em'; - - // colors used by the current theme - var dark_color = '#AAAAAA'; - var light_color = '#CCCCCC'; - - function get_viewport_height() { - if (window.innerHeight) - return window.innerHeight; - else - return jwindow.height(); - } - - function sidebar_is_collapsed() { - return sidebarwrapper.is(':not(:visible)'); - } - - function toggle_sidebar() { - if (sidebar_is_collapsed()) - expand_sidebar(); - else - collapse_sidebar(); - // adjust the scrolling of the sidebar - scroll_sidebar(); - } - - function collapse_sidebar() { - sidebarwrapper.hide(); - sidebar.css('width', ssb_width_collapsed); - bodywrapper.css('margin-left', bw_margin_collapsed); - sidebarbutton.css({ - 'margin-left': '0', - 'height': bodywrapper.height(), - 'border-radius': '5px' - }); - sidebarbutton.find('span').text('»'); - sidebarbutton.attr('title', _('Expand sidebar')); - document.cookie = 'sidebar=collapsed'; - } - - function expand_sidebar() { - bodywrapper.css('margin-left', bw_margin_expanded); - sidebar.css('width', ssb_width_expanded); - sidebarwrapper.show(); - sidebarbutton.css({ - 'margin-left': ssb_width_expanded-12, - 'height': bodywrapper.height(), - 'border-radius': '0 5px 5px 0' - }); - sidebarbutton.find('span').text('«'); - sidebarbutton.attr('title', _('Collapse sidebar')); - //sidebarwrapper.css({'padding-top': - // Math.max(window.pageYOffset - sidebarwrapper.offset().top, 10)}); - document.cookie = 'sidebar=expanded'; - } - - function add_sidebar_button() { - sidebarwrapper.css({ - 'float': 'left', - 'margin-right': '0', - 'width': ssb_width_expanded - 28 - }); - // create the button - sidebar.append( - '
    «
    ' - ); - var sidebarbutton = $('#sidebarbutton'); - // find the height of the viewport to center the '<<' in the page - var viewport_height = get_viewport_height(); - var sidebar_offset = sidebar.offset().top; - var sidebar_height = Math.max(bodywrapper.height(), sidebar.height()); - sidebarbutton.find('span').css({ - 'display': 'block', - 'position': 'fixed', - 'top': Math.min(viewport_height/2, sidebar_height/2 + sidebar_offset) - 10 - }); - - sidebarbutton.click(toggle_sidebar); - sidebarbutton.attr('title', _('Collapse sidebar')); - sidebarbutton.css({ - 'border-radius': '0 5px 5px 0', - 'color': '#444444', - 'background-color': '#CCCCCC', - 'font-size': '1.2em', - 'cursor': 'pointer', - 'height': sidebar_height, - 'padding-top': '1px', - 'padding-left': '1px', - 'margin-left': ssb_width_expanded - 12 - }); - - sidebarbutton.hover( - function () { - $(this).css('background-color', dark_color); - }, - function () { - $(this).css('background-color', light_color); - } - ); - } - - function set_position_from_cookie() { - if (!document.cookie) - return; - var items = document.cookie.split(';'); - for(var k=0; k wintop && curbot > winbot) { - sidebarwrapper.css('top', $u.max([wintop - offset - 10, 0])); - } - else if (curtop < wintop && curbot < winbot) { - sidebarwrapper.css('top', $u.min([winbot - sidebar_height - offset - 20, - jdocument.height() - sidebar_height - 200])); - } - } - } - jwindow.scroll(scroll_sidebar); -}); diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/static/version_switch.js --- a/Doc/tools/static/version_switch.js Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,68 +0,0 @@ -(function() { - 'use strict'; - - var all_versions = { - '3.6': 'dev (3.6)', - '3.5': '3.5', - '3.4': '3.4', - '3.3': '3.3', - '3.2': '3.2', - '2.7': '2.7', - '2.6': '2.6' - }; - - function build_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function patch_url(url, new_version) { - var url_re = /\.org\/(\d|py3k|dev|((release\/)?\d\.\d[\w\d\.]*))\//, - new_url = url.replace(url_re, '.org/' + new_version + '/'); - - if (new_url == url && !new_url.match(url_re)) { - // python 2 url without version? - new_url = url.replace(/\.org\//, '.org/' + new_version + '/'); - } - return new_url; - } - - function on_switch() { - var selected = $(this).children('option:selected').attr('value'); - - var url = window.location.href, - new_url = patch_url(url, selected); - - if (new_url != url) { - // check beforehand if url exists, else redirect to version's start page - $.ajax({ - url: new_url, - success: function() { - window.location.href = new_url; - }, - error: function() { - window.location.href = 'https://docs.python.org/' + selected; - } - }); - } - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var version = release.substr(0, 3); - var select = build_select(version, release); - - $('.version_switcher_placeholder').html(select); - $('.version_switcher_placeholder select').bind('change', on_switch); - }); -})(); diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/susp-ignored.csv --- a/Doc/tools/susp-ignored.csv Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,297 +0,0 @@ -c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)" -c-api/list,,:high,list[low:high] -c-api/sequence,,:i2,del o[i1:i2] -c-api/sequence,,:i2,o[i1:i2] -c-api/unicode,,:end,str[start:end] -c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,267,`,This is the description of the ``foobar`` package. -distutils/setupscript,,::, -extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" -extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" -extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {" -extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {" -faq/programming,,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" -faq/programming,,::,for x in sequence[::-1]: -faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," -faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," -faq/windows,,:bd8afb90ebf2,"Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32" -howto/cporting,,:encode,"if (!PyArg_ParseTuple(args, ""O:encode_object"", &myobj))" -howto/cporting,,:say,"if (!PyArg_ParseTuple(args, ""U:say_hello"", &name))" -howto/curses,,:black,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:red,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:green,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:yellow,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:blue,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:magenta,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:cyan,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:white,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/ipaddress,,:DB8,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,::,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,:db8,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('::1') -howto/ipaddress,,:db8,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/96') -howto/ipaddress,,::,IPv6Network('2001:db8::/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/128') -howto/ipaddress,,::,IPv6Network('2001:db8::/128') -howto/ipaddress,,:db8,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,::,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,::,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,:db8,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:ffff,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('::ffff:ffff') -howto/ipaddress,,:db8,'2001:db8::/96' -howto/ipaddress,,::,'2001:db8::/96' -howto/ipaddress,,:db8,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,'2001:db8::1' -howto/ipaddress,,::,'2001:db8::1' -howto/ipaddress,,:db8,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,::,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('2001:db8::ffff:ffff') -howto/logging,,:And,"WARNING:And this, too" -howto/logging,,:And,"WARNING:root:And this, too" -howto/logging,,:Doing,INFO:root:Doing something -howto/logging,,:Finished,INFO:root:Finished -howto/logging,,:logger,severity:logger name:message -howto/logging,,:Look,WARNING:root:Look before you leap! -howto/logging,,:message,severity:logger name:message -howto/logging,,:root,DEBUG:root:This message should go to the log file -howto/logging,,:root,INFO:root:Doing something -howto/logging,,:root,INFO:root:Finished -howto/logging,,:root,INFO:root:So should this -howto/logging,,:root,INFO:root:Started -howto/logging,,:root,"WARNING:root:And this, too" -howto/logging,,:root,WARNING:root:Look before you leap! -howto/logging,,:root,WARNING:root:Watch out! -howto/logging,,:So,INFO:root:So should this -howto/logging,,:So,INFO:So should this -howto/logging,,:Started,INFO:root:Started -howto/logging,,:This,DEBUG:root:This message should go to the log file -howto/logging,,:This,DEBUG:This message should appear on the console -howto/logging,,:Watch,WARNING:root:Watch out! -howto/pyporting,,::,Programming Language :: Python :: 2 -howto/pyporting,,::,Programming Language :: Python :: 3 -howto/regex,,::, -howto/regex,,:foo,(?:foo) -howto/urllib2,,:example,"for example ""joe@password:example.com""" -library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," -library/bisect,32,:hi,all(val >= x for val in a[i:hi]) -library/bisect,42,:hi,all(val > x for val in a[i:hi]) -library/configparser,,:home,my_dir: ${Common:home_dir}/twosheds -library/configparser,,:option,${section:option} -library/configparser,,:path,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:Python,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:system,path: ${Common:system_dir}/Library/Frameworks/ -library/datetime,,:MM, -library/datetime,,:SS, -library/decimal,,:optional,"trailneg:optional trailing minus indicator" -library/difflib,,:ahi,a[alo:ahi] -library/difflib,,:bhi,b[blo:bhi] -library/difflib,,:i1, -library/difflib,,:i2, -library/difflib,,:j2, -library/doctest,,`,``factorial`` from the ``example`` module: -library/doctest,,`,The ``example`` module -library/doctest,,`,Using ``factorial`` -library/exceptions,,:err,err.object[err.start:err.end] -library/functions,,:step,a[start:stop:step] -library/functions,,:stop,"a[start:stop, i]" -library/functions,,:stop,a[start:stop:step] -library/http.client,,:port,host:port -library/http.cookies,,`,!#$%&'*+-.^_`|~: -library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS" -library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS" -library/inspect,,:int,">>> def foo(a, *, b:int, **kwargs):" -library/inspect,,:int,"'(a, *, b:int, **kwargs)'" -library/inspect,,:int,'b:int' -library/ipaddress,,:db8,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,::,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,:db8,IPv6Address('2001:db8::') -library/ipaddress,,::,IPv6Address('2001:db8::') -library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,IPv6Address('2001:db8::1000') -library/ipaddress,,::,IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,"""::abc:7:def""" -library/ipaddress,,:def,"""::abc:7:def""" -library/ipaddress,,::,::FFFF/96 -library/ipaddress,,::,2002::/16 -library/ipaddress,,::,2001::/32 -library/ipaddress,,::,>>> str(ipaddress.IPv6Address('::1')) -library/ipaddress,,::,'::1' -library/ipaddress,,:ff00,ffff:ff00:: -library/ipaddress,,:db00,2001:db00::0/24 -library/ipaddress,,::,2001:db00::0/24 -library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: -library/ipaddress,,::,2001:db00::0/ffff:ff00:: -library/itertools,,:step,elements from seq[start:stop:step] -library/itertools,,:stop,elements from seq[start:stop:step] -library/logging.handlers,,:port,host:port -library/mmap,,:i2,obj[i1:i2] -library/multiprocessing,,`,# Add more tasks using `put()` -library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)" -library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy -library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy -library/multiprocessing,,`,# register the generator function baz; use `GeneratorProxy` to make proxies -library/nntplib,,:bytes,:bytes -library/nntplib,,:lines,:lines -library/optparse,,:len,"del parser.rargs[:len(value)]" -library/os.path,,:foo,c:foo -library/pathlib,,:bar,">>> PureWindowsPath('c:/Windows', 'd:bar')" -library/pathlib,,:bar,PureWindowsPath('d:bar') -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').root -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').anchor -library/pdb,,:lineno,filename:lineno -library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")" -library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS""" -library/pprint,,::,"'Programming Language :: Python :: 2 :: Only']," -library/pprint,,::,"'Programming Language :: Python :: 2.6'," -library/pprint,,::,"'Programming Language :: Python :: 2.7'," -library/profile,,:lineno,filename:lineno(function) -library/pyexpat,,:elem1, -library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">" -library/smtplib,,:port,method must support that as well as a regular host:port -library/socket,,::,'5aef:2b::8' -library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" -library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" -library/sqlite3,,:memory, -library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" -library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" -library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" -library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." -library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" -library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState -library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com -library/ssl,,:Some,"Locality Name (eg, city) []:Some City" -library/ssl,,:US,Country Name (2 letter code) [AU]:US -library/stdtypes,,:end,s[start:end] -library/stdtypes,,::,>>> hash(v[::-2]) == hash(b'abcefg'[::-2]) -library/stdtypes,,:len,s[len(s):len(s)] -library/stdtypes,,::,>>> y = m[::2] -library/stdtypes,,::,>>> z = y[::-2] -library/subprocess,,`,"output=`dmesg | grep hda`" -library/subprocess,,`,"output=`mycmd myarg`" -library/tarfile,,:bz2, -library/tarfile,,:compression,filemode[:compression] -library/tarfile,,:gz, -library/tarfile,,:xz,'a:xz' -library/tarfile,,:xz,'r:xz' -library/tarfile,,:xz,'w:xz' -library/time,,:mm, -library/time,,:ss, -library/tracemalloc,,:limit,"for index, stat in enumerate(top_stats[:limit], 1):" -library/turtle,,::,Example:: -library/unittest,,:foo,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:first,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:foo,'ERROR:foo.bar:second message']) -library/unittest,,:second,'ERROR:foo.bar:second message']) -library/urllib.request,,:close,Connection:close -library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" -library/urllib.request,,:password,"""joe:password@python.org""" -library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 -library/venv,,:param,":param nodist: If True, setuptools and pip are not installed into the" -library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" -library/venv,,:param,":param nopip: If True, pip is not installed into the created" -library/venv,,:param,:param context: The information for the environment creation request -library/xmlrpc.client,,:pass,http://user:pass@host:port/path -library/xmlrpc.client,,:pass,user:pass -library/xmlrpc.client,,:port,http://user:pass@host:port/path -license,,`,"``Software''), to deal in the Software without restriction, including" -license,,`,"THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND," -license,,`,* THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND -license,,`,THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -license,,`,* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -license,,`,THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND -license,,:zooko,mailto:zooko@zooko.com -reference/expressions,,:index,x[index:index] -reference/lexical_analysis,,`,$ ? ` -reference/lexical_analysis,,:fileencoding,# vim:fileencoding= -tutorial/datastructures,,:value,It is also possible to delete a key:value -tutorial/datastructures,,:value,key:value pairs within the braces adds initial key:value pairs -tutorial/stdlib2,,:config,"logging.warning('Warning:config file %s not found', 'server.conf')" -tutorial/stdlib2,,:config,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:Critical,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:Error,ERROR:root:Error occurred -tutorial/stdlib2,,:root,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:root,ERROR:root:Error occurred -tutorial/stdlib2,,:root,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:start,extra = data[start:start+extra_size] -tutorial/stdlib2,,:start,"fields = struct.unpack('>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo') -whatsnew/2.7,,:Sunday,'2009:4:Sunday' -whatsnew/2.7,,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/2.7,,::,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/3.2,,:affe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:affe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:beef,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:beef,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:cafe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:cafe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:deaf,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:deaf,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:directory,${buildout:directory}/downloads/dist -whatsnew/3.2,,::,"$ export PYTHONWARNINGS='ignore::RuntimeWarning::,once::UnicodeWarning::'" -whatsnew/3.2,,:feed,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" -whatsnew/3.2,,:location,zope9-location = ${zope9:location} -whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf -whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't" -whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as" -library/tarfile,149,:xz,'x:xz' -library/xml.etree.elementtree,290,:sometag,prefix:sometag -library/xml.etree.elementtree,301,:fictional,"Lancelot -library/xml.etree.elementtree,301,:character,Archie Leach -library/xml.etree.elementtree,301,:character,Sir Robin -library/xml.etree.elementtree,301,:character,Gunther -library/xml.etree.elementtree,301,:character,Commander Clement -library/xml.etree.elementtree,332,:actor,"for actor in root.findall('real_person:actor', ns):" -library/xml.etree.elementtree,332,:name,"name = actor.find('real_person:name', ns)" -library/xml.etree.elementtree,332,:character,"for char in actor.findall('role:character', ns):" -library/zipapp,31,:main,"$ python -m zipapp myapp -m ""myapp:main""" -library/zipapp,82,:fn,"argument should have the form ""pkg.mod:fn"", where ""pkg.mod"" is a" -library/zipapp,155,:callable,"""pkg.module:callable"" and the archive will be run by importing" -library/stdtypes,,::,>>> m[::2].tolist() -library/sys,,`,# ``wrapper`` creates a ``wrap(coro)`` coroutine: -tutorial/venv,77,:c7b9645a6f35,"Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25)" -whatsnew/3.5,,:root,'WARNING:root:warning\n' -whatsnew/3.5,,:warning,'WARNING:root:warning\n' -whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') -whatsnew/3.5,,:root,ERROR:root:exception -whatsnew/3.5,,:exception,ERROR:root:exception diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/templates/download.html --- a/Doc/tools/templates/download.html Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,65 +0,0 @@ -{% extends "layout.html" %} -{% set title = 'Download' %} -{% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} -{% else %} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} -{% endif %} - -{% block body %} -

    Download Python {{ release }} Documentation

    - -{% if last_updated %}

    Last updated on: {{ last_updated }}.

    {% endif %} - -

    To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table. The numbers -in the table are the size of the download files in megabytes.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FormatPacked as .zipPacked as .tar.bz2
    PDF (US-Letter paper size)Download (ca. 8 MB)Download (ca. 8 MB)
    PDF (A4 paper size)Download (ca. 8 MB)Download (ca. 8 MB)
    HTMLDownload (ca. 6 MB)Download (ca. 4 MB)
    Plain TextDownload (ca. 2 MB)Download (ca. 1.5 MB)
    EPUBDownload (ca. 4.5 MB)
    - -

    These archives contain all the content in the documentation.

    - -

    HTML Help (.chm) files are made available in the "Windows" section -on the Python -download page.

    - - -

    Unpacking

    - -

    Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The InfoZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.

    - -

    Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the InfoZIP zip program.

    - - -

    Problems

    - -

    If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.

    -{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/templates/indexcontent.html --- a/Doc/tools/templates/indexcontent.html Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,59 +0,0 @@ -{% extends "defindex.html" %} -{% block tables %} -

    {% trans %}Parts of the documentation:{% endtrans %}

    - - -
    -

    {% trans %}What's new in Python {{ version }}?{% endtrans %}
    - {% trans whatsnew_index=pathto("whatsnew/index") %}or all "What's new" documents since 2.0{% endtrans %}

    -

    {% trans %}Tutorial{% endtrans %}
    - {% trans %}start here{% endtrans %}

    -

    {% trans %}Library Reference{% endtrans %}
    - {% trans %}keep this under your pillow{% endtrans %}

    -

    {% trans %}Language Reference{% endtrans %}
    - {% trans %}describes syntax and language elements{% endtrans %}

    -

    {% trans %}Python Setup and Usage{% endtrans %}
    - {% trans %}how to use Python on different platforms{% endtrans %}

    -

    {% trans %}Python HOWTOs{% endtrans %}
    - {% trans %}in-depth documents on specific topics{% endtrans %}

    -     		-

    {% trans %}Installing Python Modules{% endtrans %}
    - {% trans %}installing from the Python Package Index & other sources{% endtrans %}

    -

    {% trans %}Distributing Python Modules{% endtrans %}
    - {% trans %}publishing modules for installation by others{% endtrans %}

    -

    {% trans %}Extending and Embedding{% endtrans %}
    - {% trans %}tutorial for C/C++ programmers{% endtrans %}

    -

    {% trans %}Python/C API{% endtrans %}
    - {% trans %}reference for C/C++ programmers{% endtrans %}

    -

    {% trans %}FAQs{% endtrans %}
    - {% trans %}frequently asked questions (with answers!){% endtrans %}

    -
    - -

    {% trans %}Indices and tables:{% endtrans %}

    - - -
    -

    {% trans %}Global Module Index{% endtrans %}
    - {% trans %}quick access to all modules{% endtrans %}

    -

    {% trans %}General Index{% endtrans %}
    - {% trans %}all functions, classes, terms{% endtrans %}

    -

    {% trans %}Glossary{% endtrans %}
    - {% trans %}the most important terms explained{% endtrans %}

    -     		-

    {% trans %}Search page{% endtrans %}
    - {% trans %}search this documentation{% endtrans %}

    -

    {% trans %}Complete Table of Contents{% endtrans %}
    - {% trans %}lists all sections and subsections{% endtrans %}

    -
    - -

    {% trans %}Meta information:{% endtrans %}

    - - -
    -

    {% trans %}Reporting bugs{% endtrans %}

    -

    {% trans %}About the documentation{% endtrans %}

    -     		-

    {% trans %}History and License of Python{% endtrans %}

    -

    {% trans %}Copyright{% endtrans %}

    -
    -{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/templates/indexsidebar.html --- a/Doc/tools/templates/indexsidebar.html Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -

    {% trans %}Download{% endtrans %}

    -

    {% trans %}Download these documents{% endtrans %}

    -

    {% trans %}Docs for other versions{% endtrans %}

    - - -

    {% trans %}Other resources{% endtrans %}

    - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/templates/layout.html --- a/Doc/tools/templates/layout.html Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -{% extends "!layout.html" %} -{% block rootrellink %} -
    • -
  • Python{{ reldelim1 }}
    • -
  • - {%- if versionswitcher is defined %} - {{ release }} - {% trans %}Documentation {% endtrans %}{{ reldelim1 }} - {%- else %} - {{ shorttitle }}{{ reldelim1 }} - {%- endif %} -
    • -{% endblock %} -{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %} -{% block extrahead %} - - {% if not embedded %}{% endif %} - {% if versionswitcher is defined and not embedded %}{% endif %} - {% if pagename == 'whatsnew/changelog' and not embedded %} - - {% endif %} -{{ super() }} -{% endblock %} -{% block footer %} -
    - © {% trans %}Copyright{% endtrans %} {{ copyright|e }}. -
    - {% trans %}The Python Software Foundation is a non-profit corporation.{% endtrans %} - {% trans %}Please donate.{% endtrans %} -
    - {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} - {% trans pathto_bugs=pathto('bugs') %}Found a bug?{% endtrans %} -
    - {% trans sphinx_version=sphinx_version|e %}Created using Sphinx {{ sphinx_version }}.{% endtrans %} -
    -{% endblock %} -{% block sidebarsourcelink %} -{%- if show_source and has_source and sourcename %} -

    {{ _('This Page') }}

    - -{%- endif %} -{% endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tools/templates/opensearch.xml --- a/Doc/tools/templates/opensearch.xml Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -{% extends "!opensearch.xml" %} -{% block extra -%} -https://www.python.org/images/favicon16x16.ico -{%- endblock %} diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/appendix.rst --- a/Doc/tutorial/appendix.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,124 +0,0 @@ -.. _tut-appendix: - -******** -Appendix -******** - - -.. _tut-interac: - -Interactive Mode -================ - -.. _tut-error: - -Error Handling --------------- - -When an error occurs, the interpreter prints an error message and a stack trace. -In interactive mode, it then returns to the primary prompt; when input came from -a file, it exits with a nonzero exit status after printing the stack trace. -(Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement -are not errors in this context.) Some errors are unconditionally fatal and -cause an exit with a nonzero exit; this applies to internal inconsistencies and -some cases of running out of memory. All error messages are written to the -standard error stream; normal output from executed commands is written to -standard output. - -Typing the interrupt character (usually :kbd:`Control-C` or :kbd:`Delete`) to the primary or -secondary prompt cancels the input and returns to the primary prompt. [#]_ -Typing an interrupt while a command is executing raises the -:exc:`KeyboardInterrupt` exception, which may be handled by a :keyword:`try` -statement. - - -.. _tut-scripts: - -Executable Python Scripts -------------------------- - -On BSD'ish Unix systems, Python scripts can be made directly executable, like -shell scripts, by putting the line :: - - #!/usr/bin/env python3.5 - -(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning -of the script and giving the file an executable mode. The ``#!`` must be the -first two characters of the file. On some platforms, this first line must end -with a Unix-style line ending (``'\n'``), not a Windows (``'\r\n'``) line -ending. Note that the hash, or pound, character, ``'#'``, is used to start a -comment in Python. - -The script can be given an executable mode, or permission, using the -:program:`chmod` command. - -.. code-block:: bash - - $ chmod +x myscript.py - -On Windows systems, there is no notion of an "executable mode". The Python -installer automatically associates ``.py`` files with ``python.exe`` so that -a double-click on a Python file will run it as a script. The extension can -also be ``.pyw``, in that case, the console window that normally appears is -suppressed. - - -.. _tut-startup: - -The Interactive Startup File ----------------------------- - -When you use Python interactively, it is frequently handy to have some standard -commands executed every time the interpreter is started. You can do this by -setting an environment variable named :envvar:`PYTHONSTARTUP` to the name of a -file containing your start-up commands. This is similar to the :file:`.profile` -feature of the Unix shells. - -This file is only read in interactive sessions, not when Python reads commands -from a script, and not when :file:`/dev/tty` is given as the explicit source of -commands (which otherwise behaves like an interactive session). It is executed -in the same namespace where interactive commands are executed, so that objects -that it defines or imports can be used without qualification in the interactive -session. You can also change the prompts ``sys.ps1`` and ``sys.ps2`` in this -file. - -If you want to read an additional start-up file from the current directory, you -can program this in the global start-up file using code like ``if -os.path.isfile('.pythonrc.py'): exec(open('.pythonrc.py').read())``. -If you want to use the startup file in a script, you must do this explicitly -in the script:: - - import os - filename = os.environ.get('PYTHONSTARTUP') - if filename and os.path.isfile(filename): - with open(filename) as fobj: - startup_file = fobj.read() - exec(startup_file) - - -.. _tut-customize: - -The Customization Modules -------------------------- - -Python provides two hooks to let you customize it: :mod:`sitecustomize` and -:mod:`usercustomize`. To see how it works, you need first to find the location -of your user site-packages directory. Start Python and run this code:: - - >>> import site - >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' - -Now you can create a file named :file:`usercustomize.py` in that directory and -put anything you want in it. It will affect every invocation of Python, unless -it is started with the :option:`-s` option to disable the automatic import. - -:mod:`sitecustomize` works in the same way, but is typically created by an -administrator of the computer in the global site-packages directory, and is -imported before :mod:`usercustomize`. See the documentation of the :mod:`site` -module for more details. - - -.. rubric:: Footnotes - -.. [#] A problem with the GNU Readline package may prevent this. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/classes.rst --- a/Doc/tutorial/classes.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/classes.rst Mon Jan 25 17:05:13 2016 +0100 @@ -168,6 +168,7 @@ def do_global(): global spam spam = "global spam" + spam = "test spam" do_local() print("After local assignment:", spam) @@ -387,77 +388,6 @@ argument list. -.. _tut-class-and-instance-variables: - -Class and Instance Variables ----------------------------- - -Generally speaking, instance variables are for data unique to each instance -and class variables are for attributes and methods shared by all instances -of the class:: - - class Dog: - - kind = 'canine' # class variable shared by all instances - - def __init__(self, name): - self.name = name # instance variable unique to each instance - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.kind # shared by all dogs - 'canine' - >>> e.kind # shared by all dogs - 'canine' - >>> d.name # unique to d - 'Fido' - >>> e.name # unique to e - 'Buddy' - -As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. -For example, the *tricks* list in the following code should not be used as a -class variable because just a single list would be shared by all *Dog* -instances:: - - class Dog: - - tricks = [] # mistaken use of a class variable - - def __init__(self, name): - self.name = name - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks # unexpectedly shared by all dogs - ['roll over', 'play dead'] - -Correct design of the class should use an instance variable instead:: - - class Dog: - - def __init__(self, name): - self.name = name - self.tricks = [] # creates a new empty list for each dog - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks - ['roll over'] - >>> e.tricks - ['play dead'] - - .. _tut-remarks: Random Remarks @@ -643,7 +573,7 @@ class can be subclassed without affecting the precedence order of its parents). Taken together, these properties make it possible to design reliable and extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. +http://www.python.org/download/releases/2.3/mro/. .. _tut-private: @@ -723,7 +653,7 @@ A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`read` and :meth:`readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -802,16 +732,16 @@ for char in "123": print(char) for line in open("myfile.txt"): - print(line, end='') + print(line) This style of access is clear, concise, and convenient. The use of iterators pervades and unifies Python. Behind the scenes, the :keyword:`for` statement calls :func:`iter` on the container object. The function returns an iterator -object that defines the method :meth:`~iterator.__next__` which accesses -elements in the container one at a time. When there are no more elements, -:meth:`~iterator.__next__` raises a :exc:`StopIteration` exception which tells the -:keyword:`for` loop to terminate. You can call the :meth:`~iterator.__next__` method -using the :func:`next` built-in function; this example shows how it all works:: +object that defines the method :meth:`__next__` which accesses elements in the +container one at a time. When there are no more elements, :meth:`__next__` +raises a :exc:`StopIteration` exception which tells the :keyword:`for` loop to +terminate. You can call the :meth:`__next__` method using the :func:`next` +built-in function; this example shows how it all works:: >>> s = 'abc' >>> it = iter(s) @@ -831,8 +761,8 @@ Having seen the mechanics behind the iterator protocol, it is easy to add iterator behavior to your classes. Define an :meth:`__iter__` method which -returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +returns an object with a :meth:`__next__` method. If the class defines +:meth:`__next__`, then :meth:`__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -869,7 +799,7 @@ :term:`Generator`\s are a simple and powerful tool for creating iterators. They are written like regular functions but use the :keyword:`yield` statement whenever they want to return data. Each time :func:`next` is called on it, the -generator resumes where it left off (it remembers all the data values and which +generator resumes where it left-off (it remembers all the data values and which statement was last executed). An example shows that generators can be trivially easy to create:: @@ -887,10 +817,10 @@ o g -Anything that can be done with generators can also be done with class-based +Anything that can be done with generators can also be done with class based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods -are created automatically. +compact is that the :meth:`__iter__` and :meth:`__next__` methods are created +automatically. Another key feature is that the local variables and execution state are automatically saved between calls. This made the function easier to write and diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/controlflow.rst --- a/Doc/tutorial/controlflow.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/controlflow.rst Mon Jan 25 17:05:13 2016 +0100 @@ -19,14 +19,14 @@ >>> x = int(input("Please enter an integer: ")) Please enter an integer: 42 >>> if x < 0: - ... x = 0 - ... print('Negative changed to zero') + ... x = 0 + ... print('Negative changed to zero') ... elif x == 0: - ... print('Zero') + ... print('Zero') ... elif x == 1: - ... print('Single') + ... print('Single') ... else: - ... print('More') + ... print('More') ... More @@ -58,24 +58,24 @@ :: >>> # Measure some strings: - ... words = ['cat', 'window', 'defenestrate'] - >>> for w in words: - ... print(w, len(w)) + ... a = ['cat', 'window', 'defenestrate'] + >>> for x in a: + ... print(x, len(x)) ... cat 3 window 6 defenestrate 12 -If you need to modify the sequence you are iterating over while inside the loop -(for example to duplicate selected items), it is recommended that you first -make a copy. Iterating over a sequence does not implicitly make a copy. The -slice notation makes this especially convenient:: +It is not safe to modify the sequence being iterated over in the loop (this can +only happen for mutable sequence types, such as lists). If you need to modify +the list you are iterating over (for example, to duplicate selected items) you +must iterate over a copy. The slice notation makes this particularly +convenient:: - >>> for w in words[:]: # Loop over a slice copy of the entire list. - ... if len(w) > 6: - ... words.insert(0, w) + >>> for x in a[:]: # make a slice copy of the entire list + ... if len(x) > 6: a.insert(0, x) ... - >>> words + >>> a ['defenestrate', 'cat', 'window', 'defenestrate'] @@ -157,6 +157,9 @@ The :keyword:`break` statement, like in C, breaks out of the smallest enclosing :keyword:`for` or :keyword:`while` loop. +The :keyword:`continue` statement, also borrowed from C, continues with the next +iteration of the loop. + Loop statements may have an ``else`` clause; it is executed when the loop terminates through exhaustion of the list (with :keyword:`for`) or when the condition becomes false (with :keyword:`while`), but not when the loop is @@ -184,29 +187,6 @@ (Yes, this is the correct code. Look closely: the ``else`` clause belongs to the :keyword:`for` loop, **not** the :keyword:`if` statement.) -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: - - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found a number", num) - Found an even number 2 - Found a number 3 - Found an even number 4 - Found a number 5 - Found an even number 6 - Found a number 7 - Found an even number 8 - Found a number 9 .. _tut-pass: @@ -370,7 +350,7 @@ return False retries = retries - 1 if retries < 0: - raise OSError('uncooperative user') + raise IOError('refusenik user') print(complaint) This function can be called in several ways: @@ -583,16 +563,17 @@ .. _tut-lambda: -Lambda Expressions ------------------- +Lambda Forms +------------ -Small anonymous functions can be created with the :keyword:`lambda` keyword. -This function returns the sum of its two arguments: ``lambda a, b: a+b``. -Lambda functions can be used wherever function objects are required. They are -syntactically restricted to a single expression. Semantically, they are just -syntactic sugar for a normal function definition. Like nested function -definitions, lambda functions can reference variables from the containing -scope:: +By popular demand, a few features commonly found in functional programming +languages like Lisp have been added to Python. With the :keyword:`lambda` +keyword, small anonymous functions can be created. Here's a function that +returns the sum of its two arguments: ``lambda a, b: a+b``. Lambda forms can be +used wherever function objects are required. They are syntactically restricted +to a single expression. Semantically, they are just syntactic sugar for a +normal function definition. Like nested function definitions, lambda forms can +reference variables from the containing scope:: >>> def make_incrementor(n): ... return lambda x: x + n @@ -603,14 +584,6 @@ >>> f(1) 43 -The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: - - >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] - >>> pairs.sort(key=lambda pair: pair[1]) - >>> pairs - [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')] - .. _tut-docstrings: @@ -663,39 +636,6 @@ No, really, it doesn't do anything. -.. _tut-annotations: - -Function Annotations --------------------- - -.. sectionauthor:: Zachary Ware -.. index:: - pair: function; annotations - single: -> (return annotation assignment) - -:ref:`Function annotations ` are completely optional metadata -information about the types used by user-defined functions (see :pep:`484` -for more information). - -Annotations are stored in the :attr:`__annotations__` attribute of the function -as a dictionary and have no effect on any other part of the function. Parameter -annotations are defined by a colon after the parameter name, followed by an -expression evaluating to the value of the annotation. Return annotations are -defined by a literal ``->``, followed by an expression, between the parameter -list and the colon denoting the end of the :keyword:`def` statement. The -following example has a positional argument, a keyword argument, and the return -value annotated:: - - >>> def f(ham: str, eggs: str = 'eggs') -> str: - ... print("Annotations:", f.__annotations__) - ... print("Arguments:", ham, eggs) - ... return ham + ' and ' + eggs - ... - >>> f('spam') - Annotations: {'ham': , 'return': , 'eggs': } - Arguments: spam eggs - 'spam and eggs' - .. _tut-codingstyle: Intermezzo: Coding Style @@ -755,3 +695,4 @@ .. [#] Actually, *call by object reference* would be a better description, since if a mutable object is passed, the caller will see any changes the callee makes to it (items inserted into a list). + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/datastructures.rst --- a/Doc/tutorial/datastructures.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/datastructures.rst Mon Jan 25 17:05:13 2016 +0100 @@ -54,12 +54,6 @@ will see this notation frequently in the Python Library Reference.) -.. method:: list.clear() - :noindex: - - Remove all items from the list. Equivalent to ``del a[:]``. - - .. method:: list.index(x) :noindex: @@ -73,11 +67,10 @@ Return the number of times *x* appears in the list. -.. method:: list.sort(key=None, reverse=False) +.. method:: list.sort() :noindex: - Sort the items of the list in place (the arguments can be used for sort - customization, see :func:`sorted` for their explanation). + Sort the items of the list in place. .. method:: list.reverse() @@ -86,12 +79,6 @@ Reverse the elements of the list in place. -.. method:: list.copy() - :noindex: - - Return a shallow copy of the list. Equivalent to ``a[:]``. - - An example that uses most of the list methods:: >>> a = [66.25, 333, 333, 1, 1234.5] @@ -112,15 +99,10 @@ >>> a.sort() >>> a [-1, 1, 66.25, 333, 333, 1234.5] - >>> a.pop() - 1234.5 - >>> a - [-1, 1, 66.25, 333, 333] You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that -only modify the list have no return value printed -- they return the default -``None``. [1]_ This is a design principle for all mutable data structures in -Python. +modify the list have no return value printed -- they return ``None``. [1]_ This +is a design principle for all mutable data structures in Python. .. _tut-lists-as-stacks: @@ -200,17 +182,12 @@ >>> squares [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] -Note that this creates (or overwrites) a variable named ``x`` that still exists -after the loop completes. We can calculate the list of squares without any -side effects using:: - - squares = list(map(lambda x: x**2, range(10))) - -or, equivalently:: +We can obtain the same result with:: squares = [x**2 for x in range(10)] -which is more concise and readable. +This is also equivalent to ``squares = map(lambda x: x**2, range(10))``, +but it's more concise and readable. A list comprehension consists of brackets containing an expression followed by a :keyword:`for` clause, then zero or more :keyword:`for` or :keyword:`if` @@ -320,7 +297,7 @@ In the real world, you should prefer built-in functions to complex flow statements. The :func:`zip` function would do a great job for this use case:: - >>> list(zip(*matrix)) + >>> zip(*matrix) [(1, 5, 9), (2, 6, 10), (3, 7, 11), (4, 8, 12)] See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. @@ -377,31 +354,17 @@ ... u = t, (1, 2, 3, 4, 5) >>> u ((12345, 54321, 'hello!'), (1, 2, 3, 4, 5)) - >>> # Tuples are immutable: - ... t[0] = 88888 - Traceback (most recent call last): - File "", line 1, in - TypeError: 'tuple' object does not support item assignment - >>> # but they can contain mutable objects: - ... v = ([1, 2, 3], [3, 2, 1]) - >>> v - ([1, 2, 3], [3, 2, 1]) - As you see, on output tuples are always enclosed in parentheses, so that nested tuples are interpreted correctly; they may be input with or without surrounding parentheses, although often parentheses are necessary anyway (if the tuple is -part of a larger expression). It is not possible to assign to the individual -items of a tuple, however it is possible to create tuples which contain mutable -objects, such as lists. +part of a larger expression). -Though tuples may seem similar to lists, they are often used in different -situations and for different purposes. -Tuples are :term:`immutable`, and usually contain an heterogeneous sequence of -elements that are accessed via unpacking (see later in this section) or indexing -(or even by attribute in the case of :func:`namedtuples `). -Lists are :term:`mutable`, and their elements are usually homogeneous and are -accessed by iterating over the list. +Tuples have many uses. For example: (x, y) coordinate pairs, employee records +from a database, etc. Tuples, like strings, are immutable: it is not possible +to assign to the individual items of a tuple (you can simulate much of the same +effect with slicing and concatenation, though). It is also possible to create +tuples which contain mutable objects, such as lists. A special problem is the construction of tuples containing 0 or 1 items: the syntax has some extra quirks to accommodate these. Empty tuples are constructed @@ -430,6 +393,8 @@ sequence. Note that multiple assignment is really just a combination of tuple packing and sequence unpacking. +.. XXX Add a bit on the difference between tuples and lists. + .. _tut-sets: @@ -441,7 +406,7 @@ eliminating duplicate entries. Set objects also support mathematical operations like union, intersection, difference, and symmetric difference. -Curly braces or the :func:`set` function can be used to create sets. Note: to +Curly braces or the :func:`set` function can be used to create sets. Note: To create an empty set you have to use ``set()``, not ``{}``; the latter creates an empty dictionary, a data structure that we discuss in the next section. @@ -470,14 +435,14 @@ >>> a ^ b # letters in a or b but not both {'r', 'd', 'b', 'm', 'z', 'l'} -Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: +Like :ref:`for lists `, there is a set comprehension syntax:: >>> a = {x for x in 'abracadabra' if x not in 'abc'} >>> a {'r', 'd'} + .. _tut-dictionaries: Dictionaries @@ -612,19 +577,6 @@ orange pear -It is sometimes tempting to change a list while you are looping over it; -however, it is often simpler and safer to create a new list instead. :: - - >>> import math - >>> raw_data = [56.2, float('NaN'), 51.7, 55.3, 52.5, float('NaN'), 47.8] - >>> filtered_data = [] - >>> for value in raw_data: - ... if not math.isnan(value): - ... filtered_data.append(value) - ... - >>> filtered_data - [56.2, 51.7, 55.3, 52.5, 47.8] - .. _tut-conditions: @@ -685,7 +637,7 @@ all items of two sequences compare equal, the sequences are considered equal. If one sequence is an initial sub-sequence of the other, the shorter sequence is the smaller (lesser) one. Lexicographical ordering for strings uses the Unicode -code point number to order individual characters. Some examples of comparisons +codepoint number to order individual characters. Some examples of comparisons between sequences of the same type:: (1, 2, 3) < (1, 2, 4) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/errors.rst --- a/Doc/tutorial/errors.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/errors.rst Mon Jan 25 17:05:13 2016 +0100 @@ -45,7 +45,7 @@ >>> 10 * (1/0) Traceback (most recent call last): File "", line 1, in ? - ZeroDivisionError: division by zero + ZeroDivisionError: int division or modulo by zero >>> 4 + spam*3 Traceback (most recent call last): File "", line 1, in ? @@ -131,8 +131,8 @@ f = open('myfile.txt') s = f.readline() i = int(s.strip()) - except OSError as err: - print("OS error: {0}".format(err)) + except IOError as err: + print("I/O error: {0}".format(err)) except ValueError: print("Could not convert data to an integer.") except: @@ -336,7 +336,7 @@ A *finally clause* is always executed before leaving the :keyword:`try` statement, whether an exception has occurred or not. When an exception has occurred in the :keyword:`try` clause and has not been handled by an -:keyword:`except` clause (or it has occurred in an :keyword:`except` or +:keyword:`except` clause (or it has occurred in a :keyword:`except` or :keyword:`else` clause), it is re-raised after the :keyword:`finally` clause has been executed. The :keyword:`finally` clause is also executed "on the way out" when any other clause of the :keyword:`try` statement is left via a @@ -387,7 +387,7 @@ and print its contents to the screen. :: for line in open("myfile.txt"): - print(line, end="") + print(line) The problem with this code is that it leaves the file open for an indeterminate amount of time after this part of the code has finished executing. @@ -397,7 +397,7 @@ with open("myfile.txt") as f: for line in f: - print(line, end="") + print(line) After the statement is executed, the file *f* is always closed, even if a problem was encountered while processing the lines. Objects which, like files, diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/index.rst --- a/Doc/tutorial/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,9 @@ The Python Tutorial ###################### +:Release: |version| +:Date: |today| + Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, @@ -12,7 +15,7 @@ The Python interpreter and the extensive standard library are freely available in source or binary form for all major platforms from the Python Web site, -https://www.python.org/, and may be freely distributed. The same site also +http://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. @@ -53,8 +56,6 @@ classes.rst stdlib.rst stdlib2.rst - venv.rst whatnow.rst interactive.rst floatingpoint.rst - appendix.rst diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/inputoutput.rst --- a/Doc/tutorial/inputoutput.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/inputoutput.rst Mon Jan 25 17:05:13 2016 +0100 @@ -37,7 +37,7 @@ The :func:`str` function is meant to return representations of values which are fairly human-readable, while :func:`repr` is meant to generate representations which can be read by the interpreter (or will force a :exc:`SyntaxError` if -there is no equivalent syntax). For objects which don't have a particular +there is not equivalent syntax). For objects which don't have a particular representation for human consumption, :func:`str` will return the same value as :func:`repr`. Many values, such as numbers or structures like lists and dictionaries, have the same representation using either function. Strings, in @@ -184,7 +184,7 @@ >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' - ... 'Dcab: {0[Dcab]:d}'.format(table)) + 'Dcab: {0[Dcab]:d}'.format(table)) Jack: 4098; Sjoerd: 4127; Dcab: 8637678 This could also be done by passing the table as keyword arguments with the '**' @@ -213,6 +213,10 @@ >>> print('The value of PI is approximately %5.3f.' % math.pi) The value of PI is approximately 3.142. +Since :meth:`str.format` is quite new, a lot of Python code still uses the ``%`` +operator. However, because this old style of formatting will eventually be +removed from the language, :meth:`str.format` should generally be used. + More information can be found in the :ref:`old-string-formatting` section. @@ -230,12 +234,12 @@ :: - >>> f = open('workfile', 'w') + >>> f = open('/tmp/workfile', 'w') .. XXX str(f) is >>> print(f) - + The first argument is a string containing the filename. The second argument is another string containing a few characters describing the way in which the file @@ -247,16 +251,14 @@ omitted. Normally, files are opened in :dfn:`text mode`, that means, you read and write -strings from and to the file, which are encoded in a specific encoding. If -encoding is not specified, the default is platform dependent (see -:func:`open`). ``'b'`` appended to the mode opens the file in +strings from and to the file, which are encoded in a specific encoding (the +default being UTF-8). ``'b'`` appended to the mode opens the file in :dfn:`binary mode`: now the data is read and written in the form of bytes objects. This mode should be used for all files that don't contain text. -In text mode, the default when reading is to convert platform-specific line -endings (``\n`` on Unix, ``\r\n`` on Windows) to just ``\n``. When writing in -text mode, the default is to convert occurrences of ``\n`` back to -platform-specific line endings. This behind-the-scenes modification +In text mode, the default is to convert platform-specific line endings (``\n`` +on Unix, ``\r\n`` on Windows) to just ``\n`` on reading and ``\n`` back to +platform-specific line endings on writing. This behind-the-scenes modification to file data is fine for text files, but will corrupt binary data like that in :file:`JPEG` or :file:`EXE` files. Be very careful to use binary mode when reading and writing such files. @@ -271,11 +273,10 @@ ``f`` has already been created. To read a file's contents, call ``f.read(size)``, which reads some quantity of -data and returns it as a string (in text mode) or bytes object (in binary mode). -*size* is an optional numeric argument. When *size* is omitted or negative, the -entire contents of the file will be read and returned; it's your problem if the -file is twice as large as your machine's memory. Otherwise, at most *size* bytes -are read and returned. +data and returns it as a string or bytes object. *size* is an optional numeric +argument. When *size* is omitted or negative, the entire contents of the file +will be read and returned; it's your problem if the file is twice as large as +your machine's memory. Otherwise, at most *size* bytes are read and returned. If the end of the file has been reached, ``f.read()`` will return an empty string (``''``). :: @@ -298,8 +299,18 @@ >>> f.readline() '' -For reading lines from a file, you can loop over the file object. This is memory -efficient, fast, and leads to simple code:: +``f.readlines()`` returns a list containing all the lines of data in the file. +If given an optional parameter *sizehint*, it reads that many bytes from the +file and enough more to complete a line, and returns the lines from that. This +is often used to allow efficient reading of a large file by lines, but without +having to load the entire file in memory. Only complete lines will be returned. +:: + + >>> f.readlines() + ['This is the first line of the file.\n', 'Second line of the file\n'] + +An alternative approach to reading lines is to loop over the file object. This is +memory efficient, fast, and leads to simpler code:: >>> for line in f: ... print(line, end='') @@ -307,8 +318,9 @@ This is the first line of the file. Second line of the file -If you want to read all the lines of a file in a list you can also use -``list(f)`` or ``f.readlines()``. +The alternative approach is simpler but does not provide as fine-grained +control. Since the two approaches manage line buffering differently, they +should not be mixed. ``f.write(string)`` writes the contents of *string* to the file, returning the number of characters written. :: @@ -316,26 +328,24 @@ >>> f.write('This is a test\n') 15 -Other types of objects need to be converted -- either to a string (in text mode) -or a bytes object (in binary mode) -- before writing them:: +To write something other than a string, it needs to be converted to a string +first:: >>> value = ('the answer', 42) - >>> s = str(value) # convert the tuple to string + >>> s = str(value) >>> f.write(s) 18 -``f.tell()`` returns an integer giving the file object's current position in the file -represented as number of bytes from the beginning of the file when in binary mode and -an opaque number when in text mode. - -To change the file object's position, use ``f.seek(offset, from_what)``. The position is computed +``f.tell()`` returns an integer giving the file object's current position in the +file, measured in bytes from the beginning of the file. To change the file +object's position, use ``f.seek(offset, from_what)``. The position is computed from adding *offset* to a reference point; the reference point is selected by the *from_what* argument. A *from_what* value of 0 measures from the beginning of the file, 1 uses the current file position, and 2 uses the end of the file as the reference point. *from_what* can be omitted and defaults to 0, using the beginning of the file as the reference point. :: - >>> f = open('workfile', 'rb+') + >>> f = open('/tmp/workfile', 'rb+') >>> f.write(b'0123456789abcdef') 16 >>> f.seek(5) # Go to the 6th byte in the file @@ -349,10 +359,7 @@ In text files (those opened without a ``b`` in the mode string), only seeks relative to the beginning of the file are allowed (the exception being seeking -to the very file end with ``seek(0, 2)``) and the only valid *offset* values are -those returned from the ``f.tell()``, or zero. Any other *offset* value produces -undefined behaviour. - +to the very file end with ``seek(0, 2)``). When you're done with a file, call ``f.close()`` to close it and free up any system resources taken up by the open file. After calling ``f.close()``, @@ -369,7 +376,7 @@ suite finishes, even if an exception is raised on the way. It is also much shorter than writing equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - >>> with open('workfile', 'r') as f: + >>> with open('/tmp/workfile', 'r') as f: ... read_data = f.read() >>> f.closed True @@ -379,63 +386,47 @@ Reference for a complete guide to file objects. -.. _tut-json: +.. _tut-pickle: -Saving structured data with :mod:`json` ---------------------------------------- +The :mod:`pickle` Module +------------------------ -.. index:: module: json +.. index:: module: pickle -Strings can easily be written to and read from a file. Numbers take a bit more +Strings can easily be written to and read from a file. Numbers take a bit more effort, since the :meth:`read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` -and returns its numeric value 123. When you want to save more complex data -types like nested lists and dictionaries, parsing and serializing by hand -becomes complicated. +and returns its numeric value 123. However, when you want to save more complex +data types like lists, dictionaries, or class instances, things get a lot more +complicated. -Rather than having users constantly writing and debugging code to save -complicated data types to files, Python allows you to use the popular data -interchange format called `JSON (JavaScript Object Notation) -`_. The standard module called :mod:`json` can take Python -data hierarchies, and convert them to string representations; this process is -called :dfn:`serializing`. Reconstructing the data from the string representation -is called :dfn:`deserializing`. Between serializing and deserializing, the -string representing the object may have been stored in a file or data, or +Rather than have users be constantly writing and debugging code to save +complicated data types, Python provides a standard module called :mod:`pickle`. +This is an amazing module that can take almost any Python object (even some +forms of Python code!), and convert it to a string representation; this process +is called :dfn:`pickling`. Reconstructing the object from the string +representation is called :dfn:`unpickling`. Between pickling and unpickling, +the string representing the object may have been stored in a file or data, or sent over a network connection to some distant machine. -.. note:: - The JSON format is commonly used by modern applications to allow for data - exchange. Many programmers are already familiar with it, which makes - it a good choice for interoperability. +If you have an object ``x``, and a file object ``f`` that's been opened for +writing, the simplest way to pickle the object takes only one line of code:: -If you have an object ``x``, you can view its JSON string representation with a -simple line of code:: + pickle.dump(x, f) - >>> json.dumps([1, 'simple', 'list']) - '[1, "simple", "list"]' +To unpickle the object again, if ``f`` is a file object which has been opened +for reading:: -Another variant of the :func:`~json.dumps` function, called :func:`~json.dump`, -simply serializes the object to a :term:`text file`. So if ``f`` is a -:term:`text file` object opened for writing, we can do this:: + x = pickle.load(f) - json.dump(x, f) +(There are other variants of this, used when pickling many objects or when you +don't want to write the pickled data to a file; consult the complete +documentation for :mod:`pickle` in the Python Library Reference.) -To decode the object again, if ``f`` is a :term:`text file` object which has -been opened for reading:: +:mod:`pickle` is the standard way to make Python objects which can be stored and +reused by other programs or by a future invocation of the same program; the +technical term for this is a :dfn:`persistent` object. Because :mod:`pickle` is +so widely used, many authors who write Python extensions take care to ensure +that new data types such as matrices can be properly pickled and unpickled. - x = json.load(f) -This simple serialization technique can handle lists and dictionaries, but -serializing arbitrary class instances in JSON requires a bit of extra effort. -The reference for the :mod:`json` module contains an explanation of this. - -.. seealso:: - - :mod:`pickle` - the pickle module - - Contrary to :ref:`JSON `, *pickle* is a protocol which allows - the serialization of arbitrarily complex Python objects. As such, it is - specific to Python and cannot be used to communicate with applications - written in other languages. It is also insecure by default: - deserializing pickle data coming from an untrusted source can execute - arbitrary code, if the data was crafted by a skilled attacker. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/interactive.rst --- a/Doc/tutorial/interactive.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/interactive.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,27 +7,140 @@ Some versions of the Python interpreter support editing of the current input line and history substitution, similar to facilities found in the Korn shell and the GNU Bash shell. This is implemented using the `GNU Readline`_ library, -which supports various styles of editing. This library has its own -documentation which we won't duplicate here. +which supports Emacs-style and vi-style editing. This library has its own +documentation which I won't duplicate here; however, the basics are easily +explained. The interactive editing and history described here are optionally +available in the Unix and Cygwin versions of the interpreter. + +This chapter does *not* document the editing facilities of Mark Hammond's +PythonWin package or the Tk-based environment, IDLE, distributed with Python. +The command line history recall which operates within DOS boxes on NT and some +other DOS and Windows flavors is yet another beast. + + +.. _tut-lineediting: + +Line Editing +============ + +If supported, input line editing is active whenever the interpreter prints a +primary or secondary prompt. The current line can be edited using the +conventional Emacs control characters. The most important of these are: +:kbd:`C-A` (Control-A) moves the cursor to the beginning of the line, :kbd:`C-E` +to the end, :kbd:`C-B` moves it one position to the left, :kbd:`C-F` to the +right. Backspace erases the character to the left of the cursor, :kbd:`C-D` the +character to its right. :kbd:`C-K` kills (erases) the rest of the line to the +right of the cursor, :kbd:`C-Y` yanks back the last killed string. +:kbd:`C-underscore` undoes the last change you made; it can be repeated for +cumulative effect. + + +.. _tut-history: + +History Substitution +==================== + +History substitution works as follows. All non-empty input lines issued are +saved in a history buffer, and when a new prompt is given you are positioned on +a new line at the bottom of this buffer. :kbd:`C-P` moves one line up (back) in +the history buffer, :kbd:`C-N` moves one down. Any line in the history buffer +can be edited; an asterisk appears in front of the prompt to mark a line as +modified. Pressing the :kbd:`Return` key passes the current line to the +interpreter. :kbd:`C-R` starts an incremental reverse search; :kbd:`C-S` starts +a forward search. .. _tut-keybindings: -Tab Completion and History Editing -================================== +Key Bindings +============ -Completion of variable and module names is -:ref:`automatically enabled ` at interpreter startup so -that the :kbd:`Tab` key invokes the completion function; it looks at -Python statement names, the current local variables, and the available -module names. For dotted expressions such as ``string.a``, it will evaluate -the expression up to the final ``'.'`` and then suggest completions from -the attributes of the resulting object. Note that this may execute -application-defined code if an object with a :meth:`__getattr__` method -is part of the expression. The default configuration also saves your -history into a file named :file:`.python_history` in your user directory. -The history will be available again during the next interactive interpreter -session. +The key bindings and some other parameters of the Readline library can be +customized by placing commands in an initialization file called +:file:`~/.inputrc`. Key bindings have the form :: + + key-name: function-name + +or :: + + "string": function-name + +and options can be set with :: + + set option-name value + +For example:: + + # I prefer vi-style editing: + set editing-mode vi + + # Edit using a single line: + set horizontal-scroll-mode On + + # Rebind some keys: + Meta-h: backward-kill-word + "\C-u": universal-argument + "\C-x\C-r": re-read-init-file + +Note that the default binding for :kbd:`Tab` in Python is to insert a :kbd:`Tab` +character instead of Readline's default filename completion function. If you +insist, you can override this by putting :: + + Tab: complete + +in your :file:`~/.inputrc`. (Of course, this makes it harder to type indented +continuation lines if you're accustomed to using :kbd:`Tab` for that purpose.) + +.. index:: + module: rlcompleter + module: readline + +Automatic completion of variable and module names is optionally available. To +enable it in the interpreter's interactive mode, add the following to your +startup file: [#]_ :: + + import rlcompleter, readline + readline.parse_and_bind('tab: complete') + +This binds the :kbd:`Tab` key to the completion function, so hitting the +:kbd:`Tab` key twice suggests completions; it looks at Python statement names, +the current local variables, and the available module names. For dotted +expressions such as ``string.a``, it will evaluate the expression up to the +final ``'.'`` and then suggest completions from the attributes of the resulting +object. Note that this may execute application-defined code if an object with a +:meth:`__getattr__` method is part of the expression. + +A more capable startup file might look like this example. Note that this +deletes the names it creates once they are no longer needed; this is done since +the startup file is executed in the same namespace as the interactive commands, +and removing the names avoids creating side effects in the interactive +environment. You may find it convenient to keep some of the imported modules, +such as :mod:`os`, which turn out to be needed in most sessions with the +interpreter. :: + + # Add auto-completion and a stored history file of commands to your Python + # interactive interpreter. Requires Python 2.0+, readline. Autocomplete is + # bound to the Esc key by default (you can change it - see readline docs). + # + # Store the file in ~/.pystartup, and set an environment variable to point + # to it: "export PYTHONSTARTUP=~/.pystartup" in bash. + + import atexit + import os + import readline + import rlcompleter + + historyPath = os.path.expanduser("~/.pyhistory") + + def save_history(historyPath=historyPath): + import readline + readline.write_history_file(historyPath) + + if os.path.exists(historyPath): + readline.read_history_file(historyPath) + + atexit.register(save_history) + del os, atexit, readline, rlcompleter, save_history, historyPath .. _tut-commentary: @@ -49,6 +162,14 @@ bpython_. +.. rubric:: Footnotes + +.. [#] Python will execute the contents of a file identified by the + :envvar:`PYTHONSTARTUP` environment variable when you start an interactive + interpreter. To customize Python even for non-interactive mode, see + :ref:`tut-customize`. + + .. _GNU Readline: http://tiswww.case.edu/php/chet/readline/rltop.html .. _IPython: http://ipython.scipy.org/ .. _bpython: http://www.bpython-interpreter.org/ diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/interpreter.rst --- a/Doc/tutorial/interpreter.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/interpreter.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -3.6.. _tut-using: +.. _tut-using: **************************** Using the Python Interpreter @@ -10,13 +10,11 @@ Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.6` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.3` on those machines where it is available; putting :file:`/usr/local/bin` in your -Unix shell's search path makes it possible to start it by typing the command: +Unix shell's search path makes it possible to start it by typing the command :: -.. code-block:: text - - python3.6 + python3.3 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,25 +22,26 @@ popular alternative location.) On Windows machines, the Python installation is usually placed in -:file:`C:\\Python36`, though you can change this when you're running the +:file:`C:\\Python33`, though you can change this when you're running the installer. To add this directory to your path, you can type the following command into the command prompt in a DOS box:: - set path=%path%;C:\python36 + set path=%path%;C:\python33 Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on Windows) at the primary prompt causes the interpreter to exit with a zero exit status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. -The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support readline. Perhaps the -quickest check to see whether command line editing is supported is typing -:kbd:`Control-P` to the first Python prompt you get. If it beeps, you have command -line editing; see Appendix :ref:`tut-interacting` for an introduction to the -keys. If nothing appears to happen, or if ``^P`` is echoed, command line -editing isn't available; you'll only be able to use backspace to remove -characters from the current line. +The interpreter's line-editing features usually aren't very sophisticated. On +Unix, whoever installed the interpreter may have enabled support for the GNU +readline library, which adds more elaborate interactive editing and history +features. Perhaps the quickest check to see whether command line editing is +supported is typing Control-P to the first Python prompt you get. If it beeps, +you have command line editing; see Appendix :ref:`tut-interacting` for an +introduction to the keys. If nothing appears to happen, or if ``^P`` is echoed, +command line editing isn't available; you'll only be able to use backspace to +remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard input connected to a tty device, it reads and executes commands interactively; @@ -63,8 +62,6 @@ and enter interactive mode afterwards. This can be done by passing :option:`-i` before the script. -All command line options are described in :ref:`using-on-general`. - .. _tut-argpassing: @@ -96,9 +93,9 @@ prints a welcome message stating its version number and a copyright notice before printing the first prompt:: - $ python3.6 - Python 3.6 (default, Sep 16 2015, 09:25:04) - [GCC 4.8.2] on linux + $ python3.3 + Python 3.3 (py3k, Sep 12 2007, 12:21:02) + [GCC 3.4.6 20060404 (Red Hat 3.4.6-8)] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> @@ -107,22 +104,70 @@ Continuation lines are needed when entering a multi-line construct. As an example, take a look at this :keyword:`if` statement:: - >>> the_world_is_flat = True + >>> the_world_is_flat = 1 >>> if the_world_is_flat: ... print("Be careful not to fall off!") ... Be careful not to fall off! -For more on interactive mode, see :ref:`tut-interac`. - - .. _tut-interp: The Interpreter and Its Environment =================================== +.. _tut-error: + +Error Handling +-------------- + +When an error occurs, the interpreter prints an error message and a stack trace. +In interactive mode, it then returns to the primary prompt; when input came from +a file, it exits with a nonzero exit status after printing the stack trace. +(Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement +are not errors in this context.) Some errors are unconditionally fatal and +cause an exit with a nonzero exit; this applies to internal inconsistencies and +some cases of running out of memory. All error messages are written to the +standard error stream; normal output from executed commands is written to +standard output. + +Typing the interrupt character (usually Control-C or DEL) to the primary or +secondary prompt cancels the input and returns to the primary prompt. [#]_ +Typing an interrupt while a command is executing raises the +:exc:`KeyboardInterrupt` exception, which may be handled by a :keyword:`try` +statement. + + +.. _tut-scripts: + +Executable Python Scripts +------------------------- + +On BSD'ish Unix systems, Python scripts can be made directly executable, like +shell scripts, by putting the line :: + + #! /usr/bin/env python3.3 + +(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning +of the script and giving the file an executable mode. The ``#!`` must be the +first two characters of the file. On some platforms, this first line must end +with a Unix-style line ending (``'\n'``), not a Windows (``'\r\n'``) line +ending. Note that the hash, or pound, character, ``'#'``, is used to start a +comment in Python. + +The script can be given an executable mode, or permission, using the +:program:`chmod` command:: + + $ chmod +x myscript.py + +On Windows systems, there is no notion of an "executable mode". The Python +installer automatically associates ``.py`` files with ``python.exe`` so that +a double-click on a Python file will run it as a script. The extension can +also be ``.pyw``, in that case, the console window that normally appears is +suppressed. + + .. _tut-source-encoding: Source Code Encoding @@ -156,8 +201,67 @@ within the file. +.. _tut-startup: + +The Interactive Startup File +---------------------------- + +When you use Python interactively, it is frequently handy to have some standard +commands executed every time the interpreter is started. You can do this by +setting an environment variable named :envvar:`PYTHONSTARTUP` to the name of a +file containing your start-up commands. This is similar to the :file:`.profile` +feature of the Unix shells. + +.. XXX This should probably be dumped in an appendix, since most people + don't use Python interactively in non-trivial ways. + +This file is only read in interactive sessions, not when Python reads commands +from a script, and not when :file:`/dev/tty` is given as the explicit source of +commands (which otherwise behaves like an interactive session). It is executed +in the same namespace where interactive commands are executed, so that objects +that it defines or imports can be used without qualification in the interactive +session. You can also change the prompts ``sys.ps1`` and ``sys.ps2`` in this +file. + +If you want to read an additional start-up file from the current directory, you +can program this in the global start-up file using code like ``if +os.path.isfile('.pythonrc.py'): exec(open('.pythonrc.py').read())``. +If you want to use the startup file in a script, you must do this explicitly +in the script:: + + import os + filename = os.environ.get('PYTHONSTARTUP') + if filename and os.path.isfile(filename): + exec(open(filename).read()) + + +.. _tut-customize: + +The Customization Modules +------------------------- + +Python provides two hooks to let you customize it: :mod:`sitecustomize` and +:mod:`usercustomize`. To see how it works, you need first to find the location +of your user site-packages directory. Start Python and run this code: + + >>> import site + >>> site.getusersitepackages() + '/home/user/.local/lib/python3.2/site-packages' + +Now you can create a file named :file:`usercustomize.py` in that directory and +put anything you want in it. It will affect every invocation of Python, unless +it is started with the :option:`-s` option to disable the automatic import. + +:mod:`sitecustomize` works in the same way, but is typically created by an +administrator of the computer in the global site-packages directory, and is +imported before :mod:`usercustomize`. See the documentation of the :mod:`site` +module for more details. + + .. rubric:: Footnotes .. [#] On Unix, the Python 3.x interpreter is by default not installed with the executable named ``python``, so that it does not conflict with a simultaneously installed Python 2.x executable. + +.. [#] A problem with the GNU Readline package may prevent this. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/introduction.rst --- a/Doc/tutorial/introduction.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/introduction.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,11 @@ -.. _tut-informal: +.. _tut-informal: ********************************** An Informal Introduction to Python ********************************** In the following examples, input and output are distinguished by the presence or -absence of prompts (:term:`>>>` and :term:`...`): to repeat the example, you must type +absence of prompts (``>>>`` and ``...``): to repeat the example, you must type everything after the prompt, when the prompt appears; lines that do not begin with a prompt are output from the interpreter. Note that a secondary prompt on a line by itself in an example means you must type a blank line; this is used to @@ -22,9 +22,9 @@ Some examples:: # this is the first comment - spam = 1 # and this is the second comment - # ... and now a third! - text = "# This is not a comment because it's inside quotes." + SPAM = 1 # and this is the second comment + # ... and now a third! + STRING = "# This is not a comment." .. _tut-calculator: @@ -44,55 +44,58 @@ The interpreter acts as a simple calculator: you can type an expression at it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` work just like in most other languages -(for example, Pascal or C); parentheses (``()``) can be used for grouping. -For example:: +(for example, Pascal or C); parentheses can be used for grouping. For example:: - >>> 2 + 2 + >>> 2+2 4 - >>> 50 - 5*6 - 20 - >>> (50 - 5*6) / 4 + >>> # This is a comment + ... 2+2 + 4 + >>> 2+2 # and a comment on the same line as code + 4 + >>> (50-5*6)/4 5.0 - >>> 8 / 5 # division always returns a floating point number + >>> 8/5 # Fractions aren't lost when dividing integers 1.6 -The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, -the ones with a fractional part (e.g. ``5.0``, ``1.6``) have type -:class:`float`. We will see more about numeric types later in the tutorial. +Note: You might not see exactly the same result; floating point results can +differ from one machine to another. We will say more later about controlling +the appearance of floating point output. See also :ref:`tut-fp-issues` for a +full discussion of some of the subtleties of floating point numbers and their +representations. -Division (``/``) always returns a float. To do :term:`floor division` and -get an integer result (discarding any fractional result) you can use the ``//`` -operator; to calculate the remainder you can use ``%``:: +To do integer division and get an integer result, +discarding any fractional result, there is another operator, ``//``:: - >>> 17 / 3 # classic division returns a float - 5.666666666666667 - >>> - >>> 17 // 3 # floor division discards the fractional part - 5 - >>> 17 % 3 # the % operator returns the remainder of the division + >>> # Integer division returns the floor: + ... 7//3 2 - >>> 5 * 3 + 2 # result * divisor + remainder - 17 + >>> 7//-3 + -3 -With Python, it is possible to use the ``**`` operator to calculate powers [#]_:: - - >>> 5 ** 2 # 5 squared - 25 - >>> 2 ** 7 # 2 to the power of 7 - 128 - -The equal sign (``=``) is used to assign a value to a variable. Afterwards, no +The equal sign (``'='``) is used to assign a value to a variable. Afterwards, no result is displayed before the next interactive prompt:: >>> width = 20 - >>> height = 5 * 9 + >>> height = 5*9 >>> width * height 900 -If a variable is not "defined" (assigned a value), trying to use it will -give you an error:: +A value can be assigned to several variables simultaneously:: - >>> n # try to access an undefined variable + >>> x = y = z = 0 # Zero x, y and z + >>> x + 0 + >>> y + 0 + >>> z + 0 + +Variables must be "defined" (assigned a value) before they can be used, or an +error will occur:: + + >>> # try to access an undefined variable + ... n Traceback (most recent call last): File "", line 1, in NameError: name 'n' is not defined @@ -105,6 +108,49 @@ >>> 7.0 / 2 3.5 +Complex numbers are also supported; imaginary numbers are written with a suffix +of ``j`` or ``J``. Complex numbers with a nonzero real component are written as +``(real+imagj)``, or can be created with the ``complex(real, imag)`` function. +:: + + >>> 1j * 1J + (-1+0j) + >>> 1j * complex(0, 1) + (-1+0j) + >>> 3+1j*3 + (3+3j) + >>> (3+1j)*3 + (9+3j) + >>> (1+2j)/(1+1j) + (1.5+0.5j) + +Complex numbers are always represented as two floating point numbers, the real +and imaginary part. To extract these parts from a complex number *z*, use +``z.real`` and ``z.imag``. :: + + >>> a=1.5+0.5j + >>> a.real + 1.5 + >>> a.imag + 0.5 + +The conversion functions to floating point and integer (:func:`float`, +:func:`int`) don't work for complex numbers --- there is not one correct way to +convert a complex number to a real number. Use ``abs(z)`` to get its magnitude +(as a float) or ``z.real`` to get its real part:: + + >>> a=3.0+4.0j + >>> float(a) + Traceback (most recent call last): + File "", line 1, in ? + TypeError: can't convert complex to float; use abs(z) + >>> a.real + 3.0 + >>> a.imag + 4.0 + >>> abs(a) # sqrt(a.real**2 + a.imag**2) + 5.0 + In interactive mode, the last printed expression is assigned to the variable ``_``. This means that when you are using Python as a desk calculator, it is somewhat easier to continue calculations, for example:: @@ -122,28 +168,20 @@ assign a value to it --- you would create an independent local variable with the same name masking the built-in variable with its magic behavior. -In addition to :class:`int` and :class:`float`, Python supports other types of -numbers, such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Python also has built-in support for :ref:`complex numbers `, -and uses the ``j`` or ``J`` suffix to indicate the imaginary part -(e.g. ``3+5j``). - .. _tut-strings: Strings ------- -Besides numbers, Python can also manipulate strings, which can be expressed -in several ways. They can be enclosed in single quotes (``'...'``) or -double quotes (``"..."``) with the same result [#]_. ``\`` can be used -to escape quotes:: +Besides numbers, Python can also manipulate strings, which can be expressed in +several ways. They can be enclosed in single quotes or double quotes:: - >>> 'spam eggs' # single quotes + >>> 'spam eggs' 'spam eggs' - >>> 'doesn\'t' # use \' to escape the single quote... + >>> 'doesn\'t' "doesn't" - >>> "doesn't" # ...or use double quotes instead + >>> "doesn't" "doesn't" >>> '"Yes," he said.' '"Yes," he said.' @@ -152,40 +190,38 @@ >>> '"Isn\'t," she said.' '"Isn\'t," she said.' -In the interactive interpreter, the output string is enclosed in quotes and -special characters are escaped with backslashes. While this might sometimes -look different from the input (the enclosing quotes could change), the two -strings are equivalent. The string is enclosed in double quotes if -the string contains a single quote and no double quotes, otherwise it is -enclosed in single quotes. The :func:`print` function produces a more -readable output, by omitting the enclosing quotes and by printing escaped -and special characters:: +The interpreter prints the result of string operations in the same way as they +are typed for input: inside quotes, and with quotes and other funny characters +escaped by backslashes, to show the precise value. The string is enclosed in +double quotes if the string contains a single quote and no double quotes, else +it's enclosed in single quotes. The :func:`print` function produces a more +readable output for such input strings. - >>> '"Isn\'t," she said.' - '"Isn\'t," she said.' - >>> print('"Isn\'t," she said.') - "Isn't," she said. - >>> s = 'First line.\nSecond line.' # \n means newline - >>> s # without print(), \n is included in the output - 'First line.\nSecond line.' - >>> print(s) # with print(), \n produces a new line - First line. - Second line. +String literals can span multiple lines in several ways. Continuation lines can +be used, with a backslash as the last character on the line indicating that the +next line is a logical continuation of the line:: -If you don't want characters prefaced by ``\`` to be interpreted as -special characters, you can use *raw strings* by adding an ``r`` before -the first quote:: + hello = "This is a rather long string containing\n\ + several lines of text just as you would do in C.\n\ + Note that whitespace at the beginning of the line is\ + significant." - >>> print('C:\some\name') # here \n means newline! - C:\some - ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + print(hello) -String literals can span multiple lines. One way is using triple-quotes: -``"""..."""`` or ``'''...'''``. End of lines are automatically -included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: +Note that newlines still need to be embedded in the string using ``\n`` -- the +newline following the trailing backslash is discarded. This example would print +the following: + +.. code-block:: text + + This is a rather long string containing + several lines of text just as you would do in C. + Note that whitespace at the beginning of the line is significant. + +Or, strings can be surrounded in a pair of matching triple-quotes: ``"""`` or +``'''``. End of lines do not need to be escaped when using triple-quotes, but +they will be included in the string. So the following uses one escape to +avoid an unwanted initial blank line. :: print("""\ Usage: thingy [OPTIONS] @@ -193,7 +229,7 @@ -H hostname Hostname to connect to """) -produces the following output (note that the initial newline is not included): +produces the following output: .. code-block:: text @@ -201,100 +237,143 @@ -h Display this usage message -H hostname Hostname to connect to +If we make the string literal a "raw" string, ``\n`` sequences are not converted +to newlines, but the backslash at the end of the line, and the newline character +in the source, are both included in the string as data. Thus, the example:: + + hello = r"This is a rather long string containing\n\ + several lines of text much as you would do in C." + + print(hello) + +would print: + +.. code-block:: text + + This is a rather long string containing\n\ + several lines of text much as you would do in C. + Strings can be concatenated (glued together) with the ``+`` operator, and repeated with ``*``:: - >>> # 3 times 'un', followed by 'ium' - >>> 3 * 'un' + 'ium' - 'unununium' + >>> word = 'Help' + 'A' + >>> word + 'HelpA' + >>> '<' + word*5 + '>' + '' -Two or more *string literals* (i.e. the ones enclosed between quotes) next -to each other are automatically concatenated. :: +Two string literals next to each other are automatically concatenated; the first +line above could also have been written ``word = 'Help' 'A'``; this only works +with two literals, not with arbitrary string expressions:: - >>> 'Py' 'thon' - 'Python' - -This only works with two literals though, not with variables or expressions:: - - >>> prefix = 'Py' - >>> prefix 'thon' # can't concatenate a variable and a string literal - ... - SyntaxError: invalid syntax - >>> ('un' * 3) 'ium' - ... + >>> 'str' 'ing' # <- This is ok + 'string' + >>> 'str'.strip() + 'ing' # <- This is ok + 'string' + >>> 'str'.strip() 'ing' # <- This is invalid + File "", line 1, in ? + 'str'.strip() 'ing' + ^ SyntaxError: invalid syntax -If you want to concatenate variables or a variable and a literal, use ``+``:: +Strings can be subscripted (indexed); like in C, the first character of a string +has subscript (index) 0. There is no separate character type; a character is +simply a string of size one. As in the Icon programming language, substrings +can be specified with the *slice notation*: two indices separated by a colon. +:: - >>> prefix + 'thon' - 'Python' - -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - -Strings can be *indexed* (subscripted), with the first character having index 0. -There is no separate character type; a character is simply a string of size -one:: - - >>> word = 'Python' - >>> word[0] # character in position 0 - 'P' - >>> word[5] # character in position 5 - 'n' - -Indices may also be negative numbers, to start counting from the right:: - - >>> word[-1] # last character - 'n' - >>> word[-2] # second-last character - 'o' - >>> word[-6] - 'P' - -Note that since -0 is the same as 0, negative indices start from -1. - -In addition to indexing, *slicing* is also supported. While indexing is used -to obtain individual characters, *slicing* allows you to obtain substring:: - - >>> word[0:2] # characters from position 0 (included) to 2 (excluded) - 'Py' - >>> word[2:5] # characters from position 2 (included) to 5 (excluded) - 'tho' - -Note how the start is always included, and the end always excluded. This -makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: - - >>> word[:2] + word[2:] - 'Python' - >>> word[:4] + word[4:] - 'Python' + >>> word[4] + 'A' + >>> word[0:2] + 'He' + >>> word[2:4] + 'lp' Slice indices have useful defaults; an omitted first index defaults to zero, an omitted second index defaults to the size of the string being sliced. :: - >>> word[:2] # character from the beginning to position 2 (excluded) - 'Py' - >>> word[4:] # characters from position 4 (included) to the end - 'on' - >>> word[-2:] # characters from the second-last (included) to the end - 'on' + >>> word[:2] # The first two characters + 'He' + >>> word[2:] # Everything except the first two characters + 'lpA' + +Unlike a C string, Python strings cannot be changed. Assigning to an indexed +position in the string results in an error:: + + >>> word[0] = 'x' + Traceback (most recent call last): + File "", line 1, in ? + TypeError: 'str' object does not support item assignment + >>> word[:1] = 'Splat' + Traceback (most recent call last): + File "", line 1, in ? + TypeError: 'str' object does not support slice assignment + +However, creating a new string with the combined content is easy and efficient:: + + >>> 'x' + word[1:] + 'xelpA' + >>> 'Splat' + word[4] + 'SplatA' + +Here's a useful invariant of slice operations: ``s[:i] + s[i:]`` equals ``s``. +:: + + >>> word[:2] + word[2:] + 'HelpA' + >>> word[:3] + word[3:] + 'HelpA' + +Degenerate slice indices are handled gracefully: an index that is too large is +replaced by the string size, an upper bound smaller than the lower bound returns +an empty string. :: + + >>> word[1:100] + 'elpA' + >>> word[10:] + '' + >>> word[2:1] + '' + +Indices may be negative numbers, to start counting from the right. For example:: + + >>> word[-1] # The last character + 'A' + >>> word[-2] # The last-but-one character + 'p' + >>> word[-2:] # The last two characters + 'pA' + >>> word[:-2] # Everything except the last two characters + 'Hel' + +But note that -0 is really the same as 0, so it does not count from the right! +:: + + >>> word[-0] # (since -0 equals 0) + 'H' + +Out-of-range negative slice indices are truncated, but don't try this for +single-element (non-slice) indices:: + + >>> word[-100:] + 'HelpA' + >>> word[-10] # error + Traceback (most recent call last): + File "", line 1, in ? + IndexError: string index out of range One way to remember how slices work is to think of the indices as pointing *between* characters, with the left edge of the first character numbered 0. Then the right edge of the last character of a string of *n* characters has index *n*, for example:: - +---+---+---+---+---+---+ - | P | y | t | h | o | n | - +---+---+---+---+---+---+ - 0 1 2 3 4 5 6 - -6 -5 -4 -3 -2 -1 + +---+---+---+---+---+ + | H | e | l | p | A | + +---+---+---+---+---+ + 0 1 2 3 4 5 + -5 -4 -3 -2 -1 -The first row of numbers gives the position of the indices 0...6 in the string; +The first row of numbers gives the position of the indices 0...5 in the string; the second row gives the corresponding negative indices. The slice from *i* to *j* consists of all characters between the edges labeled *i* and *j*, respectively. @@ -303,38 +382,6 @@ indices, if both are within bounds. For example, the length of ``word[1:3]`` is 2. -Attempting to use an index that is too large will result in an error:: - - >>> word[42] # the word only has 6 characters - Traceback (most recent call last): - File "", line 1, in - IndexError: string index out of range - -However, out of range slice indexes are handled gracefully when used for -slicing:: - - >>> word[4:42] - 'on' - >>> word[42:] - '' - -Python strings cannot be changed --- they are :term:`immutable`. -Therefore, assigning to an indexed position in the string results in an error:: - - >>> word[0] = 'J' - ... - TypeError: 'str' object does not support item assignment - >>> word[2:] = 'py' - ... - TypeError: 'str' object does not support item assignment - -If you need a different string, you should create a new one:: - - >>> 'J' + word[1:] - 'Jython' - >>> word[:2] + 'py' - 'Pypy' - The built-in function :func:`len` returns the length of a string:: >>> s = 'supercalifragilisticexpialidocious' @@ -344,7 +391,7 @@ .. seealso:: - :ref:`textseq` + :ref:`typesseq` Strings are examples of *sequence types*, and support the common operations supported by such types. @@ -361,6 +408,51 @@ the left operand of the ``%`` operator are described in more detail here. +.. _tut-unicodestrings: + +About Unicode +------------- + +.. sectionauthor:: Marc-Andre Lemburg + + +Starting with Python 3.0 all strings support Unicode (see +http://www.unicode.org/). + +Unicode has the advantage of providing one ordinal for every character in every +script used in modern and ancient texts. Previously, there were only 256 +possible ordinals for script characters. Texts were typically bound to a code +page which mapped the ordinals to script characters. This lead to very much +confusion especially with respect to internationalization (usually written as +``i18n`` --- ``'i'`` + 18 characters + ``'n'``) of software. Unicode solves +these problems by defining one code page for all scripts. + +If you want to include special characters in a string, +you can do so by using the Python *Unicode-Escape* encoding. The following +example shows how:: + + >>> 'Hello\u0020World !' + 'Hello World !' + +The escape sequence ``\u0020`` indicates to insert the Unicode character with +the ordinal value 0x0020 (the space character) at the given position. + +Other characters are interpreted by using their respective ordinal values +directly as Unicode ordinals. If you have literal strings in the standard +Latin-1 encoding that is used in many Western countries, you will find it +convenient that the lower 256 characters of Unicode are the same as the 256 +characters of Latin-1. + +Apart from these standard encodings, Python provides a whole set of other ways +of creating Unicode strings on the basis of a known encoding. + +To convert a string into a sequence of bytes using a specific encoding, +string objects provide an :func:`encode` method that takes one argument, the +name of the encoding. Lowercase names for encodings are preferred. :: + + >>> "Äpfel".encode('utf-8') + b'\xc3\x84pfel' + .. _tut-lists: Lists @@ -368,89 +460,97 @@ Python knows a number of *compound* data types, used to group together other values. The most versatile is the *list*, which can be written as a list of -comma-separated values (items) between square brackets. Lists might contain -items of different types, but usually the items all have the same type. :: +comma-separated values (items) between square brackets. List items need not all +have the same type. :: - >>> squares = [1, 4, 9, 16, 25] - >>> squares - [1, 4, 9, 16, 25] + >>> a = ['spam', 'eggs', 100, 1234] + >>> a + ['spam', 'eggs', 100, 1234] -Like strings (and all other built-in :term:`sequence` type), lists can be -indexed and sliced:: +Like string indices, list indices start at 0, and lists can be sliced, +concatenated and so on:: - >>> squares[0] # indexing returns the item - 1 - >>> squares[-1] - 25 - >>> squares[-3:] # slicing returns a new list - [9, 16, 25] + >>> a[0] + 'spam' + >>> a[3] + 1234 + >>> a[-2] + 100 + >>> a[1:-1] + ['eggs', 100] + >>> a[:2] + ['bacon', 2*2] + ['spam', 'eggs', 'bacon', 4] + >>> 3*a[:3] + ['Boo!'] + ['spam', 'eggs', 100, 'spam', 'eggs', 100, 'spam', 'eggs', 100, 'Boo!'] All slice operations return a new list containing the requested elements. This -means that the following slice returns a new (shallow) copy of the list:: +means that the following slice returns a shallow copy of the list *a*:: - >>> squares[:] - [1, 4, 9, 16, 25] + >>> a[:] + ['spam', 'eggs', 100, 1234] -Lists also support operations like concatenation:: +Unlike strings, which are *immutable*, it is possible to change individual +elements of a list:: - >>> squares + [36, 49, 64, 81, 100] - [1, 4, 9, 16, 25, 36, 49, 64, 81, 100] - -Unlike strings, which are :term:`immutable`, lists are a :term:`mutable` -type, i.e. it is possible to change their content:: - - >>> cubes = [1, 8, 27, 65, 125] # something's wrong here - >>> 4 ** 3 # the cube of 4 is 64, not 65! - 64 - >>> cubes[3] = 64 # replace the wrong value - >>> cubes - [1, 8, 27, 64, 125] - -You can also add new items at the end of the list, by using -the :meth:`~list.append` *method* (we will see more about methods later):: - - >>> cubes.append(216) # add the cube of 6 - >>> cubes.append(7 ** 3) # and the cube of 7 - >>> cubes - [1, 8, 27, 64, 125, 216, 343] + >>> a + ['spam', 'eggs', 100, 1234] + >>> a[2] = a[2] + 23 + >>> a + ['spam', 'eggs', 123, 1234] Assignment to slices is also possible, and this can even change the size of the list or clear it entirely:: - >>> letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> letters - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> # replace some values - >>> letters[2:5] = ['C', 'D', 'E'] - >>> letters - ['a', 'b', 'C', 'D', 'E', 'f', 'g'] - >>> # now remove them - >>> letters[2:5] = [] - >>> letters - ['a', 'b', 'f', 'g'] - >>> # clear the list by replacing all the elements with an empty list - >>> letters[:] = [] - >>> letters + >>> # Replace some items: + ... a[0:2] = [1, 12] + >>> a + [1, 12, 123, 1234] + >>> # Remove some: + ... a[0:2] = [] + >>> a + [123, 1234] + >>> # Insert some: + ... a[1:1] = ['bletch', 'xyzzy'] + >>> a + [123, 'bletch', 'xyzzy', 1234] + >>> # Insert (a copy of) itself at the beginning + >>> a[:0] = a + >>> a + [123, 'bletch', 'xyzzy', 1234, 123, 'bletch', 'xyzzy', 1234] + >>> # Clear the list: replace all items with an empty list + >>> a[:] = [] + >>> a [] The built-in function :func:`len` also applies to lists:: - >>> letters = ['a', 'b', 'c', 'd'] - >>> len(letters) + >>> a = ['a', 'b', 'c', 'd'] + >>> len(a) 4 It is possible to nest lists (create lists containing other lists), for example:: - >>> a = ['a', 'b', 'c'] - >>> n = [1, 2, 3] - >>> x = [a, n] - >>> x - [['a', 'b', 'c'], [1, 2, 3]] - >>> x[0] - ['a', 'b', 'c'] - >>> x[0][1] - 'b' + >>> q = [2, 3] + >>> p = [1, q, 4] + >>> len(p) + 3 + >>> p[1] + [2, 3] + >>> p[1][0] + 2 + +You can add something to the end of the list:: + + >>> p[1].append('xtra') + >>> p + [1, [2, 3, 'xtra'], 4] + >>> q + [2, 3, 'xtra'] + +Note that in the last example, ``p[1]`` and ``q`` really refer to the same +object! We'll come back to *object semantics* later. + .. _tut-firststeps: @@ -501,19 +601,19 @@ guess when you have typed the last line). Note that each line within a basic block must be indented by the same amount. -* The :func:`print` function writes the value of the argument(s) it is given. - It differs from just writing the expression you want to write (as we did - earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, - and a space is inserted between items, so you can format things nicely, like - this:: +* The :func:`print` function writes the value of the expression(s) it is + given. It differs from just writing the expression you want to write (as we did + earlier in the calculator examples) in the way it handles multiple + expressions, floating point quantities, + and strings. Strings are printed without quotes, and a space is inserted + between items, so you can format things nicely, like this:: >>> i = 256*256 >>> print('The value of i is', i) The value of i is 65536 - The keyword argument *end* can be used to avoid the newline after the output, - or end the output with a different string:: + The keyword *end* can be used to avoid the newline after the output, or end + the output with a different string:: >>> a, b = 0, 1 >>> while b < 1000: @@ -521,15 +621,3 @@ ... a, b = b, a+b ... 1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987, - - -.. rubric:: Footnotes - -.. [#] Since ``**`` has higher precedence than ``-``, ``-3**2`` will be - interpreted as ``-(3**2)`` and thus result in ``-9``. To avoid this - and get ``9``, you can use ``(-3)**2``. - -.. [#] Unlike other languages, special characters such as ``\n`` have the - same meaning with both single (``'...'``) and double (``"..."``) quotes. - The only difference between the two is that within single quotes you don't - need to escape ``"`` (but you have to escape ``\'``) and vice versa. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/modules.rst --- a/Doc/tutorial/modules.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/modules.rst Mon Jan 25 17:05:13 2016 +0100 @@ -72,8 +72,7 @@ A module can contain executable statements as well as function definitions. These statements are intended to initialize the module. They are executed only -the *first* time the module name is encountered in an import statement. [#]_ -(They are also run if the file is executed as a script.) +the *first* time the module is imported somewhere. [#]_ Each module has its own private symbol table, which is used as the global symbol table by all functions defined in the module. Thus, the author of a module can @@ -117,8 +116,7 @@ For efficiency reasons, each module is only imported once per interpreter session. Therefore, if you change your modules, you must restart the interpreter -- or, if it's just one module you want to test interactively, - use :func:`importlib.reload`, e.g. ``import importlib; - importlib.reload(modulename)``. + use :func:`imp.reload`, e.g. ``import imp; imp.reload(modulename)``. .. _tut-modulesasscripts: @@ -166,16 +164,10 @@ named :file:`spam.py` in a list of directories given by the variable :data:`sys.path`. :data:`sys.path` is initialized from these locations: -* The directory containing the input script (or the current directory when no - file is specified). +* the directory containing the input script (or the current directory). * :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the shell variable :envvar:`PATH`). -* The installation-dependent default. - -.. note:: - On file systems which support symlinks, the directory containing the input - script is calculated after the symlink is followed. In other words the - directory containing the symlink is **not** added to the module search path. +* the installation-dependent default. After initialization, Python programs can modify :data:`sys.path`. The directory containing the script being run is placed at the beginning of the @@ -190,45 +182,57 @@ "Compiled" Python files ----------------------- -To speed up loading modules, Python caches the compiled version of each module -in the ``__pycache__`` directory under the name :file:`module.{version}.pyc`, -where the version encodes the format of the compiled file; it generally contains -the Python version number. For example, in CPython release 3.3 the compiled -version of spam.py would be cached as ``__pycache__/spam.cpython-33.pyc``. This -naming convention allows compiled modules from different releases and different -versions of Python to coexist. +As an important speed-up of the start-up time for short programs that use a lot +of standard modules, if a file called :file:`spam.pyc` exists in the directory +where :file:`spam.py` is found, this is assumed to contain an +already-"byte-compiled" version of the module :mod:`spam`. The modification time +of the version of :file:`spam.py` used to create :file:`spam.pyc` is recorded in +:file:`spam.pyc`, and the :file:`.pyc` file is ignored if these don't match. -Python checks the modification date of the source against the compiled version -to see if it's out of date and needs to be recompiled. This is a completely -automatic process. Also, the compiled modules are platform-independent, so the -same library can be shared among systems with different architectures. - -Python does not check the cache in two circumstances. First, it always -recompiles and does not store the result for the module that's loaded directly -from the command line. Second, it does not check the cache if there is no -source module. To support a non-source (compiled only) distribution, the -compiled module must be in the source directory, and there must not be a source -module. +Normally, you don't need to do anything to create the :file:`spam.pyc` file. +Whenever :file:`spam.py` is successfully compiled, an attempt is made to write +the compiled version to :file:`spam.pyc`. It is not an error if this attempt +fails; if for any reason the file is not written completely, the resulting +:file:`spam.pyc` file will be recognized as invalid and thus ignored later. The +contents of the :file:`spam.pyc` file are platform independent, so a Python +module directory can be shared by machines of different architectures. Some tips for experts: -* You can use the :option:`-O` or :option:`-OO` switches on the Python command - to reduce the size of a compiled module. The ``-O`` switch removes assert - statements, the ``-OO`` switch removes both assert statements and __doc__ - strings. Since some programs may rely on having these available, you should - only use this option if you know what you're doing. "Optimized" modules have - an ``opt-`` tag and are usually smaller. Future releases may - change the effects of optimization. +* When the Python interpreter is invoked with the :option:`-O` flag, optimized + code is generated and stored in :file:`.pyo` files. The optimizer currently + doesn't help much; it only removes :keyword:`assert` statements. When + :option:`-O` is used, *all* :term:`bytecode` is optimized; ``.pyc`` files are + ignored and ``.py`` files are compiled to optimized bytecode. -* A program doesn't run any faster when it is read from a ``.pyc`` - file than when it is read from a ``.py`` file; the only thing that's faster - about ``.pyc`` files is the speed with which they are loaded. +* Passing two :option:`-O` flags to the Python interpreter (:option:`-OO`) will + cause the bytecode compiler to perform optimizations that could in some rare + cases result in malfunctioning programs. Currently only ``__doc__`` strings are + removed from the bytecode, resulting in more compact :file:`.pyo` files. Since + some programs may rely on having these available, you should only use this + option if you know what you're doing. -* The module :mod:`compileall` can create .pyc files for all modules in a - directory. +* A program doesn't run any faster when it is read from a :file:`.pyc` or + :file:`.pyo` file than when it is read from a :file:`.py` file; the only thing + that's faster about :file:`.pyc` or :file:`.pyo` files is the speed with which + they are loaded. -* There is more detail on this process, including a flow chart of the - decisions, in PEP 3147. +* When a script is run by giving its name on the command line, the bytecode for + the script is never written to a :file:`.pyc` or :file:`.pyo` file. Thus, the + startup time of a script may be reduced by moving most of its code to a module + and having a small bootstrap script that imports that module. It is also + possible to name a :file:`.pyc` or :file:`.pyo` file directly on the command + line. + +* It is possible to have a file called :file:`spam.pyc` (or :file:`spam.pyo` + when :option:`-O` is used) without a file :file:`spam.py` for the same module. + This can be used to distribute a library of Python code in a form that is + moderately hard to reverse engineer. + + .. index:: module: compileall + +* The module :mod:`compileall` can create :file:`.pyc` files (or :file:`.pyo` + files when :option:`-O` is used) for all modules in a directory. .. _tut-standardmodules: @@ -244,7 +248,7 @@ are not part of the core of the language but are nevertheless built in, either for efficiency or to provide access to operating system primitives such as system calls. The set of such modules is a configuration option which also -depends on the underlying platform. For example, the :mod:`winreg` module is only +depends on the underlying platform For example, the :mod:`winreg` module is only provided on Windows systems. One particular module deserves some attention: :mod:`sys`, which is built into every Python interpreter. The variables ``sys.ps1`` and ``sys.ps2`` define the strings used as primary and secondary @@ -284,24 +288,17 @@ >>> import fibo, sys >>> dir(fibo) ['__name__', 'fib', 'fib2'] - >>> dir(sys) # doctest: +NORMALIZE_WHITESPACE - ['__displayhook__', '__doc__', '__excepthook__', '__loader__', '__name__', - '__package__', '__stderr__', '__stdin__', '__stdout__', - '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe', - '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv', - 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder', - 'call_tracing', 'callstats', 'copyright', 'displayhook', - 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix', - 'executable', 'exit', 'flags', 'float_info', 'float_repr_style', - 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags', - 'getfilesystemencoding', 'getobjects', 'getprofile', 'getrecursionlimit', - 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettotalrefcount', - 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info', - 'intern', 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', - 'path_hooks', 'path_importer_cache', 'platform', 'prefix', 'ps1', - 'setcheckinterval', 'setdlopenflags', 'setprofile', 'setrecursionlimit', - 'setswitchinterval', 'settrace', 'stderr', 'stdin', 'stdout', - 'thread_info', 'version', 'version_info', 'warnoptions'] + >>> dir(sys) + ['__displayhook__', '__doc__', '__excepthook__', '__name__', '__stderr__', + '__stdin__', '__stdout__', '_getframe', 'api_version', 'argv', + 'builtin_module_names', 'byteorder', 'callstats', 'copyright', + 'displayhook', 'exc_info', 'excepthook', + 'exec_prefix', 'executable', 'exit', 'getdefaultencoding', 'getdlopenflags', + 'getrecursionlimit', 'getrefcount', 'hexversion', 'maxint', 'maxunicode', + 'meta_path', 'modules', 'path', 'path_hooks', 'path_importer_cache', + 'platform', 'prefix', 'ps1', 'ps2', 'setcheckinterval', 'setdlopenflags', + 'setprofile', 'setrecursionlimit', 'settrace', 'stderr', 'stdin', 'stdout', + 'version', 'version_info', 'warnoptions'] Without arguments, :func:`dir` lists the names you have defined currently:: @@ -309,7 +306,7 @@ >>> import fibo >>> fib = fibo.fib >>> dir() - ['__builtins__', '__name__', 'a', 'fib', 'fibo', 'sys'] + ['__builtins__', '__doc__', '__file__', '__name__', 'a', 'fib', 'fibo', 'sys'] Note that it lists all types of names: variables, modules, functions, etc. @@ -320,36 +317,28 @@ :mod:`builtins`:: >>> import builtins - >>> dir(builtins) # doctest: +NORMALIZE_WHITESPACE - ['ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', - 'BlockingIOError', 'BrokenPipeError', 'BufferError', 'BytesWarning', - 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', - 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning', - 'EOFError', 'Ellipsis', 'EnvironmentError', 'Exception', 'False', - 'FileExistsError', 'FileNotFoundError', 'FloatingPointError', - 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', - 'ImportWarning', 'IndentationError', 'IndexError', 'InterruptedError', - 'IsADirectoryError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'None', 'NotADirectoryError', 'NotImplemented', - 'NotImplementedError', 'OSError', 'OverflowError', - 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError', - 'ReferenceError', 'ResourceWarning', 'RuntimeError', 'RuntimeWarning', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', - 'SystemExit', 'TabError', 'TimeoutError', 'True', 'TypeError', - 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', - 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', - 'ValueError', 'Warning', 'ZeroDivisionError', '_', '__build_class__', - '__debug__', '__doc__', '__import__', '__name__', '__package__', 'abs', - 'all', 'any', 'ascii', 'bin', 'bool', 'bytearray', 'bytes', 'callable', - 'chr', 'classmethod', 'compile', 'complex', 'copyright', 'credits', - 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'exec', 'exit', - 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', - 'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', - 'iter', 'len', 'license', 'list', 'locals', 'map', 'max', 'memoryview', - 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property', - 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', - 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', - 'zip'] + >>> dir(builtins) + + ['ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', 'Buffer + Error', 'BytesWarning', 'DeprecationWarning', 'EOFError', 'Ellipsis', 'Environme + ntError', 'Exception', 'False', 'FloatingPointError', 'FutureWarning', 'Generato + rExit', 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 'IndexErr + or', 'KeyError', 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', + 'None', 'NotImplemented', 'NotImplementedError', 'OSError', 'OverflowError', 'P + endingDeprecationWarning', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', ' + StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'Ta + bError', 'True', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', 'Unicod + eEncodeError', 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserW + arning', 'ValueError', 'Warning', 'ZeroDivisionError', '__build_class__', '__deb + ug__', '__doc__', '__import__', '__name__', '__package__', 'abs', 'all', 'any', + 'ascii', 'bin', 'bool', 'bytearray', 'bytes', 'chr', 'classmethod', 'compile', ' + complex', 'copyright', 'credits', 'delattr', 'dict', 'dir', 'divmod', 'enumerate + ', 'eval', 'exec', 'exit', 'filter', 'float', 'format', 'frozenset', 'getattr', + 'globals', 'hasattr', 'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', + 'issubclass', 'iter', 'len', 'license', 'list', 'locals', 'map', 'max', 'memory + view', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property' + , 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', 'sort + ed', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', 'zip'] .. _tut-packages: @@ -373,9 +362,7 @@ (such as mixing, adding echo, applying an equalizer function, creating an artificial stereo effect), so in addition you will be writing a never-ending stream of modules to perform these operations. Here's a possible structure for -your package (expressed in terms of a hierarchical filesystem): - -.. code-block:: text +your package (expressed in terms of a hierarchical filesystem):: sound/ Top-level package __init__.py Initialize the sound package @@ -472,7 +459,7 @@ encountered. It is up to the package author to keep this list up-to-date when a new version of the package is released. Package authors may also decide not to support it, if they don't see a use for importing \* from their package. For -example, the file :file:`sound/effects/__init__.py` could contain the following +example, the file :file:`sounds/effects/__init__.py` could contain the following code:: __all__ = ["echo", "surround", "reverse"] @@ -547,5 +534,6 @@ .. rubric:: Footnotes .. [#] In fact function definitions are also 'statements' that are 'executed'; the - execution of a module-level function definition enters the function name in - the module's global symbol table. + execution of a module-level function enters the function name in the module's + global symbol table. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/stdlib.rst --- a/Doc/tutorial/stdlib.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/stdlib.rst Mon Jan 25 17:05:13 2016 +0100 @@ -15,7 +15,7 @@ >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python36' + 'C:\\Python33' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -40,9 +40,7 @@ >>> import shutil >>> shutil.copyfile('data.db', 'archive.db') - 'archive.db' >>> shutil.move('/build/executables', 'installdir') - 'installdir' .. _tut-file-wildcards: @@ -140,18 +138,6 @@ >>> random.randrange(6) # random integer chosen from range(6) 4 -The :mod:`statistics` module calculates basic statistical properties -(the mean, median, variance, etc.) of numeric data:: - - >>> import statistics - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> statistics.mean(data) - 1.6071428571428572 - >>> statistics.median(data) - 1.25 - >>> statistics.variance(data) - 1.3720238095238095 - The SciPy project has many other modules for numerical computations. @@ -162,14 +148,13 @@ There are a number of modules for accessing the internet and processing internet protocols. Two of the simplest are :mod:`urllib.request` for retrieving data -from URLs and :mod:`smtplib` for sending mail:: +from urls and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://tycho.usno.navy.mil/cgi-bin/timer.pl') as response: - ... for line in response: - ... line = line.decode('utf-8') # Decoding the binary data to text. - ... if 'EST' in line or 'EDT' in line: # look for Eastern Time - ... print(line) + >>> for line in urlopen('http://tycho.usno.navy.mil/cgi-bin/timer.pl'): + ... line = line.decode('utf-8') # Decoding the binary data to text. + ... if 'EST' in line or 'EDT' in line: # look for Eastern Time + ... print(line)
    Nov. 25, 09:43:32 PM EST @@ -218,7 +203,7 @@ ================ Common data archiving and compression formats are directly supported by modules -including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and +including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`zipfile` and :mod:`tarfile`. :: >>> import zlib @@ -296,10 +281,8 @@ def test_average(self): self.assertEqual(average([20, 30, 70]), 40.0) self.assertEqual(round(average([1, 5, 7]), 1), 4.3) - with self.assertRaises(ZeroDivisionError): - average([]) - with self.assertRaises(TypeError): - average(20, 30, 70) + self.assertRaises(ZeroDivisionError, average, []) + self.assertRaises(TypeError, average, 20, 30, 70) unittest.main() # Calling from the command line invokes all tests @@ -323,18 +306,13 @@ (including attachments) and for implementing internet encoding and header protocols. -* The :mod:`json` package provides robust support for parsing this - popular data interchange format. The :mod:`csv` module supports - direct reading and writing of files in Comma-Separated Value format, - commonly supported by databases and spreadsheets. XML processing is - supported by the :mod:`xml.etree.ElementTree`, :mod:`xml.dom` and - :mod:`xml.sax` packages. Together, these modules and packages - greatly simplify data interchange between Python applications and - other tools. - -* The :mod:`sqlite3` module is a wrapper for the SQLite database - library, providing a persistent database that can be updated and - accessed using slightly nonstandard SQL syntax. +* The :mod:`xml.dom` and :mod:`xml.sax` packages provide robust support for + parsing this popular data interchange format. Likewise, the :mod:`csv` module + supports direct reads and writes in a common database format. Together, these + modules and packages greatly simplify data interchange between Python + applications and other tools. * Internationalization is supported by a number of modules including :mod:`gettext`, :mod:`locale`, and the :mod:`codecs` package. + + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/stdlib2.rst --- a/Doc/tutorial/stdlib2.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/stdlib2.rst Mon Jan 25 17:05:13 2016 +0100 @@ -18,7 +18,7 @@ >>> import reprlib >>> reprlib.repr(set('supercalifragilisticexpialidocious')) - "{'a', 'c', 'd', 'e', 'f', 'g', ...}" + "set(['a', 'c', 'd', 'e', 'f', 'g', ...])" The :mod:`pprint` module offers more sophisticated control over printing both built-in and user defined objects in a way that is readable by the interpreter. @@ -71,9 +71,9 @@ Templating ========== -The :mod:`string` module includes a versatile :class:`~string.Template` class -with a simplified syntax suitable for editing by end-users. This allows users -to customize their applications without having to alter the application. +The :mod:`string` module includes a versatile :class:`Template` class with a +simplified syntax suitable for editing by end-users. This allows users to +customize their applications without having to alter the application. The format uses placeholder names formed by ``$`` with valid Python identifiers (alphanumeric characters and underscores). Surrounding the placeholder with @@ -85,17 +85,17 @@ >>> t.substitute(village='Nottingham', cause='the ditch fund') 'Nottinghamfolk send $10 to the ditch fund.' -The :meth:`~string.Template.substitute` method raises a :exc:`KeyError` when a -placeholder is not supplied in a dictionary or a keyword argument. For -mail-merge style applications, user supplied data may be incomplete and the -:meth:`~string.Template.safe_substitute` method may be more appropriate --- -it will leave placeholders unchanged if data is missing:: +The :meth:`substitute` method raises a :exc:`KeyError` when a placeholder is not +supplied in a dictionary or a keyword argument. For mail-merge style +applications, user supplied data may be incomplete and the +:meth:`safe_substitute` method may be more appropriate --- it will leave +placeholders unchanged if data is missing:: >>> t = Template('Return the $item to $owner.') >>> d = dict(item='unladen swallow') >>> t.substitute(d) Traceback (most recent call last): - ... + . . . KeyError: 'owner' >>> t.safe_substitute(d) 'Return the unladen swallow to $owner.' @@ -132,9 +132,8 @@ Working with Binary Data Record Layouts ======================================= -The :mod:`struct` module provides :func:`~struct.pack` and -:func:`~struct.unpack` functions for working with variable length binary -record formats. The following example shows +The :mod:`struct` module provides :func:`pack` and :func:`unpack` functions for +working with variable length binary record formats. The following example shows how to loop through header information in a ZIP file without using the :mod:`zipfile` module. Pack codes ``"H"`` and ``"I"`` represent two and four byte unsigned numbers respectively. The ``"<"`` indicates that they are @@ -202,7 +201,7 @@ are difficult to reproduce. So, the preferred approach to task coordination is to concentrate all access to a resource in a single thread and then use the :mod:`queue` module to feed that thread with requests from other threads. -Applications using :class:`~queue.Queue` objects for inter-thread communication and +Applications using :class:`Queue` objects for inter-thread communication and coordination are easier to design, more readable, and more reliable. @@ -221,9 +220,7 @@ logging.error('Error occurred') logging.critical('Critical error -- shutting down') -This produces the following output: - -.. code-block:: none +This produces the following output:: WARNING:root:Warning:config file server.conf not found ERROR:root:Error occurred @@ -232,9 +229,8 @@ By default, informational and debugging messages are suppressed and the output is sent to standard error. Other output options include routing messages through email, datagrams, sockets, or to an HTTP Server. New filters can select -different routing based on message priority: :const:`~logging.DEBUG`, -:const:`~logging.INFO`, :const:`~logging.WARNING`, :const:`~logging.ERROR`, -and :const:`~logging.CRITICAL`. +different routing based on message priority: :const:`DEBUG`, :const:`INFO`, +:const:`WARNING`, :const:`ERROR`, and :const:`CRITICAL`. The logging system can be configured directly from Python or can be loaded from a user editable configuration file for customized logging without altering the @@ -261,9 +257,9 @@ >>> import weakref, gc >>> class A: ... def __init__(self, value): - ... self.value = value + ... self.value = value ... def __repr__(self): - ... return str(self.value) + ... return str(self.value) ... >>> a = A(10) # create a reference >>> d = weakref.WeakValueDictionary() @@ -277,7 +273,7 @@ Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python36/lib/weakref.py", line 46, in __getitem__ + File "C:/python33/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' @@ -291,11 +287,11 @@ sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`~array.array()` object that is like -a list that stores only homogeneous data and stores it more compactly. The -following example shows an array of numbers stored as two byte unsigned binary -numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular -lists of Python int objects:: +The :mod:`array` module provides an :class:`array()` object that is like a list +that stores only homogeneous data and stores it more compactly. The following +example shows an array of numbers stored as two byte unsigned binary numbers +(typecode ``"H"``) rather than the usual 16 bytes per entry for regular lists of +Python int objects:: >>> from array import array >>> a = array('H', [4000, 10, 700, 22222]) @@ -304,10 +300,10 @@ >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`~collections.deque()` object -that is like a list with faster appends and pops from the left side but slower -lookups in the middle. These objects are well suited for implementing queues -and breadth first tree searches:: +The :mod:`collections` module provides a :class:`deque()` object that is like a +list with faster appends and pops from the left side but slower lookups in the +middle. These objects are well suited for implementing queues and breadth first +tree searches:: >>> from collections import deque >>> d = deque(["task1", "task2", "task3"]) @@ -315,8 +311,6 @@ >>> print("Handling", d.popleft()) Handling task1 -:: - unsearched = deque([starting_node]) def breadth_first_search(unsearched): node = unsearched.popleft() @@ -353,8 +347,8 @@ Decimal Floating Point Arithmetic ================================= -The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` +The :mod:`decimal` module offers a :class:`Decimal` datatype for decimal +floating point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal @@ -375,15 +369,13 @@ >>> round(.70 * 1.05, 2) 0.73 -The :class:`~decimal.Decimal` result keeps a trailing zero, automatically -inferring four place significance from multiplicands with two place -significance. Decimal reproduces mathematics as done by hand and avoids -issues that can arise when binary floating point cannot exactly represent -decimal quantities. +The :class:`Decimal` result keeps a trailing zero, automatically inferring four +place significance from multiplicands with two place significance. Decimal +reproduces mathematics as done by hand and avoids issues that can arise when +binary floating point cannot exactly represent decimal quantities. -Exact representation enables the :class:`~decimal.Decimal` class to perform -modulo calculations and equality tests that are unsuitable for binary floating -point:: +Exact representation enables the :class:`Decimal` class to perform modulo +calculations and equality tests that are unsuitable for binary floating point:: >>> Decimal('1.00') % Decimal('.10') Decimal('0.00') diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/venv.rst --- a/Doc/tutorial/venv.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,197 +0,0 @@ - -.. _tut-venv: - -********************************* -Virtual Environments and Packages -********************************* - -Introduction -============ - -Python applications will often use packages and modules that don't -come as part of the standard library. Applications will sometimes -need a specific version of a library, because the application may -require that a particular bug has been fixed or the application may be -written using an obsolete version of the library's interface. - -This means it may not be possible for one Python installation to meet -the requirements of every application. If application A needs version -1.0 of a particular module but application B needs version 2.0, then -the requirements are in conflict and installing either version 1.0 or 2.0 -will leave one application unable to run. - -The solution for this problem is to create a :term:`virtual -environment` (often shortened to "virtualenv"), a self-contained -directory tree that contains a Python installation for a particular -version of Python, plus a number of additional packages. - -Different applications can then use different virtual environments. -To resolve the earlier example of conflicting requirements, -application A can have its own virtual environment with version 1.0 -installed while application B has another virtualenv with version 2.0. -If application B requires a library be upgraded to version 3.0, this will -not affect application A's environment. - - -Creating Virtual Environments -============================= - -The script used to create and manage virtual environments is called -:program:`pyvenv`. :program:`pyvenv` will usually install the most -recent version of Python that you have available; the script is also -installed with a version number, so if you have multiple versions of -Python on your system you can select a specific Python version by -running ``pyvenv-3.4`` or whichever version you want. - -To create a virtualenv, decide upon a directory -where you want to place it and run :program:`pyvenv` with the -directory path:: - - pyvenv tutorial-env - -This will create the ``tutorial-env`` directory if it doesn't exist, -and also create directories inside it containing a copy of the Python -interpreter, the standard library, and various supporting files. - -Once you've created a virtual environment, you need to -activate it. - -On Windows, run:: - - tutorial-env/Scripts/activate - -On Unix or MacOS, run:: - - source tutorial-env/bin/activate - -(This script is written for the bash shell. If you use the -:program:`csh` or :program:`fish` shells, there are alternate -``activate.csh`` and ``activate.fish`` scripts you should use -instead.) - -Activating the virtualenv will change your shell's prompt to show what -virtualenv you're using, and modify the environment so that running -``python`` will get you that particular version and installation of -Python. For example:: - - -> source ~/envs/tutorial-env/bin/activate - (tutorial-env) -> python - Python 3.4.3+ (3.4:c7b9645a6f35+, May 22 2015, 09:31:25) - ... - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python34.zip', ..., - '~/envs/tutorial-env/lib/python3.4/site-packages'] - >>> - - -Managing Packages with pip -========================== - -Once you've activated a virtual environment, you can install, upgrade, -and remove packages using a program called :program:`pip`. By default -``pip`` will install packages from the Python Package Index, -. You can browse the Python Package Index -by going to it in your web browser, or you can use ``pip``'s -limited search feature:: - - (tutorial-env) -> pip search astronomy - skyfield - Elegant astronomy for Python - gary - Galactic astronomy and gravitational dynamics. - novas - The United States Naval Observatory NOVAS astronomy library - astroobs - Provides astronomy ephemeris to plan telescope observations - PyAstronomy - A collection of astronomy related tools for Python. - ... - -``pip`` has a number of subcommands: "search", "install", "uninstall", -"freeze", etc. (Consult the :ref:`installing-index` guide for -complete documentation for ``pip``.) - -You can install the latest version of a package by specifying a package's name:: - - -> pip install novas - Collecting novas - Downloading novas-3.1.1.3.tar.gz (136kB) - Installing collected packages: novas - Running setup.py install for novas - Successfully installed novas-3.1.1.3 - -You can also install a specific version of a package by giving the -package name followed by ``==`` and the version number:: - - -> pip install requests==2.6.0 - Collecting requests==2.6.0 - Using cached requests-2.6.0-py2.py3-none-any.whl - Installing collected packages: requests - Successfully installed requests-2.6.0 - -If you re-run this command, ``pip`` will notice that the requested -version is already installed and do nothing. You can supply a -different version number to get that version, or you can run ``pip -install --upgrade`` to upgrade the package to the latest version:: - - -> pip install --upgrade requests - Collecting requests - Installing collected packages: requests - Found existing installation: requests 2.6.0 - Uninstalling requests-2.6.0: - Successfully uninstalled requests-2.6.0 - Successfully installed requests-2.7.0 - -``pip uninstall`` followed by one or more package names will remove the -packages from the virtual environment. - -``pip show`` will display information about a particular package:: - - (tutorial-env) -> pip show requests - --- - Metadata-Version: 2.0 - Name: requests - Version: 2.7.0 - Summary: Python HTTP for Humans. - Home-page: http://python-requests.org - Author: Kenneth Reitz - Author-email: me@kennethreitz.com - License: Apache 2.0 - Location: /Users/akuchling/envs/tutorial-env/lib/python3.4/site-packages - Requires: - -``pip list`` will display all of the packages installed in the virtual -environment:: - - (tutorial-env) -> pip list - novas (3.1.1.3) - numpy (1.9.2) - pip (7.0.3) - requests (2.7.0) - setuptools (16.0) - -``pip freeze`` will produce a similar list of the installed packages, -but the output uses the format that ``pip install`` expects. -A common convention is to put this list in a ``requirements.txt`` file:: - - (tutorial-env) -> pip freeze > requirements.txt - (tutorial-env) -> cat requirements.txt - novas==3.1.1.3 - numpy==1.9.2 - requests==2.7.0 - -The ``requirements.txt`` can then be committed to version control and -shipped as part of an application. Users can then install all the -necessary packages with ``install -r``:: - - -> pip install -r requirements.txt - Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) - ... - Collecting numpy==1.9.2 (from -r requirements.txt (line 2)) - ... - Collecting requests==2.7.0 (from -r requirements.txt (line 3)) - ... - Installing collected packages: novas, numpy, requests - Running setup.py install for novas - Successfully installed novas-3.1.1.3 numpy-1.9.2 requests-2.7.0 - -``pip`` has many more options. Consult the :ref:`installing-index` -guide for complete documentation for ``pip``. When you've written -a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/tutorial/whatnow.rst --- a/Doc/tutorial/whatnow.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/tutorial/whatnow.rst Mon Jan 25 17:05:13 2016 +0100 @@ -21,8 +21,8 @@ and many other tasks. Skimming through the Library Reference will give you an idea of what's available. -* :ref:`installing-index` explains how to install additional modules written - by other Python users. +* :ref:`install-index` explains how to install external modules written by other + Python users. * :ref:`reference-index`: A detailed explanation of Python's syntax and semantics. It's heavy reading, but is useful as a complete guide to the @@ -30,27 +30,24 @@ More Python resources: -* https://www.python.org: The major Python Web site. It contains code, +* http://www.python.org: The major Python Web site. It contains code, documentation, and pointers to Python-related pages around the Web. This Web site is mirrored in various places around the world, such as Europe, Japan, and Australia; a mirror may be faster than the main site, depending on your geographical location. -* https://docs.python.org: Fast access to Python's documentation. +* http://docs.python.org: Fast access to Python's documentation. -* https://pypi.python.org/pypi: The Python Package Index, previously also nicknamed +* http://pypi.python.org: The Python Package Index, previously also nicknamed the Cheese Shop, is an index of user-created Python modules that are available for download. Once you begin releasing code, you can register it here so that others can find it. -* http://code.activestate.com/recipes/langs/python/: The Python Cookbook is a +* http://aspn.activestate.com/ASPN/Python/Cookbook/: The Python Cookbook is a sizable collection of code examples, larger modules, and useful scripts. Particularly notable contributions are collected in a book also titled Python Cookbook (O'Reilly & Associates, ISBN 0-596-00797-3.) -* http://www.pyvideo.org collects links to Python-related videos from - conferences and user-group meetings. - * http://scipy.org: The Scientific Python project includes modules for fast array computations and manipulations plus a host of packages for such things as linear algebra, Fourier transforms, non-linear solvers, @@ -60,11 +57,17 @@ :newsgroup:`comp.lang.python`, or send them to the mailing list at python-list@python.org. The newsgroup and mailing list are gatewayed, so messages posted to one will automatically be forwarded to the other. There are -hundreds of postings a day, asking (and +around 120 postings a day (with peaks up to several hundred), asking (and answering) questions, suggesting new features, and announcing new modules. -Mailing list archives are available at https://mail.python.org/pipermail/. +Before posting, be sure to check the list of `Frequently Asked Questions +`_ (also called the FAQ), or look for it in the +:file:`Misc/` directory of the Python source distribution. Mailing list +archives are available at http://mail.python.org/pipermail/. The FAQ answers +many of the questions that come up again and again, and may already contain the +solution for your problem. -Before posting, be sure to check the list of -:ref:`Frequently Asked Questions ` (also called the FAQ). The -FAQ answers many of the questions that come up again and again, and may -already contain the solution for your problem. +.. Postings figure based on average of last six months activity as + reported by www.egroups.com; Jan. 2000 - June 2000: 21272 msgs / 182 + days = 116.9 msgs / day and steadily increasing. (XXX up to date figures?) + + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/using/cmdline.rst Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,7 @@ -.. highlightlang:: sh +.. highlightlang:: none .. ATTENTION: You probably should update Misc/python.man, too, if you modify - this file. +.. this file. .. _using-on-general: @@ -24,7 +24,7 @@ When invoking Python, you may specify any of these options:: - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] + python [-bBdEhiOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: @@ -41,7 +41,7 @@ * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can - produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + produce that with *Ctrl-D* on UNIX or *Ctrl-Z, Enter* on Windows) is read. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -77,12 +77,11 @@ the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension - (``.py``). The module name should be a valid absolute Python module name, but + (``.py``). The ``module-name`` should be a valid Python module name, but the implementation may not always enforce this (e.g. it may allow you to use a name that includes a hyphen). - Package names (including namespace packages) are also permitted. When a - package name is supplied instead + Package names are also permitted. When a package name is supplied instead of a normal module, the interpreter will execute ``.__main__`` as the main module. This behaviour is deliberately similar to the handling of directories and zipfiles that are passed to the interpreter as the @@ -116,9 +115,6 @@ .. versionchanged:: 3.1 Supply the package name to run a ``__main__`` submodule. - .. versionchanged:: 3.4 - namespace packages are also supported - .. describe:: - @@ -148,22 +144,13 @@ added to the start of :data:`sys.path` and the ``__main__.py`` file in that location is executed as the :mod:`__main__` module. - .. seealso:: - :func:`runpy.run_path` - Equivalent functionality directly available to Python code - If no interface option is given, :option:`-i` is implied, ``sys.argv[0]`` is an empty string (``""``) and the current directory will be added to the -start of :data:`sys.path`. Also, tab-completion and history editing is -automatically enabled, if available on your platform (see -:ref:`rlcompleter-config`). +start of :data:`sys.path`. .. seealso:: :ref:`tut-invoking` -.. versionchanged:: 3.4 - Automatic enabling of tab-completion and history editing. - Generic options ~~~~~~~~~~~~~~~ @@ -183,23 +170,18 @@ Python 3.0 -.. _using-on-misc-options: - Miscellaneous options ~~~~~~~~~~~~~~~~~~~~~ .. cmdoption:: -b - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error when the + Issue a warning when comparing str and bytes. Issue an error when the option is given twice (:option:`-bb`). - .. versionchanged: 3.5 - Affects comparisons of :class:`bytes` with :class:`int`. .. cmdoption:: -B - If given, Python won't try to write ``.pyc`` files on the + If given, Python won't try to write ``.pyc`` or ``.pyo`` files on the import of source modules. See also :envvar:`PYTHONDONTWRITEBYTECODE`. @@ -226,17 +208,6 @@ raises an exception. See also :envvar:`PYTHONINSPECT`. -.. cmdoption:: -I - - Run Python in isolated mode. This also implies -E and -s. - In isolated mode :data:`sys.path` contains neither the script's directory nor - the user's site-packages directory. All :envvar:`PYTHON*` environment - variables are ignored, too. Further restrictions may be imposed to prevent - the user from injecting malicious code. - - .. versionadded:: 3.4 - - .. cmdoption:: -O Turn on basic optimizations. This changes the filename extension for @@ -258,22 +229,23 @@ .. cmdoption:: -R - Kept for compatibility. On Python 3.3 and greater, hash randomization is - turned on by default. + Turn on hash randomization, so that the :meth:`__hash__` values of str, bytes + and datetime objects are "salted" with an unpredictable random value. + Although they remain constant within an individual Python process, they are + not predictable between repeated invocations of Python. - On previous versions of Python, this option turns on hash randomization, - so that the :meth:`__hash__` values of str, bytes and datetime - are "salted" with an unpredictable random value. Although they remain - constant within an individual Python process, they are not predictable - between repeated invocations of Python. - - Hash randomization is intended to provide protection against a - denial-of-service caused by carefully-chosen inputs that exploit the worst - case performance of a dict construction, O(n^2) complexity. See + This is intended to provide protection against a denial-of-service caused by + carefully-chosen inputs that exploit the worst case performance of a dict + construction, O(n^2) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. - :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash - seed secret. + Changing hash values affects the order in which keys are retrieved from a + dict. Although Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds), enough real-world code + implicitly relies on this non-guaranteed behavior that the randomization is + disabled by default. + + See also :envvar:`PYTHONHASHSEED`. .. versionadded:: 3.2.3 @@ -298,8 +270,8 @@ .. cmdoption:: -u - Force the binary layer of the stdout and stderr streams (which is - available as their ``buffer`` attribute) to be unbuffered. The text I/O + Force the binary layer of the stdin, stdout and stderr streams (which is + available as their ``buffer`` attribute) to be unbuffered. The text I/O layer will still be line-buffered if writing to the console, or block-buffered if redirected to a non-interactive file. @@ -387,29 +359,12 @@ .. cmdoption:: -X Reserved for various implementation-specific options. CPython currently - defines the following possible values: - - * ``-X faulthandler`` to enable :mod:`faulthandler`; - * ``-X showrefcount`` to enable the output of the total reference count - and memory blocks (only works on debug builds); - * ``-X tracemalloc`` to start tracing Python memory allocations using the - :mod:`tracemalloc` module. By default, only the most recent frame is - stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start - tracing with a traceback limit of *NFRAME* frames. See the - :func:`tracemalloc.start` for more information. - - It also allows to pass arbitrary values and retrieve them through the - :data:`sys._xoptions` dictionary. + defines none of them, but allows to pass arbitrary values and retrieve + them through the :data:`sys._xoptions` dictionary. .. versionchanged:: 3.2 It is now allowed to pass :option:`-X` with CPython. - .. versionadded:: 3.3 - The ``-X faulthandler`` option. - - .. versionadded:: 3.4 - The ``-X showrefcount`` and ``-X tracemalloc`` options. - Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -418,7 +373,7 @@ Reserved for use by Jython_. -.. _Jython: http://www.jython.org/ +.. _Jython: http://jython.org .. _using-on-envvars: @@ -426,10 +381,7 @@ Environment variables --------------------- -These environment variables influence Python's behavior, they are processed -before the command-line switches other than -E or -I. It is customary that -command-line switches override environmental variables where there is a -conflict. +These environment variables influence Python's behavior. .. envvar:: PYTHONHOME @@ -472,7 +424,15 @@ is executed in the same namespace where interactive commands are executed so that objects defined or imported in it can be used without qualification in the interactive session. You can also change the prompts :data:`sys.ps1` and - :data:`sys.ps2` and the hook :data:`sys.__interactivehook__` in this file. + :data:`sys.ps2` in this file. + + +.. envvar:: PYTHONY2K + + Set this to a non-empty string to cause the :mod:`time` module to require + dates specified as strings to include 4-digit years, otherwise 2-digit years + are converted based on rules described in the :mod:`time` module + documentation. .. envvar:: PYTHONOPTIMIZE @@ -519,15 +479,15 @@ .. envvar:: PYTHONDONTWRITEBYTECODE - If this is set to a non-empty string, Python won't try to write ``.pyc`` or - ``.pyo`` files on the import of source modules. This is equivalent to - specifying the :option:`-B` option. + If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the + import of source modules. This is equivalent to specifying the :option:`-B` + option. .. envvar:: PYTHONHASHSEED - If this variable is not set or set to ``random``, a random value is used - to seed the hashes of str, bytes and datetime objects. + If this variable is set to ``random``, a random value is used to seed the + hashes of str, bytes and datetime objects. If :envvar:`PYTHONHASHSEED` is set to an integer value, it is used as a fixed seed for generating the hash() of the types covered by the hash @@ -546,16 +506,13 @@ .. envvar:: PYTHONIOENCODING If this is set before running the interpreter, it overrides the encoding used - for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. Both - the ``encodingname`` and the ``:errorhandler`` parts are optional and have - the same meaning as in :func:`str.encode`. + for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. The + ``:errorhandler`` part is optional and has the same meaning as in + :func:`str.encode`. For stderr, the ``:errorhandler`` part is ignored; the handler will always be ``'backslashreplace'``. - .. versionchanged:: 3.4 - The ``encodingname`` part is now optional. - .. envvar:: PYTHONNOUSERSITE @@ -571,8 +528,8 @@ Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`Distutils installation paths ` for - ``python setup.py install --user``. + and :ref:`Packaging installation paths ` for + ``pysetup run install_dist --user``. .. seealso:: @@ -593,32 +550,11 @@ .. envvar:: PYTHONFAULTHANDLER - If this environment variable is set to a non-empty string, - :func:`faulthandler.enable` is called at startup: install a handler for - :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and - :const:`SIGILL` signals to dump the Python traceback. This is equivalent to - :option:`-X` ``faulthandler`` option. - - .. versionadded:: 3.3 - - -.. envvar:: PYTHONTRACEMALLOC - - If this environment variable is set to a non-empty string, start tracing - Python memory allocations using the :mod:`tracemalloc` module. The value of - the variable is the maximum number of frames stored in a traceback of a - trace. For example, ``PYTHONTRACEMALLOC=1`` stores only the most recent - frame. See the :func:`tracemalloc.start` for more information. - - .. versionadded:: 3.4 - - -.. envvar:: PYTHONASYNCIODEBUG - - If this environment variable is set to a non-empty string, enable the - :ref:`debug mode ` of the :mod:`asyncio` module. - - .. versionadded:: 3.4 + If this environment variable is set, :func:`faulthandler.enable` is called + at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`, + :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the + Python traceback. This is equivalent to :option:`-X` ``faulthandler`` + option. Debug-mode variables @@ -642,3 +578,4 @@ If set, Python will print memory allocation statistics every time a new object arena is created, and on shutdown. + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/index.rst --- a/Doc/using/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/using/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,4 +17,4 @@ unix.rst windows.rst mac.rst - scripts.rst + diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/mac.rst --- a/Doc/using/mac.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/using/mac.rst Mon Jan 25 17:05:13 2016 +0100 @@ -17,15 +17,14 @@ Getting and Installing MacPython ================================ -Mac OS X 10.8 comes with Python 2.7 pre-installed by Apple. If you wish, you -are invited to install the most recent version of Python 3 from the Python -website (https://www.python.org). A current "universal binary" build of Python, -which runs natively on the Mac's new Intel and legacy PPC CPU's, is available -there. +Mac OS X 10.5 comes with Python 2.5.1 pre-installed by Apple. If you wish, you +are invited to install the most recent version of Python from the Python website +(http://www.python.org). A current "universal binary" build of Python, which +runs natively on the Mac's new Intel and legacy PPC CPU's, is available there. What you get after installing is a number of things: -* A :file:`MacPython 3.4` folder in your :file:`Applications` folder. In here +* A :file:`MacPython 2.5` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official Python distributions; PythonLauncher, which handles double-clicking Python scripts from the Finder; and the "Build Applet" tool, which allows you to @@ -64,7 +63,7 @@ number of standard Unix command line editors, :program:`vim` and :program:`emacs` among them. If you want a more Mac-like editor, :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see -http://www.barebones.com/products/bbedit/index.html) are good choices, as is +http://www.barebones.com/products/bbedit/index.shtml) are good choices, as is :program:`TextMate` (see http://macromates.com/). Other editors include :program:`Gvim` (http://macvim.org) and :program:`Aquamacs` (http://aquamacs.org/). @@ -93,7 +92,7 @@ anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. -With Python 3.4, you can use either :program:`python` or :program:`pythonw`. +With Python 2.5, you can use either :program:`python` or :program:`pythonw`. Configuration @@ -102,9 +101,8 @@ Python on OS X honors all standard Unix environment variables such as :envvar:`PYTHONPATH`, but setting these variables for programs started from the Finder is non-standard as the Finder does not read your :file:`.profile` or -:file:`.cshrc` at startup. You need to create a file -:file:`~/.MacOSX/environment.plist`. See Apple's Technical Document QA1067 for -details. +:file:`.cshrc` at startup. You need to create a file :file:`~ +/.MacOSX/environment.plist`. See Apple's Technical Document QA1067 for details. For more information on installation Python packages in MacPython, see section :ref:`mac-package-manager`. @@ -117,7 +115,7 @@ MacPython ships with the standard IDLE development environment. A good introduction to using IDLE can be found at -https://hkn.eecs.berkeley.edu/~dyoo/python/idle_intro/index.html. +http://hkn.eecs.berkeley.edu/~dyoo/python/idle_intro/index.html. .. _mac-package-manager: @@ -127,11 +125,13 @@ There are several methods to install additional Python packages: +* http://pythonmac.org/packages/ contains selected compiled packages for Python + 2.5, 2.4, and 2.3. + * Packages can be installed via the standard Python distutils mode (``python setup.py install``). -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +* Many packages can also be installed via the :program:`setuptools` extension. GUI Programming on the Mac @@ -141,7 +141,7 @@ *PyObjC* is a Python binding to Apple's Objective-C/Cocoa framework, which is the foundation of most modern Mac development. Information on PyObjC is -available from https://pythonhosted.org/pyobjc/. +available from http://pyobjc.sourceforge.net. The standard Python GUI toolkit is :mod:`tkinter`, based on the cross-platform Tk toolkit (http://www.tcl.tk). An Aqua-native version of Tk is bundled with OS @@ -159,7 +159,7 @@ Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 3.4 folder is fine for +The "Build Applet" tool that is placed in the MacPython 2.5 folder is fine for packaging small Python scripts on your own machine to run as a standard Mac application. This tool, however, is not robust enough to distribute Python applications to other users. @@ -169,15 +169,29 @@ at http://undefined.org/python/#py2app. +Application Scripting +===================== + +Python can also be used to script other Mac applications via Apple's Open +Scripting Architecture (OSA); see http://appscript.sourceforge.net. Appscript is +a high-level, user-friendly Apple event bridge that allows you to control +scriptable Mac OS X applications using ordinary Python scripts. Appscript makes +Python a serious alternative to Apple's own *AppleScript* language for +automating your Mac. A related package, *PyOSA*, is an OSA language component +for the Python scripting language, allowing Python code to be executed by any +OSA-enabled application (Script Editor, Mail, iTunes, etc.). PyOSA makes Python +a full peer to AppleScript. + + Other Resources =============== The MacPython mailing list is an excellent support resource for Python users and developers on the Mac: -https://www.python.org/community/sigs/current/pythonmac-sig/ +http://www.python.org/community/sigs/current/pythonmac-sig/ Another useful resource is the MacPython wiki: -https://wiki.python.org/moin/MacPython +http://wiki.python.org/moin/MacPython diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/scripts.rst --- a/Doc/using/scripts.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ -.. _tools-and-scripts: - -Additional Tools and Scripts -============================ - -.. _scripts-pyvenv: - -pyvenv - Creating virtual environments --------------------------------------- - -.. include:: venv-create.inc - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/unix.rst --- a/Doc/using/unix.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/using/unix.rst Mon Jan 25 17:05:13 2016 +0100 @@ -28,7 +28,7 @@ http://www.debian.org/doc/manuals/maint-guide/first.en.html for Debian users - http://en.opensuse.org/Portal:Packaging + http://linuxmafia.com/pub/linux/suse-linux-internals/chapter35.html for OpenSuse users http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html for Fedora users @@ -55,19 +55,18 @@ On OpenSolaris -------------- -You can get Python from `OpenCSW `_. Various versions -of Python are available and can be installed with e.g. ``pkgutil -i python27``. +To install the newest Python versions on OpenSolaris, install `blastwave +`_ and type ``pkg_get -i python`` at the +prompt. -.. _building-python-on-unix: - Building Python =============== If you want to compile CPython yourself, first thing you should do is get the -`source `_. You can download either the +`source `_. You can download either the latest release's source or just grab a fresh `clone -`_. (If you want +`_. (If you want to contribute patches, you will need a clone.) The build process consists in the usual :: @@ -142,7 +141,8 @@ * http://sourceforge.net/projects/python-mode Geany is an excellent IDE with support for a lot of languages. For more -information, read: http://www.geany.org/ +information, read: http://geany.uvena.de/ Komodo edit is another extremely good IDE. It also has support for a lot of -languages. For more information, read http://komodoide.com/. +languages. For more information, read: +http://www.activestate.com/store/productdetail.aspx?prdGuid=20f4ed15-6684-4118-a78b-d37ff4058c5f diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/venv-create.inc --- a/Doc/using/venv-create.inc Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,110 +0,0 @@ -Creation of :ref:`virtual environments ` is done by executing the -``pyvenv`` script:: - - pyvenv /path/to/new/virtual/environment - -Running this command creates the target directory (creating any parent -directories that don't exist already) and places a ``pyvenv.cfg`` file in it -with a ``home`` key pointing to the Python installation the command was run -from. It also creates a ``bin`` (or ``Scripts`` on Windows) subdirectory -containing a copy of the ``python`` binary (or binaries, in the case of -Windows). It also creates an (initially empty) ``lib/pythonX.Y/site-packages`` -subdirectory (on Windows, this is ``Lib\site-packages``). - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - -.. highlight:: none - -On Windows, you may have to invoke the ``pyvenv`` script as follows, if you -don't have the relevant PATH and PATHEXT settings:: - - c:\Temp>c:\Python35\python c:\Python35\Tools\Scripts\pyvenv.py myenv - -or equivalently:: - - c:\Temp>c:\Python35\python -m venv myenv - -The command, if run with ``-h``, will show the available options:: - - usage: venv [-h] [--system-site-packages] [--symlinks] [--clear] - [--upgrade] [--without-pip] ENV_DIR [ENV_DIR ...] - - Creates virtual Python environments in one or more target directories. - - positional arguments: - ENV_DIR A directory to create the environment in. - - optional arguments: - -h, --help show this help message and exit - --system-site-packages Give access to the global site-packages dir to the - virtual environment. - --symlinks Try to use symlinks rather than copies, when symlinks - are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the environment directory if it already exists. - If not specified and the directory exists, an error is - raised. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - -Depending on how the ``venv`` functionality has been invoked, the usage message -may vary slightly, e.g. referencing ``pyvenv`` rather than ``venv``. - -.. versionchanged:: 3.4 - Installs pip by default, added the ``--without-pip`` and ``--copies`` - options - -.. versionchanged:: 3.4 - In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. Now, - if an existing directory is specified, its contents are removed and - the directory is processed as if it had been newly created. - -The created ``pyvenv.cfg`` file also includes the -``include-system-site-packages`` key, set to ``true`` if ``venv`` is -run with the ``--system-site-packages`` option, ``false`` otherwise. - -Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be -invoked to bootstrap ``pip`` into the virtual environment. - -Multiple paths can be given to ``pyvenv``, in which case an identical -virtualenv will be created, according to the given options, at each -provided path. - -Once a venv has been created, it can be "activated" using a script in the -venv's binary directory. The invocation of the script is platform-specific: - -+-------------+-----------------+-----------------------------------------+ -| Platform | Shell | Command to activate virtual environment | -+=============+=================+=========================================+ -| Posix | bash/zsh | $ source /bin/activate | -+-------------+-----------------+-----------------------------------------+ -| | fish | $ . /bin/activate.fish | -+-------------+-----------------+-----------------------------------------+ -| | csh/tcsh | $ source /bin/activate.csh | -+-------------+-----------------+-----------------------------------------+ -| Windows | cmd.exe | C:\> /Scripts/activate.bat | -+-------------+-----------------+-----------------------------------------+ -| | PowerShell | PS C:\> /Scripts/Activate.ps1 | -+-------------+-----------------+-----------------------------------------+ - -You don't specifically *need* to activate an environment; activation just -prepends the venv's binary directory to your path, so that "python" invokes the -venv's Python interpreter and you can run installed scripts without having to -use their full path. However, all scripts installed in a venv should be -runnable without activating it, and run with the venv's Python automatically. - -You can deactivate a venv by typing "deactivate" in your shell. The exact -mechanism is platform-specific: for example, the Bash activation script defines -a "deactivate" function, whereas on Windows there are separate scripts called -``deactivate.bat`` and ``Deactivate.ps1`` which are installed when the venv is -created. - -.. versionadded:: 3.4 - ``fish`` and ``csh`` activation scripts. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/win_installer.png Binary file Doc/using/win_installer.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Doc/using/windows.rst --- a/Doc/using/windows.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/using/windows.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,252 +7,37 @@ ************************* .. sectionauthor:: Robert Lehmann -.. sectionauthor:: Steve Dower This document aims to give an overview of Windows-specific behaviour you should know about when using Python on Microsoft Windows. + Installing Python ================= -Unlike most Unix systems and services, Windows does not include a system -supported installation of Python. To make Python available, the CPython team +Unlike most Unix systems and services, Windows does not require Python natively +and thus does not pre-install a version of Python. However, the CPython team has compiled Windows installers (MSI packages) with every `release -`_ for many years. These installers -are primarily intended to add a per-user installation of Python, with the -core interpreter and library being used by a single user. The installer is also -able to install for all users of a single machine, and a separate ZIP file is -available for application-local distributions. - -Supported Versions ------------------- - -As specified in :pep:`11`, a Python release only supports a Windows platform -while Microsoft considers the platform under extended support. This means that -Python 3.5 supports Windows Vista and newer. If you require Windows XP support -then please install Python 3.4. - -Installation Steps ------------------- - -Four Python 3.5 installers are available for download - two each for the 32-bit -and 64-bit versions of the interpreter. The *web installer* is a small initial -download, and it will automatically download the required components as -necessary. The *offline installer* includes the components necessary for a -default installation and only requires an internet connection for optional -features. See :ref:`install-layout-option` for other ways to avoid downloading -during installation. - -After starting the installer, one of two options may be selected: - -.. image:: win_installer.png - -If you select "Install Now": - -* You will *not* need to be an administrator (unless a system update for the - C Runtime Library is required or you install the :ref:`launcher` for all - users) -* Python will be installed into your user directory -* The :ref:`launcher` will be installed according to the option at the bottom - of the first pace -* The standard library, test suite, launcher and pip will be installed -* If selected, the install directory will be added to your :envvar:`PATH` -* Shortcuts will only be visible for the current user - -Selecting "Customize installation" will allow you to select the features to -install, the installation location and other options or post-install actions. -To install debugging symbols or binaries, you will need to use this option. - -To perform an all-users installation, you should select "Customize -installation". In this case: - -* You may be required to provide administrative credentials or approval -* Python will be installed into the Program Files directory -* The :ref:`launcher` will be installed into the Windows directory -* Optional features may be selected during installation -* The standard library can be pre-compiled to bytecode -* If selected, the install directory will be added to the system :envvar:`PATH` -* Shortcuts are available for all users - -.. _install-quiet-option: - -Installing Without UI ---------------------- - -All of the options available in the installer UI can also be specified from the -command line, allowing scripted installers to replicate an installation on many -machines without user interaction. These options may also be set without -suppressing the UI in order to change some of the defaults. - -To completely hide the installer UI and install Python silently, pass the -``/quiet`` option. To skip past the user interaction but still display -progress and errors, pass the ``/passive`` option. The ``/uninstall`` -option may be passed to immediately begin removing Python - no prompt will be -displayed. - -All other options are passed as ``name=value``, where the value is usually -``0`` to disable a feature, ``1`` to enable a feature, or a path. The full list -of available options is shown below. - -+---------------------------+--------------------------------------+--------------------------+ -| Name | Description | Default | -+===========================+======================================+==========================+ -| InstallAllUsers | Perform a system-wide installation. | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| TargetDir | The installation directory | Selected based on | -| | | InstallAllUsers | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultAllUsersTargetDir | The default installation directory | :file:`%ProgramFiles%\\\ | -| | for all-user installs | Python X.Y` or :file:`\ | -| | | %ProgramFiles(x86)%\\\ | -| | | Python X.Y` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultJustForMeTargetDir | The default install directory for | :file:`%LocalAppData%\\\ | -| | just-for-me installs | Programs\\PythonXY` or | -| | | :file:`%LocalAppData%\\\ | -| | | Programs\\PythonXY-32` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultCustomTargetDir | The default custom install directory | (empty) | -| | displayed in the UI | | -+---------------------------+--------------------------------------+--------------------------+ -| AssociateFiles | Create file associations if the | 1 | -| | launcher is also installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| CompileAll | Compile all ``.py`` files to | 0 | -| | ``.pyc``. | | -+---------------------------+--------------------------------------+--------------------------+ -| PrependPath | Add install and Scripts directories | 0 | -| | tho :envvar:`PATH` and ``.PY`` to | | -| | :envvar:`PATHEXT` | | -+---------------------------+--------------------------------------+--------------------------+ -| Shortcuts | Create shortcuts for the interpreter,| 1 | -| | documentation and IDLE if installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_doc | Install Python manual | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_debug | Install debug binaries | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_dev | Install developer headers and | 1 | -| | libraries | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_exe | Install :file:`python.exe` and | 1 | -| | related files | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_launcher | Install :ref:`launcher`. | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| InstallLauncherAllUsers | Installs :ref:`launcher` for all | 1 | -| | users. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_lib | Install standard library and | 1 | -| | extension modules | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_pip | Install bundled pip and setuptools | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_symbols | Install debugging symbols (`*`.pdb) | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tcltk | Install Tcl/Tk support and IDLE | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_test | Install standard library test suite | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tools | Install utility scripts | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| LauncherOnly | Only installs the launcher. This | 0 | -| | will override most other options. | | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstall | Disable most install UI | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstallDescription | A custom message to display when the | (empty) | -| | simplified install UI is used. | | -+---------------------------+--------------------------------------+--------------------------+ - -For example, to silently install a default, system-wide Python installation, -you could use the following command (from an elevated command prompt):: - - python-3.5.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 - -To allow users to easily install a personal copy of Python without the test -suite, you could provide a shortcut with the following command. This will -display a simplified initial page and disallow customization:: - - python-3.5.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0 - SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite." - -(Note that omitting the launcher also omits file associations, and is only -recommended for per-user installs when there is also a system-wide installation -that included the launcher.) - -The options listed above can also be provided in a file named ``unattend.xml`` -alongside the executable. This file specifies a list of options and values. -When a value is provided as an attribute, it will be converted to a number if -possible. Values provided as element text are always left as strings. This -example file sets the same options and the previous example:: - - - - - -.. _install-layout-option: - -Installing Without Downloading ------------------------------- - -As some features of Python are not included in the initial installer download, -selecting those features may require an internet connection. To avoid this -need, all possible components may be downloaded on-demand to create a complete -*layout* that will no longer require an internet connection regardless of the -selected features. Note that this download may be bigger than required, but -where a large number of installations are going to be performed it is very -useful to have a locally cached copy. - -Execute the following command from Command Prompt to download all possible -required files. Remember to substitute ``python-3.5.0.exe`` for the actual -name of your installer, and to create layouts in their own directories to -avoid collisions between files with the same name. - -:: - - python-3.5.0.exe /layout [optional target directory] - -You may also specify the ``/quiet`` option to hide the progress display. - -Modifying an install --------------------- - -Once Python has been installed, you can add or remove features through the -Programs and Features tool that is part of Windows. Select the Python entry and -choose "Uninstall/Change" to open the installer in maintenance mode. - -"Modify" allows you to add or remove features by modifying the checkboxes - -unchanged checkboxes will not install or remove anything. Some options cannot be -changed in this mode, such as the install directory; to modify these, you will -need to remove and then reinstall Python completely. - -"Repair" will verify all the files that should be installed using the current -settings and replace any that have been removed or modified. - -"Uninstall" will remove Python entirely, with the exception of the -:ref:`launcher`, which has its own entry in Programs and Features. - -Other Platforms ---------------- +`_ for many years. With ongoing development of Python, some platforms that used to be supported earlier are no longer supported (due to the lack of users or developers). Check :pep:`11` for details on all unsupported platforms. +* Up to 2.5, Python was still compatible with Windows 95, 98 and ME (but already + raised a deprecation warning on installation). For Python 2.6 (and all + following releases), this support was dropped and new releases are just + expected to work on the Windows NT family. * `Windows CE `_ is still supported. -* The `Cygwin `_ installer offers to install the Python - interpreter as well (cf. `Cygwin package source +* The `Cygwin `_ installer offers to install the `Python + interpreter `_ as well; it is located under + "Interpreters." (cf. `Cygwin package source `_, `Maintainer releases `_) -See `Python for Windows `_ -for detailed information about platforms with pre-compiled installers. +See `Python for Windows (and DOS) `_ +for detailed information about platforms with precompiled installers. .. seealso:: @@ -260,15 +45,15 @@ "7 Minutes to "Hello World!"" by Richard Dooling, 2006 - `Installing on Windows `_ + `Installing on Windows `_ in "`Dive into Python: Python from novice to pro - `_" + `_" by Mark Pilgrim, 2004, ISBN 1-59059-356-1 - `For Windows users `_ + `For Windows users `_ in "Installing Python" - in "`A Byte of Python `_" + in "`A Byte of Python `_" by Swaroop C H, 2003 @@ -279,407 +64,91 @@ additional functionality. The following is a list of popular versions and their key features: -`ActivePython `_ +`ActivePython `_ Installer with multi-platform compatibility, documentation, PyWin32 -`Anaconda `_ - Popular scientific modules (such as numpy, scipy and pandas) and the - ``conda`` package manager. +`Enthought Python Distribution `_ + Popular modules (such as PyWin32) with their respective documentation, tool + suite for building extensible Python applications -`Canopy `_ - A "comprehensive Python analysis environment" with editors and other - development tools. - -`WinPython `_ - Windows-specific distribution with prebuilt scientific packages and - tools for building packages. - -Note that these packages may not include the latest versions of Python or -other libraries, and are not maintained or supported by the core Python team. +Notice that these packages are likely to install *older* versions of Python. Configuring Python ================== -To run Python conveniently from a command prompt, you might consider changing -some default environment variables in Windows. While the installer provides an -option to configure the PATH and PATHEXT variables for you, this is only -reliable for a single, system-wide installation. If you regularly use multiple -versions of Python, consider using the :ref:`launcher`. +In order to run Python flawlessly, you might have to change certain environment +settings in Windows. -.. _setting-envvars: - Excursus: Setting environment variables --------------------------------------- -Windows allows environment variables to be configured permanently at both the -User level and the System level, or temporarily in a command prompt. +Windows has a built-in dialog for changing environment variables (following +guide applies to XP classical view): Right-click the icon for your machine +(usually located on your Desktop and called "My Computer") and choose +:menuselection:`Properties` there. Then, open the :guilabel:`Advanced` tab +and click the :guilabel:`Environment Variables` button. -To temporarily set environment variables, open Command Prompt and use the -:command:`set` command:: +In short, your path is: - C:\>set PATH=C:\Program Files\Python 3.5;%PATH% - C:\>set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib - C:\>python + :menuselection:`My Computer + --> Properties + --> Advanced + --> Environment Variables` -These changes will apply to any further commands executed in that console, and -will be inherited by any applications started from the console. - -Including the variable name within percent signs will expand to the existing -value, allowing you to add your new value at either the start or the end. -Modifying :envvar:`PATH` by adding the directory containing -:program:`python.exe` to the start is a common way to ensure the correct version -of Python is launched. - -To permanently modify the default environment variables, click Start and search -for 'edit environment variables', or open System properties, :guilabel:`Advanced -system settings` and click the :guilabel:`Environment Variables` button. In this dialog, you can add or modify User and System variables. To change System variables, you need non-restricted access to your machine (i.e. Administrator rights). -.. note:: +Another way of adding variables to your environment is using the :command:`set` +command:: - Windows will concatenate User variables *after* System variables, which may - cause unexpected results when modifying :envvar:`PATH`. + set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib - The :envvar:`PYTHONPATH` variable is used by all versions of Python 2 and - Python 3, so you should not permanently configure this variable unless it - only includes code that is compatible with all of your installed Python - versions. +To make this setting permanent, you could add the corresponding command line to +your :file:`autoexec.bat`. :program:`msconfig` is a graphical interface to this +file. + +Viewing environment variables can also be done more straight-forward: The +command prompt will expand strings wrapped into percent signs automatically:: + + echo %PATH% + +Consult :command:`set /?` for details on this behaviour. .. seealso:: - http://support.microsoft.com/kb/100843 + http://support.microsoft.com/kb/100843 Environment variables in Windows NT - http://technet.microsoft.com/en-us/library/cc754250.aspx - The SET command, for temporarily modifying environment variables - - http://technet.microsoft.com/en-us/library/cc755104.aspx - The SETX command, for permanently modifying environment variables - - http://support.microsoft.com/kb/310519 + http://support.microsoft.com/kb/310519 How To Manage Environment Variables in Windows XP - http://www.chem.gla.ac.uk/~louis/software/faq/q1.html + http://www.chem.gla.ac.uk/~louis/software/faq/q1.html Setting Environment variables, Louis J. Farrugia -.. _windows-path-mod: Finding the Python executable ----------------------------- -.. versionchanged:: 3.5 +Besides using the automatically created start menu entry for the Python +interpreter, you might want to start Python in the DOS prompt. To make this +work, you need to set your :envvar:`%PATH%` environment variable to include the +directory of your Python distribution, delimited by a semicolon from other +entries. An example variable could look like this (assuming the first two +entries are Windows' default):: -Besides using the automatically created start menu entry for the Python -interpreter, you might want to start Python in the command prompt. The -installer for Python 3.5 and later has an option to set that up for you. + C:\WINDOWS\system32;C:\WINDOWS;C:\Python25 -On the first page of the installer, an option labelled "Add Python 3.5 to -PATH" can be selected to have the installer add the install location into the -:envvar:`PATH`. The location of the :file:`Scripts\\` folder is also added. -This allows you to type :command:`python` to run the interpreter, and -:command:`pip` or . Thus, you can also execute your -scripts with command line options, see :ref:`using-on-cmdline` documentation. +Typing :command:`python` on your command prompt will now fire up the Python +interpreter. Thus, you can also execute your scripts with command line options, +see :ref:`using-on-cmdline` documentation. -If you don't enable this option at install time, you can always re-run the -installer, select Modify, and enable it. Alternatively, you can manually -modify the :envvar:`PATH` using the directions in :ref:`setting-envvars`. You -need to set your :envvar:`PATH` environment variable to include the directory -of your Python installation, delimited by a semicolon from other entries. An -example variable could look like this (assuming the first two entries already -existed):: - - C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.5 - -.. _launcher: - -Python Launcher for Windows -=========================== - -.. versionadded:: 3.3 - -The Python launcher for Windows is a utility which aids in locating and -executing of different Python versions. It allows scripts (or the -command-line) to indicate a preference for a specific Python version, and -will locate and execute that version. - -Unlike the :envvar:`PATH` variable, the launcher will correctly select the most -appropriate version of Python. It will prefer per-user installations over -system-wide ones, and orders by language version rather than using the most -recently installed version. - -Getting started ---------------- - -From the command-line -^^^^^^^^^^^^^^^^^^^^^ - -System-wide installations of Python 3.3 and later will put the launcher on your -:envvar:`PATH`. The launcher is compatible with all available versions of -Python, so it does not matter which version is installed. To check that the -launcher is available, execute the following command in Command Prompt: - -:: - - py - -You should find that the latest version of Python 2.x you have installed is -started - it can be exited as normal, and any additional command-line -arguments specified will be sent directly to Python. - -If you have multiple versions of Python 2.x installed (e.g., 2.6 and 2.7) you -will have noticed that Python 2.7 was started - to launch Python 2.6, try the -command: - -:: - - py -2.6 - -If you have a Python 3.x installed, try the command: - -:: - - py -3 - -You should find the latest version of Python 3.x starts. - -If you see the following error, you do not have the launcher installed: - -:: - - 'py' is not recognized as an internal or external command, - operable program or batch file. - -Per-user installations of Python do not add the launcher to :envvar:`PATH` -unless the option was selected on installation. - -Virtual environments -^^^^^^^^^^^^^^^^^^^^ - -.. versionadded:: 3.5 - -If the launcher is run with no explicit Python version specification, and a -virtual environment (created with the standard library :mod:`venv` module or -the external ``virtualenv`` tool) active, the launcher will run the virtual -environment's interpreter rather than the global one. To run the global -interpreter, either deactivate the virtual environment, or explicitly specify -the global Python version. - -From a script -^^^^^^^^^^^^^ - -Let's create a test Python script - create a file called ``hello.py`` with the -following contents - -:: - - #! python - import sys - sys.stdout.write("hello from Python %s\n" % (sys.version,)) - -From the directory in which hello.py lives, execute the command: - -:: - - py hello.py - -You should notice the version number of your latest Python 2.x installation -is printed. Now try changing the first line to be: - -:: - - #! python3 - -Re-executing the command should now print the latest Python 3.x information. -As with the above command-line examples, you can specify a more explicit -version qualifier. Assuming you have Python 2.6 installed, try changing the -first line to ``#! python2.6`` and you should find the 2.6 version -information printed. - -From file associations -^^^^^^^^^^^^^^^^^^^^^^ - -The launcher should have been associated with Python files (i.e. ``.py``, -``.pyw``, ``.pyc`` files) when it was installed. This means that -when you double-click on one of these files from Windows explorer the launcher -will be used, and therefore you can use the same facilities described above to -have the script specify the version which should be used. - -The key benefit of this is that a single launcher can support multiple Python -versions at the same time depending on the contents of the first line. - -Shebang Lines -------------- - -If the first line of a script file starts with ``#!``, it is known as a -"shebang" line. Linux and other Unix like operating systems have native -support for such lines and are commonly used on such systems to indicate how -a script should be executed. This launcher allows the same facilities to be -using with Python scripts on Windows and the examples above demonstrate their -use. - -To allow shebang lines in Python scripts to be portable between Unix and -Windows, this launcher supports a number of 'virtual' commands to specify -which interpreter to use. The supported virtual commands are: - -* ``/usr/bin/env python`` -* ``/usr/bin/python`` -* ``/usr/local/bin/python`` -* ``python`` - -For example, if the first line of your script starts with - -:: - - #! /usr/bin/python - -The default Python will be located and used. As many Python scripts written -to work on Unix will already have this line, you should find these scripts can -be used by the launcher without modification. If you are writing a new script -on Windows which you hope will be useful on Unix, you should use one of the -shebang lines starting with ``/usr``. - -Any of the above virtual commands can be suffixed with an explicit version -(either just the major version, or the major and minor version) - for example -``/usr/bin/python2.7`` - which will cause that specific version to be located -and used. - -The ``/usr/bin/env`` form of shebang line has one further special property. -Before looking for installed Python interpreters, this form will search the -executable :envvar:`PATH` for a Python executable. This corresponds to the -behaviour of the Unix ``env`` program, which performs a :envvar:`PATH` search. - -Arguments in shebang lines --------------------------- - -The shebang lines can also specify additional options to be passed to the -Python interpreter. For example, if you have a shebang line: - -:: - - #! /usr/bin/python -v - -Then Python will be started with the ``-v`` option - -Customization -------------- - -Customization via INI files -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Two .ini files will be searched by the launcher - ``py.ini`` in the current -user's "application data" directory (i.e. the directory returned by calling the -Windows function SHGetFolderPath with CSIDL_LOCAL_APPDATA) and ``py.ini`` in the -same directory as the launcher. The same .ini files are used for both the -'console' version of the launcher (i.e. py.exe) and for the 'windows' version -(i.e. pyw.exe) - -Customization specified in the "application directory" will have precedence over -the one next to the executable, so a user, who may not have write access to the -.ini file next to the launcher, can override commands in that global .ini file) - -Customizing default Python versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In some cases, a version qualifier can be included in a command to dictate -which version of Python will be used by the command. A version qualifier -starts with a major version number and can optionally be followed by a period -('.') and a minor version specifier. If the minor qualifier is specified, it -may optionally be followed by "-32" to indicate the 32-bit implementation of -that version be used. - -For example, a shebang line of ``#!python`` has no version qualifier, while -``#!python3`` has a version qualifier which specifies only a major version. - -If no version qualifiers are found in a command, the environment variable -``PY_PYTHON`` can be set to specify the default version qualifier - the default -value is "2". Note this value could specify just a major version (e.g. "2") or -a major.minor qualifier (e.g. "2.6"), or even major.minor-32. - -If no minor version qualifiers are found, the environment variable -``PY_PYTHON{major}`` (where ``{major}`` is the current major version qualifier -as determined above) can be set to specify the full version. If no such option -is found, the launcher will enumerate the installed Python versions and use -the latest minor release found for the major version, which is likely, -although not guaranteed, to be the most recently installed version in that -family. - -On 64-bit Windows with both 32-bit and 64-bit implementations of the same -(major.minor) Python version installed, the 64-bit version will always be -preferred. This will be true for both 32-bit and 64-bit implementations of the -launcher - a 32-bit launcher will prefer to execute a 64-bit Python installation -of the specified version if available. This is so the behavior of the launcher -can be predicted knowing only what versions are installed on the PC and -without regard to the order in which they were installed (i.e., without knowing -whether a 32 or 64-bit version of Python and corresponding launcher was -installed last). As noted above, an optional "-32" suffix can be used on a -version specifier to change this behaviour. - -Examples: - -* If no relevant options are set, the commands ``python`` and - ``python2`` will use the latest Python 2.x version installed and - the command ``python3`` will use the latest Python 3.x installed. - -* The commands ``python3.1`` and ``python2.7`` will not consult any - options at all as the versions are fully specified. - -* If ``PY_PYTHON=3``, the commands ``python`` and ``python3`` will both use - the latest installed Python 3 version. - -* If ``PY_PYTHON=3.1-32``, the command ``python`` will use the 32-bit - implementation of 3.1 whereas the command ``python3`` will use the latest - installed Python (PY_PYTHON was not considered at all as a major - version was specified.) - -* If ``PY_PYTHON=3`` and ``PY_PYTHON3=3.1``, the commands - ``python`` and ``python3`` will both use specifically 3.1 - -In addition to environment variables, the same settings can be configured -in the .INI file used by the launcher. The section in the INI file is -called ``[defaults]`` and the key name will be the same as the -environment variables without the leading ``PY_`` prefix (and note that -the key names in the INI file are case insensitive.) The contents of -an environment variable will override things specified in the INI file. - -For example: - -* Setting ``PY_PYTHON=3.1`` is equivalent to the INI file containing: - -:: - - [defaults] - python=3.1 - -* Setting ``PY_PYTHON=3`` and ``PY_PYTHON3=3.1`` is equivalent to the INI file - containing: - -:: - - [defaults] - python=3 - python3=3.1 - -Diagnostics ------------ - -If an environment variable ``PYLAUNCH_DEBUG`` is set (to any value), the -launcher will print diagnostic information to stderr (i.e. to the console). -While this information manages to be simultaneously verbose *and* terse, it -should allow you to see what versions of Python were located, why a -particular version was chosen and the exact command-line used to execute the -target Python. - - - -.. finding_modules: Finding modules -=============== +--------------- Python usually stores its library (and thereby your site-packages folder) in the installation directory. So, if you had installed Python to @@ -715,17 +184,6 @@ the environment, and no registry entries can be found, a default path with relative entries is used (e.g. ``.\Lib;.\plat-win``, etc). -If a ``pyvenv.cfg`` file is found alongside the main executable or in the -directory one level above the executable, the following variations apply: - -* If ``home`` is an absolute path and :envvar:`PYTHONHOME` is not set, this - path is used instead of the path to the main executable when deducing the - home location. - -* If ``applocal`` is set to true, the ``home`` property or the main executable - is always used as the home path, and all environment variables or registry - values affecting the path are ignored. The landmark file is not checked. - The end result of all this is: * When running :file:`python.exe`, or any other .exe in the main Python @@ -737,36 +195,35 @@ etc), the "Python Home" will not be deduced, so the core path from the registry is used. Other "application paths" in the registry are always read. -* If Python can't find its home and there are no registry value (frozen .exe, - some very strange installation setup) you get a path with some default, but +* If Python can't find its home and there is no registry (eg, frozen .exe, some + very strange installation setup) you get a path with some default, but relative, paths. -For those who want to bundle Python into their application or distribution, the -following advice will prevent conflicts with other installations: -* Include a ``pyvenv.cfg`` file alongside your executable containing - ``applocal = true``. This will ensure that your own directory will be used to - resolve paths even if you have included the standard library in a ZIP file. - It will also ignore user site-packages and other paths listed in the - registry. +Executing scripts +----------------- -* If you are loading :file:`python3.dll` or :file:`python35.dll` in your own - executable, explicitly call :c:func:`Py_SetPath` or (at least) - :c:func:`Py_SetProgramName` before :c:func:`Py_Initialize`. +Python scripts (files with the extension ``.py``) will be executed by +:program:`python.exe` by default. This executable opens a terminal, which stays +open even if the program uses a GUI. If you do not want this to happen, use the +extension ``.pyw`` which will cause the script to be executed by +:program:`pythonw.exe` by default (both executables are located in the top-level +of your Python installation directory). This suppresses the terminal window on +startup. -* Clear and/or overwrite :envvar:`PYTHONPATH` and set :envvar:`PYTHONHOME` - before launching :file:`python.exe` from your application. +You can also make all ``.py`` scripts execute with :program:`pythonw.exe`, +setting this through the usual facilities, for example (might require +administrative rights): -* If you cannot use the previous suggestions (for example, you are a - distribution that allows people to run :file:`python.exe` directly), ensure - that the landmark file (:file:`Lib\\os.py`) exists in your install directory. - (Note that it will not be detected inside a ZIP file.) +#. Launch a command prompt. +#. Associate the correct file group with ``.py`` scripts:: -These will ensure that the files in a system-wide installation will not take -precedence over the copy of the standard library bundled with your application. -Otherwise, your users may experience problems using your application. Note that -the first suggestion is the best, as the other may still be susceptible to -non-standard paths in the registry and user site-packages. + assoc .py=Python.File + +#. Redirect all Python files to the new executable:: + + ftype Python.File=C:\Path\to\pythonw.exe "%1" %* + Additional modules ================== @@ -778,6 +235,7 @@ The Windows-specific standard modules are documented in :ref:`mswin-specific-services`. + PyWin32 ------- @@ -793,7 +251,7 @@ user interfaces `PythonWin `_ is a sample MFC application +http://www.python.org/windows/pythonwin/>`_ is a sample MFC application shipped with PyWin32. It is an embeddable IDE with a built-in debugger. .. seealso:: @@ -805,14 +263,13 @@ by David and Paul Boddie -cx_Freeze ---------- +Py2exe +------ -`cx_Freeze `_ is a :mod:`distutils` -extension (see :ref:`extending-distutils`) which wraps Python scripts into -executable Windows programs (:file:`{*}.exe` files). When you have done this, -you can distribute your application without requiring your users to install -Python. +`Py2exe `_ is a :mod:`distutils` extension (see +:ref:`extending-distutils`) which wraps Python scripts into executable Windows +programs (:file:`{*}.exe` files). When you have done this, you can distribute +your application without requiring your users to install Python. WConio @@ -831,15 +288,31 @@ =========================== If you want to compile CPython yourself, first thing you should do is get the -`source `_. You can download either the +`source `_. You can download either the latest release's source or just grab a fresh `checkout -`_. +`_. -The source tree contains a build solution and project files for Microsoft -Visual Studio 2015, which is the compiler used to build the official Python -releases. These files are in the :file:`PCbuild` directory. +For Microsoft Visual C++, which is the compiler with which official Python +releases are built, the source tree contains solutions/project files. View the +:file:`readme.txt` in their respective directories: -Check :file:`PCbuild/readme.txt` for general information on the build process. ++--------------------+--------------+-----------------------+ +| Directory | MSVC version | Visual Studio version | ++====================+==============+=======================+ +| :file:`PC/VC6/` | 6.0 | 97 | ++--------------------+--------------+-----------------------+ +| :file:`PC/VS7.1/` | 7.1 | 2003 | ++--------------------+--------------+-----------------------+ +| :file:`PC/VS8.0/` | 8.0 | 2005 | ++--------------------+--------------+-----------------------+ +| :file:`PCbuild/` | 9.0 | 2008 | ++--------------------+--------------+-----------------------+ + +Note that not all of these build directories are fully supported. Read the +release notes to see which compiler version the official releases for your +version are built with. + +Check :file:`PC/readme.txt` for general information on the build process. For extension modules, consult :ref:`building-on-windows`. @@ -855,89 +328,12 @@ by Trent Apted et al, 2007 -Embedded Distribution -===================== - -.. versionadded:: 3.5 - -The embedded distribution is a ZIP file containing a minimal Python environment. -It is intended for acting as part of another application, rather than being -directly accessed by end-users. - -When extracted, the embedded distribution is (almost) fully isolated from the -user's system, including environment variables, system registry settings, and -installed packages. The standard library is included as pre-compiled and -optimized ``.pyc`` files in a ZIP, and ``python3.dll``, ``python35.dll``, -``python.exe`` and ``pythonw.exe`` are all provided. Tcl/tk (including all -dependants, such as Idle), pip and the Python documentation are not included. - -.. note:: - - The embedded distribution does not include the `Microsoft C Runtime - `_ and it is - the responsibility of the application installer to provide this. The - runtime may have already been installed on a user's system previously or - automatically via Windows Update, and can be detected by finding - ``ucrtbase.dll`` in the system directory. - -Third-party packages should be installed by the application installer alongside -the embedded distribution. Using pip to manage dependencies as for a regular -Python installation is not supported with this distribution, though with some -care it may be possible to include and use pip for automatic updates. In -general, third-party packages should be treated as part of the application -("vendoring") so that the developer can ensure compatibility with newer -versions before providing updates to users. - -The two recommended use cases for this distribution are described below. - -Python Application ------------------- - -An application written in Python does not necessarily require users to be aware -of that fact. The embedded distribution may be used in this case to include a -private version of Python in an install package. Depending on how transparent it -should be (or conversely, how professional it should appear), there are two -options. - -Using a specialized executable as a launcher requires some coding, but provides -the most transparent experience for users. With a customized launcher, there are -no obvious indications that the program is running on Python: icons can be -customized, company and version information can be specified, and file -associations behave properly. In most cases, a custom launcher should simply be -able to call ``Py_Main`` with a hard-coded command line. - -The simpler approach is to provide a batch file or generated shortcut that -directly calls the ``python.exe`` or ``pythonw.exe`` with the required -command-line arguments. In this case, the application will appear to be Python -and not its actual name, and users may have trouble distinguishing it from other -running Python processes or file associations. - -With the latter approach, packages should be installed as directories alongside -the Python executable to ensure they are available on the path. With the -specialized launcher, packages can be located in other locations as there is an -opportunity to specify the search path before launching the application. - -Embedding Python ----------------- - -Applications written in native code often require some form of scripting -language, and the embedded Python distribution can be used for this purpose. In -general, the majority of the application is in native code, and some part will -either invoke ``python.exe`` or directly use ``python3.dll``. For either case, -extracting the embedded distribution to a subdirectory of the application -installation is sufficient to provide a loadable Python interpreter. - -As with the application use, packages can be installed to any location as there -is an opportunity to specify search paths before initializing the interpreter. -Otherwise, there is no fundamental differences between using the embedded -distribution and a regular installation. - Other resources =============== .. seealso:: - `Python Programming On Win32 `_ + `Python Programming On Win32 `_ "Help for Windows Programmers" by Mark Hammond and Andy Robinson, O'Reilly Media, 2000, ISBN 1-56592-621-8 @@ -945,5 +341,3 @@ `A Python for Windows Tutorial `_ by Amanda Birmingham, 2004 - :pep:`397` - Python launcher for Windows - The proposal for the launcher to be included in the Python distribution. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.0.rst --- a/Doc/whatsnew/2.0.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.0.rst Mon Jan 25 17:05:13 2016 +0100 @@ -130,7 +130,7 @@ Read the rest of PEP 1 for the details of the PEP editorial process, style, and format. PEPs are kept in the Python CVS tree on SourceForge, though they're not part of the Python 2.0 distribution, and are also available in HTML form from -https://www.python.org/peps/. As of September 2000, there are 25 PEPS, ranging +http://www.python.org/peps/. As of September 2000, there are 25 PEPS, ranging from PEP 201, "Lockstep Iteration", to PEP 225, "Elementwise/Objectwise Operators". @@ -166,7 +166,7 @@ registering new encodings that are then available throughout a Python program. If an encoding isn't specified, the default encoding is usually 7-bit ASCII, though it can be changed for your Python installation by calling the -``sys.setdefaultencoding(encoding)`` function in a customised version of +:func:`sys.setdefaultencoding(encoding)` function in a customised version of :file:`site.py`. Combining 8-bit and Unicode strings always coerces to Unicode, using the default @@ -203,7 +203,7 @@ The :mod:`codecs` module contains functions to look up existing encodings and register new ones. Unless you want to implement a new encoding, you'll most -often use the ``codecs.lookup(encoding)`` function, which returns a +often use the :func:`codecs.lookup(encoding)` function, which returns a 4-element tuple: ``(encode_func, decode_func, stream_reader, stream_writer)``. * *encode_func* is a function that takes a Unicode string, and returns a 2-tuple @@ -566,7 +566,7 @@ simply be silently swallowed. .. Starting URL: -.. https://www.python.org/pipermail/python-dev/2000-April/004834.html +.. http://www.python.org/pipermail/python-dev/2000-April/004834.html Work has been done on porting Python to 64-bit Windows on the Itanium processor, mostly by Trent Mick of ActiveState. (Confusingly, ``sys.platform`` is still @@ -600,7 +600,7 @@ Changes to Built-in Functions ----------------------------- -A new built-in, ``zip(seq1, seq2, ...)``, has been added. :func:`zip` +A new built-in, :func:`zip(seq1, seq2, ...)`, has been added. :func:`zip` returns a list of tuples where each tuple contains the i-th element from each of the argument sequences. The difference between :func:`zip` and ``map(None, seq1, seq2)`` is that :func:`map` pads the sequences with ``None`` if the @@ -619,7 +619,7 @@ would be ``(2, 0, 1, 'beta', 1)``. *level* is a string such as ``"alpha"``, ``"beta"``, or ``"final"`` for a final release. -Dictionaries have an odd new method, ``setdefault(key, default)``, which +Dictionaries have an odd new method, :meth:`setdefault(key, default)`, which behaves similarly to the existing :meth:`get` method. However, if the key is missing, :meth:`setdefault` both returns the value of *default* as :meth:`get` would do, and also inserts it into the dictionary as the value for *key*. Thus, @@ -1003,7 +1003,7 @@ The XML Special Interest Group has been working on XML-related Python code for a while. Its code distribution, called PyXML, is available from the SIG's Web -pages at https://www.python.org/community/sigs/current/xml-sig. The PyXML distribution also used +pages at http://www.python.org/sigs/xml-sig/. The PyXML distribution also used the package name ``xml``. If you've written programs that used PyXML, you're probably wondering about its compatibility with the 2.0 :mod:`xml` package. @@ -1038,7 +1038,7 @@ is an implementation of the Secure Socket Layer, which encrypts the data being sent over a socket. When compiling Python, you can edit :file:`Modules/Setup` to include SSL support, which adds an additional function to the :mod:`socket` -module: ``socket.ssl(socket, keyfile, certfile)``, which takes a socket +module: :func:`socket.ssl(socket, keyfile, certfile)`, which takes a socket object and returns an SSL socket. The :mod:`httplib` and :mod:`urllib` modules were also changed to support ``https://`` URLs, though no one has implemented FTP or SMTP over SSL. @@ -1174,8 +1174,8 @@ * In the editor window, there is now a line/column bar at the bottom. -* Three new keystroke commands: Check module (:kbd:`Alt-F5`), Import module (:kbd:`F5`) and - Run script (:kbd:`Ctrl-F5`). +* Three new keystroke commands: Check module (Alt-F5), Import module (F5) and + Run script (Ctrl-F5). .. ====================================================================== diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.1.rst --- a/Doc/whatsnew/2.1.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.1.rst Mon Jan 25 17:05:13 2016 +0100 @@ -204,7 +204,7 @@ list, or any other Python object. Alternatively they can raise an exception if the comparison is impossible, inconsistent, or otherwise meaningless. -The built-in ``cmp(A,B)`` function can use the rich comparison machinery, +The built-in :func:`cmp(A,B)` function can use the rich comparison machinery, and now accepts an optional argument specifying which comparison operation to use; this is given as one of the strings ``"<"``, ``"<="``, ``">"``, ``">="``, ``"=="``, or ``"!="``. If called without the optional third argument, @@ -219,7 +219,7 @@ .. seealso:: - :pep:`207` - Rich Comparisons + :pep:`207` - Rich Comparisions Written by Guido van Rossum, heavily based on earlier work by David Ascher, and implemented by Guido van Rossum. @@ -350,7 +350,7 @@ and another being circular references in data structures such as trees. For example, consider a memoizing function that caches the results of another -function ``f(x)`` by storing the function's argument and its result in a +function :func:`f(x)` by storing the function's argument and its result in a dictionary:: _cache = {} @@ -555,14 +555,14 @@ and experiment with them. With the result experience, perhaps it'll be possible to design a really good catalog and then build support for it into Python 2.2. For example, the Distutils :command:`sdist` and :command:`bdist_\*` commands -could support an ``upload`` option that would automatically upload your +could support a :option:`upload` option that would automatically upload your package to a catalog server. You can start creating packages containing :file:`PKG-INFO` even if you're not using Python 2.1, since a new release of the Distutils will be made for users of earlier Python versions. Version 1.0.2 of the Distutils includes the changes described in PEP 241, as well as various bugfixes and enhancements. It will be -available from the Distutils SIG at https://www.python.org/sigs/distutils-sig/. +available from the Distutils SIG at http://www.python.org/sigs/distutils-sig/. .. seealso:: @@ -656,7 +656,7 @@ use :mod:`ftplib` to retrieve files and then don't work from behind a firewall. It's deemed unlikely that this will cause problems for anyone, because Netscape defaults to passive mode and few people complain, but if passive mode is - unsuitable for your application or network setup, call ``set_pasv(0)`` on + unsuitable for your application or network setup, call :meth:`set_pasv(0)` on FTP objects to disable passive mode. * Support for raw socket access has been added to the :mod:`socket` module, @@ -666,7 +666,7 @@ for displaying timing profiles for Python programs, invoked when the module is run as a script. Contributed by Eric S. Raymond. -* A new implementation-dependent function, ``sys._getframe([depth])``, has +* A new implementation-dependent function, :func:`sys._getframe([depth])`, has been added to return a given frame object from the current call stack. :func:`sys._getframe` returns the frame at the top of the call stack; if the optional integer argument *depth* is supplied, the function returns the frame @@ -731,7 +731,7 @@ ... For a fuller discussion of the line I/O changes, see the python-dev summary for - January 1-15, 2001 at https://www.python.org/dev/summary/2001-01-1/. + January 1-15, 2001 at http://www.python.org/dev/summary/2001-01-1/. * A new method, :meth:`popitem`, was added to dictionaries to enable destructively iterating through the contents of a dictionary; this can be faster diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.2.rst --- a/Doc/whatsnew/2.2.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.2.rst Mon Jan 25 17:05:13 2016 +0100 @@ -24,8 +24,8 @@ This article doesn't attempt to provide a complete specification of the new features, but instead provides a convenient overview. For full details, you should refer to the documentation for Python 2.2, such as the `Python Library -Reference `_ and the `Python -Reference Manual `_. If you want to +Reference `_ and the `Python +Reference Manual `_. If you want to understand the complete implementation and design rationale for a change, refer to the PEP for a particular new feature. @@ -173,12 +173,12 @@ * :attr:`__doc__` is the attribute's docstring. -* ``__get__(object)`` is a method that retrieves the attribute value from +* :meth:`__get__(object)` is a method that retrieves the attribute value from *object*. -* ``__set__(object, value)`` sets the attribute on *object* to *value*. +* :meth:`__set__(object, value)` sets the attribute on *object* to *value*. -* ``__delete__(object, value)`` deletes the *value* attribute of *object*. +* :meth:`__delete__(object, value)` deletes the *value* attribute of *object*. For example, when you write ``obj.x``, the steps that Python actually performs are:: @@ -288,7 +288,7 @@ which is the behaviour we're after. This lookup rule is the same as the one followed by Common Lisp. A new built-in function, :func:`super`, provides a way to get at a class's superclasses without having to reimplement Python's -algorithm. The most commonly used form will be ``super(class, obj)``, which +algorithm. The most commonly used form will be :func:`super(class, obj)`, which returns a bound superclass object (not the actual class object). This form will be used in methods to call a method in the superclass; for example, :class:`D`'s :meth:`save` method would look like this:: @@ -301,7 +301,7 @@ ... :func:`super` can also return unbound superclass objects when called as -``super(class)`` or ``super(class1, class2)``, but this probably won't +:func:`super(class)` or :func:`super(class1, class2)`, but this probably won't often be useful. @@ -314,13 +314,13 @@ ``obj.parent`` into a method call such as ``obj.get_parent``. Python 2.2 adds some new ways of controlling attribute access. -First, ``__getattr__(attr_name)`` is still supported by new-style classes, +First, :meth:`__getattr__(attr_name)` is still supported by new-style classes, and nothing about it has changed. As before, it will be called when an attempt is made to access ``obj.foo`` and no attribute named ``foo`` is found in the instance's dictionary. New-style classes also support a new method, -``__getattribute__(attr_name)``. The difference between the two methods is +:meth:`__getattribute__(attr_name)`. The difference between the two methods is that :meth:`__getattribute__` is *always* called whenever any attribute is accessed, while the old :meth:`__getattr__` is only called if ``foo`` isn't found in the instance's dictionary. @@ -395,7 +395,7 @@ of an explanation to start you programming, but many details have been simplified or ignored. Where should you go to get a more complete picture? -https://www.python.org/2.2/descrintro.html is a lengthy tutorial introduction to +http://www.python.org/2.2/descrintro.html is a lengthy tutorial introduction to the descriptor features, written by Guido van Rossum. If my description has whetted your appetite, go read this tutorial next, because it goes into much more detail about the new features while still remaining quite easy to read. @@ -441,8 +441,8 @@ In Python 2.2, iteration can be implemented separately, and :meth:`__getitem__` methods can be limited to classes that really do support random access. The -basic idea of iterators is simple. A new built-in function, ``iter(obj)`` -or ``iter(C, sentinel)``, is used to get an iterator. ``iter(obj)`` returns +basic idea of iterators is simple. A new built-in function, :func:`iter(obj)` +or ``iter(C, sentinel)``, is used to get an iterator. :func:`iter(obj)` returns an iterator for the object *obj*, while ``iter(C, sentinel)`` returns an iterator that will invoke the callable object *C* until it returns *sentinel* to signal that the iterator is done. @@ -450,9 +450,9 @@ Python classes can define an :meth:`__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their -own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` +own iterators. Extension types implemented in C can implement a :attr:`tp_iter` function in order to return an iterator, and extension types that want to behave -as iterators can define a :c:member:`~PyTypeObject.tp_iternext` function. +as iterators can define a :attr:`tp_iternext` function. So, after all this, what do iterators actually do? They have one required method, :meth:`next`, which takes no arguments and returns the next value. When @@ -478,7 +478,7 @@ In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so +sequences that don't implement :meth:`__iter__` or a :attr:`tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: @@ -793,7 +793,7 @@ Another change is simpler to explain. Since their introduction, Unicode strings have supported an :meth:`encode` method to convert the string to a selected -encoding such as UTF-8 or Latin-1. A symmetric ``decode([*encoding*])`` +encoding such as UTF-8 or Latin-1. A symmetric :meth:`decode([*encoding*])` method has been added to 8-bit strings (though not to Unicode strings) in 2.2. :meth:`decode` assumes that the string is in the specified encoding and decodes it, returning whatever is returned by the codec. @@ -1203,7 +1203,7 @@ to an MBCS encoded string, as used by the Microsoft file APIs. As MBCS is explicitly used by the file APIs, Python's choice of ASCII as the default encoding turns out to be an annoyance. On Unix, the locale's character set is - used if ``locale.nl_langinfo(CODESET)`` is available. (Windows support was + used if :func:`locale.nl_langinfo(CODESET)` is available. (Windows support was contributed by Mark Hammond with assistance from Marc-André Lemburg. Unix support was added by Martin von Löwis.) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.3.rst --- a/Doc/whatsnew/2.3.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.3.rst Mon Jan 25 17:05:13 2016 +0100 @@ -366,9 +366,6 @@ .. ====================================================================== -.. index:: - single: universal newlines; What's new - PEP 278: Universal Newline Support ================================== @@ -379,12 +376,12 @@ 10), MacOS uses the carriage return (ASCII character 13), and Windows uses a two-character sequence of a carriage return plus a newline. -Python's file objects can now support end of line conventions other than the -one followed by the platform on which Python is running. Opening a file with -the mode ``'U'`` or ``'rU'`` will open a file for reading in :term:`universal -newlines` mode. All three line ending conventions will be translated to a -``'\n'`` in the strings returned by the various file methods such as -:meth:`read` and :meth:`readline`. +Python's file objects can now support end of line conventions other than the one +followed by the platform on which Python is running. Opening a file with the +mode ``'U'`` or ``'rU'`` will open a file for reading in universal newline mode. +All three line ending conventions will be translated to a ``'\n'`` in the +strings returned by the various file methods such as :meth:`read` and +:meth:`readline`. Universal newline support is also used when importing modules and when executing a file with the :func:`execfile` function. This means that Python modules can @@ -411,7 +408,7 @@ A new built-in function, :func:`enumerate`, will make certain loops a bit clearer. ``enumerate(thing)``, where *thing* is either an iterator or a -sequence, returns an iterator that will return ``(0, thing[0])``, ``(1, +sequence, returns a iterator that will return ``(0, thing[0])``, ``(1, thing[1])``, ``(2, thing[2])``, and so forth. A common idiom to change every element of a list looks like this:: @@ -504,8 +501,8 @@ ZeroDivisionError: integer division or modulo by zero Slightly more advanced programs will use a logger other than the root logger. -The ``getLogger(name)`` function is used to get a particular log, creating -it if it doesn't exist yet. ``getLogger(None)`` returns the root logger. :: +The :func:`getLogger(name)` function is used to get a particular log, creating +it if it doesn't exist yet. :func:`getLogger(None)` returns the root logger. :: log = logging.getLogger('server') ... @@ -657,7 +654,7 @@ Running ``python setup.py register`` will collect the metadata describing a package, such as its name, version, maintainer, description, &c., and send it to a central catalog server. The resulting catalog is available from -https://pypi.python.org/pypi. +http://www.python.org/pypi. To make the catalog a bit more useful, a new optional *classifiers* keyword argument has been added to the Distutils :func:`setup` function. A list of @@ -724,10 +721,10 @@ objects to it. Additional built-in and frozen modules can be imported by an object added to this list. -Importer objects must have a single method, ``find_module(fullname, -path=None)``. *fullname* will be a module or package name, e.g. ``string`` or +Importer objects must have a single method, :meth:`find_module(fullname, +path=None)`. *fullname* will be a module or package name, e.g. ``string`` or ``distutils.core``. :meth:`find_module` must return a loader object that has a -single method, ``load_module(fullname)``, that creates and returns the +single method, :meth:`load_module(fullname)`, that creates and returns the corresponding module object. Pseudo-code for Python's new import logic, therefore, looks something like this @@ -935,7 +932,7 @@ [0, 2, 4] To simplify implementing sequences that support extended slicing, slice objects -now have a method ``indices(length)`` which, given the length of a sequence, +now have a method :meth:`indices(length)` which, given the length of a sequence, returns a ``(start, stop, step)`` tuple that can be passed directly to :func:`range`. :meth:`indices` handles omitted and out-of-bounds indices in a manner consistent with regular slices (and this innocuous phrase hides a welter @@ -984,7 +981,7 @@ * Built-in types now support the extended slicing syntax, as described in section :ref:`section-slices` of this document. -* A new built-in function, ``sum(iterable, start=0)``, adds up the numeric +* A new built-in function, :func:`sum(iterable, start=0)`, adds up the numeric items in the iterable object and returns their sum. :func:`sum` only accepts numbers, meaning that you can't use it to concatenate a bunch of strings. (Contributed by Alex Martelli.) @@ -998,7 +995,7 @@ its index, now takes optional *start* and *stop* arguments to limit the search to only part of the list. -* Dictionaries have a new method, ``pop(key[, *default*])``, that returns +* Dictionaries have a new method, :meth:`pop(key[, *default*])`, that returns the value corresponding to *key* and removes that key/value pair from the dictionary. If the requested key isn't present in the dictionary, *default* is returned if it's specified and :exc:`KeyError` raised if it isn't. :: @@ -1020,7 +1017,7 @@ {} >>> - There's also a new class method, ``dict.fromkeys(iterable, value)``, that + There's also a new class method, :meth:`dict.fromkeys(iterable, value)`, that creates a dictionary with keys taken from the supplied iterator *iterable* and all values set to *value*, defaulting to ``None``. @@ -1082,9 +1079,9 @@ C3 algorithm as described in the paper `"A Monotonic Superclass Linearization for Dylan" `_. To understand the motivation for this change, read Michele Simionato's article - `"Python 2.3 Method Resolution Order" `_, or + `"Python 2.3 Method Resolution Order" `_, or read the thread on python-dev starting with the message at - https://mail.python.org/pipermail/python-dev/2002-October/029035.html. Samuele + http://mail.python.org/pipermail/python-dev/2002-October/029035.html. Samuele Pedroni first pointed out the problem and also implemented the fix by coding the C3 algorithm. @@ -1093,7 +1090,7 @@ 100 bytecodes, speeding up single-threaded applications by reducing the switching overhead. Some multithreaded applications may suffer slower response time, but that's easily fixed by setting the limit back to a lower number using - ``sys.setcheckinterval(N)``. The limit can be retrieved with the new + :func:`sys.setcheckinterval(N)`. The limit can be retrieved with the new :func:`sys.getcheckinterval` function. * One minor but far-reaching change is that the names of extension types defined @@ -1272,10 +1269,10 @@ * Previously the :mod:`doctest` module would only search the docstrings of public methods and functions for test cases, but it now also examines private - ones as well. The :func:`DocTestSuite` function creates a + ones as well. The :func:`DocTestSuite(` function creates a :class:`unittest.TestSuite` object from a set of :mod:`doctest` tests. -* The new ``gc.get_referents(object)`` function returns a list of all the +* The new :func:`gc.get_referents(object)` function returns a list of all the objects referenced by *object*. * The :mod:`getopt` module gained a new function, :func:`gnu_getopt`, that @@ -1330,7 +1327,7 @@ (Contributed by Kevin O'Connor.) * The IDLE integrated development environment has been updated using the code - from the IDLEfork project (http://idlefork.sourceforge.net). The most notable feature is + from the IDLEfork project (http://idlefork.sf.net). The most notable feature is that the code being developed is now executed in a subprocess, meaning that there's no longer any need for manual ``reload()`` operations. IDLE's core code has been incorporated into the standard library as the :mod:`idlelib` package. @@ -1347,8 +1344,8 @@ documentation for details. (Contributed by Raymond Hettinger.) -* Two new functions in the :mod:`math` module, ``degrees(rads)`` and - ``radians(degs)``, convert between radians and degrees. Other functions in +* Two new functions in the :mod:`math` module, :func:`degrees(rads)` and + :func:`radians(degs)`, convert between radians and degrees. Other functions in the :mod:`math` module such as :func:`math.sin` and :func:`math.cos` have always required input values measured in radians. Also, an optional *base* argument was added to :func:`math.log` to make it easier to compute logarithms for bases @@ -1405,7 +1402,7 @@ and therefore faster performance. Setting the parser object's :attr:`buffer_text` attribute to :const:`True` will enable buffering. -* The ``sample(population, k)`` function was added to the :mod:`random` +* The :func:`sample(population, k)` function was added to the :mod:`random` module. *population* is a sequence or :class:`xrange` object containing the elements of a population, and :func:`sample` chooses *k* elements from the population without replacing chosen elements. *k* can be any value up to @@ -1451,7 +1448,7 @@ encryption is not believed to be secure. If you need encryption, use one of the several AES Python modules that are available separately. -* The :mod:`shutil` module gained a ``move(src, dest)`` function that +* The :mod:`shutil` module gained a :func:`move(src, dest)` function that recursively moves a file or directory to a new location. * Support for more advanced POSIX signal handling was added to the :mod:`signal` @@ -1459,7 +1456,7 @@ platforms. * The :mod:`socket` module now supports timeouts. You can call the - ``settimeout(t)`` method on a socket object to set a timeout of *t* seconds. + :meth:`settimeout(t)` method on a socket object to set a timeout of *t* seconds. Subsequent socket operations that take longer than *t* seconds to complete will abort and raise a :exc:`socket.timeout` exception. @@ -1480,9 +1477,9 @@ :program:`tar`\ -format archive files. (Contributed by Lars Gustäbel.) * The new :mod:`textwrap` module contains functions for wrapping strings - containing paragraphs of text. The ``wrap(text, width)`` function takes a + containing paragraphs of text. The :func:`wrap(text, width)` function takes a string and returns a list containing the text split into lines of no more than - the chosen width. The ``fill(text, width)`` function returns a single + the chosen width. The :func:`fill(text, width)` function returns a single string, reformatted to fit into lines no longer than the chosen width. (As you can guess, :func:`fill` is built on top of :func:`wrap`. For example:: @@ -1564,7 +1561,7 @@ to the correct thread, and waiting for the results. Other interfaces can't be handled automatically but :mod:`Tkinter` will now raise an exception on such an access so that you can at least find out about the problem. See - https://mail.python.org/pipermail/python-dev/2002-December/031107.html for a more + http://mail.python.org/pipermail/python-dev/2002-December/031107.html for a more detailed explanation of this change. (Implemented by Martin von Löwis.) * Calling Tcl methods through :mod:`_tkinter` no longer returns only strings. @@ -1858,7 +1855,7 @@ .. seealso:: - https://svn.python.org/view/python/trunk/Objects/obmalloc.c + http://svn.python.org/view/python/trunk/Objects/obmalloc.c For the full details of the pymalloc implementation, see the comments at the top of the file :file:`Objects/obmalloc.c` in the Python source code. The above link points to the file within the python.org SVN browser. @@ -1903,7 +1900,7 @@ short int`, ``I`` for :c:type:`unsigned int`, and ``K`` for :c:type:`unsigned long long`. -* A new function, ``PyObject_DelItemString(mapping, char *key)`` was added +* A new function, :c:func:`PyObject_DelItemString(mapping, char \*key)` was added as shorthand for ``PyObject_DelItem(mapping, PyString_New(key))``. * File objects now manage their internal string buffer differently, increasing diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.4.rst --- a/Doc/whatsnew/2.4.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.4.rst Mon Jan 25 17:05:13 2016 +0100 @@ -37,7 +37,7 @@ Python 2.3 introduced the :mod:`sets` module. C implementations of set data types have now been added to the Python core as two new built-in types, -``set(iterable)`` and ``frozenset(iterable)``. They provide high speed +:func:`set(iterable)` and :func:`frozenset(iterable)`. They provide high speed operations for membership testing, for eliminating duplicates from sequences, and for mathematical operations like unions, intersections, differences, and symmetric differences. :: @@ -337,7 +337,7 @@ wrote patches implementing function decorators, but the one that was actually checked in was patch #979728, written by Mark Russell. - https://www.python.org/moin/PythonDecoratorLibrary + http://www.python.org/moin/PythonDecoratorLibrary This Wiki page contains several examples of decorators. .. ====================================================================== @@ -346,7 +346,7 @@ PEP 322: Reverse Iteration ========================== -A new built-in function, ``reversed(seq)``, takes a sequence and returns an +A new built-in function, :func:`reversed(seq)`, takes a sequence and returns an iterator that loops over the elements of the sequence in reverse order. :: >>> for i in reversed(xrange(1,4)): @@ -384,7 +384,7 @@ The standard library provides a number of ways to execute a subprocess, offering different features and different levels of complexity. -``os.system(command)`` is easy to use, but slow (it runs a shell process +:func:`os.system(command)` is easy to use, but slow (it runs a shell process which executes the command) and dangerous (you have to be careful about escaping the shell's metacharacters). The :mod:`popen2` module offers classes that can capture standard output and standard error from the subprocess, but the naming @@ -411,9 +411,6 @@ you can use the constant ``subprocess.PIPE`` to create a pipe between the subprocess and the parent. -.. index:: - single: universal newlines; What's new - The constructor has a number of handy options: * *close_fds* requests that all file descriptors be closed before running the @@ -427,12 +424,12 @@ * *preexec_fn* is a function that gets called before the child is started. * *universal_newlines* opens the child's input and output using Python's - :term:`universal newlines` feature. + universal newline feature. Once you've created the :class:`Popen` instance, you can call its :meth:`wait` method to pause until the subprocess has exited, :meth:`poll` to check if it's -exited without pausing, or ``communicate(data)`` to send the string *data* -to the subprocess's standard input. ``communicate(data)`` then reads any +exited without pausing, or :meth:`communicate(data)` to send the string *data* +to the subprocess's standard input. :meth:`communicate(data)` then reads any data that the subprocess has sent to its standard output or standard error, returning a tuple ``(stdout_data, stderr_data)``. @@ -749,10 +746,10 @@ The solution described in the PEP is to add three new functions to the Python API that perform ASCII-only conversions, ignoring the locale setting: -* ``PyOS_ascii_strtod(str, ptr)`` and ``PyOS_ascii_atof(str, ptr)`` +* :c:func:`PyOS_ascii_strtod(str, ptr)` and :c:func:`PyOS_ascii_atof(str, ptr)` both convert a string to a C :c:type:`double`. -* ``PyOS_ascii_formatd(buffer, buf_len, format, d)`` converts a +* :c:func:`PyOS_ascii_formatd(buffer, buf_len, format, d)` converts a :c:type:`double` to an ASCII string. The code for these functions came from the GLib library @@ -778,7 +775,7 @@ * Decorators for functions and methods were added (:pep:`318`). * Built-in :func:`set` and :func:`frozenset` types were added (:pep:`218`). - Other new built-ins include the ``reversed(seq)`` function (:pep:`322`). + Other new built-ins include the :func:`reversed(seq)` function (:pep:`322`). * Generator expressions were added (:pep:`289`). @@ -846,7 +843,7 @@ ['A', 'b', 'c', 'D'] Finally, the *reverse* parameter takes a Boolean value. If the value is true, - the list will be sorted into reverse order. Instead of ``L.sort(); + the list will be sorted into reverse order. Instead of ``L.sort() ; L.reverse()``, you can now write ``L.sort(reverse=True)``. The results of sorting are now guaranteed to be stable. This means that two @@ -857,7 +854,7 @@ (All changes to :meth:`sort` contributed by Raymond Hettinger.) -* There is a new built-in function ``sorted(iterable)`` that works like the +* There is a new built-in function :func:`sorted(iterable)` that works like the in-place :meth:`list.sort` method but can be used in expressions. The differences are: @@ -898,8 +895,8 @@ For example, you can now run the Python profiler with ``python -m profile``. (Contributed by Nick Coghlan.) -* The ``eval(expr, globals, locals)`` and ``execfile(filename, globals, - locals)`` functions and the ``exec`` statement now accept any mapping type +* The :func:`eval(expr, globals, locals)` and :func:`execfile(filename, globals, + locals)` functions and the ``exec`` statement now accept any mapping type for the *locals* parameter. Previously this had to be a regular Python dictionary. (Contributed by Raymond Hettinger.) @@ -1090,7 +1087,7 @@ Yves Dionne) and new :meth:`deleteacl` and :meth:`myrights` methods (contributed by Arnaud Mazin). -* The :mod:`itertools` module gained a ``groupby(iterable[, *func*])`` +* The :mod:`itertools` module gained a :func:`groupby(iterable[, *func*])` function. *iterable* is something that can be iterated over to return a stream of elements, and the optional *func* parameter is a function that takes an element and returns a key value; if omitted, the key is simply the element @@ -1139,7 +1136,7 @@ (Contributed by Hye-Shik Chang.) -* :mod:`itertools` also gained a function named ``tee(iterator, N)`` that +* :mod:`itertools` also gained a function named :func:`tee(iterator, N)` that returns *N* independent iterators that replicate *iterator*. If *N* is omitted, the default is 2. :: @@ -1177,7 +1174,7 @@ level=0, # Log all messages format='%(levelname):%(process):%(thread):%(message)') - Other additions to the :mod:`logging` package include a ``log(level, msg)`` + Other additions to the :mod:`logging` package include a :meth:`log(level, msg)` convenience method, as well as a :class:`TimedRotatingFileHandler` class that rotates its log files at a timed interval. The module already had :class:`RotatingFileHandler`, which rotated logs once the file exceeded a @@ -1196,7 +1193,7 @@ group or for a range of groups. (Contributed by Jürgen A. Erhard.) * Two new functions were added to the :mod:`operator` module, - ``attrgetter(attr)`` and ``itemgetter(index)``. Both functions return + :func:`attrgetter(attr)` and :func:`itemgetter(index)`. Both functions return callables that take a single argument and return the corresponding attribute or item; these callables make excellent data extractors when used with :func:`map` or :func:`sorted`. For example:: @@ -1223,14 +1220,14 @@ replacement for :func:`rfc822.formatdate`. You may want to write new e-mail processing code with this in mind. (Change implemented by Anthony Baxter.) -* A new ``urandom(n)`` function was added to the :mod:`os` module, returning +* A new :func:`urandom(n)` function was added to the :mod:`os` module, returning a string containing *n* bytes of random data. This function provides access to platform-specific sources of randomness such as :file:`/dev/urandom` on Linux or the Windows CryptoAPI. (Contributed by Trevor Perrin.) -* Another new function: ``os.path.lexists(path)`` returns true if the file +* Another new function: :func:`os.path.lexists(path)` returns true if the file specified by *path* exists, whether or not it's a symbolic link. This differs - from the existing ``os.path.exists(path)`` function, which returns false if + from the existing :func:`os.path.exists(path)` function, which returns false if *path* is a symlink that points to a destination that doesn't exist. (Contributed by Beni Cherniavsky.) @@ -1243,7 +1240,7 @@ * The :mod:`profile` module can now profile C extension functions. (Contributed by Nick Bastin.) -* The :mod:`random` module has a new method called ``getrandbits(N)`` that +* The :mod:`random` module has a new method called :meth:`getrandbits(N)` that returns a long integer *N* bits in length. The existing :meth:`randrange` method now uses :meth:`getrandbits` where appropriate, making generation of arbitrarily large random numbers more efficient. (Contributed by Raymond @@ -1272,7 +1269,7 @@ this, but 2.4 will raise a :exc:`RuntimeError` exception. * Two new functions were added to the :mod:`socket` module. :func:`socketpair` - returns a pair of connected sockets and ``getservbyport(port)`` looks up the + returns a pair of connected sockets and :func:`getservbyport(port)` looks up the service name for a given port number. (Contributed by Dave Cole and Barry Warsaw.) @@ -1282,7 +1279,7 @@ interface, accessed only by :mod:`atexit`. * The :mod:`tarfile` module now generates GNU-format tar files by default. - (Contributed by Lars Gustäbel.) + (Contributed by Lars Gustaebel.) * The :mod:`threading` module now has an elegantly simple way to support thread-local data. The module contains a :class:`local` class whose attribute @@ -1454,11 +1451,11 @@ * Another new macro, :c:macro:`Py_CLEAR(obj)`, decreases the reference count of *obj* and sets *obj* to the null pointer. (Contributed by Jim Fulton.) -* A new function, ``PyTuple_Pack(N, obj1, obj2, ..., objN)``, constructs +* A new function, :c:func:`PyTuple_Pack(N, obj1, obj2, ..., objN)`, constructs tuples from a variable length argument list of Python objects. (Contributed by Raymond Hettinger.) -* A new function, ``PyDict_Contains(d, k)``, implements fast dictionary +* A new function, :c:func:`PyDict_Contains(d, k)`, implements fast dictionary lookups without masking exceptions raised during the look-up process. (Contributed by Raymond Hettinger.) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.5.rst --- a/Doc/whatsnew/2.5.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.5.rst Mon Jan 25 17:05:13 2016 +0100 @@ -39,7 +39,7 @@ This article doesn't try to be a complete specification of the new features; instead changes are briefly introduced using helpful examples. For full details, you should always refer to the documentation for Python 2.5 at -https://docs.python.org. If you want to understand the complete implementation +http://docs.python.org. If you want to understand the complete implementation and design rationale, refer to the PEP for a particular new feature. Comments, suggestions, and error reports for this document are welcome; please @@ -171,7 +171,7 @@ popup_menu.append( ("Open", open_func, 1) ) Another function in the :mod:`functools` module is the -``update_wrapper(wrapper, wrapped)`` function that helps you write well- +:func:`update_wrapper(wrapper, wrapped)` function that helps you write well- behaved decorators. :func:`update_wrapper` copies the name, module, and docstring attribute to a wrapper function so that tracebacks inside the wrapped function are easier to understand. For example, you might write:: @@ -229,7 +229,7 @@ ) Another new enhancement to the Python package index at -https://pypi.python.org is storing source and binary archives for a +http://cheeseshop.python.org is storing source and binary archives for a package. The new :command:`upload` Distutils command will upload a package to the repository. @@ -286,7 +286,7 @@ :mod:`pkg.string` and look for the standard module; generally you had to look at the contents of ``sys.modules``, which is slightly unclean. Holger Krekel's :mod:`py.std` package provides a tidier way to perform imports from the standard -library, ``import py; py.std.string.join()``, but that package isn't available +library, ``import py ; py.std.string.join()``, but that package isn't available on all Python installations. Reading code which relies on relative imports is also less clear, because a @@ -454,7 +454,7 @@ ``val = yield i`` but have to use parentheses when there's an operation, as in ``val = (yield i) + 12``.) -Values are sent into a generator by calling its ``send(value)`` method. The +Values are sent into a generator by calling its :meth:`send(value)` method. The generator's code is then resumed and the :keyword:`yield` expression returns the specified *value*. If the regular :meth:`next` method is called, the :keyword:`yield` returns :const:`None`. @@ -496,7 +496,7 @@ In addition to :meth:`send`, there are two other new methods on generators: -* ``throw(type, value=None, traceback=None)`` is used to raise an exception +* :meth:`throw(type, value=None, traceback=None)` is used to raise an exception inside the generator; the exception is raised by the :keyword:`yield` expression where the generator's execution is paused. @@ -660,7 +660,7 @@ * The code in *BLOCK* is executed. -* If *BLOCK* raises an exception, the ``__exit__(type, value, traceback)`` +* If *BLOCK* raises an exception, the :meth:`__exit__(type, value, traceback)` is called with the exception details, the same values returned by :func:`sys.exc_info`. The method's return value controls whether the exception is re-raised: any false value re-raises the exception, and ``True`` will result @@ -773,7 +773,7 @@ with db_transaction(db) as cursor: ... -The :mod:`contextlib` module also has a ``nested(mgr1, mgr2, ...)`` function +The :mod:`contextlib` module also has a :func:`nested(mgr1, mgr2, ...)` function that combines a number of context managers so you don't need to write nested ':keyword:`with`' statements. In this example, the single ':keyword:`with`' statement both starts a database transaction and acquires a thread lock:: @@ -782,7 +782,7 @@ with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the ``closing(object)`` function returns *object* so that it can be +Finally, the :func:`closing(object)` function returns *object* so that it can be bound to a variable, and calls ``object.close`` at the end of the block. :: import urllib, sys @@ -827,7 +827,7 @@ This rearrangement was done because people often want to catch all exceptions that indicate program errors. :exc:`KeyboardInterrupt` and :exc:`SystemExit` aren't errors, though, and usually represent an explicit action such as the user -hitting :kbd:`Control-C` or code calling :func:`sys.exit`. A bare ``except:`` will +hitting Control-C or code calling :func:`sys.exit`. A bare ``except:`` will catch all exceptions, so you commonly need to list :exc:`KeyboardInterrupt` and :exc:`SystemExit` in order to re-raise them. The usual pattern is:: @@ -955,7 +955,7 @@ A corresponding :attr:`nb_index` slot was added to the C-level :c:type:`PyNumberMethods` structure to let C extensions implement this protocol. -``PyNumber_Index(obj)`` can be used in extension code to call the +:c:func:`PyNumber_Index(obj)` can be used in extension code to call the :meth:`__index__` function and retrieve its result. @@ -976,7 +976,7 @@ * The :class:`dict` type has a new hook for letting subclasses provide a default value when a key isn't contained in the dictionary. When a key isn't found, the - dictionary's ``__missing__(key)`` method will be called. This hook is used + dictionary's :meth:`__missing__(key)` method will be called. This hook is used to implement the new :class:`defaultdict` class in the :mod:`collections` module. The following example defines a dictionary that returns zero for any missing key:: @@ -989,16 +989,16 @@ print d[1], d[2] # Prints 1, 2 print d[3], d[4] # Prints 0, 0 -* Both 8-bit and Unicode strings have new ``partition(sep)`` and - ``rpartition(sep)`` methods that simplify a common use case. - - The ``find(S)`` method is often used to get an index which is then used to +* Both 8-bit and Unicode strings have new :meth:`partition(sep)` and + :meth:`rpartition(sep)` methods that simplify a common use case. + + The :meth:`find(S)` method is often used to get an index which is then used to slice the string and obtain the pieces that are before and after the separator. - ``partition(sep)`` condenses this pattern into a single method call that + :meth:`partition(sep)` condenses this pattern into a single method call that returns a 3-tuple containing the substring before the separator, the separator itself, and the substring after the separator. If the separator isn't found, the first element of the tuple is the entire string and the other two elements - are empty. ``rpartition(sep)`` also returns a 3-tuple but starts searching + are empty. :meth:`rpartition(sep)` also returns a 3-tuple but starts searching from the end of the string; the ``r`` stands for 'reverse'. Some examples:: @@ -1157,7 +1157,7 @@ .. Patch 1313939, 1359618 -* The ``long(str, base)`` function is now faster on long digit strings +* The :func:`long(str, base)` function is now faster on long digit strings because fewer intermediate results are calculated. The peak is for strings of around 800--1000 digits where the function is 6 times faster. (Contributed by Alan McIntyre and committed at the NeedForSpeed sprint.) @@ -1268,7 +1268,7 @@ (Contributed by Guido van Rossum.) * The :class:`deque` double-ended queue type supplied by the :mod:`collections` - module now has a ``remove(value)`` method that removes the first occurrence + module now has a :meth:`remove(value)` method that removes the first occurrence of *value* in the queue, raising :exc:`ValueError` if the value isn't found. (Contributed by Raymond Hettinger.) @@ -1291,7 +1291,7 @@ * The :mod:`csv` module, which parses files in comma-separated value format, received several enhancements and a number of bugfixes. You can now set the maximum size in bytes of a field by calling the - ``csv.field_size_limit(new_limit)`` function; omitting the *new_limit* + :meth:`csv.field_size_limit(new_limit)` function; omitting the *new_limit* argument will return the currently-set limit. The :class:`reader` class now has a :attr:`line_num` attribute that counts the number of physical lines read from the source; records can span multiple physical lines, so :attr:`line_num` is not @@ -1308,7 +1308,7 @@ (Contributed by Skip Montanaro and Andrew McNamara.) * The :class:`datetime` class in the :mod:`datetime` module now has a - ``strptime(string, format)`` method for parsing date strings, contributed + :meth:`strptime(string, format)` method for parsing date strings, contributed by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: @@ -1338,17 +1338,13 @@ .. XXX need to provide some more detail here - .. index:: - single: universal newlines; What's new - * The :mod:`fileinput` module was made more flexible. Unicode filenames are now supported, and a *mode* parameter that defaults to ``"r"`` was added to the - :func:`input` function to allow opening files in binary or :term:`universal - newlines` mode. Another new parameter, *openhook*, lets you use a function - other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`fileno` returns - the file descriptor for the currently opened file. (Contributed by Georg - Brandl.) + :func:`input` function to allow opening files in binary or universal-newline + mode. Another new parameter, *openhook*, lets you use a function other than + :func:`open` to open the input files. Once you're iterating over the set of + files, the :class:`FileInput` object's new :meth:`fileno` returns the file + descriptor for the currently opened file. (Contributed by Georg Brandl.) * In the :mod:`gc` module, the new :func:`get_count` function returns a 3-tuple containing the current collection counts for the three GC generations. This is @@ -1403,7 +1399,7 @@ * The :mod:`mailbox` module underwent a massive rewrite to add the capability to modify mailboxes in addition to reading them. A new set of classes that include :class:`mbox`, :class:`MH`, and :class:`Maildir` are used to read mailboxes, and - have an ``add(message)`` method to add messages, ``remove(key)`` to + have an :meth:`add(message)` method to add messages, :meth:`remove(key)` to remove messages, and :meth:`lock`/:meth:`unlock` to lock/unlock the mailbox. The following example converts a maildir-format mailbox into an mbox-format one:: @@ -1458,7 +1454,7 @@ :func:`wait4` return additional information. :func:`wait3` doesn't take a process ID as input, so it waits for any child process to exit and returns a 3-tuple of *process-id*, *exit-status*, *resource-usage* as returned from the - :func:`resource.getrusage` function. ``wait4(pid)`` does take a process ID. + :func:`resource.getrusage` function. :func:`wait4(pid)` does take a process ID. (Contributed by Chad J. Schroeder.) On FreeBSD, the :func:`os.stat` function now returns times with nanosecond @@ -1532,8 +1528,8 @@ In Python code, netlink addresses are represented as a tuple of 2 integers, ``(pid, group_mask)``. - Two new methods on socket objects, ``recv_into(buffer)`` and - ``recvfrom_into(buffer)``, store the received data in an object that + Two new methods on socket objects, :meth:`recv_into(buffer)` and + :meth:`recvfrom_into(buffer)`, store the received data in an object that supports the buffer protocol instead of returning the data as a string. This means you can put the data directly into an array or a memory-mapped file. @@ -1557,8 +1553,8 @@ year, number, name = s.unpack(data) You can also pack and unpack data to and from buffer objects directly using the - ``pack_into(buffer, offset, v1, v2, ...)`` and ``unpack_from(buffer, - offset)`` methods. This lets you store data directly into an array or a memory- + :meth:`pack_into(buffer, offset, v1, v2, ...)` and :meth:`unpack_from(buffer, + offset)` methods. This lets you store data directly into an array or a memory- mapped file. (:class:`Struct` objects were implemented by Bob Ippolito at the NeedForSpeed @@ -1592,7 +1588,7 @@ .. patch 918101 * The :mod:`threading` module now lets you set the stack size used when new - threads are created. The ``stack_size([*size*])`` function returns the + threads are created. The :func:`stack_size([*size*])` function returns the currently configured stack size, and supplying the optional *size* parameter sets a new value. Not all platforms support changing the stack size, but Windows, POSIX threading, and OS/2 all do. (Contributed by Andrew MacIntyre.) @@ -1911,7 +1907,7 @@ h = hashlib.new('md5') # Provide algorithm as a string Once a hash object has been created, its methods are the same as before: -``update(string)`` hashes the specified string into the current digest +:meth:`update(string)` hashes the specified string into the current digest state, :meth:`digest` and :meth:`hexdigest` return the digest value as a binary string or a string of hex digits, and :meth:`copy` returns a new hashing object with the same digest state. @@ -2130,7 +2126,7 @@ such as PyCon. .. List of names taken from Jeremy's python-dev post at - .. https://mail.python.org/pipermail/python-dev/2005-October/057500.html + .. http://mail.python.org/pipermail/python-dev/2005-October/057500.html * Evan Jones's patch to obmalloc, first described in a talk at PyCon DC 2005, was applied. Python 2.4 allocated small objects in 256K-sized arenas, but never @@ -2168,20 +2164,20 @@ * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:func:`Py_LOCAL(type)` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:func:`Py_LOCAL_INLINE(type)` does the same thing and also requests the function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at the NeedForSpeed sprint.) -* ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base +* :c:func:`PyErr_NewException(name, base, dict)` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) * The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in - favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you + favour of :c:func:`PyErr_WarnEx(category, message, stacklevel)` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, and so forth. (Added by Neal Norwitz.) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.6.rst --- a/Doc/whatsnew/2.6.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.6.rst Mon Jan 25 17:05:13 2016 +0100 @@ -7,6 +7,8 @@ .. XXX add trademark info for Apple, Microsoft, SourceForge. :Author: A.M. Kuchling (amk at amk.ca) +:Release: |release| +:Date: |today| .. $Id$ Rules for maintenance: @@ -164,7 +166,7 @@ to administer it and a server to host it. After posting a call for volunteers, a new Roundup installation was -set up at https://bugs.python.org. One installation of Roundup can +set up at http://bugs.python.org. One installation of Roundup can host multiple trackers, and this server now also hosts issue trackers for Jython and for the Python web site. It will surely find other uses in the future. Where possible, @@ -173,7 +175,7 @@ Hosting of the Python bug tracker is kindly provided by `Upfront Systems `__ -of Stellenbosch, South Africa. Martin von Löwis put a +of Stellenbosch, South Africa. Martin von Loewis put a lot of effort into importing existing bugs and patches from SourceForge; his scripts for this import operation are at http://svn.python.org/view/tracker/importer/ and may be useful to @@ -181,7 +183,7 @@ .. seealso:: - https://bugs.python.org + http://bugs.python.org The Python bug tracker. http://bugs.jython.org: @@ -191,7 +193,7 @@ Roundup downloads and documentation. http://svn.python.org/view/tracker/importer/ - Martin von Löwis's conversion scripts. + Martin von Loewis's conversion scripts. New Documentation Format: reStructuredText Using Sphinx ----------------------------------------------------------- @@ -227,18 +229,18 @@ Sphinx is a standalone package that can be used for writing, and almost two dozen other projects -(`listed on the Sphinx web site `__) +(`listed on the Sphinx web site `__) have adopted Sphinx as their documentation tool. .. seealso:: - `Documenting Python `__ + `Documenting Python `__ Describes how to write for Python's documentation. - `Sphinx `__ + `Sphinx `__ Documentation and code for the Sphinx toolchain. - `Docutils `__ + `Docutils `__ The underlying reStructuredText parser and toolset. @@ -1069,12 +1071,9 @@ The :class:`BytesIO` class supports reading, writing, and seeking over an in-memory buffer. - .. index:: - single: universal newlines; What's new - * :class:`TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting universal newlines. :class:`TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. @@ -1101,7 +1100,7 @@ :pep:`3116` - New I/O PEP written by Daniel Stutzbach, Mike Verdone, and Guido van Rossum. Code by Guido van Rossum, Georg Brandl, Walter Doerwald, - Jeremy Hylton, Martin von Löwis, Tony Lownds, and others. + Jeremy Hylton, Martin von Loewis, Tony Lownds, and others. .. ====================================================================== @@ -1775,7 +1774,7 @@ ``latin-1``; the optional *errorhandler* part specifies what to do with characters that can't be handled by the encoding, and should be one of "error", "ignore", or "replace". (Contributed -by Martin von Löwis.) +by Martin von Loewis.) .. ====================================================================== @@ -1793,7 +1792,7 @@ were applied. (Maintained by Josiah Carlson; see :issue:`1736190` for one patch.) -* The :mod:`bsddb` module also has a new maintainer, Jesús Cea Avión, and the package +* The :mod:`bsddb` module also has a new maintainer, Jesús Cea Avion, and the package is now available as a standalone package. The web page for the package is `www.jcea.es/programacion/pybsddb.htm `__. @@ -1891,7 +1890,7 @@ >>> dq=deque(maxlen=3) >>> dq deque([], maxlen=3) - >>> dq.append(1); dq.append(2); dq.append(3) + >>> dq.append(1) ; dq.append(2) ; dq.append(3) >>> dq deque([1, 2, 3], maxlen=3) >>> dq.append(4) @@ -2363,7 +2362,7 @@ negotiation itself. (Patch contributed by Bill Fenner; :issue:`829951`.) -* The :mod:`socket` module now supports TIPC (http://tipc.sourceforge.net/), +* The :mod:`socket` module now supports TIPC (http://tipc.sf.net), a high-performance non-IP-based protocol designed for use in clustered environments. TIPC addresses are 4- or 5-tuples. (Contributed by Alberto Bertogli; :issue:`1646`.) @@ -2385,7 +2384,7 @@ (Contributed by Pedro Werneck and Jeffrey Yasskin; :issue:`742598`, :issue:`1193577`.) -* The :mod:`sqlite3` module, maintained by Gerhard Häring, +* The :mod:`sqlite3` module, maintained by Gerhard Haering, has been updated from version 2.3.2 in Python 2.5 to version 2.4.1. @@ -2598,7 +2597,7 @@ * The Unicode database provided by the :mod:`unicodedata` module has been updated to version 5.1.0. (Updated by - Martin von Löwis; :issue:`3811`.) + Martin von Loewis; :issue:`3811`.) * The :mod:`warnings` module's :func:`formatwarning` and :func:`showwarning` gained an optional *line* argument that can be used to supply the @@ -2783,12 +2782,12 @@ types. The following example encodes and decodes a dictionary:: >>> import json - >>> data = {"spam": "foo", "parrot": 42} + >>> data = {"spam" : "foo", "parrot" : 42} >>> in_json = json.dumps(data) # Encode the data >>> in_json '{"parrot": 42, "spam": "foo"}' >>> json.loads(in_json) # Decode into a Python object - {"spam": "foo", "parrot": 42} + {"spam" : "foo", "parrot" : 42} It's also possible to write your own decoders and encoders to support more types. Pretty-printing of the JSON strings is also supported. @@ -3105,7 +3104,7 @@ :file:`PCbuild` directory supports cross compilation for X64, debug builds and Profile Guided Optimization (PGO). PGO builds are roughly 10% faster than normal builds. (Contributed by Christian Heimes - with help from Amaury Forgeot d'Arc and Martin von Löwis.) + with help from Amaury Forgeot d'Arc and Martin von Loewis.) * The :mod:`msvcrt` module now supports both the normal and wide char variants of the console I/O diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/2.7.rst --- a/Doc/whatsnew/2.7.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/2.7.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,10 +3,13 @@ **************************** :Author: A.M. Kuchling (amk at amk.ca) +:Release: |release| +:Date: |today| .. hyperlink all the methods & functions. .. T_STRING_INPLACE not described in main docs +.. "Format String Syntax" in string.rst could use many more examples. .. $Id$ Rules for maintenance: @@ -49,16 +52,17 @@ This saves the maintainer some effort going through the SVN logs when researching a change. -This article explains the new features in Python 2.7. Python 2.7 was released -on July 3, 2010. +This article explains the new features in Python 2.7. The final +release of 2.7 is currently scheduled for July 2010; the detailed +schedule is described in :pep:`373`. Numeric handling has been improved in many ways, for both -floating-point numbers and for the :class:`~decimal.Decimal` class. -There are some useful additions to the standard library, such as a -greatly enhanced :mod:`unittest` module, the :mod:`argparse` module -for parsing command-line options, convenient :class:`~collections.OrderedDict` -and :class:`~collections.Counter` classes in the :mod:`collections` module, -and many other improvements. +floating-point numbers and for the :class:`Decimal` class. There are +some useful additions to the standard library, such as a greatly +enhanced :mod:`unittest` module, the :mod:`argparse` module for +parsing command-line options, convenient ordered-dictionary and +:class:`Counter` classes in the :mod:`collections` module, and many +other improvements. Python 2.7 is planned to be the last of the 2.x releases, so we worked on making it a good release for the long term. To help with porting @@ -68,9 +72,9 @@ This article doesn't attempt to provide a complete specification of the new features, but instead provides a convenient overview. For full details, you should refer to the documentation for Python 2.7 at -https://docs.python.org. If you want to understand the rationale for +http://docs.python.org. If you want to understand the rationale for the design and implementation, refer to the PEP for a particular new -feature or the issue on https://bugs.python.org in which a change was +feature or the issue on http://bugs.python.org in which a change was discussed. Whenever possible, "What's New in Python" links to the bug/patch item for each change. @@ -79,91 +83,45 @@ The Future for Python 2.x ========================= -Python 2.7 is the last major release in the 2.x series, as the Python -maintainers have shifted the focus of their new feature development efforts -to the Python 3.x series. This means that while Python 2 continues to -receive bug fixes, and to be updated to build correctly on new hardware and -versions of supported operated systems, there will be no new full feature -releases for the language or standard library. - -However, while there is a large common subset between Python 2.7 and Python -3, and many of the changes involved in migrating to that common subset, or -directly to Python 3, can be safely automated, some other changes (notably -those associated with Unicode handling) may require careful consideration, -and preferably robust automated regression test suites, to migrate -effectively. - -This means that Python 2.7 will remain in place for a long time, providing a -stable and supported base platform for production systems that have not yet -been ported to Python 3. The full expected lifecycle of the Python 2.7 -series is detailed in :pep:`373`. - -Some key consequences of the long-term significance of 2.7 are: - -* As noted above, the 2.7 release has a much longer period of maintenance - when compared to earlier 2.x versions. Python 2.7 is currently expected to - remain supported by the core development team (receiving security updates - and other bug fixes) until at least 2020 (10 years after its initial - release, compared to the more typical support period of 18-24 months). - -* As the Python 2.7 standard library ages, making effective use of the - Python Package Index (either directly or via a redistributor) becomes - more important for Python 2 users. In addition to a wide variety of third - party packages for various tasks, the available packages include backports - of new modules and features from the Python 3 standard library that are - compatible with Python 2, as well as various tools and libraries that can - make it easier to migrate to Python 3. The `Python Packaging User Guide - `__ provides guidance on downloading and - installing software from the Python Package Index. - -* While the preferred approach to enhancing Python 2 is now the publication - of new packages on the Python Package Index, this approach doesn't - necessarily work in all cases, especially those related to network - security. In exceptional cases that cannot be handled adequately by - publishing new or updated packages on PyPI, the Python Enhancement - Proposal process may be used to make the case for adding new features - directly to the Python 2 standard library. Any such additions, and the - maintenance releases where they were added, will be noted in the - :ref:`py27-maintenance-enhancements` section below. - -For projects wishing to migrate from Python 2 to Python 3, or for library -and framework developers wishing to support users on both Python 2 and -Python 3, there are a variety of tools and guides available to help decide -on a suitable approach and manage some of the technical details involved. -The recommended starting point is the :ref:`pyporting-howto` HOWTO guide. - - -Changes to the Handling of Deprecation Warnings -=============================================== - -For Python 2.7, a policy decision was made to silence warnings only of -interest to developers by default. :exc:`DeprecationWarning` and its -descendants are now ignored unless otherwise requested, preventing -users from seeing warnings triggered by an application. This change -was also made in the branch that became Python 3.2. (Discussed -on stdlib-sig and carried out in :issue:`7319`.) - -In previous releases, :exc:`DeprecationWarning` messages were -enabled by default, providing Python developers with a clear -indication of where their code may break in a future major version -of Python. - -However, there are increasingly many users of Python-based -applications who are not directly involved in the development of -those applications. :exc:`DeprecationWarning` messages are -irrelevant to such users, making them worry about an application -that's actually working correctly and burdening application developers -with responding to these concerns. - -You can re-enable display of :exc:`DeprecationWarning` messages by -running Python with the :option:`-Wdefault <-W>` (short form: -:option:`-Wd <-W>`) switch, or by setting the :envvar:`PYTHONWARNINGS` -environment variable to ``"default"`` (or ``"d"``) before running -Python. Python code can also re-enable them -by calling ``warnings.simplefilter('default')``. - -The ``unittest`` module also automatically reenables deprecation warnings -when running tests. +Python 2.7 is intended to be the last major release in the 2.x series. +The Python maintainers are planning to focus their future efforts on +the Python 3.x series. + +This means that 2.7 will remain in place for a long time, running +production systems that have not been ported to Python 3.x. +Two consequences of the long-term significance of 2.7 are: + +* It's very likely the 2.7 release will have a longer period of + maintenance compared to earlier 2.x versions. Python 2.7 will + continue to be maintained while the transition to 3.x continues, and + the developers are planning to support Python 2.7 with bug-fix + releases beyond the typical two years. + +* A policy decision was made to silence warnings only of interest to + developers. :exc:`DeprecationWarning` and its + descendants are now ignored unless otherwise requested, preventing + users from seeing warnings triggered by an application. This change + was also made in the branch that will become Python 3.2. (Discussed + on stdlib-sig and carried out in :issue:`7319`.) + + In previous releases, :exc:`DeprecationWarning` messages were + enabled by default, providing Python developers with a clear + indication of where their code may break in a future major version + of Python. + + However, there are increasingly many users of Python-based + applications who are not directly involved in the development of + those applications. :exc:`DeprecationWarning` messages are + irrelevant to such users, making them worry about an application + that's actually working correctly and burdening application developers + with responding to these concerns. + + You can re-enable display of :exc:`DeprecationWarning` messages by + running Python with the :option:`-Wdefault` (short form: + :option:`-Wd`) switch, or by setting the :envvar:`PYTHONWARNINGS` + environment variable to ``"default"`` (or ``"d"``) before running + Python. Python code can also re-enable them + by calling ``warnings.simplefilter('default')``. Python 3.1 Features @@ -177,7 +135,7 @@ A partial list of 3.1 features that were backported to 2.7: * The syntax for set literals (``{1,2,3}`` is a mutable set). -* Dictionary and set comprehensions (``{i: i*2 for i in range(3)}``). +* Dictionary and set comprehensions (``{ i: i*2 for i in range(3)}``). * Multiple context managers in a single :keyword:`with` statement. * A new version of the :mod:`io` library, rewritten in C for performance. * The ordered-dictionary type described in :ref:`pep-0372`. @@ -199,7 +157,7 @@ * :func:`operator.isCallable` and :func:`operator.sequenceIncludes`, which are not supported in 3.x, now trigger warnings. * The :option:`-3` switch now automatically - enables the :option:`-Qwarn <-Q>` switch that causes warnings + enables the :option:`-Qwarn` switch that causes warnings about using classic division with integers and long integers. @@ -434,10 +392,9 @@ .. seealso:: - :mod:`argparse` documentation - The documentation page of the argparse module. - - :ref:`upgrading-optparse-code` + `argparse module documentation `__ + + `Upgrading optparse code to use argparse `__ Part of the Python documentation, describing how to convert code that uses :mod:`optparse`. @@ -447,6 +404,8 @@ PEP 391: Dictionary-Based Configuration For Logging ==================================================== +.. XXX not documented in library reference yet; add link here once it's added. + The :mod:`logging` module is very flexible; applications can define a tree of logging subsystems, and each logger in this tree can filter out certain messages, format them differently, and direct messages to @@ -455,21 +414,21 @@ All this flexibility can require a lot of configuration. You can write Python statements to create objects and set their properties, but a complex set-up requires verbose but boring code. -:mod:`logging` also supports a :func:`~logging.fileConfig` +:mod:`logging` also supports a :func:`~logging.config.fileConfig` function that parses a file, but the file format doesn't support configuring filters, and it's messier to generate programmatically. -Python 2.7 adds a :func:`~logging.dictConfig` function that +Python 2.7 adds a :func:`~logging.config.dictConfig` function that uses a dictionary to configure logging. There are many ways to produce a dictionary from different sources: construct one with code; parse a file containing JSON; or use a YAML parsing library if one is -installed. For more information see :ref:`logging-config-api`. +installed. The following example configures two loggers, the root logger and a -logger named "network". Messages sent to the root logger will be +logger named "network". Messages sent to the root logger will be sent to the system log using the syslog protocol, and messages to the "network" logger will be written to a :file:`network.log` file -that will be rotated once the log reaches 1MB. +that will be rotated once the log reaches 1Mb. :: @@ -488,7 +447,7 @@ 'filename': '/logs/network.log', 'formatter': 'standard', 'level': 'INFO', - 'maxBytes': 1000000}, + 'maxBytes': 1024*1024}, 'syslog': {'class': 'logging.handlers.SysLogHandler', 'formatter': 'standard', 'level': 'ERROR'}}, @@ -526,19 +485,16 @@ for UDP or :const:`socket.SOCK_STREAM` for TCP. The default protocol remains UDP. -* :class:`~logging.Logger` instances gained a :meth:`~logging.Logger.getChild` - method that retrieves a descendant logger using a relative path. - For example, once you retrieve a logger by doing ``log = getLogger('app')``, +* :class:`Logger` instances gained a :meth:`getChild` method that retrieves a + descendant logger using a relative path. For example, + once you retrieve a logger by doing ``log = getLogger('app')``, calling ``log.getChild('network.listen')`` is equivalent to ``getLogger('app.network.listen')``. -* The :class:`~logging.LoggerAdapter` class gained a - :meth:`~logging.LoggerAdapter.isEnabledFor` method that takes a - *level* and returns whether the underlying logger would +* The :class:`LoggerAdapter` class gained a :meth:`isEnabledFor` method + that takes a *level* and returns whether the underlying logger would process a message of that level of importance. -.. XXX: Logger objects don't have a class declaration so the link don't work - .. seealso:: :pep:`391` - Dictionary-Based Configuration For Logging @@ -547,15 +503,14 @@ PEP 3106: Dictionary Views ==================================================== -The dictionary methods :meth:`~dict.keys`, :meth:`~dict.values`, and -:meth:`~dict.items` are different in Python 3.x. They return an object -called a :dfn:`view` instead of a fully materialized list. - -It's not possible to change the return values of :meth:`~dict.keys`, -:meth:`~dict.values`, and :meth:`~dict.items` in Python 2.7 because -too much code would break. Instead the 3.x versions were added -under the new names :meth:`~dict.viewkeys`, :meth:`~dict.viewvalues`, -and :meth:`~dict.viewitems`. +The dictionary methods :meth:`keys`, :meth:`values`, and :meth:`items` +are different in Python 3.x. They return an object called a :dfn:`view` +instead of a fully materialized list. + +It's not possible to change the return values of :meth:`keys`, +:meth:`values`, and :meth:`items` in Python 2.7 because too much code +would break. Instead the 3.x versions were added under the new names +:meth:`viewkeys`, :meth:`viewvalues`, and :meth:`viewitems`. :: @@ -597,8 +552,8 @@ RuntimeError: dictionary changed size during iteration You can use the view methods in Python 2.x code, and the 2to3 -converter will change them to the standard :meth:`~dict.keys`, -:meth:`~dict.values`, and :meth:`~dict.items` methods. +converter will change them to the standard :meth:`keys`, +:meth:`values`, and :meth:`items` methods. .. seealso:: @@ -671,7 +626,7 @@ ``{}`` continues to represent an empty dictionary; use ``set()`` for an empty set. - >>> {1, 2, 3, 4, 5} + >>> {1,2,3,4,5} set([1, 2, 3, 4, 5]) >>> set() # empty set set([]) @@ -708,7 +663,7 @@ The :func:`contextlib.nested` function provides a very similar function, so it's no longer necessary and has been deprecated. - (Proposed in https://codereview.appspot.com/53094; implemented by + (Proposed in http://codereview.appspot.com/53094; implemented by Georg Brandl.) * Conversions between floating-point numbers and strings are @@ -841,7 +796,7 @@ ``None`` as its first argument. (Fixed by Georg Brandl; :issue:`4759`.) - .. XXX bytearray doesn't seem to be documented + .. bytearray doesn't seem to be documented * When using ``@classmethod`` and ``@staticmethod`` to wrap methods as class or static methods, the wrapper object now @@ -914,6 +869,12 @@ Several performance enhancements have been added: +.. * A new :program:`configure` option, :option:`--with-computed-gotos`, + compiles the main bytecode interpreter loop using a new dispatch + mechanism that gives speedups of up to 20%, depending on the system + and benchmark. The new mechanism is only supported on certain + compilers, such as gcc, SunPro, and icc. + * A new opcode was added to perform the initial setup for :keyword:`with` statements, looking up the :meth:`__enter__` and :meth:`__exit__` methods. (Contributed by Benjamin Peterson.) @@ -1095,7 +1056,7 @@ :meth:`~collections.deque.count` method that returns the number of contained elements equal to the supplied argument *x*, and a :meth:`~collections.deque.reverse` method that reverses the elements - of the deque in-place. :class:`~collections.deque` also exposes its maximum + of the deque in-place. :class:`deque` also exposes its maximum length as the read-only :attr:`~collections.deque.maxlen` attribute. (Both features added by Raymond Hettinger.) @@ -1118,7 +1079,7 @@ (Fixed by Daniel Stutzbach; :issue:`8729`.) * Constructors for the parsing classes in the :mod:`ConfigParser` module now - take an *allow_no_value* parameter, defaulting to false; if true, + take a *allow_no_value* parameter, defaulting to false; if true, options without values will be allowed. For example:: >>> import ConfigParser, StringIO @@ -1176,14 +1137,15 @@ ``Decimal('0.1000000000000000055511151231257827021181583404541015625')``. (Implemented by Raymond Hettinger; :issue:`4796`.) - Comparing instances of :class:`~decimal.Decimal` with floating-point + Comparing instances of :class:`Decimal` with floating-point numbers now produces sensible results based on the numeric values of the operands. Previously such comparisons would fall back to Python's default rules for comparing objects, which produced arbitrary results based on their type. Note that you still cannot combine :class:`Decimal` and floating-point in other operations such as addition, since you should be explicitly choosing how to convert between float and - :class:`~decimal.Decimal`. (Fixed by Mark Dickinson; :issue:`2531`.) + :class:`Decimal`. + (Fixed by Mark Dickinson; :issue:`2531`.) The constructor for :class:`~decimal.Decimal` now accepts floating-point numbers (added by Raymond Hettinger; :issue:`8257`) @@ -1235,8 +1197,8 @@ Ordering comparisons (``<``, ``<=``, ``>``, ``>=``) between fractions and complex numbers now raise a :exc:`TypeError`. - This fixes an oversight, making the :class:`~fractions.Fraction` - match the other numeric types. + This fixes an oversight, making the :class:`Fraction` match the other + numeric types. .. revision 79455 @@ -1250,7 +1212,7 @@ uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` +* New class decorator: :func:`total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`__eq__` method and one of :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, and generates the missing comparison methods. Since the @@ -1258,7 +1220,7 @@ this decorator makes it easier to define ordered classes. (Added by Raymond Hettinger; :issue:`5479`.) - New function: :func:`~functools.cmp_to_key` will take an old-style comparison + New function: :func:`cmp_to_key` will take an old-style comparison function that expects two arguments and return a new callable that can be used as the *key* parameter to functions such as :func:`sorted`, :func:`min` and :func:`max`, etc. The primary @@ -1385,7 +1347,7 @@ with any object literal that decodes to a list of pairs. (Contributed by Raymond Hettinger; :issue:`5381`.) -* The :mod:`mailbox` module's :class:`~mailbox.Maildir` class now records the +* The :mod:`mailbox` module's :class:`Maildir` class now records the timestamp on the directories it reads, and only re-reads them if the modification time has subsequently changed. This improves performance by avoiding unneeded directory scans. (Fixed by @@ -1506,10 +1468,10 @@ defaults to False; if overridden to be True, new request connections will have the TCP_NODELAY option set to prevent buffering many small sends into a single TCP packet. - The :attr:`~SocketServer.BaseServer.timeout` class attribute can hold + The :attr:`~SocketServer.TCPServer.timeout` class attribute can hold a timeout in seconds that will be applied to the request socket; if - no request is received within that time, :meth:`~SocketServer.BaseServer.handle_timeout` - will be called and :meth:`~SocketServer.BaseServer.handle_request` will return. + no request is received within that time, :meth:`handle_timeout` + will be called and :meth:`handle_request` will return. (Contributed by Kristján Valur Jónsson; :issue:`6192` and :issue:`6267`.) * Updated module: the :mod:`sqlite3` module has been updated to @@ -1519,7 +1481,7 @@ and then call :meth:`~sqlite3.Connection.load_extension` to load a particular shared library. (Updated by Gerhard Häring.) -* The :mod:`ssl` module's :class:`~ssl.SSLSocket` objects now support the +* The :mod:`ssl` module's :class:`ssl.SSLSocket` objects now support the buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set OpenSSL's :c:macro:`SSL_MODE_AUTO_RETRY`, which will prevent an error @@ -1575,7 +1537,7 @@ on receiving an :const:`EINTR` signal. (Reported by several people; final patch by Gregory P. Smith in :issue:`1068268`.) -* New function: :func:`~symtable.Symbol.is_declared_global` in the :mod:`symtable` module +* New function: :func:`~symtable.is_declared_global` in the :mod:`symtable` module returns true for variables that are explicitly declared to be global, false for ones that are implicitly global. (Contributed by Jeremy Hylton.) @@ -1612,7 +1574,7 @@ resulting archive. This is more powerful than the existing *exclude* argument, which has therefore been deprecated. (Added by Lars Gustäbel; :issue:`6856`.) - The :class:`~tarfile.TarFile` class also now supports the context management protocol. + The :class:`~tarfile.TarFile` class also now supports the context manager protocol. (Added by Lars Gustäbel; :issue:`7232`.) * The :meth:`~threading.Event.wait` method of the :class:`threading.Event` class @@ -1756,7 +1718,7 @@ Makefile and the :file:`pyconfig.h` file. * :func:`~sysconfig.get_config_vars` returns a dictionary containing all of the configuration variables. -* :func:`~sysconfig.get_path` returns the configured path for +* :func:`~sysconfig.getpath` returns the configured path for a particular type of module: the standard library, site-specific modules, platform-specific modules, etc. * :func:`~sysconfig.is_python_build` returns true if you're running a @@ -1767,7 +1729,7 @@ The Distutils package and :mod:`sysconfig` are now maintained by Tarek Ziadé, who has also started a Distutils2 package (source repository at -https://hg.python.org/distutils2/) for developing a next-generation +http://hg.python.org/distutils2/) for developing a next-generation version of Distutils. @@ -1804,7 +1766,7 @@ by Michael Foord, unless otherwise noted. The enhanced version of the module is downloadable separately for use with Python versions 2.4 to 2.6, packaged as the :mod:`unittest2` package, from -https://pypi.python.org/pypi/unittest2. +http://pypi.python.org/pypi/unittest2. When used from the command line, the module can automatically discover tests. It's not as fancy as `py.test `__ or @@ -1818,7 +1780,7 @@ Consult the :mod:`unittest` module documentation for more details. (Developed in :issue:`6001`.) -The :func:`~unittest.main` function supports some other new options: +The :func:`main` function supports some other new options: * :option:`-b` or :option:`--buffer` will buffer the standard output and standard error streams during each test. If the test passes, @@ -1836,7 +1798,7 @@ being tested or the tests being run have defined a signal handler of their own, by noticing that a signal handler was already set and calling it. If this doesn't work for you, there's a - :func:`~unittest.removeHandler` decorator that can be used to mark tests that + :func:`removeHandler` decorator that can be used to mark tests that should have the control-C handling disabled. * :option:`-f` or :option:`--failfast` makes @@ -1963,7 +1925,7 @@ :func:`unittest.main` now takes an optional ``exit`` argument. If False, :func:`~unittest.main` doesn't call :func:`sys.exit`, allowing -:func:`~unittest.main` to be used from the interactive interpreter. +:func:`main` to be used from the interactive interpreter. (Contributed by J. Pablo Fernández; :issue:`3379`.) :class:`~unittest.TestResult` has new :meth:`~unittest.TestResult.startTestRun` and @@ -2160,7 +2122,7 @@ :c:macro:`Py_ISSPACE`, :c:macro:`Py_ISUPPER`, :c:macro:`Py_ISXDIGIT`, - :c:macro:`Py_TOLOWER`, and :c:macro:`Py_TOUPPER`. + and :c:macro:`Py_TOLOWER`, :c:macro:`Py_TOUPPER`. All of these functions are analogous to the C standard macros for classifying characters, but ignore the current locale setting, because in @@ -2306,11 +2268,11 @@ (Contributed by David Cournapeau; :issue:`4365`.) * The :mod:`_winreg` module for accessing the registry now implements - the :func:`~_winreg.CreateKeyEx` and :func:`~_winreg.DeleteKeyEx` - functions, extended versions of previously-supported functions that - take several extra arguments. The :func:`~_winreg.DisableReflectionKey`, - :func:`~_winreg.EnableReflectionKey`, and :func:`~_winreg.QueryReflectionKey` - were also tested and documented. + the :func:`CreateKeyEx` and :func:`DeleteKeyEx` functions, extended + versions of previously-supported functions that take several extra + arguments. The :func:`DisableReflectionKey`, + :func:`EnableReflectionKey`, and :func:`QueryReflectionKey` were also + tested and documented. (Implemented by Brian Curtin: :issue:`7347`.) * The new :c:func:`_beginthreadex` API is used to start threads, and @@ -2320,7 +2282,7 @@ * The :func:`os.kill` function now works on Windows. The signal value can be the constants :const:`CTRL_C_EVENT`, :const:`CTRL_BREAK_EVENT`, or any integer. The first two constants - will send :kbd:`Control-C` and :kbd:`Control-Break` keystroke events to + will send Control-C and Control-Break keystroke events to subprocesses; any other value will use the :c:func:`TerminateProcess` API. (Contributed by Miki Tebeka; :issue:`1220212`.) @@ -2427,20 +2389,20 @@ In the standard library: -* Operations with :class:`~datetime.datetime` instances that resulted in a year +* Operations with :class:`datetime` instances that resulted in a year falling outside the supported range didn't always raise :exc:`OverflowError`. Such errors are now checked more carefully and will now raise the exception. (Reported by Mark Leander, patch by Anand B. Pillai and Alexander Belopolsky; :issue:`7150`.) -* When using :class:`~decimal.Decimal` instances with a string's +* When using :class:`Decimal` instances with a string's :meth:`format` method, the default alignment was previously left-alignment. This has been changed to right-alignment, which might change the output of your programs. (Changed by Mark Dickinson; :issue:`6857`.) Comparisons involving a signaling NaN value (or ``sNAN``) now signal - :const:`~decimal.InvalidOperation` instead of silently returning a true or + :const:`InvalidOperation` instead of silently returning a true or false value depending on the comparison operator. Quiet NaN values (or ``NaN``) are now hashable. (Fixed by Mark Dickinson; :issue:`7279`.) @@ -2451,7 +2413,7 @@ or comment (which looks like ``). (Patch by Neil Muller; :issue:`2746`.) -* The :meth:`~StringIO.StringIO.readline` method of :class:`~StringIO.StringIO` objects now does +* The :meth:`readline` method of :class:`StringIO` objects now does nothing when a negative length is requested, as other file-like objects do. (:issue:`7348`). @@ -2510,73 +2472,6 @@ .. ====================================================================== -.. _py27-maintenance-enhancements: - -New Features Added to Python 2.7 Maintenance Releases -===================================================== - -New features may be added to Python 2.7 maintenance releases when the -situation genuinely calls for it. Any such additions must go through -the Python Enhancement Proposal process, and make a compelling case for why -they can't be adequately addressed by either adding the new feature solely to -Python 3, or else by publishing it on the Python Package Index. - -In addition to the specific proposals listed below, there is a general -exemption allowing new ``-3`` warnings to be added in any Python 2.7 -maintenance release. - - -PEP 434: IDLE Enhancement Exception for All Branches ----------------------------------------------------- - -:pep:`434` describes a general exemption for changes made to the IDLE -development environment shipped along with Python. This exemption makes it -possible for the IDLE developers to provide a more consistent user -experience across all supported versions of Python 2 and 3. - -For details of any IDLE changes, refer to the NEWS file for the specific -release. - - -PEP 466: Network Security Enhancements for Python 2.7 ------------------------------------------------------ - -:pep:`466` describes a number of network security enhancement proposals -that have been approved for inclusion in Python 2.7 maintenance releases, -with the first of those changes appearing in the Python 2.7.7 release. - -:pep:`466` related features added in Python 2.7.7: - -* :func:`hmac.compare_digest` was backported from Python 3 to make a timing - attack resistant comparison operation available to Python 2 applications. - (Contributed by Alex Gaynor; :issue:`21306`.) - -* OpenSSL 1.0.1g was upgraded in the official Windows installers published on - python.org. (Contributed by Zachary Ware; :issue:`21462`.) - -:pep:`466` related features added in Python 2.7.8: - -* :func:`hashlib.pbkdf2_hmac` was backported from Python 3 to make a hashing - algorithm suitable for secure password storage broadly available to Python - 2 applications. (Contributed by Alex Gaynor; :issue:`21304`.) - -* OpenSSL 1.0.1h was upgraded for the official Windows installers published on - python.org. (contributed by Zachary Ware in :issue:`21671` for CVE-2014-0224) - -:pep:`466` related features added in Python 2.7.9: - -* Most of Python 3.4's :mod:`ssl` module was backported. This means :mod:`ssl` - now supports Server Name Indication, TLS1.x settings, access to the platform - certificate store, the :class:`~ssl.SSLContext` class, and other - features. (Contributed by Alex Gaynor and David Reid; :issue:`21308`.) - -* :func:`os.urandom` was changed to cache a file descriptor to ``/dev/urandom`` - instead of reopening ``/dev/urandom`` on every call. (Contributed by Alex - Gaynor; :issue:`21305`.) - - -.. ====================================================================== - .. _acks27: Acknowledgements diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.0.rst --- a/Doc/whatsnew/3.0.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/3.0.rst Mon Jan 25 17:05:13 2016 +0100 @@ -5,6 +5,8 @@ .. XXX Add trademark info for Apple, Microsoft. :Author: Guido van Rossum +:Release: |release| +:Date: |today| .. $Id$ Rules for maintenance: @@ -164,7 +166,7 @@ If the input sequences are not of equal length, :func:`map` will stop at the termination of the shortest of the sequences. For full - compatibility with :func:`map` from Python 2.x, also wrap the sequences in + compatibility with `map` from Python 2.x, also wrap the sequences in :func:`itertools.zip_longest`, e.g. ``map(func, *sequences)`` becomes ``list(map(func, itertools.zip_longest(*sequences)))``. @@ -769,7 +771,7 @@ respectively). * :pep:`3114`: the standard :meth:`next` method has been renamed to - :meth:`~iterator.__next__`. + :meth:`__next__`. * The :meth:`__oct__` and :meth:`__hex__` special methods are removed -- :func:`oct` and :func:`hex` use :meth:`__index__` now to convert @@ -805,12 +807,12 @@ To get the old behavior of :func:`input`, use ``eval(input())``. * A new built-in function :func:`next` was added to call the - :meth:`~iterator.__next__` method on an object. + :meth:`__next__` method on an object. * The :func:`round` function rounding strategy and return type have changed. Exact halfway cases are now rounded to the nearest even result instead of away from zero. (For example, ``round(2.5)`` now - returns ``2`` rather than ``3``.) ``round(x[, n])`` now + returns ``2`` rather than ``3``.) :func:`round(x[, n])` now delegates to ``x.__round__([n])`` instead of always returning a float. It generally returns an integer when called with a single argument and a value of the same type as ``x`` when called with two diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.1.rst --- a/Doc/whatsnew/3.1.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/3.1.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,6 +3,8 @@ **************************** :Author: Raymond Hettinger +:Release: |release| +:Date: |today| .. $Id$ Rules for maintenance: @@ -172,7 +174,7 @@ needed and is now deprecated. (Contributed by Georg Brandl and Mattias Brändström; - `appspot issue 53094 `_.) + `appspot issue 53094 `_.) * ``round(x, n)`` now returns an integer if *x* is an integer. Previously it returned a float:: @@ -238,7 +240,7 @@ (Contributed by Guilherme Polo; :issue:`2983`.) * The :class:`gzip.GzipFile` and :class:`bz2.BZ2File` classes now support - the context management protocol:: + the context manager protocol:: >>> # Automatically close file after writing >>> with gzip.GzipFile(filename, "wb") as f: @@ -336,7 +338,7 @@ (Contributed by David Laban; :issue:`4739`.) * The :mod:`unittest` module now supports skipping individual tests or classes - of tests. And it supports marking a test as an expected failure, a test that + of tests. And it supports marking a test as a expected failure, a test that is known to be broken, but shouldn't be counted as a failure on a TestResult:: diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.2.rst --- a/Doc/whatsnew/3.2.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/3.2.rst Mon Jan 25 17:05:13 2016 +0100 @@ -3,6 +3,8 @@ **************************** :Author: Raymond Hettinger +:Release: |release| +:Date: |today| .. $Id$ Rules for maintenance: @@ -50,7 +52,7 @@ This article explains the new features in Python 3.2 as compared to 3.1. It focuses on a few highlights and gives a few examples. For full details, see the -`Misc/NEWS `_ file. +:source:`Misc/NEWS ` file. .. seealso:: @@ -368,9 +370,9 @@ module:: >>> import sysconfig - >>> sysconfig.get_config_var('SOABI') # find the version tag + >>> sysconfig.get_config_var('SOABI') # find the version tag 'cpython-32mu' - >>> sysconfig.get_config_var('EXT_SUFFIX') # find the full filename extension + >>> sysconfig.get_config_var('SO') # find the full filename extension '.cpython-32mu.so' .. seealso:: @@ -522,7 +524,7 @@ (Proposed and implemented by Mark Dickinson; :issue:`9337`.) * :class:`memoryview` objects now have a :meth:`~memoryview.release()` method - and they also now support the context management protocol. This allows timely + and they also now support the context manager protocol. This allows timely release of any resources that were acquired when requesting a buffer from the original object. @@ -788,7 +790,7 @@ :issue:`8814`.) * To help write classes with rich comparison methods, a new decorator - :func:`functools.total_ordering` will use existing equality and inequality + :func:`functools.total_ordering` will use a existing equality and inequality methods to fill in the remaining methods. For example, supplying *__eq__* and *__lt__* will enable @@ -816,7 +818,7 @@ >>> sorted(iterable, key=cmp_to_key(locale.strcoll)) For sorting examples and a brief sorting tutorial, see the `Sorting HowTo - `_ tutorial. + `_ tutorial. (Contributed by Raymond Hettinger.) @@ -1020,7 +1022,7 @@ :issue:`5094`, :issue:`6641`, :issue:`2706`, :issue:`1777412`, :issue:`8013`, and :issue:`10827`.) -.. XXX https://bugs.python.org/issue?%40search_text=datetime&%40sort=-activity +.. XXX http://bugs.python.org/issue?%40search_text=datetime&%40sort=-activity math ---- @@ -1315,7 +1317,7 @@ ftp --- -The :class:`ftplib.FTP` class now supports the context management protocol to +The :class:`ftplib.FTP` class now supports the context manager protocol to unconditionally consume :exc:`socket.error` exceptions and to close the FTP connection when done:: @@ -1399,7 +1401,7 @@ Also, the :class:`zipfile.ZipExtFile` class was reworked internally to represent files stored inside an archive. The new implementation is significantly faster -and can be wrapped in an :class:`io.BufferedReader` object for more speedups. It +and can be wrapped in a :class:`io.BufferedReader` object for more speedups. It also solves an issue where interleaved calls to *read* and *readline* gave the wrong results. @@ -1595,7 +1597,7 @@ descriptor. The latter can then be reused for other purposes. (Added by Antoine Pitrou; :issue:`8524`.) -* :func:`socket.create_connection` now supports the context management protocol +* :func:`socket.create_connection` now supports the context manager protocol to unconditionally consume :exc:`socket.error` exceptions and to close the socket when done. (Contributed by Giampaolo Rodolà; :issue:`9794`.) @@ -2283,7 +2285,7 @@ Additional details about the implementation can be read from a `python-dev mailing-list message - `_ + `_ (however, "priority requests" as exposed in this message have not been kept for inclusion). @@ -2469,7 +2471,7 @@ In addition to the existing Subversion code repository at http://svn.python.org there is now a `Mercurial `_ repository at -https://hg.python.org/\ . +http://hg.python.org/ . After the 3.2 release, there are plans to switch to Mercurial as the primary repository. This distributed version control system should make it easier for @@ -2478,7 +2480,7 @@ To learn to use the new version control system, see the `tutorial by Joel Spolsky `_ or the `Guide to Mercurial Workflows -`_. +`_. Build and C API Changes @@ -2560,7 +2562,7 @@ build, there is a known problem with the default Tcl/Tk on Mac OS X 10.6. Accordingly, we recommend installing an updated alternative such as `ActiveState Tcl/Tk 8.5.9 `_\. -See https://www.python.org/download/mac/tcltk/ for additional details. +See http://www.python.org/download/mac/tcltk/ for additional details. Porting to Python 3.2 ===================== @@ -2649,7 +2651,7 @@ outfile.write(line) (Contributed by Georg Brandl and Mattias Brändström; - `appspot issue 53094 `_.) + `appspot issue 53094 `_.) * :func:`struct.pack` now only allows bytes for the ``s`` string pack code. Formerly, it would accept text arguments and implicitly encode them to bytes diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.3.rst --- a/Doc/whatsnew/3.3.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/3.3.rst Mon Jan 25 17:05:13 2016 +0100 @@ -2,6 +2,10 @@ What's New In Python 3.3 **************************** +:Author: Raymond Hettinger +:Release: |release| +:Date: |today| + .. Rules for maintenance: * Anyone can add text to this document. Do not spend very much time @@ -43,103 +47,15 @@ when researching a change. This article explains the new features in Python 3.3, compared to 3.2. -Python 3.3 was released on September 29, 2012. For full details, -see the `changelog `_. -.. seealso:: - :pep:`398` - Python 3.3 Release Schedule - - -Summary -- Release highlights -============================= - -.. This section singles out the most important changes in Python 3.3. - Brevity is key. - -New syntax features: - -* New ``yield from`` expression for :ref:`generator delegation `. -* The ``u'unicode'`` syntax is accepted again for :class:`str` objects. - -New library modules: - -* :mod:`faulthandler` (helps debugging low-level crashes) -* :mod:`ipaddress` (high-level objects representing IP addresses and masks) -* :mod:`lzma` (compress data using the XZ / LZMA algorithm) -* :mod:`unittest.mock` (replace parts of your system under test with mock objects) -* :mod:`venv` (Python :ref:`virtual environments `, as in the - popular ``virtualenv`` package) - -New built-in features: - -* Reworked :ref:`I/O exception hierarchy `. - -Implementation improvements: - -* Rewritten :ref:`import machinery ` based on :mod:`importlib`. -* More compact :ref:`unicode strings `. -* More compact :ref:`attribute dictionaries `. - -Significantly Improved Library Modules: - -* C Accelerator for the :ref:`decimal ` module. -* Better unicode handling in the :ref:`email ` module - (:term:`provisional `). - -Security improvements: - -* Hash randomization is switched on by default. - -Please read on for a comprehensive list of user-facing changes. - - -.. _pep-405: - -PEP 405: Virtual Environments -============================= - -Virtual environments help create separate Python setups while sharing a -system-wide base install, for ease of maintenance. Virtual environments -have their own set of private site packages (i.e. locally-installed -libraries), and are optionally segregated from the system-wide site -packages. Their concept and implementation are inspired by the popular -``virtualenv`` third-party package, but benefit from tighter integration -with the interpreter core. - -This PEP adds the :mod:`venv` module for programmatic access, and the -:ref:`pyvenv ` script for command-line access and -administration. The Python interpreter checks for a ``pyvenv.cfg``, -file whose existence signals the base of a virtual environment's directory -tree. - -.. seealso:: - - :pep:`405` - Python Virtual Environments - PEP written by Carl Meyer; implementation by Carl Meyer and Vinay Sajip - - -PEP 420: Implicit Namespace Packages -==================================== - -Native support for package directories that don't require ``__init__.py`` -marker files and can automatically span multiple path segments (inspired by -various third party approaches to namespace packages, as described in -:pep:`420`) - -.. seealso:: - - :pep:`420` - Implicit Namespace Packages - PEP written by Eric V. Smith; implementation by Eric V. Smith - and Barry Warsaw - - -.. _pep-3118-update: +.. pep-3118-update: PEP 3118: New memoryview implementation and buffer protocol documentation ========================================================================= -The implementation of :pep:`3118` has been significantly improved. +:issue:`10181` - memoryview bug fixes and features. + Written by Stefan Krah. The new memoryview implementation comprehensively fixes all ownership and lifetime issues of dynamically allocated fields in the Py_buffer struct @@ -170,8 +86,10 @@ * Multi-dimensional comparisons are supported for any array type. -* One-dimensional memoryviews of hashable (read-only) types with formats B, - b or c are now hashable. (Contributed by Antoine Pitrou in :issue:`13411`.) +* All array types are hashable if the exporting object is hashable + and the view is read-only. (Contributed by Antoine Pitrou in + :issue:`13411`) + * Arbitrary slicing of any 1-D arrays type is supported. For example, it is now possible to reverse a memoryview in O(1) by using a negative step. @@ -188,20 +106,7 @@ now returns an integer (in accordance with the struct module syntax). For returning a bytes object the view must be cast to 'c' first. -* memoryview comparisons now use the logical structure of the operands - and compare all array elements by value. All format strings in struct - module syntax are supported. Views with unrecognised format strings - are still permitted, but will always compare as unequal, regardless - of view contents. - -* For further changes see `Build and C API Changes`_ and `Porting C code`_. - -(Contributed by Stefan Krah in :issue:`10181`.) - -.. seealso:: - - :pep:`3118` - Revising the Buffer Protocol - +* For further changes see `Build and C API Changes`_ and `Porting C code`_ . .. _pep-393: @@ -228,7 +133,7 @@ Changes introduced by :pep:`393` are the following: -* Python now always supports the full range of Unicode code points, including +* Python now always supports the full range of Unicode codepoints, including non-BMP ones (i.e. from ``U+0000`` to ``U+10FFFF``). The distinction between narrow and wide builds no longer exists and Python now behaves like a wide build, even under Windows. @@ -246,7 +151,7 @@ so ``'\U0010FFFF'[0]`` now returns ``'\U0010FFFF'`` and not ``'\uDBFF'``; * all other functions in the standard library now correctly handle - non-BMP code points. + non-BMP codepoints. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns @@ -258,13 +163,13 @@ Performance and resource usage ------------------------------ -The storage of Unicode strings now depends on the highest code point in the string: +The storage of Unicode strings now depends on the highest codepoint in the string: -* pure ASCII and Latin1 strings (``U+0000-U+00FF``) use 1 byte per code point; +* pure ASCII and Latin1 strings (``U+0000-U+00FF``) use 1 byte per codepoint; -* BMP strings (``U+0000-U+FFFF``) use 2 bytes per code point; +* BMP strings (``U+0000-U+FFFF``) use 2 bytes per codepoint; -* non-BMP strings (``U+10000-U+10FFFF``) use 4 bytes per code point. +* non-BMP strings (``U+10000-U+10FFFF``) use 4 bytes per codepoint. The net effect is that for most applications, memory usage of string storage should decrease significantly - especially compared to former @@ -277,55 +182,13 @@ bit better than Python 2.7, on a Django benchmark (see the PEP for details). -.. seealso:: - - :pep:`393` - Flexible String Representation - PEP written by Martin von Löwis; implementation by Torsten Becker - and Martin von Löwis. - - -.. _pep-397: - -PEP 397: Python Launcher for Windows -==================================== - -The Python 3.3 Windows installer now includes a ``py`` launcher application -that can be used to launch Python applications in a version independent -fashion. - -This launcher is invoked implicitly when double-clicking ``*.py`` files. -If only a single Python version is installed on the system, that version -will be used to run the file. If multiple versions are installed, the most -recent version is used by default, but this can be overridden by including -a Unix-style "shebang line" in the Python script. - -The launcher can also be used explicitly from the command line as the ``py`` -application. Running ``py`` follows the same version selection rules as -implicitly launching scripts, but a more specific version can be selected -by passing appropriate arguments (such as ``-3`` to request Python 3 when -Python 2 is also installed, or ``-2.6`` to specifclly request an earlier -Python version when a more recent version is installed). - -In addition to the launcher, the Windows installer now includes an -option to add the newly installed Python to the system PATH. (Contributed -by Brian Curtin in :issue:`3561`.) - -.. seealso:: - - :pep:`397` - Python Launcher for Windows - PEP written by Mark Hammond and Martin v. Löwis; implementation by - Vinay Sajip. - - Launcher documentation: :ref:`launcher` - - Installer PATH modification: :ref:`windows-path-mod` - - -.. _pep-3151: PEP 3151: Reworking the OS and IO exception hierarchy ===================================================== +:pep:`3151` - Reworking the OS and IO exception hierarchy + PEP written and implemented by Antoine Pitrou. + The hierarchy of exceptions raised by operating system errors is now both simplified and finer-grained. @@ -387,24 +250,16 @@ except PermissionError: print("You are not allowed to read document.txt") -.. seealso:: - - :pep:`3151` - Reworking the OS and IO Exception Hierarchy - PEP written and implemented by Antoine Pitrou - - -.. index:: - single: yield; yield from (in What's New) - -.. _pep-380: PEP 380: Syntax for Delegating to a Subgenerator ================================================ -PEP 380 adds the ``yield from`` expression, allowing a :term:`generator` to -delegate +:pep:`380` - Syntax for Delegating to a Subgenerator + PEP written by Greg Ewing. + +PEP 380 adds the ``yield from`` expression, allowing a generator to delegate part of its operations to another generator. This allows a section of code -containing :keyword:`yield` to be factored out and placed in another generator. +containing 'yield' to be factored out and placed in another generator. Additionally, the subgenerator is allowed to return with a value, and the value is made available to the delegating generator. @@ -425,15 +280,15 @@ receive sent and thrown values directly from the calling scope, and return a final value to the outer generator:: - >>> def accumulate(): - ... tally = 0 + >>> def accumulate(start=0): + ... tally = start ... while 1: ... next = yield ... if next is None: ... return tally ... tally += next ... - >>> def gather_tallies(tallies): + >>> def gather_tallies(tallies, start=0): ... while 1: ... tally = yield from accumulate() ... tallies.append(tally) @@ -441,7 +296,7 @@ >>> tallies = [] >>> acc = gather_tallies(tallies) >>> next(acc) # Ensure the accumulator is ready to accept values - >>> for i in range(4): + >>> for i in range(10): ... acc.send(i) ... >>> acc.send(None) # Finish the first tally @@ -450,24 +305,24 @@ ... >>> acc.send(None) # Finish the second tally >>> tallies - [6, 10] + [45, 10] The main principle driving this change is to allow even generators that are designed to be used with the ``send`` and ``throw`` methods to be split into multiple subgenerators as easily as a single large function can be split into multiple subfunctions. -.. seealso:: - - :pep:`380` - Syntax for Delegating to a Subgenerator - PEP written by Greg Ewing; implementation by Greg Ewing, integrated into - 3.3 by Renaud Blanch, Ryan Kelly and Nick Coghlan; documentation by - Zbigniew Jędrzejewski-Szmek and Nick Coghlan +(Implementation by Greg Ewing, integrated into 3.3 by Renaud Blanch, Ryan +Kelly and Nick Coghlan, documentation by Zbigniew Jędrzejewski-Szmek and +Nick Coghlan) PEP 409: Suppressing exception context ====================================== +:pep:`409` - Suppressing exception context + PEP written by Ethan Furman, implemented by Ethan Furman and Nick Coghlan. + PEP 409 introduces new syntax that allows the display of the chained exception context to be disabled. This allows cleaner error messages in applications that convert between exception types:: @@ -522,16 +377,13 @@ ... KeyError('x',) -.. seealso:: - - :pep:`409` - Suppressing exception context - PEP written by Ethan Furman; implemented by Ethan Furman and Nick - Coghlan. - PEP 414: Explicit Unicode literals ====================================== +:pep:`414` - Explicit Unicode literals + PEP written by Armin Ronacher. + To ease the transition from Python 2 for Unicode aware Python applications that make heavy use of Unicode literals, Python 3.3 once again supports the "``u``" prefix for string literals. This prefix has no semantic significance @@ -540,15 +392,13 @@ the more significant semantic changes (such as the stricter default separation of binary and text data). -.. seealso:: - - :pep:`414` - Explicit Unicode literals - PEP written by Armin Ronacher. - PEP 3155: Qualified name for classes and functions ================================================== +:pep:`3155` - Qualified name for classes and functions + PEP written and implemented by Antoine Pitrou. + Functions and class objects have a new ``__qualname__`` attribute representing the "path" from the module top-level to their definition. For global functions and classes, this is the same as ``__name__``. For other functions and classes, @@ -601,176 +451,6 @@ >>> str(C.D.meth) '' -.. seealso:: - - :pep:`3155` - Qualified name for classes and functions - PEP written and implemented by Antoine Pitrou. - - -.. _pep-412: - -PEP 412: Key-Sharing Dictionary -=============================== - -Dictionaries used for the storage of objects' attributes are now able to -share part of their internal storage between each other (namely, the part -which stores the keys and their respective hashes). This reduces the memory -consumption of programs creating many instances of non-builtin types. - -.. seealso:: - - :pep:`412` - Key-Sharing Dictionary - PEP written and implemented by Mark Shannon. - - -PEP 362: Function Signature Object -================================== - -A new function :func:`inspect.signature` makes introspection of python -callables easy and straightforward. A broad range of callables is supported: -python functions, decorated or not, classes, and :func:`functools.partial` -objects. New classes :class:`inspect.Signature`, :class:`inspect.Parameter` -and :class:`inspect.BoundArguments` hold information about the call signatures, -such as, annotations, default values, parameters kinds, and bound arguments, -which considerably simplifies writing decorators and any code that validates -or amends calling signatures or arguments. - -.. seealso:: - - :pep:`362`: - Function Signature Object - PEP written by Brett Cannon, Yury Selivanov, Larry Hastings, Jiwon Seo; - implemented by Yury Selivanov. - - -PEP 421: Adding sys.implementation -================================== - -A new attribute on the :mod:`sys` module exposes details specific to the -implementation of the currently running interpreter. The initial set of -attributes on :attr:`sys.implementation` are ``name``, ``version``, -``hexversion``, and ``cache_tag``. - -The intention of ``sys.implementation`` is to consolidate into one namespace -the implementation-specific data used by the standard library. This allows -different Python implementations to share a single standard library code base -much more easily. In its initial state, ``sys.implementation`` holds only a -small portion of the implementation-specific data. Over time that ratio will -shift in order to make the standard library more portable. - -One example of improved standard library portability is ``cache_tag``. As of -Python 3.3, ``sys.implementation.cache_tag`` is used by :mod:`importlib` to -support :pep:`3147` compliance. Any Python implementation that uses -``importlib`` for its built-in import system may use ``cache_tag`` to control -the caching behavior for modules. - -SimpleNamespace ---------------- - -The implementation of ``sys.implementation`` also introduces a new type to -Python: :class:`types.SimpleNamespace`. In contrast to a mapping-based -namespace, like :class:`dict`, ``SimpleNamespace`` is attribute-based, like -:class:`object`. However, unlike ``object``, ``SimpleNamespace`` instances -are writable. This means that you can add, remove, and modify the namespace -through normal attribute access. - -.. seealso:: - - :pep:`421` - Adding sys.implementation - PEP written and implemented by Eric Snow. - - -.. _importlib: - -Using importlib as the Implementation of Import -=============================================== -:issue:`2377` - Replace __import__ w/ importlib.__import__ -:issue:`13959` - Re-implement parts of :mod:`imp` in pure Python -:issue:`14605` - Make import machinery explicit -:issue:`14646` - Require loaders set __loader__ and __package__ - -The :func:`__import__` function is now powered by :func:`importlib.__import__`. -This work leads to the completion of "phase 2" of :pep:`302`. There are -multiple benefits to this change. First, it has allowed for more of the -machinery powering import to be exposed instead of being implicit and hidden -within the C code. It also provides a single implementation for all Python VMs -supporting Python 3.3 to use, helping to end any VM-specific deviations in -import semantics. And finally it eases the maintenance of import, allowing for -future growth to occur. - -For the common user, there should be no visible change in semantics. For -those whose code currently manipulates import or calls import -programmatically, the code changes that might possibly be required are covered -in the `Porting Python code`_ section of this document. - -New APIs --------- -One of the large benefits of this work is the exposure of what goes into -making the import statement work. That means the various importers that were -once implicit are now fully exposed as part of the :mod:`importlib` package. - -The abstract base classes defined in :mod:`importlib.abc` have been expanded -to properly delineate between :term:`meta path finders ` -and :term:`path entry finders ` by introducing -:class:`importlib.abc.MetaPathFinder` and -:class:`importlib.abc.PathEntryFinder`, respectively. The old ABC of -:class:`importlib.abc.Finder` is now only provided for backwards-compatibility -and does not enforce any method requirements. - -In terms of finders, :class:`importlib.machinery.FileFinder` exposes the -mechanism used to search for source and bytecode files of a module. Previously -this class was an implicit member of :attr:`sys.path_hooks`. - -For loaders, the new abstract base class :class:`importlib.abc.FileLoader` helps -write a loader that uses the file system as the storage mechanism for a module's -code. The loader for source files -(:class:`importlib.machinery.SourceFileLoader`), sourceless bytecode files -(:class:`importlib.machinery.SourcelessFileLoader`), and extension modules -(:class:`importlib.machinery.ExtensionFileLoader`) are now available for -direct use. - -:exc:`ImportError` now has ``name`` and ``path`` attributes which are set when -there is relevant data to provide. The message for failed imports will also -provide the full name of the module now instead of just the tail end of the -module's name. - -The :func:`importlib.invalidate_caches` function will now call the method with -the same name on all finders cached in :attr:`sys.path_importer_cache` to help -clean up any stored state as necessary. - -Visible Changes ---------------- - -For potential required changes to code, see the `Porting Python code`_ -section. - -Beyond the expanse of what :mod:`importlib` now exposes, there are other -visible changes to import. The biggest is that :attr:`sys.meta_path` and -:attr:`sys.path_hooks` now store all of the meta path finders and path entry -hooks used by import. Previously the finders were implicit and hidden within -the C code of import instead of being directly exposed. This means that one can -now easily remove or change the order of the various finders to fit one's needs. - -Another change is that all modules have a ``__loader__`` attribute, storing the -loader used to create the module. :pep:`302` has been updated to make this -attribute mandatory for loaders to implement, so in the future once 3rd-party -loaders have been updated people will be able to rely on the existence of the -attribute. Until such time, though, import is setting the module post-load. - -Loaders are also now expected to set the ``__package__`` attribute from -:pep:`366`. Once again, import itself is already setting this on all loaders -from :mod:`importlib` and import itself is setting the attribute post-load. - -``None`` is now inserted into :attr:`sys.path_importer_cache` when no finder -can be found on :attr:`sys.path_hooks`. Since :class:`imp.NullImporter` is not -directly exposed on :attr:`sys.path_hooks` it could no longer be relied upon to -always be available to use as a value representing no finder found. - -All other changes relate to semantic changes which should be taken into -consideration when updating code for Python 3.3, and thus should be read about -in the `Porting Python code`_ section of this document. - -(Implementation by Brett Cannon) - Other Language Changes ====================== @@ -781,130 +461,33 @@ Both :func:`unicodedata.lookup()` and ``'\N{...}'`` now resolve name aliases, and :func:`unicodedata.lookup()` resolves named sequences too. - (Contributed by Ezio Melotti in :issue:`12753`.) - -* Unicode database updated to UCD version 6.1.0 + (Contributed by Ezio Melotti in :issue:`12753`) * Equality comparisons on :func:`range` objects now return a result reflecting the equality of the underlying sequences generated by those range objects. + (:issue:`13201`) * The ``count()``, ``find()``, ``rfind()``, ``index()`` and ``rindex()`` methods of :class:`bytes` and :class:`bytearray` objects now accept an integer between 0 and 255 as their first argument. - (Contributed by Petri Lehtinen in :issue:`12170`.) + (:issue:`12170`) -* The ``rjust()``, ``ljust()``, and ``center()`` methods of :class:`bytes` - and :class:`bytearray` now accept a :class:`bytearray` for the ``fill`` - argument. (Contributed by Petri Lehtinen in :issue:`12380`.) +* A dict lookup now raises a :exc:`RuntimeError` if the dict is modified during + the lookup. If you implement your own comparison function for objects used + as dict keys and the dict is shared by multiple threads, access to the dict + should be protected by a lock. + + (:issue:`14205`) * New methods have been added to :class:`list` and :class:`bytearray`: - ``copy()`` and ``clear()`` (:issue:`10516`). Consequently, - :class:`~collections.abc.MutableSequence` now also defines a - :meth:`~collections.abc.MutableSequence.clear` method (:issue:`11388`). + ``copy()`` and ``clear()``. -* Raw bytes literals can now be written ``rb"..."`` as well as ``br"..."``. + (:issue:`10516`) - (Contributed by Antoine Pitrou in :issue:`13748`.) - -* :meth:`dict.setdefault` now does only one lookup for the given key, making - it atomic when used with built-in types. - - (Contributed by Filip Gruszczyński in :issue:`13521`.) - -* The error messages produced when a function call does not match the function - signature have been significantly improved. - - (Contributed by Benjamin Peterson.) - - -A Finer-Grained Import Lock -=========================== - -Previous versions of CPython have always relied on a global import lock. -This led to unexpected annoyances, such as deadlocks when importing a module -would trigger code execution in a different thread as a side-effect. -Clumsy workarounds were sometimes employed, such as the -:c:func:`PyImport_ImportModuleNoBlock` C API function. - -In Python 3.3, importing a module takes a per-module lock. This correctly -serializes importation of a given module from multiple threads (preventing -the exposure of incompletely initialized modules), while eliminating the -aforementioned annoyances. - -(Contributed by Antoine Pitrou in :issue:`9260`.) - - -Builtin functions and types -=========================== - -* :func:`open` gets a new *opener* parameter: the underlying file descriptor - for the file object is then obtained by calling *opener* with (*file*, - *flags*). It can be used to use custom flags like :data:`os.O_CLOEXEC` for - example. The ``'x'`` mode was added: open for exclusive creation, failing if - the file already exists. -* :func:`print`: added the *flush* keyword argument. If the *flush* keyword - argument is true, the stream is forcibly flushed. -* :func:`hash`: hash randomization is enabled by default, see - :meth:`object.__hash__` and :envvar:`PYTHONHASHSEED`. -* The :class:`str` type gets a new :meth:`~str.casefold` method: return a - casefolded copy of the string, casefolded strings may be used for caseless - matching. For example, ``'ß'.casefold()`` returns ``'ss'``. -* The sequence documentation has been substantially rewritten to better - explain the binary/text sequence distinction and to provide specific - documentation sections for the individual builtin sequence types - (:issue:`4966`). - - -New Modules -=========== - -faulthandler ------------- - -This new debug module :mod:`faulthandler` contains functions to dump Python tracebacks explicitly, -on a fault (a crash like a segmentation fault), after a timeout, or on a user -signal. Call :func:`faulthandler.enable` to install fault handlers for the -:const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and -:const:`SIGILL` signals. You can also enable them at startup by setting the -:envvar:`PYTHONFAULTHANDLER` environment variable or by using :option:`-X` -``faulthandler`` command line option. - -Example of a segmentation fault on Linux: :: - - $ python -q -X faulthandler - >>> import ctypes - >>> ctypes.string_at(0) - Fatal Python error: Segmentation fault - - Current thread 0x00007fb899f39700: - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at - File "", line 1 in - Segmentation fault - - -ipaddress ---------- - -The new :mod:`ipaddress` module provides tools for creating and manipulating -objects representing IPv4 and IPv6 addresses, networks and interfaces (i.e. -an IP address associated with a specific IP subnet). - -(Contributed by Google and Peter Moody in :pep:`3144`.) - -lzma ----- - -The newly-added :mod:`lzma` module provides data compression and decompression -using the LZMA algorithm, including support for the ``.xz`` and ``.lzma`` -file formats. - -(Contributed by Nadeem Vawda and Per Øyvind Karlsen in :issue:`6715`.) - - -Improved Modules -================ +New and Improved Modules +======================== abc --- @@ -921,11 +504,7 @@ * :class:`abc.abstractstaticmethod` has been deprecated, use :class:`staticmethod` with :func:`abc.abstractmethod` instead. -(Contributed by Darren Dale in :issue:`11610`.) - -:meth:`abc.ABCMeta.register` now returns the registered subclass, which means -it can now be used as a class decorator (:issue:`10868`). - +(Contributed by Darren Dale in :issue:`11610`) array ----- @@ -933,23 +512,7 @@ The :mod:`array` module supports the :c:type:`long long` type using ``q`` and ``Q`` type codes. -(Contributed by Oren Tirosh and Hirokazu Yamamoto in :issue:`1172711`.) - - -base64 ------- - -ASCII-only Unicode strings are now accepted by the decoding functions of the -:mod:`base64` modern interface. For example, ``base64.b64decode('YWJj')`` -returns ``b'abc'``. (Contributed by Catalin Iacob in :issue:`13641`.) - - -binascii --------- - -In addition to the binary objects they normally accept, the ``a2b_`` functions -now all also accept ASCII-only strings as input. (Contributed by Antoine -Pitrou in :issue:`13637`.) +(Contributed by Oren Tirosh and Hirokazu Yamamoto in :issue:`1172711`) bz2 @@ -958,20 +521,17 @@ The :mod:`bz2` module has been rewritten from scratch. In the process, several new features have been added: -* New :func:`bz2.open` function: open a bzip2-compressed file in binary or - text mode. - * :class:`bz2.BZ2File` can now read from and write to arbitrary file-like objects, by means of its constructor's *fileobj* argument. - (Contributed by Nadeem Vawda in :issue:`5863`.) + (Contributed by Nadeem Vawda in :issue:`5863`) * :class:`bz2.BZ2File` and :func:`bz2.decompress` can now decompress multi-stream inputs (such as those produced by the :program:`pbzip2` tool). :class:`bz2.BZ2File` can now also be used to create this type of file, using the ``'a'`` (append) mode. - (Contributed by Nir Aides in :issue:`1625`.) + (Contributed by Nir Aides in :issue:`1625`) * :class:`bz2.BZ2File` now implements all of the :class:`io.BufferedIOBase` API, except for the :meth:`detach` and :meth:`truncate` methods. @@ -1012,45 +572,10 @@ The ``unicode_internal`` codec has been deprecated. - -collections ------------ - -Addition of a new :class:`~collections.ChainMap` class to allow treating a -number of mappings as a single unit. (Written by Raymond Hettinger for -:issue:`11089`, made public in :issue:`11297`.) - -The abstract base classes have been moved in a new :mod:`collections.abc` -module, to better differentiate between the abstract and the concrete -collections classes. Aliases for ABCs are still present in the -:mod:`collections` module to preserve existing imports. (:issue:`11085`) - -.. XXX addition of __slots__ to ABCs not recorded here: internal detail - -The :class:`~collections.Counter` class now supports the unary ``+`` and ``-`` -operators, as well as the in-place operators ``+=``, ``-=``, ``|=``, and -``&=``. (Contributed by Raymond Hettinger in :issue:`13121`.) - - -contextlib ----------- - -:class:`~contextlib.ExitStack` now provides a solid foundation for -programmatic manipulation of context managers and similar cleanup -functionality. Unlike the previous ``contextlib.nested`` API (which was -deprecated and removed), the new API is designed to work correctly -regardless of whether context managers acquire their resources in -their ``__init__`` method (for example, file objects) or in their -``__enter__`` method (for example, synchronisation objects from the -:mod:`threading` module). - -(:issue:`13585`) - - crypt ----- -Addition of salt and modular crypt format (hashing method) and the :func:`~crypt.mksalt` +Addition of salt and modular crypt format and the :func:`~crypt.mksalt` function to the :mod:`crypt` module. (:issue:`10924`) @@ -1069,24 +594,7 @@ push a wide character so the next :meth:`~curses.window.get_wch` will return it -(Contributed by Iñigo Serna in :issue:`6755`.) - -datetime --------- - - * Equality comparisons between naive and aware :class:`~datetime.datetime` - instances now return :const:`False` instead of raising :exc:`TypeError` - (:issue:`15006`). - * New :meth:`datetime.datetime.timestamp` method: Return POSIX timestamp - corresponding to the :class:`~datetime.datetime` instance. - * The :meth:`datetime.datetime.strftime` method supports formatting years - older than 1000. - * The :meth:`datetime.datetime.astimezone` method can now be - called without arguments to convert datetime instance to the system - timezone. - - -.. _new-decimal: +(Contributed by Iñigo Serna in :issue:`6755`) decimal ------- @@ -1095,22 +603,16 @@ C-module and libmpdec written by Stefan Krah. The new C version of the decimal module integrates the high speed libmpdec -library for arbitrary precision correctly-rounded decimal floating point -arithmetic. libmpdec conforms to IBM's General Decimal Arithmetic Specification. +library for arbitrary precision correctly-rounded decimal arithmetic. +libmpdec conforms to IBM's General Decimal Arithmetic Specification. -Performance gains range from 10x for database applications to 100x for -numerically intensive applications. These numbers are expected gains -for standard precisions used in decimal floating point arithmetic. Since -the precision is user configurable, the exact figures may vary. For example, -in integer bignum arithmetic the differences can be significantly higher. - -The following table is meant as an illustration. Benchmarks are available -at http://www.bytereef.org/mpdecimal/quickstart.html. +Performance gains range from 12x for database applications to 80x for +numerically intensive applications: +---------+-------------+--------------+-------------+ | | decimal.py | _decimal | speedup | +=========+=============+==============+=============+ - | pi | 42.02s | 0.345s | 120x | + | pi | 42.75s | 0.58s | 74x | +---------+-------------+--------------+-------------+ | telco | 172.19s | 5.68s | 30x | +---------+-------------+--------------+-------------+ @@ -1125,7 +627,7 @@ * If Python is compiled without threads, the C version automatically disables the expensive thread local context machinery. In this case, - the variable :data:`~decimal.HAVE_THREADS` is set to ``False``. + the variable :data:`~decimal.HAVE_THREADS` is set to False. API changes ~~~~~~~~~~~ @@ -1181,268 +683,23 @@ changed to match the order displayed by :func:`repr`. -* The ``watchexp`` parameter in the :meth:`~decimal.Decimal.quantize` method - is deprecated. +faulthandler +------------ +New module: :mod:`faulthandler`. -.. _new-email: - -email ------ - -Policy Framework -~~~~~~~~~~~~~~~~ - -The email package now has a :mod:`~email.policy` framework. A -:class:`~email.policy.Policy` is an object with several methods and properties -that control how the email package behaves. The primary policy for Python 3.3 -is the :class:`~email.policy.Compat32` policy, which provides backward -compatibility with the email package in Python 3.2. A ``policy`` can be -specified when an email message is parsed by a :mod:`~email.parser`, or when a -:class:`~email.message.Message` object is created, or when an email is -serialized using a :mod:`~email.generator`. Unless overridden, a policy passed -to a ``parser`` is inherited by all the ``Message`` object and sub-objects -created by the ``parser``. By default a ``generator`` will use the policy of -the ``Message`` object it is serializing. The default policy is -:data:`~email.policy.compat32`. - -The minimum set of controls implemented by all ``policy`` objects are: - - .. tabularcolumns:: |l|L| - - =============== ======================================================= - max_line_length The maximum length, excluding the linesep character(s), - individual lines may have when a ``Message`` is - serialized. Defaults to 78. - - linesep The character used to separate individual lines when a - ``Message`` is serialized. Defaults to ``\n``. - - cte_type ``7bit`` or ``8bit``. ``8bit`` applies only to a - ``Bytes`` ``generator``, and means that non-ASCII may - be used where allowed by the protocol (or where it - exists in the original input). - - raise_on_defect Causes a ``parser`` to raise error when defects are - encountered instead of adding them to the ``Message`` - object's ``defects`` list. - =============== ======================================================= - -A new policy instance, with new settings, is created using the -:meth:`~email.policy.Policy.clone` method of policy objects. ``clone`` takes -any of the above controls as keyword arguments. Any control not specified in -the call retains its default value. Thus you can create a policy that uses -``\r\n`` linesep characters like this:: - - mypolicy = compat32.clone(linesep='\r\n') - -Policies can be used to make the generation of messages in the format needed by -your application simpler. Instead of having to remember to specify -``linesep='\r\n'`` in all the places you call a ``generator``, you can specify -it once, when you set the policy used by the ``parser`` or the ``Message``, -whichever your program uses to create ``Message`` objects. On the other hand, -if you need to generate messages in multiple forms, you can still specify the -parameters in the appropriate ``generator`` call. Or you can have custom -policy instances for your different cases, and pass those in when you create -the ``generator``. - - -Provisional Policy with New Header API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -While the policy framework is worthwhile all by itself, the main motivation for -introducing it is to allow the creation of new policies that implement new -features for the email package in a way that maintains backward compatibility -for those who do not use the new policies. Because the new policies introduce a -new API, we are releasing them in Python 3.3 as a :term:`provisional policy -`. Backwards incompatible changes (up to and including -removal of the code) may occur if deemed necessary by the core developers. - -The new policies are instances of :class:`~email.policy.EmailPolicy`, -and add the following additional controls: - - .. tabularcolumns:: |l|L| - - =============== ======================================================= - refold_source Controls whether or not headers parsed by a - :mod:`~email.parser` are refolded by the - :mod:`~email.generator`. It can be ``none``, ``long``, - or ``all``. The default is ``long``, which means that - source headers with a line longer than - ``max_line_length`` get refolded. ``none`` means no - line get refolded, and ``all`` means that all lines - get refolded. - - header_factory A callable that take a ``name`` and ``value`` and - produces a custom header object. - =============== ======================================================= - -The ``header_factory`` is the key to the new features provided by the new -policies. When one of the new policies is used, any header retrieved from -a ``Message`` object is an object produced by the ``header_factory``, and any -time you set a header on a ``Message`` it becomes an object produced by -``header_factory``. All such header objects have a ``name`` attribute equal -to the header name. Address and Date headers have additional attributes -that give you access to the parsed data of the header. This means you can now -do things like this:: - - >>> m = Message(policy=SMTP) - >>> m['To'] = 'Éric ' - >>> m['to'] - 'Éric ' - >>> m['to'].addresses - (Address(display_name='Éric', username='foo', domain='example.com'),) - >>> m['to'].addresses[0].username - 'foo' - >>> m['to'].addresses[0].display_name - 'Éric' - >>> m['Date'] = email.utils.localtime() - >>> m['Date'].datetime - datetime.datetime(2012, 5, 25, 21, 39, 24, 465484, tzinfo=datetime.timezone(datetime.timedelta(-1, 72000), 'EDT')) - >>> m['Date'] - 'Fri, 25 May 2012 21:44:27 -0400' - >>> print(m) - To: =?utf-8?q?=C3=89ric?= - Date: Fri, 25 May 2012 21:44:27 -0400 - -You will note that the unicode display name is automatically encoded as -``utf-8`` when the message is serialized, but that when the header is accessed -directly, you get the unicode version. This eliminates any need to deal with -the :mod:`email.header` :meth:`~email.header.decode_header` or -:meth:`~email.header.make_header` functions. - -You can also create addresses from parts:: - - >>> m['cc'] = [Group('pals', [Address('Bob', 'bob', 'example.com'), - ... Address('Sally', 'sally', 'example.com')]), - ... Address('Bonzo', addr_spec='bonz@laugh.com')] - >>> print(m) - To: =?utf-8?q?=C3=89ric?= - Date: Fri, 25 May 2012 21:44:27 -0400 - cc: pals: Bob , Sally ;, Bonzo - -Decoding to unicode is done automatically:: - - >>> m2 = message_from_string(str(m)) - >>> m2['to'] - 'Éric ' - -When you parse a message, you can use the ``addresses`` and ``groups`` -attributes of the header objects to access the groups and individual -addresses:: - - >>> m2['cc'].addresses - (Address(display_name='Bob', username='bob', domain='example.com'), Address(display_name='Sally', username='sally', domain='example.com'), Address(display_name='Bonzo', username='bonz', domain='laugh.com')) - >>> m2['cc'].groups - (Group(display_name='pals', addresses=(Address(display_name='Bob', username='bob', domain='example.com'), Address(display_name='Sally', username='sally', domain='example.com')), Group(display_name=None, addresses=(Address(display_name='Bonzo', username='bonz', domain='laugh.com'),)) - -In summary, if you use one of the new policies, header manipulation works the -way it ought to: your application works with unicode strings, and the email -package transparently encodes and decodes the unicode to and from the RFC -standard Content Transfer Encodings. - -Other API Changes -~~~~~~~~~~~~~~~~~ - -New :class:`~email.parser.BytesHeaderParser`, added to the :mod:`~email.parser` -module to complement :class:`~email.parser.HeaderParser` and complete the Bytes -API. - -New utility functions: - - * :func:`~email.utils.format_datetime`: given a :class:`~datetime.datetime`, - produce a string formatted for use in an email header. - - * :func:`~email.utils.parsedate_to_datetime`: given a date string from - an email header, convert it into an aware :class:`~datetime.datetime`, - or a naive :class:`~datetime.datetime` if the offset is ``-0000``. - - * :func:`~email.utils.localtime`: With no argument, returns the - current local time as an aware :class:`~datetime.datetime` using the local - :class:`~datetime.timezone`. Given an aware :class:`~datetime.datetime`, - converts it into an aware :class:`~datetime.datetime` using the - local :class:`~datetime.timezone`. - + * :envvar:`PYTHONFAULTHANDLER` + * :option:`-X` ``faulthandler`` ftplib ------ -* :class:`ftplib.FTP` now accepts a ``source_address`` keyword argument to - specify the ``(host, port)`` to use as the source address in the bind call - when creating the outgoing socket. (Contributed by Giampaolo Rodolà - in :issue:`8594`.) +The :class:`~ftplib.FTP_TLS` class now provides a new +:func:`~ftplib.FTP_TLS.ccc` function to revert control channel back to +plaintext. This can be useful to take advantage of firewalls that know how to +handle NAT with non-secure FTP without opening fixed ports. -* The :class:`~ftplib.FTP_TLS` class now provides a new - :func:`~ftplib.FTP_TLS.ccc` function to revert control channel back to - plaintext. This can be useful to take advantage of firewalls that know how - to handle NAT with non-secure FTP without opening fixed ports. (Contributed - by Giampaolo Rodolà in :issue:`12139`.) - -* Added :meth:`ftplib.FTP.mlsd` method which provides a parsable directory - listing format and deprecates :meth:`ftplib.FTP.nlst` and - :meth:`ftplib.FTP.dir`. (Contributed by Giampaolo Rodolà in :issue:`11072`.) - - -functools ---------- - -The :func:`functools.lru_cache` decorator now accepts a ``typed`` keyword -argument (that defaults to ``False`` to ensure that it caches values of -different types that compare equal in separate cache slots. (Contributed -by Raymond Hettinger in :issue:`13227`.) - - -gc --- - -It is now possible to register callbacks invoked by the garbage collector -before and after collection using the new :data:`~gc.callbacks` list. - - -hmac ----- - -A new :func:`~hmac.compare_digest` function has been added to prevent side -channel attacks on digests through timing analysis. (Contributed by Nick -Coghlan and Christian Heimes in :issue:`15061`.) - - -http ----- - -:class:`http.server.BaseHTTPRequestHandler` now buffers the headers and writes -them all at once when :meth:`~http.server.BaseHTTPRequestHandler.end_headers` is -called. A new method :meth:`~http.server.BaseHTTPRequestHandler.flush_headers` -can be used to directly manage when the accumlated headers are sent. -(Contributed by Andrew Schaaf in :issue:`3709`.) - -:class:`http.server` now produces valid ``HTML 4.01 strict`` output. -(Contributed by Ezio Melotti in :issue:`13295`.) - -:class:`http.client.HTTPResponse` now has a -:meth:`~http.client.HTTPResponse.readinto` method, which means it can be used -as an :class:`io.RawIOBase` class. (Contributed by John Kuhn in -:issue:`13464`.) - - -html ----- - -:class:`html.parser.HTMLParser` is now able to parse broken markup without -raising errors, therefore the *strict* argument of the constructor and the -:exc:`~html.parser.HTMLParseError` exception are now deprecated. -The ability to parse broken markup is the result of a number of bug fixes that -are also available on the latest bug fix releases of Python 2.7/3.2. -(Contributed by Ezio Melotti in :issue:`15114`, and :issue:`14538`, -:issue:`13993`, :issue:`13960`, :issue:`13358`, :issue:`1745761`, -:issue:`755670`, :issue:`13357`, :issue:`12629`, :issue:`1200313`, -:issue:`670664`, :issue:`13273`, :issue:`12888`, :issue:`7311`.) - -A new :data:`~html.entities.html5` dictionary that maps HTML5 named character -references to the equivalent Unicode character(s) (e.g. ``html5['gt;'] == -'>'``) has been added to the :mod:`html.entities` module. The dictionary is -now also used by :class:`~html.parser.HTMLParser`. (Contributed by Ezio -Melotti in :issue:`11113` and :issue:`15156`.) +(Contributed by Giampaolo Rodolà in :issue:`12139`) imaplib @@ -1451,26 +708,9 @@ The :class:`~imaplib.IMAP4_SSL` constructor now accepts an SSLContext parameter to control parameters of the secure channel. -(Contributed by Sijin Joseph in :issue:`8808`.) +(Contributed by Sijin Joseph in :issue:`8808`) -inspect -------- - -A new :func:`~inspect.getclosurevars` function has been added. This function -reports the current binding of all names referenced from the function body and -where those names were resolved, making it easier to verify correct internal -state when testing code that relies on stateful closures. - -(Contributed by Meador Inge and Nick Coghlan in :issue:`13062`.) - -A new :func:`~inspect.getgeneratorlocals` function has been added. This -function reports the current binding of local variables in the generator's -stack frame, making it easier to verify correct internal state when testing -generators. - -(Contributed by Meador Inge in :issue:`15153`.) - io -- @@ -1478,85 +718,32 @@ exclusively create a new file, and raise a :exc:`FileExistsError` if the file already exists. It is based on the C11 'x' mode to fopen(). -(Contributed by David Townshend in :issue:`12760`.) +(Contributed by David Townshend in :issue:`12760`) -The constructor of the :class:`~io.TextIOWrapper` class has a new -*write_through* optional argument. If *write_through* is ``True``, calls to -:meth:`~io.TextIOWrapper.write` are guaranteed not to be buffered: any data -written on the :class:`~io.TextIOWrapper` object is immediately handled to its -underlying binary buffer. +lzma +---- -itertools ---------- +The newly-added :mod:`lzma` module provides data compression and decompression +using the LZMA algorithm, including support for the ``.xz`` and ``.lzma`` +file formats. -:func:`~itertools.accumulate` now takes an optional ``func`` argument for -providing a user-supplied binary function. - - -logging -------- - -The :func:`~logging.basicConfig` function now supports an optional ``handlers`` -argument taking an iterable of handlers to be added to the root logger. - -A class level attribute :attr:`~logging.handlers.SysLogHandler.append_nul` has -been added to :class:`~logging.handlers.SysLogHandler` to allow control of the -appending of the ``NUL`` (``\000``) byte to syslog records, since for some -deamons it is required while for others it is passed through to the log. - +(Contributed by Nadeem Vawda and Per Øyvind Karlsen in :issue:`6715`) math ---- -The :mod:`math` module has a new function, :func:`~math.log2`, which returns -the base-2 logarithm of *x*. +The :mod:`math` module has a new function: -(Written by Mark Dickinson in :issue:`11888`.) - - -mmap ----- - -The :meth:`~mmap.mmap.read` method is now more compatible with other file-like -objects: if the argument is omitted or specified as ``None``, it returns the -bytes from the current file position to the end of the mapping. (Contributed -by Petri Lehtinen in :issue:`12021`.) - - -multiprocessing ---------------- - -The new :func:`multiprocessing.connection.wait` function allows to poll -multiple objects (such as connections, sockets and pipes) with a timeout. -(Contributed by Richard Oudkerk in :issue:`12328`.) - -:class:`multiprocessing.Connection` objects can now be transferred over -multiprocessing connections. -(Contributed by Richard Oudkerk in :issue:`4892`.) - -:class:`multiprocessing.Process` now accepts a ``daemon`` keyword argument -to override the default behavior of inheriting the ``daemon`` flag from -the parent process (:issue:`6064`). - -New attribute :data:`multiprocessing.Process.sentinel` allows a -program to wait on multiple :class:`~multiprocessing.Process` objects at one -time using the appropriate OS primitives (for example, :mod:`select` on -posix systems). - -New methods :meth:`multiprocessing.pool.Pool.starmap` and -:meth:`~multiprocessing.pool.Pool.starmap_async` provide -:func:`itertools.starmap` equivalents to the existing -:meth:`multiprocessing.pool.Pool.map` and -:meth:`~multiprocessing.pool.Pool.map_async` functions. (Contributed by Hynek -Schlawack in :issue:`12708`.) + * :func:`~math.log2`: return the base-2 logarithm of *x* + (Written by Mark Dickinson in :issue:`11888`). nntplib ------- -The :class:`nntplib.NNTP` class now supports the context management protocol to +The :class:`nntplib.NNTP` class now supports the context manager protocol to unconditionally consume :exc:`socket.error` exceptions and to close the NNTP connection when done:: @@ -1567,7 +754,7 @@ ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') >>> -(Contributed by Giampaolo Rodolà in :issue:`9795`.) +(Contributed by Giampaolo Rodolà in :issue:`9795`) os @@ -1579,7 +766,7 @@ avoid race conditions in multi-threaded programs. * The :mod:`os` module has a new :func:`~os.sendfile` function which provides - an efficient "zero-copy" way for copying data from one file (or socket) + an efficent "zero-copy" way for copying data from one file (or socket) descriptor to another. The phrase "zero-copy" refers to the fact that all of the copying of data between the two descriptors is done entirely by the kernel, with no copying of data into userspace buffers. :func:`~os.sendfile` @@ -1588,38 +775,6 @@ (Patch submitted by Ross Lagerwall and Giampaolo Rodolà in :issue:`10882`.) -* To avoid race conditions like symlink attacks and issues with temporary - files and directories, it is more reliable (and also faster) to manipulate - file descriptors instead of file names. Python 3.3 enhances existing functions - and introduces new functions to work on file descriptors (:issue:`4761`, - :issue:`10755` and :issue:`14626`). - - - The :mod:`os` module has a new :func:`~os.fwalk` function similar to - :func:`~os.walk` except that it also yields file descriptors referring to the - directories visited. This is especially useful to avoid symlink races. - - - The following functions get new optional *dir_fd* (:ref:`paths relative to - directory descriptors `) and/or *follow_symlinks* (:ref:`not - following symlinks `): - :func:`~os.access`, :func:`~os.chflags`, :func:`~os.chmod`, :func:`~os.chown`, - :func:`~os.link`, :func:`~os.lstat`, :func:`~os.mkdir`, :func:`~os.mkfifo`, - :func:`~os.mknod`, :func:`~os.open`, :func:`~os.readlink`, :func:`~os.remove`, - :func:`~os.rename`, :func:`~os.replace`, :func:`~os.rmdir`, :func:`~os.stat`, - :func:`~os.symlink`, :func:`~os.unlink`, :func:`~os.utime`. Platform - support for using these parameters can be checked via the sets - :data:`os.supports_dir_fd` and :data:`os.supports_follows_symlinks`. - - - The following functions now support a file descriptor for their path argument: - :func:`~os.chdir`, :func:`~os.chmod`, :func:`~os.chown`, - :func:`~os.execve`, :func:`~os.listdir`, :func:`~os.pathconf`, :func:`~os.path.exists`, - :func:`~os.stat`, :func:`~os.statvfs`, :func:`~os.utime`. Platform support - for this can be checked via the :data:`os.supports_fd` set. - -* :func:`~os.access` accepts an ``effective_ids`` keyword argument to turn on - using the effective uid/gid rather than the real uid/gid in the access check. - Platform support for this can be checked via the - :data:`~os.supports_effective_ids` set. - * The :mod:`os` module has two new functions: :func:`~os.getpriority` and :func:`~os.setpriority`. They can be used to get or set process niceness/priority in a fashion similar to :func:`os.nice` but extended to all @@ -1627,98 +782,98 @@ (Patch submitted by Giampaolo Rodolà in :issue:`10784`.) -* The new :func:`os.replace` function allows cross-platform renaming of a - file with overwriting the destination. With :func:`os.rename`, an existing - destination file is overwritten under POSIX, but raises an error under - Windows. - (Contributed by Antoine Pitrou in :issue:`8828`.) +* The :mod:`os` module has a new :func:`~os.fwalk` function similar to + :func:`~os.walk` except that it also yields file descriptors referring to the + directories visited. This is especially useful to avoid symlink races. -* The stat family of functions (:func:`~os.stat`, :func:`~os.fstat`, - and :func:`~os.lstat`) now support reading a file's timestamps - with nanosecond precision. Symmetrically, :func:`~os.utime` - can now write file timestamps with nanosecond precision. (Contributed by - Larry Hastings in :issue:`14127`.) +* "at" functions (:issue:`4761`): -* The new :func:`os.get_terminal_size` function queries the size of the - terminal attached to a file descriptor. See also - :func:`shutil.get_terminal_size`. - (Contributed by Zbigniew Jędrzejewski-Szmek in :issue:`13609`.) + * :func:`~os.faccessat` + * :func:`~os.fchmodat` + * :func:`~os.fchownat` + * :func:`~os.fstatat` + * :func:`~os.futimesat` + * :func:`~os.linkat` + * :func:`~os.mkdirat` + * :func:`~os.mkfifoat` + * :func:`~os.mknodat` + * :func:`~os.openat` + * :func:`~os.readlinkat` + * :func:`~os.renameat` + * :func:`~os.symlinkat` + * :func:`~os.unlinkat` + * :func:`~os.utimensat` -.. XXX sort out this mess after beta1 +* extended attributes (:issue:`12720`): -* New functions to support Linux extended attributes (:issue:`12720`): - :func:`~os.getxattr`, :func:`~os.listxattr`, :func:`~os.removexattr`, - :func:`~os.setxattr`. + * :func:`~os.fgetxattr` + * :func:`~os.flistxattr` + * :func:`~os.fremovexattr` + * :func:`~os.fsetxattr` + * :func:`~os.getxattr` + * :func:`~os.lgetxattr` + * :func:`~os.listxattr` + * :func:`~os.llistxattr` + * :func:`~os.lremovexattr` + * :func:`~os.lsetxattr` + * :func:`~os.removexattr` + * :func:`~os.setxattr` -* New interface to the scheduler. These functions - control how a process is allocated CPU time by the operating system. New - functions: - :func:`~os.sched_get_priority_max`, :func:`~os.sched_get_priority_min`, - :func:`~os.sched_getaffinity`, :func:`~os.sched_getparam`, - :func:`~os.sched_getscheduler`, :func:`~os.sched_rr_get_interval`, - :func:`~os.sched_setaffinity`, :func:`~os.sched_setparam`, - :func:`~os.sched_setscheduler`, :func:`~os.sched_yield`, +* Scheduler functions (:issue:`12655`): -* New functions to control the file system: + * :func:`~os.sched_get_priority_max` + * :func:`~os.sched_get_priority_min` + * :func:`~os.sched_getaffinity` + * :func:`~os.sched_getparam` + * :func:`~os.sched_getscheduler` + * :func:`~os.sched_rr_get_interval` + * :func:`~os.sched_setaffinity` + * :func:`~os.sched_setparam` + * :func:`~os.sched_setscheduler` + * :func:`~os.sched_yield` - * :func:`~os.posix_fadvise`: Announces an intention to access data in a - specific pattern thus allowing the kernel to make optimizations. - * :func:`~os.posix_fallocate`: Ensures that enough disk space is allocated - for a file. - * :func:`~os.sync`: Force write of everything to disk. +* Add some extra posix functions to the os module (:issue:`10812`): -* Additional new posix functions: + * :func:`~os.fexecve` + * :func:`~os.futimens` + * :func:`~os.futimes` + * :func:`~os.lockf` + * :func:`~os.lutimes` + * :func:`~os.posix_fadvise` + * :func:`~os.posix_fallocate` + * :func:`~os.pread` + * :func:`~os.pwrite` + * :func:`~os.readv` + * :func:`~os.sync` + * :func:`~os.truncate` + * :func:`~os.waitid` + * :func:`~os.writev` - * :func:`~os.lockf`: Apply, test or remove a POSIX lock on an open file descriptor. - * :func:`~os.pread`: Read from a file descriptor at an offset, the file - offset remains unchanged. - * :func:`~os.pwrite`: Write to a file descriptor from an offset, leaving - the file offset unchanged. - * :func:`~os.readv`: Read from a file descriptor into a number of writable buffers. - * :func:`~os.truncate`: Truncate the file corresponding to *path*, so that - it is at most *length* bytes in size. - * :func:`~os.waitid`: Wait for the completion of one or more child processes. - * :func:`~os.writev`: Write the contents of *buffers* to a file descriptor, - where *buffers* is an arbitrary sequence of buffers. - * :func:`~os.getgrouplist` (:issue:`9344`): Return list of group ids that - specified user belongs to. +* Other new functions: -* :func:`~os.times` and :func:`~os.uname`: Return type changed from a tuple to - a tuple-like object with named attributes. + * :func:`~os.flistdir` (:issue:`10755`) + * :func:`~os.getgrouplist` (:issue:`9344`) -* Some platforms now support additional constants for the :func:`~os.lseek` - function, such as ``os.SEEK_HOLE`` and ``os.SEEK_DATA``. -* New constants :data:`~os.RTLD_LAZY`, :data:`~os.RTLD_NOW`, - :data:`~os.RTLD_GLOBAL`, :data:`~os.RTLD_LOCAL`, :data:`~os.RTLD_NODELETE`, - :data:`~os.RTLD_NOLOAD`, and :data:`~os.RTLD_DEEPBIND` are available on - platforms that support them. These are for use with the - :func:`sys.setdlopenflags` function, and supersede the similar constants - defined in :mod:`ctypes` and :mod:`DLFCN`. (Contributed by Victor Stinner - in :issue:`13226`.) +packaging +--------- -* :func:`os.symlink` now accepts (and ignores) the ``target_is_directory`` - keyword argument on non-Windows platforms, to ease cross-platform support. +:mod:`distutils` has undergone additions and refactoring under a new name, +:mod:`packaging`, to allow developers to break backward compatibility. +:mod:`distutils` is still provided in the standard library, but users are +encouraged to transition to :mod:`packaging`. For older versions of Python, a +backport compatible with 2.4+ and 3.1+ will be made available on PyPI under the +name :mod:`distutils2`. + +.. TODO add examples and howto to the packaging docs and link to them pdb --- -Tab-completion is now available not only for command names, but also their -arguments. For example, for the ``break`` command, function and file names -are completed. - -(Contributed by Georg Brandl in :issue:`14210`) - - -pickle ------- - -:class:`pickle.Pickler` objects now have an optional -:attr:`~pickle.Pickler.dispatch_table` attribute allowing to set per-pickler -reduction functions. - -(Contributed by Richard Oudkerk in :issue:`14166`.) +* Tab-completion is now available not only for command names, but also their + arguments. For example, for the ``break`` command, function and file names + are completed. (Contributed by Georg Brandl in :issue:`14210`) pydoc @@ -1729,14 +884,6 @@ in Python 3.2. -re --- - -:class:`str` regular expressions now support ``\u`` and ``\U`` escapes. - -(Contributed by Serhiy Storchaka in :issue:`3665`.) - - sched ----- @@ -1744,77 +891,36 @@ set to False makes the method execute the scheduled events due to expire soonest (if any) and then return immediately. This is useful in case you want to use the :class:`~sched.scheduler` in - non-blocking applications. (Contributed by Giampaolo Rodolà in :issue:`13449`.) + non-blocking applications. (Contributed by Giampaolo Rodolà in :issue:`13449`) * :class:`~sched.scheduler` class can now be safely used in multi-threaded environments. (Contributed by Josiah Carlson and Giampaolo Rodolà in - :issue:`8684`.) + :issue:`8684`) * *timefunc* and *delayfunct* parameters of :class:`~sched.scheduler` class constructor are now optional and defaults to :func:`time.time` and :func:`time.sleep` respectively. (Contributed by Chris Clark in - :issue:`13245`.) + :issue:`13245`) * :meth:`~sched.scheduler.enter` and :meth:`~sched.scheduler.enterabs` *argument* parameter is now optional. (Contributed by Chris Clark in - :issue:`13245`.) + :issue:`13245`) * :meth:`~sched.scheduler.enter` and :meth:`~sched.scheduler.enterabs` now accept a *kwargs* parameter. (Contributed by Chris Clark in - :issue:`13245`.) - - -select ------- - -Solaris and derivative platforms have a new class :class:`select.devpoll` -for high performance asynchronous sockets via :file:`/dev/poll`. -(Contributed by Jesús Cea Avión in :issue:`6397`.) - - -shlex ------ - -The previously undocumented helper function ``quote`` from the -:mod:`pipes` modules has been moved to the :mod:`shlex` module and -documented. :func:`~shlex.quote` properly escapes all characters in a string -that might be otherwise given special meaning by the shell. + :issue:`13245`) shutil ------ -* New functions: +* The :mod:`shutil` module has these new fuctions: * :func:`~shutil.disk_usage`: provides total, used and free disk space - statistics. (Contributed by Giampaolo Rodolà in :issue:`12442`.) + statistics. (Contributed by Giampaolo Rodolà in :issue:`12442`) * :func:`~shutil.chown`: allows one to change user and/or group of the given path also specifying the user/group names and not only their numeric - ids. (Contributed by Sandro Tosi in :issue:`12191`.) - * :func:`shutil.get_terminal_size`: returns the size of the terminal window - to which the interpreter is attached. (Contributed by Zbigniew - Jędrzejewski-Szmek in :issue:`13609`.) - -* :func:`~shutil.copy2` and :func:`~shutil.copystat` now preserve file - timestamps with nanosecond precision on platforms that support it. - They also preserve file "extended attributes" on Linux. (Contributed - by Larry Hastings in :issue:`14127` and :issue:`15238`.) - -* Several functions now take an optional ``symlinks`` argument: when that - parameter is true, symlinks aren't dereferenced and the operation instead - acts on the symlink itself (or creates one, if relevant). - (Contributed by Hynek Schlawack in :issue:`12715`.) - -* When copying files to a different file system, :func:`~shutil.move` now - handles symlinks the way the posix ``mv`` command does, recreating the - symlink rather than copying the target file contents. (Contributed by - Jonathan Niehof in :issue:`9993`.) :func:`~shutil.move` now also returns - the ``dst`` argument as its result. - -* :func:`~shutil.rmtree` is now resistant to symlink attacks on platforms - which support the new ``dir_fd`` parameter in :func:`os.open` and - :func:`os.unlink`. (Contributed by Martin von Löwis and Hynek Schlawack - in :issue:`4489`.) + ids. (Contributed by Sandro Tosi in :issue:`12191`) signal @@ -1823,12 +929,12 @@ * The :mod:`signal` module has new functions: * :func:`~signal.pthread_sigmask`: fetch and/or change the signal mask of the - calling thread (Contributed by Jean-Paul Calderone in :issue:`8407`); - * :func:`~signal.pthread_kill`: send a signal to a thread; - * :func:`~signal.sigpending`: examine pending functions; - * :func:`~signal.sigwait`: wait a signal; + calling thread (Contributed by Jean-Paul Calderone in :issue:`8407`) ; + * :func:`~signal.pthread_kill`: send a signal to a thread ; + * :func:`~signal.sigpending`: examine pending functions ; + * :func:`~signal.sigwait`: wait a signal. * :func:`~signal.sigwaitinfo`: wait for a signal, returning detailed - information about it; + information about it. * :func:`~signal.sigtimedwait`: like :func:`~signal.sigwaitinfo` but with a timeout. @@ -1839,35 +945,14 @@ * :func:`signal.signal` and :func:`signal.siginterrupt` raise an OSError, instead of a RuntimeError: OSError has an errno attribute. - -smtpd ------ - -The :mod:`smtpd` module now supports :rfc:`5321` (extended SMTP) and :rfc:`1870` -(size extension). Per the standard, these extensions are enabled if and only -if the client initiates the session with an ``EHLO`` command. - -(Initial ``ELHO`` support by Alberto Trevino. Size extension by Juhana -Jauhiainen. Substantial additional work on the patch contributed by Michele -Orrù and Dan Boswell. :issue:`8739`) - - smtplib ------- -The :class:`~smtplib.SMTP`, :class:`~smtplib.SMTP_SSL`, and -:class:`~smtplib.LMTP` classes now accept a ``source_address`` keyword argument -to specify the ``(host, port)`` to use as the source address in the bind call -when creating the outgoing socket. (Contributed by Paulo Scardine in -:issue:`11281`.) - -:class:`~smtplib.SMTP` now supports the context management protocol, allowing an -``SMTP`` instance to be used in a ``with`` statement. (Contributed -by Giampaolo Rodolà in :issue:`11289`.) - The :class:`~smtplib.SMTP_SSL` constructor and the :meth:`~smtplib.SMTP.starttls` method now accept an SSLContext parameter to control parameters of the secure -channel. (Contributed by Kasun Herath in :issue:`8809`.) +channel. + +(Contributed by Kasun Herath in :issue:`8809`) socket @@ -1887,37 +972,11 @@ (http://en.wikipedia.org/wiki/Socketcan), on Linux (http://lwn.net/Articles/253425). - (Contributed by Matthias Fuchs, updated by Tiago Gonçalves in :issue:`10141`.) + (Contributed by Matthias Fuchs, updated by Tiago Gonçalves in :issue:`10141`) * The :class:`~socket.socket` class now supports the PF_RDS protocol family (http://en.wikipedia.org/wiki/Reliable_Datagram_Sockets and - https://oss.oracle.com/projects/rds/). - -* The :class:`~socket.socket` class now supports the ``PF_SYSTEM`` protocol - family on OS X. (Contributed by Michael Goderbauer in :issue:`13777`.) - -* New function :func:`~socket.sethostname` allows the hostname to be set - on unix systems if the calling process has sufficient privileges. - (Contributed by Ross Lagerwall in :issue:`10866`.) - - -socketserver ------------- - -:class:`~socketserver.BaseServer` now has an overridable method -:meth:`~socketserver.BaseServer.service_actions` that is called by the -:meth:`~socketserver.BaseServer.serve_forever` method in the service loop. -:class:`~socketserver.ForkingMixIn` now uses this to clean up zombie -child processes. (Contributed by Justin Warkentin in :issue:`11109`.) - - -sqlite3 -------- - -New :class:`sqlite3.Connection` method -:meth:`~sqlite3.Connection.set_trace_callback` can be used to capture a trace of -all sql commands processed by sqlite. (Contributed by Torsten Landschoff -in :issue:`11688`.) + http://oss.oracle.com/projects/rds/). ssl @@ -1929,171 +988,55 @@ pseudo-random bytes. * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes. - (Contributed by Victor Stinner in :issue:`12049`.) + (Contributed by Victor Stinner in :issue:`12049`) * The :mod:`ssl` module now exposes a finer-grained exception hierarchy in order to make it easier to inspect the various kinds of errors. - (Contributed by Antoine Pitrou in :issue:`11183`.) + + (Contributed by Antoine Pitrou in :issue:`11183`) * :meth:`~ssl.SSLContext.load_cert_chain` now accepts a *password* argument to be used if the private key is encrypted. - (Contributed by Adam Simpkins in :issue:`12803`.) + + (Contributed by Adam Simpkins in :issue:`12803`) * Diffie-Hellman key exchange, both regular and Elliptic Curve-based, is now supported through the :meth:`~ssl.SSLContext.load_dh_params` and :meth:`~ssl.SSLContext.set_ecdh_curve` methods. - (Contributed by Antoine Pitrou in :issue:`13626` and :issue:`13627`.) + + (Contributed by Antoine Pitrou in :issue:`13626` and :issue:`13627`) * SSL sockets have a new :meth:`~ssl.SSLSocket.get_channel_binding` method allowing the implementation of certain authentication mechanisms such as - SCRAM-SHA-1-PLUS. (Contributed by Jacek Konieczny in :issue:`12551`.) + SCRAM-SHA-1-PLUS. + + (Contributed by Jacek Konieczny in :issue:`12551`) * You can query the SSL compression algorithm used by an SSL socket, thanks - to its new :meth:`~ssl.SSLSocket.compression` method. The new attribute - :attr:`~ssl.OP_NO_COMPRESSION` can be used to disable compression. - (Contributed by Antoine Pitrou in :issue:`13634`.) + to its new :meth:`~ssl.SSLSocket.compression` method. -* Support has been added for the Next Procotol Negotiation extension using - the :meth:`ssl.SSLContext.set_npn_protocols` method. - (Contributed by Colin Marc in :issue:`14204`.) - -* SSL errors can now be introspected more easily thanks to - :attr:`~ssl.SSLError.library` and :attr:`~ssl.SSLError.reason` attributes. - (Contributed by Antoine Pitrou in :issue:`14837`.) - -* The :func:`~ssl.get_server_certificate` function now supports IPv6. - (Contributed by Charles-François Natali in :issue:`11811`.) - -* New attribute :attr:`~ssl.OP_CIPHER_SERVER_PREFERENCE` allows setting - SSLv3 server sockets to use the server's cipher ordering preference rather - than the client's (:issue:`13635`). - - -stat ----- - -The undocumented tarfile.filemode function has been moved to -:func:`stat.filemode`. It can be used to convert a file's mode to a string of -the form '-rwxrwxrwx'. - -(Contributed by Giampaolo Rodolà in :issue:`14807`.) - - -struct ------- - -The :mod:`struct` module now supports ``ssize_t`` and ``size_t`` via the -new codes ``n`` and ``N``, respectively. (Contributed by Antoine Pitrou -in :issue:`3163`.) - - -subprocess ----------- - -Command strings can now be bytes objects on posix platforms. (Contributed by -Victor Stinner in :issue:`8513`.) - -A new constant :data:`~subprocess.DEVNULL` allows suppressing output in a -platform-independent fashion. (Contributed by Ross Lagerwall in -:issue:`5870`.) + (Contributed by Antoine Pitrou in :issue:`13634`) sys --- -The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct -sequence` holding informations about the thread implementation -(:issue:`11223`). +* The :mod:`sys` module has a new :data:`~sys.thread_info` :term:`struct + sequence` holding informations about the thread implementation. - -tarfile -------- - -:mod:`tarfile` now supports ``lzma`` encoding via the :mod:`lzma` module. -(Contributed by Lars Gustäbel in :issue:`5689`.) - - -tempfile --------- - -:class:`tempfile.SpooledTemporaryFile`\'s -:meth:`~tempfile.SpooledTemporaryFile.truncate` method now accepts -a ``size`` parameter. (Contributed by Ryan Kelly in :issue:`9957`.) - - -textwrap --------- - -The :mod:`textwrap` module has a new :func:`~textwrap.indent` that makes -it straightforward to add a common prefix to selected lines in a block -of text (:issue:`13857`). - - -threading ---------- - -:class:`threading.Condition`, :class:`threading.Semaphore`, -:class:`threading.BoundedSemaphore`, :class:`threading.Event`, and -:class:`threading.Timer`, all of which used to be factory functions returning a -class instance, are now classes and may be subclassed. (Contributed by Éric -Araujo in :issue:`10968`.) - -The :class:`threading.Thread` constructor now accepts a ``daemon`` keyword -argument to override the default behavior of inheriting the ``deamon`` flag -value from the parent thread (:issue:`6064`). - -The formerly private function ``_thread.get_ident`` is now available as the -public function :func:`threading.get_ident`. This eliminates several cases of -direct access to the ``_thread`` module in the stdlib. Third party code that -used ``_thread.get_ident`` should likewise be changed to use the new public -interface. + (:issue:`11223`) time ---- -The :pep:`418` added new functions to the :mod:`time` module: +The :mod:`time` module has new functions: -* :func:`~time.get_clock_info`: Get information on a clock. -* :func:`~time.monotonic`: Monotonic clock (cannot go backward), not affected - by system clock updates. -* :func:`~time.perf_counter`: Performance counter with the highest available - resolution to measure a short duration. -* :func:`~time.process_time`: Sum of the system and user CPU time of the - current process. +* :func:`~time.clock_getres` and :func:`~time.clock_gettime` functions and + ``CLOCK_xxx`` constants. +* :func:`~time.steady`. -Other new functions: - -* :func:`~time.clock_getres`, :func:`~time.clock_gettime` and - :func:`~time.clock_settime` functions with ``CLOCK_xxx`` constants. - (Contributed by Victor Stinner in :issue:`10278`.) - -To improve cross platform consistency, :func:`~time.sleep` now raises a -:exc:`ValueError` when passed a negative sleep value. Previously this was an -error on posix, but produced an infinite sleep on Windows. - - -types ------ - -Add a new :class:`types.MappingProxyType` class: Read-only proxy of a mapping. -(:issue:`14386`) - - -The new functions :func:`types.new_class` and :func:`types.prepare_class` provide support -for PEP 3115 compliant dynamic type creation. (:issue:`14588`) - - -unittest --------- - -:meth:`.assertRaises`, :meth:`.assertRaisesRegex`, :meth:`.assertWarns`, and -:meth:`.assertWarnsRegex` now accept a keyword argument *msg* when used as -context managers. (Contributed by Ezio Melotti and Winston Ewert in -:issue:`10775`.) - -:meth:`unittest.TestCase.run` now returns the :class:`~unittest.TestResult` -object. +(Contributed by Victor Stinner in :issue:`10278`) urllib @@ -2103,66 +1046,25 @@ used by :meth:`~urllib.request.Request.get_method` to determine what HTTP method should be used. For example, this will send a ``'HEAD'`` request:: - >>> urlopen(Request('https://www.python.org', method='HEAD')) + >>> urlopen(Request('http://www.python.org', method='HEAD')) (:issue:`1673007`) -webbrowser ----------- - -The :mod:`webbrowser` module supports more "browsers": Google Chrome (named -:program:`chrome`, :program:`chromium`, :program:`chrome-browser` or -:program:`chromium-browser` depending on the version and operating system), -and the generic launchers :program:`xdg-open`, from the FreeDesktop.org -project, and :program:`gvfs-open`, which is the default URI handler for GNOME -3. (The former contributed by Arnaud Calmettes in :issue:`13620`, the latter -by Matthias Klose in :issue:`14493`.) - - -xml.etree.ElementTree ---------------------- - -The :mod:`xml.etree.ElementTree` module now imports its C accelerator by -default; there is no longer a need to explicitly import -:mod:`xml.etree.cElementTree` (this module stays for backwards compatibility, -but is now deprecated). In addition, the ``iter`` family of methods of -:class:`~xml.etree.ElementTree.Element` has been optimized (rewritten in C). -The module's documentation has also been greatly improved with added examples -and a more detailed reference. - - -zlib ----- - -New attribute :attr:`zlib.Decompress.eof` makes it possible to distinguish -between a properly-formed compressed stream and an incomplete or truncated one. -(Contributed by Nadeem Vawda in :issue:`12646`.) - -New attribute :attr:`zlib.ZLIB_RUNTIME_VERSION` reports the version string of -the underlying ``zlib`` library that is loaded at runtime. (Contributed by -Torsten Landschoff in :issue:`12306`.) - - Optimizations ============= Major performance enhancements have been added: -* Thanks to :pep:`393`, some operations on Unicode strings have been optimized: +* Thanks to the :pep:`393`, some operations on Unicode strings has been optimized: * the memory footprint is divided by 2 to 4 depending on the text * encode an ASCII string to UTF-8 doesn't need to encode characters anymore, the UTF-8 representation is shared with the ASCII representation * the UTF-8 encoder has been optimized - * repeating a single ASCII letter and getting a substring of an ASCII string + * repeating a single ASCII letter and getting a substring of a ASCII strings is 4 times faster -* UTF-8 is now 2x to 4x faster. UTF-16 encoding is now up to 10x faster. - - (Contributed by Serhiy Storchaka, :issue:`14624`, :issue:`14738` and - :issue:`15026`.) - Build and C API Changes ======================= @@ -2173,7 +1075,7 @@ * :c:func:`PyMemoryView_FromMemory` -* :pep:`393` added new Unicode types, macros and functions: +* The :pep:`393` added new Unicode types, macros and functions: * High-level API: @@ -2199,9 +1101,6 @@ * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` -* :c:macro:`PyArg_ParseTuple` now accepts a :class:`bytearray` for the ``c`` - format (:issue:`12380`). - Deprecated @@ -2215,118 +1114,86 @@ Windows 2000 and Windows platforms which set ``COMSPEC`` to ``command.com`` are no longer supported due to maintenance burden. -OSF support, which was deprecated in 3.2, has been completely removed. - Deprecated Python modules, functions and methods ------------------------------------------------ -* Passing a non-empty string to ``object.__format__()`` is deprecated, and - will produce a :exc:`TypeError` in Python 3.4 (:issue:`9856`). +* The :mod:`packaging` module replaces the :mod:`distutils` module * The ``unicode_internal`` codec has been deprecated because of the :pep:`393`, use UTF-8, UTF-16 (``utf-16-le`` or ``utf-16-be``), or UTF-32 (``utf-32-le`` or ``utf-32-be``) * :meth:`ftplib.FTP.nlst` and :meth:`ftplib.FTP.dir`: use :meth:`ftplib.FTP.mlsd` * :func:`platform.popen`: use the :mod:`subprocess` module. Check especially - the :ref:`subprocess-replacements` section (:issue:`11377`). + the :ref:`subprocess-replacements` section. * :issue:`13374`: The Windows bytes API has been deprecated in the :mod:`os` module. Use Unicode filenames, instead of bytes filenames, to not depend on the ANSI code page anymore and to support any filename. * :issue:`13988`: The :mod:`xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -* The behaviour of :func:`time.clock` depends on the platform: use the new - :func:`time.perf_counter` or :func:`time.process_time` function instead, - depending on your requirements, to have a well defined behaviour. -* The :func:`os.stat_float_times` function is deprecated. -* :mod:`abc` module: - - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. - -* :mod:`importlib` package: - - * :meth:`importlib.abc.SourceLoader.path_mtime` is now deprecated in favour of - :meth:`importlib.abc.SourceLoader.path_stats` as bytecode files now store - both the modification time and size of the source file the bytecode file was - compiled from. - - - Deprecated functions and types of the C API ------------------------------------------- -The :c:type:`Py_UNICODE` has been deprecated by :pep:`393` and will be +The :c:type:`Py_UNICODE` has been deprecated by the :pep:`393` and will be removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:type:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or - :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with - :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use - :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use - ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready - strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or - :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` + * :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or + :c:func:`PyUnicode_FromKindAndData` + * :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, + :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` + * :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with + :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` + * :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use + :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` + * :c:macro:`PyUnicode_GET_DATA_SIZE`: use + ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready + strings) + * :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or + :c:func:`PyUnicode_AsWideCharString` + * :c:func:`PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or - :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use - :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` + * :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or + :c:macro:`PyUnicode_GET_LENGTH` + * :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_FromFormat` + * :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, + :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_Substring` + * :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` + * :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` + * :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use + :c:func:`PyUnicode_FindChar` + * :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` + * :c:macro:`Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or - :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape:` use - :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape:` use - :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or - :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` - - -Deprecated features -------------------- - -The :mod:`array` module's ``'u'`` format code is now deprecated and will be -removed in Python 4 together with the rest of the (:c:type:`Py_UNICODE`) API. + * :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` + * :c:func:`PyUnicode_EncodeUTF7` + * :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or + :c:func:`PyUnicode_AsUTF8String` + * :c:func:`PyUnicode_EncodeUTF32` + * :c:func:`PyUnicode_EncodeUTF16` + * :c:func:`PyUnicode_EncodeUnicodeEscape:` use + :c:func:`PyUnicode_AsUnicodeEscapeString` + * :c:func:`PyUnicode_EncodeRawUnicodeEscape:` use + :c:func:`PyUnicode_AsRawUnicodeEscapeString` + * :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` + * :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` + * :c:func:`PyUnicode_EncodeCharmap` + * :c:func:`PyUnicode_TranslateCharmap` + * :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or + :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) + * :c:func:`PyUnicode_EncodeDecimal`, + :c:func:`PyUnicode_TransformDecimalToASCII` Porting to Python 3.3 @@ -2335,14 +1202,15 @@ This section lists previously described changes and other bugfixes that may require changes to your code. -.. _portingpythoncode: - Porting Python code ------------------- -* Hash randomization is enabled by default. Set the :envvar:`PYTHONHASHSEED` - environment variable to ``0`` to disable hash randomization. See also the - :meth:`object.__hash__` method. +.. XXX add a point about hash randomization and that it's always on in 3.3 + +* :issue:`14205`: A dict lookup now raises a :exc:`RuntimeError` if the dict is + modified during the lookup. If you implement your own comparison function for + objects used as dict keys and the dict is shared by multiple threads, access + to the dict should be protected by a lock. * :issue:`12326`: On Linux, sys.platform doesn't contain the major version anymore. It is now always 'linux', instead of 'linux2' or 'linux3' depending @@ -2355,107 +1223,6 @@ timestamp is out of range. :exc:`OSError` is now raised if C functions :c:func:`gmtime` or :c:func:`localtime` failed. -* The default finders used by import now utilize a cache of what is contained - within a specific directory. If you create a Python source file or sourceless - bytecode file, make sure to call :func:`importlib.invalidate_caches` to clear - out the cache for the finders to notice the new file. - -* :exc:`ImportError` now uses the full name of the module that was attempted to - be imported. Doctests that check ImportErrors' message will need to be - updated to use the full name of the module instead of just the tail of the - name. - -* The *index* argument to :func:`__import__` now defaults to 0 instead of -1 - and no longer support negative values. It was an oversight when :pep:`328` was - implemented that the default value remained -1. If you need to continue to - perform a relative import followed by an absolute import, then perform the - relative import using an index of 1, followed by another import using an - index of 0. It is preferred, though, that you use - :func:`importlib.import_module` rather than call :func:`__import__` directly. - -* :func:`__import__` no longer allows one to use an index value other than 0 - for top-level modules. E.g. ``__import__('sys', level=1)`` is now an error. - -* Because :attr:`sys.meta_path` and :attr:`sys.path_hooks` now have finders on - them by default, you will most likely want to use :meth:`list.insert` instead - of :meth:`list.append` to add to those lists. - -* Because ``None`` is now inserted into :attr:`sys.path_importer_cache`, if you - are clearing out entries in the dictionary of paths that do not have a - finder, you will need to remove keys paired with values of ``None`` **and** - :class:`imp.NullImporter` to be backwards-compatible. This will lead to extra - overhead on older versions of Python that re-insert ``None`` into - :attr:`sys.path_importer_cache` where it repesents the use of implicit - finders, but semantically it should not change anything. - -* :class:`importlib.abc.Finder` no longer specifies a `find_module()` abstract - method that must be implemented. If you were relying on subclasses to - implement that method, make sure to check for the method's existence first. - You will probably want to check for `find_loader()` first, though, in the - case of working with :term:`path entry finders `. - -* :mod:`pkgutil` has been converted to use :mod:`importlib` internally. This - eliminates many edge cases where the old behaviour of the PEP 302 import - emulation failed to match the behaviour of the real import system. The - import emulation itself is still present, but is now deprecated. The - :func:`pkgutil.iter_importers` and :func:`pkgutil.walk_packages` functions - special case the standard import hooks so they are still supported even - though they do not provide the non-standard ``iter_modules()`` method. - -* A longstanding RFC-compliance bug (:issue:`1079`) in the parsing done by - :func:`email.header.decode_header` has been fixed. Code that uses the - standard idiom to convert encoded headers into unicode - (``str(make_header(decode_header(h))``) will see no change, but code that - looks at the individual tuples returned by decode_header will see that - whitespace that precedes or follows ``ASCII`` sections is now included in the - ``ASCII`` section. Code that builds headers using ``make_header`` should - also continue to work without change, since ``make_header`` continues to add - whitespace between ``ASCII`` and non-``ASCII`` sections if it is not already - present in the input strings. - -* :func:`email.utils.formataddr` now does the correct content transfer - encoding when passed non-``ASCII`` display names. Any code that depended on - the previous buggy behavior that preserved the non-``ASCII`` unicode in the - formatted output string will need to be changed (:issue:`1690608`). - -* :meth:`poplib.POP3.quit` may now raise protocol errors like all other - ``poplib`` methods. Code that assumes ``quit`` does not raise - :exc:`poplib.error_proto` errors may need to be changed if errors on ``quit`` - are encountered by a particular application (:issue:`11291`). - -* The ``strict`` argument to :class:`email.parser.Parser`, deprecated since - Python 2.4, has finally been removed. - -* The deprecated method ``unittest.TestCase.assertSameElements`` has been - removed. - -* The deprecated variable ``time.accept2dyear`` has been removed. - -* The deprecated ``Context._clamp`` attribute has been removed from the - :mod:`decimal` module. It was previously replaced by the public attribute - :attr:`~decimal.Context.clamp`. (See :issue:`8540`.) - -* The undocumented internal helper class ``SSLFakeFile`` has been removed - from :mod:`smtplib`, since its functionality has long been provided directly - by :meth:`socket.socket.makefile`. - -* Passing a negative value to :func:`time.sleep` on Windows now raises an - error instead of sleeping forever. It has always raised an error on posix. - -* The ``ast.__version__`` constant has been removed. If you need to - make decisions affected by the AST version, use :attr:`sys.version_info` - to make the decision. - -* Code that used to work around the fact that the :mod:`threading` module used - factory functions by subclassing the private classes will need to change to - subclass the now-public classes. - -* The undocumented debugging machinery in the threading module has been - removed, simplifying the code. This should have no effect on production - code, but is mentioned here in case any application debug frameworks were - interacting with it (:issue:`13550`). - - Porting C code -------------- @@ -2471,7 +1238,7 @@ functions using this type are deprecated (but will stay available for at least five years). If you were using low-level Unicode APIs to construct and access unicode objects and you want to benefit of the - memory footprint reduction provided by PEP 393, you have to convert + memory footprint reduction provided by the PEP 393, you have to convert your code to the new :doc:`Unicode API <../c-api/unicode>`. However, if you only have been using high-level functions such as @@ -2479,14 +1246,6 @@ :c:func:`PyUnicode_FromFormat()`, your code will automatically take advantage of the new unicode representations. -* :c:func:`PyImport_GetMagicNumber` now returns -1 upon failure. - -* As a negative value for the *level* argument to :func:`__import__` is no - longer valid, the same now holds for :c:func:`PyImport_ImportModuleLevel`. - This also means that the value of *level* used by - :c:func:`PyImport_ImportModuleEx` is now 0 instead of -1. - - Building C extensions --------------------- @@ -2501,16 +1260,14 @@ (implemented in :issue:`14040`.) -Command Line Switch Changes ---------------------------- +Other issues +------------ -* The -Q command-line flag and related artifacts have been removed. Code - checking sys.flags.division_warning will need updating. +.. Issue #11591: When :program:`python` was started with :option:`-S`, + ``import site`` will not add site-specific paths to the module search + paths. In previous versions, it did. See changeset for doc changes in + various files. Contributed by Carl Meyer with editions by Éric Araujo. - (:issue:`10998`, contributed by Éric Araujo.) - -* When :program:`python` is started with :option:`-S`, ``import site`` - will no longer add site-specific paths to the module search paths. In - previous versions, it did. - - (:issue:`11591`, contributed by Carl Meyer with editions by Éric Araujo.) +.. Issue #10998: the -Q command-line flag and related artifacts have been + removed. Code checking sys.flags.division_warning will need updating. + Contributed by Éric Araujo. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2545 +0,0 @@ -**************************** - What's New In Python 3.4 -**************************** - -:Author: R. David Murray (Editor) - -.. Rules for maintenance: - - * Anyone can add text to this document, but the maintainer reserves the - right to rewrite any additions. In particular, for obscure or esoteric - features, the maintainer may reduce any addition to a simple reference to - the new documentation rather than explaining the feature inline. - - * While the maintainer will periodically go through Misc/NEWS - and add changes, it's best not to rely on this. We know from experience - that any changes that aren't in the What's New documentation around the - time of the original release will remain largely unknown to the community - for years, even if they're added later. We also know from experience that - other priorities can arise, and the maintainer will run out of time to do - updates -- in such cases, end users will be much better served by partial - notifications that at least give a hint about new features to - investigate. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. The What's New should focus on changes that - are visible to Python *users* and that *require* a feature release (i.e. - most bug fixes should only be recorded in Misc/NEWS) - - * PEPs should not be marked Final until they have an entry in What's New. - A placeholder entry that is just a section header and a link to the PEP - (e.g ":pep:`397` has been implemented") is acceptable. If a PEP has been - implemented and noted in What's New, don't forget to mark it as Final! - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to add just a very brief note about a change. For - example: "The :ref:`~socket.transmogrify()` function was added to the - :mod:`socket` module." The maintainer will research the change and - write the necessary text (if appropriate). The advantage of doing this - is that even if no more descriptive text is ever added, readers will at - least have a notification that the new feature exists and a link to the - relevant documentation. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - The :ref:`~socket.transmogrify()` function was added to the - :mod:`socket` module. (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - - * Cross referencing tip: :ref:`mod.attr` will display as ``mod.attr``, - while :ref:`~mod.attr` will display as ``attr``. - -This article explains the new features in Python 3.4, compared to 3.3. -Python 3.4 was released on March 16, 2014. For full details, see the -`changelog `_. - - -.. seealso:: - - :pep:`429` -- Python 3.4 Release Schedule - - - -Summary -- Release Highlights -============================= - -.. This section singles out the most important changes in Python 3.4. - Brevity is key. - -New syntax features: - -* No new syntax features were added in Python 3.4. - -Other new features: - -* :ref:`pip should always be available ` (:pep:`453`). -* :ref:`Newly created file descriptors are non-inheritable ` - (:pep:`446`). -* command line option for :ref:`isolated mode ` - (:issue:`16499`). -* :ref:`improvements in the handling of codecs ` - that are not text encodings (multiple issues). -* :ref:`A ModuleSpec Type ` for the Import System - (:pep:`451`). (Affects importer authors.) -* The :mod:`marshal` format has been made :ref:`more compact and efficient - ` (:issue:`16475`). - -New library modules: - -* :mod:`asyncio`: :ref:`New provisional API for asynchronous IO - ` (:pep:`3156`). -* :mod:`ensurepip`: :ref:`Bootstrapping the pip installer ` - (:pep:`453`). -* :mod:`enum`: :ref:`Support for enumeration types ` - (:pep:`435`). -* :mod:`pathlib`: :ref:`Object-oriented filesystem paths ` - (:pep:`428`). -* :mod:`selectors`: :ref:`High-level and efficient I/O multiplexing - `, built upon the :mod:`select` module primitives (part - of :pep:`3156`). -* :mod:`statistics`: A basic :ref:`numerically stable statistics library - ` (:pep:`450`). -* :mod:`tracemalloc`: :ref:`Trace Python memory allocations - ` (:pep:`454`). - -Significantly improved library modules: - -* :ref:`Single-dispatch generic functions ` in - :mod:`functools` (:pep:`443`). -* New :mod:`pickle` :ref:`protocol 4 ` (:pep:`3154`). -* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork - on Unix ` (:issue:`8713`). -* :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and - a new :mod:`~email.message.Message` subclass - (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME - handling ` (:issue:`18891`). -* The :mod:`inspect` and :mod:`pydoc` modules are now capable of - correct introspection of a much wider variety of callable objects, - which improves the output of the Python :func:`help` system. -* The :mod:`ipaddress` module API has been declared stable - -Security improvements: - -* :ref:`Secure and interchangeable hash algorithm ` - (:pep:`456`). -* :ref:`Make newly created file descriptors non-inheritable ` - (:pep:`446`) to avoid leaking file descriptors to child processes. -* New command line option for :ref:`isolated mode `, - (:issue:`16499`). -* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork - on Unix `. *spawn* and *forkserver* are - more secure because they avoid sharing data with child processes. -* :mod:`multiprocessing` child processes on Windows no longer inherit - all of the parent's inheritable handles, only the necessary ones. -* A new :func:`hashlib.pbkdf2_hmac` function provides - the `PKCS#5 password-based key derivation function 2 - `_. -* :ref:`TLSv1.1 and TLSv1.2 support ` for :mod:`ssl`. -* :ref:`Retrieving certificates from the Windows system cert store support - ` for :mod:`ssl`. -* :ref:`Server-side SNI (Server Name Indication) support - ` for :mod:`ssl`. -* The :class:`ssl.SSLContext` class has a :ref:`lot of improvements - `. -* All modules in the standard library that support SSL now support server - certificate verification, including hostname matching - (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see - :func:`ssl.SSLContext.load_verify_locations`). - -CPython implementation improvements: - -* :ref:`Safe object finalization ` (:pep:`442`). -* Leveraging :pep:`442`, in most cases :ref:`module globals are no longer set - to None during finalization ` (:issue:`18214`). -* :ref:`Configurable memory allocators ` (:pep:`445`). -* :ref:`Argument Clinic ` (:pep:`436`). - -Please read on for a comprehensive list of user-facing changes, including many -other smaller improvements, CPython optimizations, deprecations, and potential -porting issues. - - - -New Features -============ - -.. _whatsnew-pep-453: - -PEP 453: Explicit Bootstrapping of PIP in Python Installations --------------------------------------------------------------- - -Bootstrapping pip By Default -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The new :mod:`ensurepip` module (defined in :pep:`453`) provides a standard -cross-platform mechanism to bootstrap the pip installer into Python -installations and virtual environments. The version of ``pip`` included -with Python 3.4.0 is ``pip`` 1.5.4, and future 3.4.x maintenance releases -will update the bundled version to the latest version of ``pip`` that is -available at the time of creating the release candidate. - -By default, the commands ``pipX`` and ``pipX.Y`` will be installed on all -platforms (where X.Y stands for the version of the Python installation), -along with the ``pip`` Python package and its dependencies. On Windows and -in virtual environments on all platforms, the unversioned ``pip`` command -will also be installed. On other platforms, the system wide unversioned -``pip`` command typically refers to the separately installed Python 2 -version. - -The :ref:`pyvenv ` command line utility and the :mod:`venv` -module make use of the :mod:`ensurepip` module to make ``pip`` readily -available in virtual environments. When using the command line utility, -``pip`` is installed by default, while when using the :mod:`venv` module -:ref:`venv-api` installation of ``pip`` must be requested explicitly. - -For CPython :ref:`source builds on POSIX systems `, -the ``make install`` and ``make altinstall`` commands bootstrap ``pip`` by -default. This behaviour can be controlled through configure options, and -overridden through Makefile options. - -On Windows and Mac OS X, the CPython installers now default to installing -``pip`` along with CPython itself (users may opt out of installing it -during the installation process). Window users will need to opt in to the -automatic ``PATH`` modifications to have ``pip`` available from the command -line by default, otherwise it can still be accessed through the Python -launcher for Windows as ``py -m pip``. - -As `discussed in the PEP`__, platform packagers may choose not to install -these commands by default, as long as, when invoked, they provide clear and -simple directions on how to install them on that platform (usually using -the system package manager). - -__ https://www.python.org/dev/peps/pep-0453/#recommendations-for-downstream-distributors - -.. note:: - - To avoid conflicts between parallel Python 2 and Python 3 installations, - only the versioned ``pip3`` and ``pip3.4`` commands are bootstrapped by - default when ``ensurepip`` is invoked directly - the ``--default-pip`` - option is needed to also request the unversioned ``pip`` command. - ``pyvenv`` and the Windows installer ensure that the unqualified ``pip`` - command is made available in those environments, and ``pip`` can always be - invoked via the ``-m`` switch rather than directly to avoid ambiguity on - systems with multiple Python installations. - - -Documentation Changes -~~~~~~~~~~~~~~~~~~~~~ - -As part of this change, the :ref:`installing-index` and -:ref:`distributing-index` sections of the documentation have been -completely redesigned as short getting started and FAQ documents. Most -packaging documentation has now been moved out to the Python Packaging -Authority maintained `Python Packaging User Guide -`__ and the documentation of the individual -projects. - -However, as this migration is currently still incomplete, the legacy -versions of those guides remaining available as :ref:`install-index` -and :ref:`distutils-index`. - -.. seealso:: - - :pep:`453` -- Explicit bootstrapping of pip in Python installations - PEP written by Donald Stufft and Nick Coghlan, implemented by - Donald Stufft, Nick Coghlan, Martin von Löwis and Ned Deily. - - -.. _whatsnew-pep-446: - -PEP 446: Newly Created File Descriptors Are Non-Inheritable ------------------------------------------------------------ - -:pep:`446` makes newly created file descriptors :ref:`non-inheritable -`. In general, this is the behavior an application will -want: when launching a new process, having currently open files also -open in the new process can lead to all sorts of hard to find bugs, -and potentially to security issues. - -However, there are occasions when inheritance is desired. To support -these cases, the following new functions and methods are available: - -* :func:`os.get_inheritable`, :func:`os.set_inheritable` -* :func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable` -* :meth:`socket.socket.get_inheritable`, :meth:`socket.socket.set_inheritable` - -.. seealso:: - - :pep:`446` -- Make newly created file descriptors non-inheritable - PEP written and implemented by Victor Stinner. - - -.. _codec-handling-improvements: - -Improvements to Codec Handling ------------------------------- - -Since it was first introduced, the :mod:`codecs` module has always been -intended to operate as a type-neutral dynamic encoding and decoding -system. However, its close coupling with the Python text model, especially -the type restricted convenience methods on the builtin :class:`str`, -:class:`bytes` and :class:`bytearray` types, has historically obscured that -fact. - -As a key step in clarifying the situation, the :meth:`codecs.encode` and -:meth:`codecs.decode` convenience functions are now properly documented in -Python 2.7, 3.3 and 3.4. These functions have existed in the :mod:`codecs` -module (and have been covered by the regression test suite) since Python 2.4, -but were previously only discoverable through runtime introspection. - -Unlike the convenience methods on :class:`str`, :class:`bytes` and -:class:`bytearray`, the :mod:`codecs` convenience functions support arbitrary -codecs in both Python 2 and Python 3, rather than being limited to Unicode text -encodings (in Python 3) or ``basestring`` <-> ``basestring`` conversions (in -Python 2). - -In Python 3.4, the interpreter is able to identify the known non-text -encodings provided in the standard library and direct users towards these -general purpose convenience functions when appropriate:: - - >>> b"abcdef".decode("hex") - Traceback (most recent call last): - File "", line 1, in - LookupError: 'hex' is not a text encoding; use codecs.decode() to handle arbitrary codecs - - >>> "hello".encode("rot13") - Traceback (most recent call last): - File "", line 1, in - LookupError: 'rot13' is not a text encoding; use codecs.encode() to handle arbitrary codecs - - >>> open("foo.txt", encoding="hex") - Traceback (most recent call last): - File "", line 1, in - LookupError: 'hex' is not a text encoding; use codecs.open() to handle arbitrary codecs - -In a related change, whenever it is feasible without breaking backwards -compatibility, exceptions raised during encoding and decoding operations -are wrapped in a chained exception of the same type that mentions the -name of the codec responsible for producing the error:: - - >>> import codecs - - >>> codecs.decode(b"abcdefgh", "hex") - Traceback (most recent call last): - File "/usr/lib/python3.4/encodings/hex_codec.py", line 20, in hex_decode - return (binascii.a2b_hex(input), len(input)) - binascii.Error: Non-hexadecimal digit found - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 1, in - binascii.Error: decoding with 'hex' codec failed (Error: Non-hexadecimal digit found) - - >>> codecs.encode("hello", "bz2") - Traceback (most recent call last): - File "/usr/lib/python3.4/encodings/bz2_codec.py", line 17, in bz2_encode - return (bz2.compress(input), len(input)) - File "/usr/lib/python3.4/bz2.py", line 498, in compress - return comp.compress(data) + comp.flush() - TypeError: 'str' does not support the buffer interface - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 1, in - TypeError: encoding with 'bz2' codec failed (TypeError: 'str' does not support the buffer interface) - -Finally, as the examples above show, these improvements have permitted -the restoration of the convenience aliases for the non-Unicode codecs that -were themselves restored in Python 3.2. This means that encoding binary data -to and from its hexadecimal representation (for example) can now be written -as:: - - >>> from codecs import encode, decode - >>> encode(b"hello", "hex") - b'68656c6c6f' - >>> decode(b"68656c6c6f", "hex") - b'hello' - -The binary and text transforms provided in the standard library are detailed -in :ref:`binary-transforms` and :ref:`text-transforms`. - -(Contributed by Nick Coghlan in :issue:`7475`, :issue:`17827`, -:issue:`17828` and :issue:`19619`.) - - -.. _whatsnew-pep-451: - -PEP 451: A ModuleSpec Type for the Import System ------------------------------------------------- - -:pep:`451` provides an encapsulation of the information about a module that the -import machinery will use to load it (that is, a module specification). This -helps simplify both the import implementation and several import-related APIs. -The change is also a stepping stone for `several future import-related -improvements`__. - -__ https://mail.python.org/pipermail/python-dev/2013-November/130111.html - -The public-facing changes from the PEP are entirely backward-compatible. -Furthermore, they should be transparent to everyone but importer authors. Key -finder and loader methods have been deprecated, but they will continue working. -New importers should use the new methods described in the PEP. Existing -importers should be updated to implement the new methods. See the -:ref:`deprecated-3.4` section for a list of methods that should be replaced and -their replacements. - - -Other Language Changes ----------------------- - -Some smaller changes made to the core Python language are: - -* Unicode database updated to UCD version 6.3. - -* :func:`min` and :func:`max` now accept a *default* keyword-only argument that - can be used to specify the value they return if the iterable they are - evaluating has no elements. (Contributed by Julian Berman in - :issue:`18111`.) - -* Module objects are now :mod:`weakref`'able. - -* Module ``__file__`` attributes (and related values) should now always - contain absolute paths by default, with the sole exception of - ``__main__.__file__`` when a script has been executed directly using - a relative path. (Contributed by Brett Cannon in :issue:`18416`.) - -* All the UTF-\* codecs (except UTF-7) now reject surrogates during both - encoding and decoding unless the ``surrogatepass`` error handler is used, - with the exception of the UTF-16 decoder (which accepts valid surrogate pairs) - and the UTF-16 encoder (which produces them while encoding non-BMP characters). - (Contributed by Victor Stinner, Kang-Hao (Kenny) Lu and Serhiy Storchaka in - :issue:`12892`.) - -* New German EBCDIC :ref:`codec ` ``cp273``. (Contributed - by Michael Bierenfeld and Andrew Kuchling in :issue:`1097797`.) - -* New Ukrainian :ref:`codec ` ``cp1125``. (Contributed by - Serhiy Storchaka in :issue:`19668`.) - -* :class:`bytes`.join() and :class:`bytearray`.join() now accept arbitrary - buffer objects as arguments. (Contributed by Antoine Pitrou in - :issue:`15958`.) - -* The :class:`int` constructor now accepts any object that has an ``__index__`` - method for its *base* argument. (Contributed by Mark Dickinson in - :issue:`16772`.) - -* Frame objects now have a :func:`~frame.clear` method that clears all - references to local variables from the frame. (Contributed by Antoine Pitrou - in :issue:`17934`.) - -* :class:`memoryview` is now registered as a :class:`Sequence `, - and supports the :func:`reversed` builtin. (Contributed by Nick Coghlan - and Claudiu Popa in :issue:`18690` and :issue:`19078`.) - -* Signatures reported by :func:`help` have been modified and improved in - several cases as a result of the introduction of Argument Clinic and other - changes to the :mod:`inspect` and :mod:`pydoc` modules. - -* :meth:`~object.__length_hint__` is now part of the formal language - specification (see :pep:`424`). (Contributed by Armin Ronacher in - :issue:`16148`.) - - -New Modules -=========== - - -.. _whatsnew-asyncio: - -asyncio -------- - -The new :mod:`asyncio` module (defined in :pep:`3156`) provides a standard -pluggable event loop model for Python, providing solid asynchronous IO -support in the standard library, and making it easier for other event loop -implementations to interoperate with the standard library and each other. - -For Python 3.4, this module is considered a :term:`provisional API`. - -.. seealso:: - - :pep:`3156` -- Asynchronous IO Support Rebooted: the "asyncio" Module - PEP written and implementation led by Guido van Rossum. - - -.. _whatsnew-ensurepip: - -ensurepip ---------- - -The new :mod:`ensurepip` module is the primary infrastructure for the -:pep:`453` implementation. In the normal course of events end users will not -need to interact with this module, but it can be used to manually bootstrap -``pip`` if the automated bootstrapping into an installation or virtual -environment was declined. - -:mod:`ensurepip` includes a bundled copy of ``pip``, up-to-date as of the first -release candidate of the release of CPython with which it ships (this applies -to both maintenance releases and feature releases). ``ensurepip`` does not -access the internet. If the installation has Internet access, after -``ensurepip`` is run the bundled ``pip`` can be used to upgrade ``pip`` to a -more recent release than the bundled one. (Note that such an upgraded version -of ``pip`` is considered to be a separately installed package and will not be -removed if Python is uninstalled.) - -The module is named *ensure*\ pip because if called when ``pip`` is already -installed, it does nothing. It also has an ``--upgrade`` option that will -cause it to install the bundled copy of ``pip`` if the existing installed -version of ``pip`` is older than the bundled copy. - - -.. _whatsnew-enum: - -enum ----- - -The new :mod:`enum` module (defined in :pep:`435`) provides a standard -implementation of enumeration types, allowing other modules (such as -:mod:`socket`) to provide more informative error messages and better -debugging support by replacing opaque integer constants with backwards -compatible enumeration values. - -.. seealso:: - - :pep:`435` -- Adding an Enum type to the Python standard library - PEP written by Barry Warsaw, Eli Bendersky and Ethan Furman, - implemented by Ethan Furman. - - -.. _whatsnew-pathlib: - -pathlib -------- - -The new :mod:`pathlib` module offers classes representing filesystem paths -with semantics appropriate for different operating systems. Path classes are -divided between *pure paths*, which provide purely computational operations -without I/O, and *concrete paths*, which inherit from pure paths but also -provide I/O operations. - -For Python 3.4, this module is considered a :term:`provisional API`. - -.. seealso:: - - :pep:`428` -- The pathlib module -- object-oriented filesystem paths - PEP written and implemented by Antoine Pitrou. - - -.. _whatsnew-selectors: - -selectors ---------- - -The new :mod:`selectors` module (created as part of implementing :pep:`3156`) -allows high-level and efficient I/O multiplexing, built upon the -:mod:`select` module primitives. - - -.. _whatsnew-statistics: - -statistics ----------- - -The new :mod:`statistics` module (defined in :pep:`450`) offers some core -statistics functionality directly in the standard library. This module -supports calculation of the mean, median, mode, variance and standard -deviation of a data series. - -.. seealso:: - - :pep:`450` -- Adding A Statistics Module To The Standard Library - PEP written and implemented by Steven D'Aprano - -.. _whatsnew-tracemalloc: - - -tracemalloc ------------ - -The new :mod:`tracemalloc` module (defined in :pep:`454`) is a debug tool to -trace memory blocks allocated by Python. It provides the following information: - -* Trace where an object was allocated -* Statistics on allocated memory blocks per filename and per line number: - total size, number and average size of allocated memory blocks -* Compute the differences between two snapshots to detect memory leaks - -.. seealso:: - - :pep:`454` -- Add a new tracemalloc module to trace Python memory allocations - PEP written and implemented by Victor Stinner - - - -Improved Modules -================ - - -abc ---- - -New function :func:`abc.get_cache_token` can be used to know when to invalidate -caches that are affected by changes in the object graph. (Contributed -by Łukasz Langa in :issue:`16832`.) - -New class :class:`~abc.ABC` has :class:`~abc.ABCMeta` as its meta class. -Using ``ABC`` as a base class has essentially the same effect as specifying -``metaclass=abc.ABCMeta``, but is simpler to type and easier to read. -(Contributed by Bruno Dupuis in :issue:`16049`.) - - -aifc ----- - -The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.) - -:func:`aifc.open` now supports the context management protocol: when used in a -:keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned -object will be called automatically at the end of the block. (Contributed by -Serhiy Storchacha in :issue:`16486`.) - -The :meth:`~aifc.aifc.writeframesraw` and :meth:`~aifc.aifc.writeframes` -methods now accept any :term:`bytes-like object`. (Contributed by Serhiy -Storchaka in :issue:`8311`.) - - -argparse --------- - -The :class:`~argparse.FileType` class now accepts *encoding* and -*errors* arguments, which are passed through to :func:`open`. (Contributed -by Lucas Maystre in :issue:`11175`.) - - -audioop -------- - -:mod:`audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka -in :issue:`12866`.) - -New :func:`~audioop.byteswap` function converts big-endian samples to -little-endian and vice versa. (Contributed by Serhiy Storchaka in -:issue:`19641`.) - -All :mod:`audioop` functions now accept any :term:`bytes-like object`. Strings -are not accepted: they didn't work before, now they raise an error right away. -(Contributed by Serhiy Storchaka in :issue:`16685`.) - - -base64 ------- - -The encoding and decoding functions in :mod:`base64` now accept any -:term:`bytes-like object` in cases where it previously required a -:class:`bytes` or :class:`bytearray` instance. (Contributed by Nick Coghlan in -:issue:`17839`.) - -New functions :func:`~base64.a85encode`, :func:`~base64.a85decode`, -:func:`~base64.b85encode`, and :func:`~base64.b85decode` provide the ability to -encode and decode binary data from and to ``Ascii85`` and the git/mercurial -``Base85`` formats, respectively. The ``a85`` functions have options that can -be used to make them compatible with the variants of the ``Ascii85`` encoding, -including the Adobe variant. (Contributed by Martin Morrison, the Mercurial -project, Serhiy Storchaka, and Antoine Pitrou in :issue:`17618`.) - - -collections ------------ - -The :meth:`.ChainMap.new_child` method now accepts an *m* argument specifying -the child map to add to the chain. This allows an existing mapping and/or a -custom mapping type to be used for the child. (Contributed by Vinay Sajip in -:issue:`16613`.) - - -colorsys --------- - -The number of digits in the coefficients for the RGB --- YIQ conversions have -been expanded so that they match the FCC NTSC versions. The change in -results should be less than 1% and may better match results found elsewhere. -(Contributed by Brian Landers and Serhiy Storchaka in :issue:`14323`.) - - -contextlib ----------- - -The new :class:`contextlib.suppress` context manager helps to clarify the -intent of code that deliberately suppresses exceptions from a single -statement. (Contributed by Raymond Hettinger in :issue:`15806` and -Zero Piraeus in :issue:`19266`.) - -The new :func:`contextlib.redirect_stdout` context manager makes it easier -for utility scripts to handle inflexible APIs that write their output to -:data:`sys.stdout` and don't provide any options to redirect it. Using the -context manager, the :data:`sys.stdout` output can be redirected to any -other stream or, in conjunction with :class:`io.StringIO`, to a string. -The latter can be especially useful, for example, to capture output -from a function that was written to implement a command line interface. -It is recommended only for utility scripts because it affects the -global state of :data:`sys.stdout`. (Contributed by Raymond Hettinger -in :issue:`15805`.) - -The :mod:`contextlib` documentation has also been updated to include a -:ref:`discussion ` of the -differences between single use, reusable and reentrant context managers. - - -dbm ---- - -:func:`dbm.open` objects now support the context management protocol. When -used in a :keyword:`with` statement, the ``close`` method of the database -object will be called automatically at the end of the block. (Contributed by -Claudiu Popa and Nick Coghlan in :issue:`19282`.) - - -dis ---- - -Functions :func:`~dis.show_code`, :func:`~dis.dis`, :func:`~dis.distb`, and -:func:`~dis.disassemble` now accept a keyword-only *file* argument that -controls where they write their output. - -The :mod:`dis` module is now built around an :class:`~dis.Instruction` class -that provides object oriented access to the details of each individual bytecode -operation. - -A new method, :func:`~dis.get_instructions`, provides an iterator that emits -the Instruction stream for a given piece of Python code. Thus it is now -possible to write a program that inspects and manipulates a bytecode -object in ways different from those provided by the :mod:`~dis` module -itself. For example:: - - >>> import dis - >>> for instr in dis.get_instructions(lambda x: x + 1): - ... print(instr.opname) - LOAD_FAST - LOAD_CONST - BINARY_ADD - RETURN_VALUE - -The various display tools in the :mod:`dis` module have been rewritten to use -these new components. - -In addition, a new application-friendly class :class:`~dis.Bytecode` provides -an object-oriented API for inspecting bytecode in both in human-readable form -and for iterating over instructions. The :class:`~dis.Bytecode` constructor -takes the same arguments that :func:`~dis.get_instruction` does (plus an -optional *current_offset*), and the resulting object can be iterated to produce -:class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis` -method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but -returned as a multi-line string:: - - >>> bytecode = dis.Bytecode(lambda x: x +1, current_offset=3) - >>> for instr in bytecode: - ... print('{} ({})'.format(instr.opname, instr.opcode)) - LOAD_FAST (124) - LOAD_CONST (100) - BINARY_ADD (23) - RETURN_VALUE (83) - >>> bytecode.dis().splitlines() # doctest: +NORMALIZE_WHITESPACE - [' 1 0 LOAD_FAST 0 (x)', - ' --> 3 LOAD_CONST 1 (1)', - ' 6 BINARY_ADD', - ' 7 RETURN_VALUE'] - -:class:`~dis.Bytecode` also has a class method, -:meth:`~dis.Bytecode.from_traceback`, that provides the ability to manipulate a -traceback (that is, ``print(Bytecode.from_traceback(tb).dis())`` is equivalent -to ``distb(tb)``). - -(Contributed by Nick Coghlan, Ryan Kelly and Thomas Kluyver in :issue:`11816` -and Claudiu Popa in :issue:`17916`.) - -New function :func:`~dis.stack_effect` computes the effect on the Python stack -of a given opcode and argument, information that is not otherwise available. -(Contributed by Larry Hastings in :issue:`19722`.) - - -doctest -------- - -A new :ref:`option flag `, :data:`~doctest.FAIL_FAST`, halts -test running as soon as the first failure is detected. (Contributed by R. -David Murray and Daniel Urban in :issue:`16522`.) - -The :mod:`doctest` command line interface now uses :mod:`argparse`, and has two -new options, ``-o`` and ``-f``. ``-o`` allows :ref:`doctest options -` to be specified on the command line, and ``-f`` is a -shorthand for ``-o FAIL_FAST`` (to parallel the similar option supported by the -:mod:`unittest` CLI). (Contributed by R. David Murray in :issue:`11390`.) - -:mod:`doctest` will now find doctests in extension module ``__doc__`` strings. -(Contributed by Zachary Ware in :issue:`3158`.) - - -email ------ - -:meth:`~email.message.Message.as_string` now accepts a *policy* argument to -override the default policy of the message when generating a string -representation of it. This means that ``as_string`` can now be used in more -circumstances, instead of having to create and use a :mod:`~email.generator` in -order to pass formatting parameters to its ``flatten`` method. (Contributed by -R. David Murray in :issue:`18600`.) - -New method :meth:`~email.message.Message.as_bytes` added to produce a bytes -representation of the message in a fashion similar to how ``as_string`` -produces a string representation. It does not accept the *maxheaderlen* -argument, but does accept the *unixfrom* and *policy* arguments. The -:class:`~email.message.Message` :meth:`~email.message.Message.__bytes__` method -calls it, meaning that ``bytes(mymsg)`` will now produce the intuitive -result: a bytes object containing the fully formatted message. (Contributed -by R. David Murray in :issue:`18600`.) - -The :meth:`.Message.set_param` message now accepts a *replace* keyword argument. -When specified, the associated header will be updated without changing -its location in the list of headers. For backward compatibility, the default -is ``False``. (Contributed by R. David Murray in :issue:`18891`.) - - -.. _whatsnew_email_contentmanager: - -A pair of new subclasses of :class:`~email.message.Message` have been added -(:class:`.EmailMessage` and :class:`.MIMEPart`), along with a new sub-module, -:mod:`~email.contentmanager` and a new :mod:`~email.policy` attribute -:attr:`~email.policy.EmailPolicy.content_manager`. All documentation is -currently in the new module, which is being added as part of email's new -:term:`provisional API`. These classes provide a number of new methods that -make extracting content from and inserting content into email messages much -easier. For details, see the :mod:`~email.contentmanager` documentation and -the :ref:`email-contentmanager-api-examples`. These API additions complete the -bulk of the work that was planned as part of the email6 project. The currently -provisional API is scheduled to become final in Python 3.5 (possibly with a few -minor additions in the area of error handling). (Contributed by R. David -Murray in :issue:`18891`.) - - -filecmp -------- - -A new :func:`~filecmp.clear_cache` function provides the ability to clear the -:mod:`filecmp` comparison cache, which uses :func:`os.stat` information to -determine if the file has changed since the last compare. This can be used, -for example, if the file might have been changed and re-checked in less time -than the resolution of a particular filesystem's file modification time field. -(Contributed by Mark Levitt in :issue:`18149`.) - -New module attribute :data:`~filecmp.DEFAULT_IGNORES` provides the list of -directories that are used as the default value for the *ignore* parameter of -the :func:`~filecmp.dircmp` function. (Contributed by Eli Bendersky in -:issue:`15442`.) - - -functools ---------- - -The new :func:`~functools.partialmethod` descriptor brings partial argument -application to descriptors, just as :func:`~functools.partial` provides -for normal callables. The new descriptor also makes it easier to get -arbitrary callables (including :func:`~functools.partial` instances) -to behave like normal instance methods when included in a class definition. -(Contributed by Alon Horev and Nick Coghlan in :issue:`4331`.) - -.. _whatsnew-singledispatch: - -The new :func:`~functools.singledispatch` decorator brings support for -single-dispatch generic functions to the Python standard library. Where -object oriented programming focuses on grouping multiple operations on a -common set of data into a class, a generic function focuses on grouping -multiple implementations of an operation that allows it to work with -*different* kinds of data. - -.. seealso:: - - :pep:`443` -- Single-dispatch generic functions - PEP written and implemented by Łukasz Langa. - -:func:`~functools.total_ordering` now supports a return value of -:const:`NotImplemented` from the underlying comparison function. (Contributed -by Katie Miller in :issue:`10042`.) - -A pure-python version of the :func:`~functools.partial` function is now in the -stdlib; in CPython it is overridden by the C accelerated version, but it is -available for other implementations to use. (Contributed by Brian Thorne in -:issue:`12428`.) - - -gc --- - -New function :func:`~gc.get_stats` returns a list of three per-generation -dictionaries containing the collections statistics since interpreter startup. -(Contributed by Antoine Pitrou in :issue:`16351`.) - - -glob ----- - -A new function :func:`~glob.escape` provides a way to escape special characters -in a filename so that they do not become part of the globbing expansion but are -instead matched literally. (Contributed by Serhiy Storchaka in :issue:`8402`.) - - -hashlib -------- - -A new :func:`hashlib.pbkdf2_hmac` function provides -the `PKCS#5 password-based key derivation function 2 -`_. (Contributed by Christian -Heimes in :issue:`18582`.) - -The :attr:`~hashlib.hash.name` attribute of :mod:`hashlib` hash objects is now -a formally supported interface. It has always existed in CPython's -:mod:`hashlib` (although it did not return lower case names for all supported -hashes), but it was not a public interface and so some other Python -implementations have not previously supported it. (Contributed by Jason R. -Coombs in :issue:`18532`.) - - -hmac ----- - -:mod:`hmac` now accepts ``bytearray`` as well as ``bytes`` for the *key* -argument to the :func:`~hmac.new` function, and the *msg* parameter to both the -:func:`~hmac.new` function and the :meth:`~hmac.HMAC.update` method now -accepts any type supported by the :mod:`hashlib` module. (Contributed -by Jonas Borgström in :issue:`18240`.) - -The *digestmod* argument to the :func:`hmac.new` function may now be any hash -digest name recognized by :mod:`hashlib`. In addition, the current behavior in -which the value of *digestmod* defaults to ``MD5`` is deprecated: in a -future version of Python there will be no default value. (Contributed by -Christian Heimes in :issue:`17276`.) - -With the addition of :attr:`~hmac.HMAC.block_size` and :attr:`~hmac.HMAC.name` -attributes (and the formal documentation of the :attr:`~hmac.HMAC.digest_size` -attribute), the :mod:`hmac` module now conforms fully to the :pep:`247` API. -(Contributed by Christian Heimes in :issue:`18775`.) - - -html ----- - -New function :func:`~html.unescape` function converts HTML5 character references to -the corresponding Unicode characters. (Contributed by Ezio Melotti in -:issue:`2927`.) - -:class:`~html.parser.HTMLParser` accepts a new keyword argument -*convert_charrefs* that, when ``True``, automatically converts all character -references. For backward-compatibility, its value defaults to ``False``, but -it will change to ``True`` in a future version of Python, so you are invited to -set it explicitly and update your code to use this new feature. (Contributed -by Ezio Melotti in :issue:`13633`.) - -The *strict* argument of :class:`~html.parser.HTMLParser` is now deprecated. -(Contributed by Ezio Melotti in :issue:`15114`.) - - -http ----- - -:meth:`~http.server.BaseHTTPRequestHandler.send_error` now accepts an -optional additional *explain* parameter which can be used to provide an -extended error description, overriding the hardcoded default if there is one. -This extended error description will be formatted using the -:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body -of the error response. (Contributed by Karl Cow in :issue:`12921`.) - -The :mod:`http.server` :ref:`command line interface ` now has -a ``-b/--bind`` option that causes the server to listen on a specific address. -(Contributed by Malte Swart in :issue:`17764`.) - - -idlelib and IDLE ----------------- - -Since idlelib implements the IDLE shell and editor and is not intended for -import by other programs, it gets improvements with every release. See -:file:`Lib/idlelib/NEWS.txt` for a cumulative list of changes since 3.3.0, -as well as changes made in future 3.4.x releases. This file is also available -from the IDLE :menuselection:`Help --> About IDLE` dialog. - - -importlib ---------- - -The :class:`~importlib.abc.InspectLoader` ABC defines a new method, -:meth:`~importlib.abc.InspectLoader.source_to_code` that accepts source -data and a path and returns a code object. The default implementation -is equivalent to ``compile(data, path, 'exec', dont_inherit=True)``. -(Contributed by Eric Snow and Brett Cannon in :issue:`15627`.) - -:class:`~importlib.abc.InspectLoader` also now has a default implementation -for the :meth:`~importlib.abc.InspectLoader.get_code` method. However, -it will normally be desirable to override the default implementation -for performance reasons. (Contributed by Brett Cannon in :issue:`18072`.) - -The :func:`~importlib.reload` function has been moved from :mod:`imp` to -:mod:`importlib` as part of the :mod:`imp` module deprecation. (Contributed by -Berker Peksag in :issue:`18193`.) - -:mod:`importlib.util` now has a :data:`~importlib.util.MAGIC_NUMBER` attribute -providing access to the bytecode version number. This replaces the -:func:`~imp.get_magic` function in the deprecated :mod:`imp` module. -(Contributed by Brett Cannon in :issue:`18192`.) - -New :mod:`importlib.util` functions :func:`~importlib.util.cache_from_source` -and :func:`~importlib.util.source_from_cache` replace the same-named functions -in the deprecated :mod:`imp` module. (Contributed by Brett Cannon in -:issue:`18194`.) - -The :mod:`importlib` bootstrap :class:`.NamespaceLoader` now conforms to -the :class:`.InspectLoader` ABC, which means that ``runpy`` and -``python -m`` can now be used with namespace packages. (Contributed -by Brett Cannon in :issue:`18058`.) - -:mod:`importlib.util` has a new function :func:`~importlib.util.decode_source` -that decodes source from bytes using universal newline processing. This is -useful for implementing :meth:`.InspectLoader.get_source` methods. - -:class:`importlib.machinery.ExtensionFileLoader` now has a -:meth:`~importlib.machinery.ExtensionFileLoader.get_filename` method. This was -inadvertently omitted in the original implementation. (Contributed by Eric -Snow in :issue:`19152`.) - - -inspect -------- - -The :mod:`inspect` module now offers a basic :ref:`command line interface -` to quickly display source code and other -information for modules, classes and functions. (Contributed by Claudiu Popa -and Nick Coghlan in :issue:`18626`.) - -:func:`~inspect.unwrap` makes it easy to unravel wrapper function chains -created by :func:`functools.wraps` (and any other API that sets the -``__wrapped__`` attribute on a wrapper function). (Contributed by -Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`.) - -As part of the implementation of the new :mod:`enum` module, the -:mod:`inspect` module now has substantially better support for custom -``__dir__`` methods and dynamic class attributes provided through -metaclasses. (Contributed by Ethan Furman in :issue:`18929` and -:issue:`19030`.) - -:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec` -now use the :func:`~inspect.signature` API. This allows them to -support a much broader range of callables, including those with -``__signature__`` attributes, those with metadata provided by argument -clinic, :func:`functools.partial` objects and more. Note that, unlike -:func:`~inspect.signature`, these functions still ignore ``__wrapped__`` -attributes, and report the already bound first argument for bound methods, -so it is still necessary to update your code to use -:func:`~inspect.signature` directly if those features are desired. -(Contributed by Yury Selivanov in :issue:`17481`.) - -:func:`~inspect.signature` now supports duck types of CPython functions, -which adds support for functions compiled with Cython. (Contributed -by Stefan Behnel and Yury Selivanov in :issue:`17159`.) - - -ipaddress ---------- - -:mod:`ipaddress` was added to the standard library in Python 3.3 as a -:term:`provisional API`. With the release of Python 3.4, this qualification -has been removed: :mod:`ipaddress` is now considered a stable API, covered -by the normal standard library requirements to maintain backwards -compatibility. - -A new :attr:`~ipaddress.IPv4Address.is_global` property is ``True`` if -an address is globally routeable. (Contributed by Peter Moody in -:issue:`17400`.) - - -logging -------- - -The :class:`~logging.handlers.TimedRotatingFileHandler` has a new *atTime* -parameter that can be used to specify the time of day when rollover should -happen. (Contributed by Ronald Oussoren in :issue:`9556`.) - -:class:`~logging.handlers.SocketHandler` and -:class:`~logging.handlers.DatagramHandler` now support Unix domain sockets (by -setting *port* to ``None``). (Contributed by Vinay Sajip in commit -ce46195b56a9.) - -:func:`~logging.config.fileConfig` now accepts a -:class:`configparser.RawConfigParser` subclass instance for the *fname* -parameter. This facilitates using a configuration file when logging -configuration is just a part of the overall application configuration, or where -the application modifies the configuration before passing it to -:func:`~logging.config.fileConfig`. (Contributed by Vinay Sajip in -:issue:`16110`.) - -Logging configuration data received from a socket via the -:func:`logging.config.listen` function can now be validated before being -processed by supplying a verification function as the argument to the new -*verify* keyword argument. (Contributed by Vinay Sajip in :issue:`15452`.) - - -.. _whatsnew-marshal-3: - -marshal -------- - -The default :mod:`marshal` version has been bumped to 3. The code implementing -the new version restores the Python2 behavior of recording only one copy of -interned strings and preserving the interning on deserialization, and extends -this "one copy" ability to any object type (including handling recursive -references). This reduces both the size of ``.pyc`` files and the amount of -memory a module occupies in memory when it is loaded from a ``.pyc`` (or -``.pyo``) file. (Contributed by Kristján Valur Jónsson in :issue:`16475`, -with additional speedups by Antoine Pitrou in :issue:`19219`.) - - -mmap ----- - -mmap objects can now be :mod:`weakref`\ ed. (Contributed by Valerie Lambert in -:issue:`4885`.) - - -multiprocessing ---------------- - -.. _whatsnew-multiprocessing-no-fork: - -On Unix two new :ref:`start methods `, -``spawn`` and ``forkserver``, have been added for starting processes using -:mod:`multiprocessing`. These make the mixing of processes with threads more -robust, and the ``spawn`` method matches the semantics that multiprocessing has -always used on Windows. New function -:func:`~multiprocessing.get_all_start_methods` reports all start methods -available on the platform, :func:`~multiprocessing.get_start_method` reports -the current start method, and :func:`~multiprocessing.set_start_method` sets -the start method. (Contributed by Richard Oudkerk in :issue:`8713`.) - -:mod:`multiprocessing` also now has the concept of a ``context``, which -determines how child processes are created. New function -:func:`~multiprocessing.get_context` returns a context that uses a specified -start method. It has the same API as the :mod:`multiprocessing` module itself, -so you can use it to create :class:`~multiprocessing.pool.Pool`\ s and other -objects that will operate within that context. This allows a framework and an -application or different parts of the same application to use multiprocessing -without interfering with each other. (Contributed by Richard Oudkerk in -:issue:`18999`.) - -Except when using the old *fork* start method, child processes no longer -inherit unneeded handles/file descriptors from their parents (part of -:issue:`8713`). - -:mod:`multiprocessing` now relies on :mod:`runpy` (which implements the -``-m`` switch) to initialise ``__main__`` appropriately in child processes -when using the ``spawn`` or ``forkserver`` start methods. This resolves some -edge cases where combining multiprocessing, the ``-m`` command line switch, -and explicit relative imports could cause obscure failures in child -processes. (Contributed by Nick Coghlan in :issue:`19946`.) - - -operator --------- - -New function :func:`~operator.length_hint` provides an implementation of the -specification for how the :meth:`~object.__length_hint__` special method should -be used, as part of the :pep:`424` formal specification of this language -feature. (Contributed by Armin Ronacher in :issue:`16148`.) - -There is now a pure-python version of the :mod:`operator` module available for -reference and for use by alternate implementations of Python. (Contributed by -Zachary Ware in :issue:`16694`.) - - -os --- - -There are new functions to get and set the :ref:`inheritable flag -` of a file descriptor (:func:`os.get_inheritable`, -:func:`os.set_inheritable`) or a Windows handle -(:func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable`). - -New function :func:`~os.cpu_count` reports the number of CPUs available on the -platform on which Python is running (or ``None`` if the count can't be -determined). The :func:`multiprocessing.cpu_count` function is now implemented -in terms of this function). (Contributed by Trent Nelson, Yogesh Chaudhari, -Victor Stinner, and Charles-François Natali in :issue:`17914`.) - -:func:`os.path.samestat` is now available on the Windows platform (and the -:func:`os.path.samefile` implementation is now shared between Unix and -Windows). (Contributed by Brian Curtin in :issue:`11939`.) - -:func:`os.path.ismount` now recognizes volumes mounted below a drive -root on Windows. (Contributed by Tim Golden in :issue:`9035`.) - -:func:`os.open` supports two new flags on platforms that provide them, -:data:`~os.O_PATH` (un-opened file descriptor), and :data:`~os.O_TMPFILE` -(unnamed temporary file; as of 3.4.0 release available only on Linux systems -with a kernel version of 3.11 or newer that have uapi headers). (Contributed -by Christian Heimes in :issue:`18673` and Benjamin Peterson, respectively.) - - -pdb ---- - -:mod:`pdb` has been enhanced to handle generators, :keyword:`yield`, and -``yield from`` in a more useful fashion. This is especially helpful when -debugging :mod:`asyncio` based programs. (Contributed by Andrew Svetlov and -Xavier de Gaye in :issue:`16596`.) - -The ``print`` command has been removed from :mod:`pdb`, restoring access to the -Python :func:`print` function from the pdb command line. Python2's ``pdb`` did -not have a ``print`` command; instead, entering ``print`` executed the -``print`` statement. In Python3 ``print`` was mistakenly made an alias for the -pdb :pdbcmd:`p` command. ``p``, however, prints the ``repr`` of its argument, -not the ``str`` like the Python2 ``print`` command did. Worse, the Python3 -``pdb print`` command shadowed the Python3 ``print`` function, making it -inaccessible at the ``pdb`` prompt. (Contributed by Connor Osborn in -:issue:`18764`.) - - -.. _whatsnew-protocol-4: - -pickle ------- - -:mod:`pickle` now supports (but does not use by default) a new pickle protocol, -protocol 4. This new protocol addresses a number of issues that were present -in previous protocols, such as the serialization of nested classes, very large -strings and containers, and classes whose :meth:`__new__` method takes -keyword-only arguments. It also provides some efficiency improvements. - -.. seealso:: - - :pep:`3154` -- Pickle protocol 4 - PEP written by Antoine Pitrou and implemented by Alexandre Vassalotti. - - -plistlib --------- - -:mod:`plistlib` now has an API that is similar to the standard pattern for -stdlib serialization protocols, with new :func:`~plistlib.load`, -:func:`~plistlib.dump`, :func:`~plistlib.loads`, and :func:`~plistlib.dumps` -functions. (The older API is now deprecated.) In addition to the already -supported XML plist format (:data:`~plistlib.FMT_XML`), it also now supports -the binary plist format (:data:`~plistlib.FMT_BINARY`). (Contributed by Ronald -Oussoren and others in :issue:`14455`.) - - -poplib ------- - -Two new methods have been added to :mod:`poplib`: :meth:`~poplib.POP3.capa`, -which returns the list of capabilities advertised by the POP server, and -:meth:`~poplib.POP3.stls`, which switches a clear-text POP3 session into an -encrypted POP3 session if the POP server supports it. (Contributed by Lorenzo -Catucci in :issue:`4473`.) - - -pprint ------- - -The :mod:`pprint` module's :class:`~pprint.PrettyPrinter` class and its -:func:`~pprint.pformat`, and :func:`~pprint.pprint` functions have a new -option, *compact*, that controls how the output is formatted. Currently -setting *compact* to ``True`` means that sequences will be printed with as many -sequence elements as will fit within *width* on each (indented) line. -(Contributed by Serhiy Storchaka in :issue:`19132`.) - -Long strings are now wrapped using Python's normal line continuation -syntax. (Contributed by Antoine Pitrou in :issue:`17150`.) - - -pty ---- - -:func:`pty.spawn` now returns the status value from :func:`os.waitpid` on -the child process, instead of ``None``. (Contributed by Gregory P. Smith.) - - -pydoc ------ - -The :mod:`pydoc` module is now based directly on the :func:`inspect.signature` -introspection API, allowing it to provide signature information for a wider -variety of callable objects. This change also means that ``__wrapped__`` -attributes are now taken into account when displaying help information. -(Contributed by Larry Hastings in :issue:`19674`.) - -The :mod:`pydoc` module no longer displays the ``self`` parameter for -already bound methods. Instead, it aims to always display the exact current -signature of the supplied callable. (Contributed by Larry Hastings in -:issue:`20710`.) - -In addition to the changes that have been made to :mod:`pydoc` directly, -its handling of custom ``__dir__`` methods and various descriptor -behaviours has also been improved substantially by the underlying changes in -the :mod:`inspect` module. - -As the :func:`help` builtin is based on :mod:`pydoc`, the above changes also -affect the behaviour of :func:`help`. - - -re --- - -New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor -the pattern at both ends of the string to match. This provides a way to be -explicit about the goal of the match, which avoids a class of subtle bugs where -``$`` characters get lost during code changes or the addition of alternatives -to an existing regular expression. (Contributed by Matthew Barnett in -:issue:`16203`.) - -The repr of :ref:`regex objects ` now includes the pattern -and the flags; the repr of :ref:`match objects ` now -includes the start, end, and the part of the string that matched. (Contributed -by Hugo Lopes Tavares and Serhiy Storchaka in :issue:`13592` and -:issue:`17087`.) - - -resource --------- - -New :func:`~resource.prlimit` function, available on Linux platforms with a -kernel version of 2.6.36 or later and glibc of 2.13 or later, provides the -ability to query or set the resource limits for processes other than the one -making the call. (Contributed by Christian Heimes in :issue:`16595`.) - -On Linux kernel version 2.6.36 or later, there are there are also some new -Linux specific constants: :attr:`~resource.RLIMIT_MSGQUEUE`, -:attr:`~resource.RLIMIT_NICE`, :attr:`~resource.RLIMIT_RTPRIO`, -:attr:`~resource.RLIMIT_RTTIME`, and :attr:`~resource.RLIMIT_SIGPENDING`. -(Contributed by Christian Heimes in :issue:`19324`.) - -On FreeBSD version 9 and later, there some new FreeBSD specific constants: -:attr:`~resource.RLIMIT_SBSIZE`, :attr:`~resource.RLIMIT_SWAP`, and -:attr:`~resource.RLIMIT_NPTS`. (Contributed by Claudiu Popa in -:issue:`19343`.) - - -select ------- - -:class:`~select.epoll` objects now support the context management protocol. -When used in a :keyword:`with` statement, the :meth:`~select.epoll.close` -method will be called automatically at the end of the block. (Contributed -by Serhiy Storchaka in :issue:`16488`.) - -:class:`~select.devpoll` objects now have :meth:`~select.devpoll.fileno` and -:meth:`~select.devpoll.close` methods, as well as a new attribute -:attr:`~select.devpoll.closed`. (Contributed by Victor Stinner in -:issue:`18794`.) - - -shelve ------- - -:class:`~shelve.Shelf` instances may now be used in :keyword:`with` statements, -and will be automatically closed at the end of the :keyword:`with` block. -(Contributed by Filip Gruszczyński in :issue:`13896`.) - - -shutil ------- - -:func:`~shutil.copyfile` now raises a specific :exc:`~shutil.Error` subclass, -:exc:`~shutil.SameFileError`, when the source and destination are the same -file, which allows an application to take appropriate action on this specific -error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in -:issue:`1492704`.) - - -smtpd ------ - -The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now -accept a *map* keyword argument which, if specified, is passed in to -:class:`asynchat.async_chat` as its *map* argument. This allows an application -to avoid affecting the global socket map. (Contributed by Vinay Sajip in -:issue:`11959`.) - - -smtplib -------- - -:exc:`~smtplib.SMTPException` is now a subclass of :exc:`OSError`, which allows -both socket level errors and SMTP protocol level errors to be caught in one -try/except statement by code that only cares whether or not an error occurred. -(Contributed by Ned Jackson Lovely in :issue:`2118`.) - - -socket ------- - -The socket module now supports the :data:`~socket.CAN_BCM` protocol on -platforms that support it. (Contributed by Brian Thorne in :issue:`15359`.) - -Socket objects have new methods to get or set their :ref:`inheritable flag -`, :meth:`~socket.socket.get_inheritable` and -:meth:`~socket.socket.set_inheritable`. - -The ``socket.AF_*`` and ``socket.SOCK_*`` constants are now enumeration values -using the new :mod:`enum` module. This allows meaningful names to be printed -during debugging, instead of integer "magic numbers". - -The :data:`~socket.AF_LINK` constant is now available on BSD and OSX. - -:func:`~socket.inet_pton` and :func:`~socket.inet_ntop` are now supported -on Windows. (Contributed by Atsuo Ishimoto in :issue:`7171`.) - - -sqlite3 -------- - -A new boolean parameter to the :func:`~sqlite3.connect` function, *uri*, can be -used to indicate that the *database* parameter is a ``uri`` (see the `SQLite -URI documentation `_). (Contributed by poq in -:issue:`13773`.) - - -ssl ---- - -.. _whatsnew-tls-11-12: - -:data:`~ssl.PROTOCOL_TLSv1_1` and :data:`~ssl.PROTOCOL_TLSv1_2` (TLSv1.1 and -TLSv1.2 support) have been added; support for these protocols is only available if -Python is linked with OpenSSL 1.0.1 or later. (Contributed by Michele Orrù and -Antoine Pitrou in :issue:`16692`.) - -.. _whatsnew34-sslcontext: - -New function :func:`~ssl.create_default_context` provides a standard way to -obtain an :class:`~ssl.SSLContext` whose settings are intended to be a -reasonable balance between compatibility and security. These settings are -more stringent than the defaults provided by the :class:`~ssl.SSLContext` -constructor, and may be adjusted in the future, without prior deprecation, if -best-practice security requirements change. The new recommended best -practice for using stdlib libraries that support SSL is to use -:func:`~ssl.create_default_context` to obtain an :class:`~ssl.SSLContext` -object, modify it if needed, and then pass it as the *context* argument -of the appropriate stdlib API. (Contributed by Christian Heimes -in :issue:`19689`.) - -:class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_verify_locations` -accepts a new optional argument *cadata*, which can be used to provide PEM or -DER encoded certificates directly via strings or bytes, respectively. -(Contributed by Christian Heimes in :issue:`18138`.) - -New function :func:`~ssl.get_default_verify_paths` returns -a named tuple of the paths and environment variables that the -:meth:`~ssl.SSLContext.set_default_verify_paths` method uses to set -OpenSSL's default ``cafile`` and ``capath``. This can be an aid in -debugging default verification issues. (Contributed by Christian Heimes -in :issue:`18143`.) - -:class:`~ssl.SSLContext` has a new method, -:meth:`~ssl.SSLContext.cert_store_stats`, that reports the number of loaded -``X.509`` certs, ``X.509 CA`` certs, and certificate revocation lists (``crl``\ -s), as well as a :meth:`~ssl.SSLContext.get_ca_certs` method that returns a -list of the loaded ``CA`` certificates. (Contributed by Christian Heimes in -:issue:`18147`.) - -If OpenSSL 0.9.8 or later is available, :class:`~ssl.SSLContext` has an new -attribute :attr:`~ssl.SSLContext.verify_flags` that can be used to control the -certificate verification process by setting it to some combination of the new -constants :data:`~ssl.VERIFY_DEFAULT`, :data:`~ssl.VERIFY_CRL_CHECK_LEAF`, -:data:`~ssl.VERIFY_CRL_CHECK_CHAIN`, or :data:`~ssl.VERIFY_X509_STRICT`. -OpenSSL does not do any CRL verification by default. (Contributed by -Christien Heimes in :issue:`8813`.) - -New :class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_default_certs` -loads a set of default "certificate authority" (CA) certificates from default -locations, which vary according to the platform. It can be used to load both -TLS web server authentication certificates -(``purpose=``:data:`~ssl.Purpose.SERVER_AUTH`) for a client to use to verify a -server, and certificates for a server to use in verifying client certificates -(``purpose=``:data:`~ssl.Purpose.CLIENT_AUTH`). (Contributed by Christian -Heimes in :issue:`19292`.) - -.. _whatsnew34-win-cert-store: - -Two new windows-only functions, :func:`~ssl.enum_certificates` and -:func:`~ssl.enum_crls` provide the ability to retrieve certificates, -certificate information, and CRLs from the Windows cert store. (Contributed -by Christian Heimes in :issue:`17134`.) - -.. _whatsnew34-sni: - -Support for server-side SNI (Server Name Indication) using the new -:meth:`ssl.SSLContext.set_servername_callback` method. -(Contributed by Daniel Black in :issue:`8109`.) - -The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional -``X509v3`` extension items: ``crlDistributionPoints``, ``calIssuers``, and -``OCSP`` URIs. (Contributed by Christian Heimes in :issue:`18379`.) - - -stat ----- - -The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C -implementation is required as most of the values aren't standardized and -are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.) - -The module supports new :mod:`~stat.ST_MODE` flags, :mod:`~stat.S_IFDOOR`, -:attr:`~stat.S_IFPORT`, and :attr:`~stat.S_IFWHT`. (Contributed by -Christian Hiemes in :issue:`11016`.) - - -struct ------- - -New function :mod:`~struct.iter_unpack` and a new -:meth:`struct.Struct.iter_unpack` method on compiled formats provide streamed -unpacking of a buffer containing repeated instances of a given format of data. -(Contributed by Antoine Pitrou in :issue:`17804`.) - - -subprocess ----------- - -:func:`~subprocess.check_output` now accepts an *input* argument that can -be used to provide the contents of ``stdin`` for the command that is run. -(Contributed by Zack Weinberg in :issue:`16624`.) - -:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now -work on Windows. This change was actually inadvertently made in 3.3.4. -(Contributed by Tim Golden in :issue:`10197`.) - - -sunau ------ - -The :meth:`~sunau.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) - -:meth:`sunau.open` now supports the context management protocol: when used in a -:keyword:`with` block, the ``close`` method of the returned object will be -called automatically at the end of the block. (Contributed by Serhiy Storchaka -in :issue:`18878`.) - -:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding -support for writing 24 sample using the module. (Contributed by -Serhiy Storchaka in :issue:`19261`.) - -The :meth:`~sunau.AU_write.writeframesraw` and -:meth:`~sunau.AU_write.writeframes` methods now accept any :term:`bytes-like -object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) - - -sys ---- - -New function :func:`sys.getallocatedblocks` returns the current number of -blocks allocated by the interpreter. (In CPython with the default -``--with-pymalloc`` setting, this is allocations made through the -:c:func:`PyObject_Malloc` API.) This can be useful for tracking memory leaks, -especially if automated via a test suite. (Contributed by Antoine Pitrou -in :issue:`13390`.) - -When the Python interpreter starts in :ref:`interactive mode -`, it checks for an :data:`~sys.__interactivehook__` attribute -on the :mod:`sys` module. If the attribute exists, its value is called with no -arguments just before interactive mode is started. The check is made after the -:envvar:`PYTHONSTARTUP` file is read, so it can be set there. The :mod:`site` -module :ref:`sets it ` to a function that enables tab -completion and history saving (in :file:`~/.python-history`) if the platform -supports :mod:`readline`. If you do not want this (new) behavior, you can -override it in :envvar:`PYTHONSTARTUP`, :mod:`sitecustomize`, or -:mod:`usercustomize` by deleting this attribute from :mod:`sys` (or setting it -to some other callable). (Contributed by Éric Araujo and Antoine Pitrou in -:issue:`5845`.) - - -tarfile -------- - -The :mod:`tarfile` module now supports a simple :ref:`tarfile-commandline` when -called as a script directly or via ``-m``. This can be used to create and -extract tarfile archives. (Contributed by Berker Peksag in :issue:`13477`.) - - -textwrap --------- - -The :class:`~textwrap.TextWrapper` class has two new attributes/constructor -arguments: :attr:`~textwrap.TextWrapper.max_lines`, which limits the number of -lines in the output, and :attr:`~textwrap.TextWrapper.placeholder`, which is a -string that will appear at the end of the output if it has been truncated -because of *max_lines*. Building on these capabilities, a new convenience -function :func:`~textwrap.shorten` collapses all of the whitespace in the input -to single spaces and produces a single line of a given *width* that ends with -the *placeholder* (by default, ``[...]``). (Contributed by Antoine Pitrou and -Serhiy Storchaka in :issue:`18585` and :issue:`18725`.) - - -threading ---------- - -The :class:`~threading.Thread` object representing the main thread can be -obtained from the new :func:`~threading.main_thread` function. In normal -conditions this will be the thread from which the Python interpreter was -started. (Contributed by Andrew Svetlov in :issue:`18882`.) - - -traceback ---------- - -A new :func:`traceback.clear_frames` function takes a traceback object -and clears the local variables in all of the frames it references, -reducing the amount of memory consumed. (Contributed by Andrew Kuchling in -:issue:`1565525`.) - - -types ------ - -A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define -an attribute that acts normally when looked up through an instance object, but -which is routed to the *class* ``__getattr__`` when looked up through the -class. This allows one to have properties active on a class, and have virtual -attributes on the class with the same name (see :mod:`Enum` for an example). -(Contributed by Ethan Furman in :issue:`19030`.) - - -urllib ------- - -:mod:`urllib.request` now supports ``data:`` URLs via the -:class:`~urllib.request.DataHandler` class. (Contributed by Mathias Panzenböck -in :issue:`16423`.) - -The http method that will be used by a :class:`~urllib.request.Request` class -can now be specified by setting a :class:`~urllib.request.Request.method` -class attribute on the subclass. (Contributed by Jason R Coombs in -:issue:`18978`.) - -:class:`~urllib.request.Request` objects are now reusable: if the -:attr:`~urllib.request.Request.full_url` or :attr:`~urllib.request.Request.data` -attributes are modified, all relevant internal properties are updated. This -means, for example, that it is now possible to use the same -:class:`~urllib.request.Request` object in more than one -:meth:`.OpenerDirector.open` call with different *data* arguments, or to -modify a :class:`~urllib.request.Request`\ 's ``url`` rather than recomputing it -from scratch. There is also a new -:meth:`~urllib.request.Request.remove_header` method that can be used to remove -headers from a :class:`~urllib.request.Request`. (Contributed by Alexey -Kachayev in :issue:`16464`, Daniel Wozniak in :issue:`17485`, and Damien Brecht -and Senthil Kumaran in :issue:`17272`.) - -:class:`~urllib.error.HTTPError` objects now have a -:attr:`~urllib.error.HTTPError.headers` attribute that provides access to the -HTTP response headers associated with the error. (Contributed by -Berker Peksag in :issue:`15701`.) - - -unittest --------- - -The :class:`~unittest.TestCase` class has a new method, -:meth:`~unittest.TestCase.subTest`, that produces a context manager whose -:keyword:`with` block becomes a "sub-test". This context manager allows a test -method to dynamically generate subtests by, say, calling the ``subTest`` -context manager inside a loop. A single test method can thereby produce an -indefinite number of separately-identified and separately-counted tests, all of -which will run even if one or more of them fail. For example:: - - class NumbersTest(unittest.TestCase): - def test_even(self): - for i in range(6): - with self.subTest(i=i): - self.assertEqual(i % 2, 0) - -will result in six subtests, each identified in the unittest verbose output -with a label consisting of the variable name ``i`` and a particular value for -that variable (``i=0``, ``i=1``, etc). See :ref:`subtests` for the full -version of this example. (Contributed by Antoine Pitrou in :issue:`16997`.) - -:func:`unittest.main` now accepts an iterable of test names for -*defaultTest*, where previously it only accepted a single test name as a -string. (Contributed by Jyrki Pulliainen in :issue:`15132`.) - -If :class:`~unittest.SkipTest` is raised during test discovery (that is, at the -module level in the test file), it is now reported as a skip instead of an -error. (Contributed by Zach Ware in :issue:`16935`.) - -:meth:`~unittest.TestLoader.discover` now sorts the discovered files to provide -consistent test ordering. (Contributed by Martin Melin and Jeff Ramnani in -:issue:`16709`.) - -:class:`~unittest.TestSuite` now drops references to tests as soon as the test -has been run, if the test is successful. On Python interpreters that do -garbage collection, this allows the tests to be garbage collected if nothing -else is holding a reference to the test. It is possible to override this -behavior by creating a :class:`~unittest.TestSuite` subclass that defines a -custom ``_removeTestAtIndex`` method. (Contributed by Tom Wardill, Matt -McClure, and Andrew Svetlov in :issue:`11798`.) - -A new test assertion context-manager, :meth:`~unittest.TestCase.assertLogs`, -will ensure that a given block of code emits a log message using the -:mod:`logging` module. By default the message can come from any logger and -have a priority of ``INFO`` or higher, but both the logger name and an -alternative minimum logging level may be specified. The object returned by the -context manager can be queried for the :class:`~logging.LogRecord`\ s and/or -formatted messages that were logged. (Contributed by Antoine Pitrou in -:issue:`18937`.) - -Test discovery now works with namespace packages (Contributed by Claudiu Popa -in :issue:`17457`.) - -:mod:`unittest.mock` objects now inspect their specification signatures when -matching calls, which means an argument can now be matched by either position -or name, instead of only by position. (Contributed by Antoine Pitrou in -:issue:`17015`.) - -:func:`~mock.mock_open` objects now have ``readline`` and ``readlines`` -methods. (Contributed by Toshio Kuratomi in :issue:`17467`.) - - -venv ----- - -:mod:`venv` now includes activation scripts for the ``csh`` and ``fish`` -shells. (Contributed by Andrew Svetlov in :issue:`15417`.) - -:class:`~venv.EnvBuilder` and the :func:`~venv.create` convenience function -take a new keyword argument *with_pip*, which defaults to ``False``, that -controls whether or not :class:`~venv.EnvBuilder` ensures that ``pip`` is -installed in the virtual environment. (Contributed by Nick Coghlan in -:issue:`19552` as part of the :pep:`453` implementation.) - - -wave ----- - -The :meth:`~wave.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) - -:meth:`wave.open` now supports the context management protocol. (Contributed -by Claudiu Popa in :issue:`17616`.) - -:mod:`wave` can now :ref:`write output to unseekable files -`. (Contributed by David Jones, Guilherme Polo, and Serhiy -Storchaka in :issue:`5202`.) - -The :meth:`~wave.Wave_write.writeframesraw` and -:meth:`~wave.Wave_write.writeframes` methods now accept any :term:`bytes-like -object`. (Contributed by Serhiy Storchaka in :issue:`8311`.) - - -weakref -------- - -New :class:`~weakref.WeakMethod` class simulates weak references to bound -methods. (Contributed by Antoine Pitrou in :issue:`14631`.) - -New :class:`~weakref.finalize` class makes it possible to register a callback -to be invoked when an object is garbage collected, without needing to -carefully manage the lifecycle of the weak reference itself. (Contributed by -Richard Oudkerk in :issue:`15528`.) - -The callback, if any, associated with a :class:`~weakref.ref` is now -exposed via the :attr:`~weakref.ref.__callback__` attribute. (Contributed -by Mark Dickinson in :issue:`17643`.) - - -xml.etree ---------- - -A new parser, :class:`~xml.etree.ElementTree.XMLPullParser`, allows a -non-blocking applications to parse XML documents. An example can be -seen at :ref:`elementtree-pull-parsing`. (Contributed by Antoine -Pitrou in :issue:`17741`.) - -The :mod:`xml.etree.ElementTree` :func:`~xml.etree.ElementTree.tostring` and -:func:`~xml.etree.ElementTree.tostringlist` functions, and the -:class:`~xml.etree.ElementTree.ElementTree` -:meth:`~xml.etree.ElementTree.ElementTree.write` method, now have a -*short_empty_elements* :ref:`keyword-only parameter ` -providing control over whether elements with no content are written in -abbreviated (````) or expanded (````) form. (Contributed by -Ariel Poliak and Serhiy Storchaka in :issue:`14377`.) - - -zipfile -------- - -The :meth:`~zipfile.PyZipFile.writepy` method of the -:class:`~zipfile.PyZipFile` class has a new *filterfunc* option that can be -used to control which directories and files are added to the archive. For -example, this could be used to exclude test files from the archive. -(Contributed by Christian Tismer in :issue:`19274`.) - -The *allowZip64* parameter to :class:`~zipfile.ZipFile` and -:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by -William Mallard in :issue:`17201`.) - - - -CPython Implementation Changes -============================== - - -.. _whatsnew-pep-445: - -PEP 445: Customization of CPython Memory Allocators ---------------------------------------------------- - -:pep:`445` adds new C level interfaces to customize memory allocation in -the CPython interpreter. - -.. seealso:: - - :pep:`445` -- Add new APIs to customize Python memory allocators - PEP written and implemented by Victor Stinner. - - -.. _whatsnew-pep-442: - -PEP 442: Safe Object Finalization ---------------------------------- - -:pep:`442` removes the current limitations and quirks of object finalization -in CPython. With it, objects with :meth:`__del__` methods, as well as -generators with :keyword:`finally` clauses, can be finalized when they are -part of a reference cycle. - -As part of this change, module globals are no longer forcibly set to -:const:`None` during interpreter shutdown in most cases, instead relying -on the normal operation of the cyclic garbage collector. This avoids a -whole class of interpreter-shutdown-time errors, usually involving -``__del__`` methods, that have plagued Python since the cyclic GC -was first introduced. - -.. seealso:: - - :pep:`442` -- Safe object finalization - PEP written and implemented by Antoine Pitrou. - - -.. _whatsnew-pep-456: - -PEP 456: Secure and Interchangeable Hash Algorithm --------------------------------------------------- - -:pep:`456` follows up on earlier security fix work done on Python's hash -algorithm to address certain DOS attacks to which public facing APIs backed by -dictionary lookups may be subject. (See :issue:`14621` for the start of the -current round of improvements.) The PEP unifies CPython's hash code to make it -easier for a packager to substitute a different hash algorithm, and switches -Python's default implementation to a SipHash implementation on platforms that -have a 64 bit data type. Any performance differences in comparison with the -older FNV algorithm are trivial. - -The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to -describe the hash algorithm in use by the currently executing binary. Otherwise, -the PEP does not alter any existing CPython APIs. - - -.. _whatsnew-pep-436: - -PEP 436: Argument Clinic ------------------------- - -"Argument Clinic" (:pep:`436`) is now part of the CPython build process -and can be used to simplify the process of defining and maintaining -accurate signatures for builtins and standard library extension modules -implemented in C. - -Some standard library extension modules have been converted to use Argument -Clinic in Python 3.4, and :mod:`pydoc` and :mod:`inspect` have been updated -accordingly. - -It is expected that signature metadata for programmatic introspection will -be added to additional callables implemented in C as part of Python 3.4 -maintenance releases. - -.. note:: - The Argument Clinic PEP is not fully up to date with the state of the - implementation. This has been deemed acceptable by the release manager - and core development team in this case, as Argument Clinic will not - be made available as a public API for third party use in Python 3.4. - -.. seealso:: - - :pep:`436` -- The Argument Clinic DSL - PEP written and implemented by Larry Hastings. - - -Other Build and C API Changes ------------------------------ - -* The new :c:func:`PyType_GetSlot` function has been added to the stable ABI, - allowing retrieval of function pointers from named type slots when using - the limited API. (Contributed by Martin von Löwis in :issue:`17162`.) - -* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API - allows applications embedding the CPython interpreter to reliably force - a particular encoding and error handler for the standard streams. - (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`.) - -* Most Python C APIs that don't mutate string arguments are now correctly - marked as accepting ``const char *`` rather than ``char *``. (Contributed - by Serhiy Storchaka in :issue:`1772673`.) - -* A new shell version of ``python-config`` can be used even when a python - interpreter is not available (for example, in cross compilation scenarios). - -* :c:func:`PyUnicode_FromFormat` now supports width and precision - specifications for ``%s``, ``%A``, ``%U``, ``%V``, ``%S``, and ``%R``. - (Contributed by Ysj Ray and Victor Stinner in :issue:`7330`.) - -* New function :c:func:`PyStructSequence_InitType2` supplements the - existing :c:func:`PyStructSequence_InitType` function. The difference - is that it returns ``0`` on success and ``-1`` on failure. - -* The CPython source can now be compiled using the address sanity checking - features of recent versions of GCC and clang: the false alarms in the small - object allocator have been silenced. (Contributed by Dhiru Kholia in - :issue:`18596`.) - -* The Windows build now uses `Address Space Layout Randomization - `_ and `Data Execution Prevention - `_. (Contributed by - Christian Heimes in :issue:`16632`.) - -* New function :c:func:`PyObject_LengthHint` is the C API equivalent - of :func:`operator.length_hint`. (Contributed by Armin Ronacher in - :issue:`16148`.) - - -.. _other-improvements-3.4: - -Other Improvements ------------------- - -.. _whatsnew-isolated-mode: - -* The :ref:`python ` command has a new :ref:`option - `, ``-I``, which causes it to run in "isolated mode", - which means that :data:`sys.path` contains neither the script's directory nor - the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment - variables are ignored (it implies both ``-s`` and ``-E``). Other - restrictions may also be applied in the future, with the goal being to - isolate the execution of a script from the user's environment. This is - appropriate, for example, when Python is used to run a system script. On - most POSIX systems it can and should be used in the ``#!`` line of system - scripts. (Contributed by Christian Heimes in :issue:`16499`.) - -* Tab-completion is now enabled by default in the interactive interpreter - on systems that support :mod:`readline`. History is also enabled by default, - and is written to (and read from) the file :file:`~/.python-history`. - (Contributed by Antoine Pitrou and Éric Araujo in :issue:`5845`.) - -* Invoking the Python interpreter with ``--version`` now outputs the version to - standard output instead of standard error (:issue:`18338`). Similar changes - were made to :mod:`argparse` (:issue:`18920`) and other modules that have - script-like invocation capabilities (:issue:`18922`). - -* The CPython Windows installer now adds ``.py`` to the :envvar:`PATHEXT` - variable when extensions are registered, allowing users to run a python - script at the windows command prompt by just typing its name without the - ``.py`` extension. (Contributed by Paul Moore in :issue:`18569`.) - -* A new ``make`` target `coverage-report - `_ - will build python, run the test suite, and generate an HTML coverage report - for the C codebase using ``gcov`` and `lcov - `_. - -* The ``-R`` option to the :ref:`python regression test suite ` now - also checks for memory allocation leaks, using - :func:`sys.getallocatedblocks()`. (Contributed by Antoine Pitrou in - :issue:`13390`.) - -* ``python -m`` now works with namespace packages. - -* The :mod:`stat` module is now implemented in C, which means it gets the - values for its constants from the C header files, instead of having the - values hard-coded in the python module as was previously the case. - -* Loading multiple python modules from a single OS module (``.so``, ``.dll``) - now works correctly (previously it silently returned the first python - module in the file). (Contributed by Václav Šmilauer in :issue:`16421`.) - -* A new opcode, :opcode:`LOAD_CLASSDEREF`, has been added to fix a bug in the - loading of free variables in class bodies that could be triggered by certain - uses of :ref:`__prepare__ `. (Contributed by Benjamin Peterson in - :issue:`17853`.) - -* A number of MemoryError-related crashes were identified and fixed by Victor - Stinner using his :pep:`445`-based ``pyfailmalloc`` tool (:issue:`18408`, - :issue:`18520`). - -* The :ref:`pyvenv ` command now accepts a ``--copies`` option - to use copies rather than symlinks even on systems where symlinks are the - default. (Contributed by Vinay Sajip in :issue:`18807`.) - -* The :ref:`pyvenv ` command also accepts a ``--without-pip`` - option to suppress the otherwise-automatic bootstrapping of pip into - the virtual environment. (Contributed by Nick Coghlan in :issue:`19552` - as part of the :pep:`453` implementation.) - -* The encoding name is now optional in the value set for the - :envvar:`PYTHONIOENCODING` environment variable. This makes it possible to - set just the error handler, without changing the default encoding. - (Contributed by Serhiy Storchaka in :issue:`18818`.) - -* The :mod:`bz2`, :mod:`lzma`, and :mod:`gzip` module ``open`` functions now - support ``x`` (exclusive creation) mode. (Contributed by Tim Heaney and - Vajrasky Kok in :issue:`19201`, :issue:`19222`, and :issue:`19223`.) - - -Significant Optimizations -------------------------- - -* The UTF-32 decoder is now 3x to 4x faster. (Contributed by Serhiy Storchaka - in :issue:`14625`.) - -* The cost of hash collisions for sets is now reduced. Each hash table - probe now checks a series of consecutive, adjacent key/hash pairs before - continuing to make random probes through the hash table. This exploits - cache locality to make collision resolution less expensive. - The collision resolution scheme can be described as a hybrid of linear - probing and open addressing. The number of additional linear probes - defaults to nine. This can be changed at compile-time by defining - LINEAR_PROBES to be any value. Set LINEAR_PROBES=0 to turn-off - linear probing entirely. (Contributed by Raymond Hettinger in - :issue:`18771`.) - -* The interpreter starts about 30% faster. A couple of measures lead to the - speedup. The interpreter loads fewer modules on startup, e.g. the :mod:`re`, - :mod:`collections` and :mod:`locale` modules and their dependencies are no - longer imported by default. The marshal module has been improved to load - compiled Python code faster. (Contributed by Antoine Pitrou, Christian - Heimes and Victor Stinner in :issue:`19219`, :issue:`19218`, :issue:`19209`, - :issue:`19205` and :issue:`9548`.) - -* :class:`bz2.BZ2File` is now as fast or faster than the Python2 version for - most cases. :class:`lzma.LZMAFile` has also been optimized. (Contributed by - Serhiy Storchaka and Nadeem Vawda in :issue:`16034`.) - -* :func:`random.getrandbits` is 20%-40% faster for small integers (the most - common use case). (Contributed by Serhiy Storchaka in :issue:`16674`.) - -* By taking advantage of the new storage format for strings, pickling of - strings is now significantly faster. (Contributed by Victor Stinner and - Antoine Pitrou in :issue:`15596`.) - -* A performance issue in :meth:`io.FileIO.readall` has been solved. This - particularly affects Windows, and significantly speeds up the case of piping - significant amounts of data through :mod:`subprocess`. (Contributed - by Richard Oudkerk in :issue:`15758`.) - -* :func:`html.escape` is now 10x faster. (Contributed by Matt Bryant in - :issue:`18020`.) - -* On Windows, the native ``VirtualAlloc`` is now used instead of the CRT - ``malloc`` in ``obmalloc``. Artificial benchmarks show about a 3% memory - savings. - -* :func:`os.urandom` now uses a lazily-opened persistent file descriptor - so as to avoid using many file descriptors when run in parallel from - multiple threads. (Contributed by Antoine Pitrou in :issue:`18756`.) - - -.. _deprecated-3.4: - -Deprecated -========== - -This section covers various APIs and other features that have been deprecated -in Python 3.4, and will be removed in Python 3.5 or later. In most (but not -all) cases, using the deprecated APIs will produce a :exc:`DeprecationWarning` -when the interpreter is run with deprecation warnings enabled (for example, by -using ``-Wd``). - - -Deprecations in the Python API ------------------------------- - -* As mentioned in :ref:`whatsnew-pep-451`, a number of :mod:`importlib` - methods and functions are deprecated: :meth:`importlib.find_loader` is - replaced by :func:`importlib.util.find_spec`; - :meth:`importlib.machinery.PathFinder.find_module` is replaced by - :meth:`importlib.machinery.PathFinder.find_spec`; - :meth:`importlib.abc.MetaPathFinder.find_module` is replaced by - :meth:`importlib.abc.MetaPathFinder.find_spec`; - :meth:`importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module` are replaced by - :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC - ``load_module`` methods (:meth:`importlib.abc.Loader.load_module`, - :meth:`importlib.abc.InspectLoader.load_module`, - :meth:`importlib.abc.FileLoader.load_module`, - :meth:`importlib.abc.SourceLoader.load_module`) should no longer be - implemented, instead loaders should implement an - ``exec_module`` method - (:meth:`importlib.abc.Loader.exec_module`, - :meth:`importlib.abc.InspectLoader.exec_module` - :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system - take care of the rest; and - :meth:`importlib.abc.Loader.module_repr`, - :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, - and :meth:`importlib.util.set_package` are no longer needed because their - functions are now handled automatically by the import system. - -* The :mod:`imp` module is pending deprecation. To keep compatibility with - Python 2/3 code bases, the module's removal is currently not scheduled. - -* The :mod:`formatter` module is pending deprecation and is slated for removal - in Python 3.6. - -* ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is - deprecated. Python 3.6 will require an explicit digest name or constructor as - *digestmod* argument. - -* The internal ``Netrc`` class in the :mod:`ftplib` module has been documented - as deprecated in its docstring for quite some time. It now emits a - :exc:`DeprecationWarning` and will be removed completely in Python 3.5. - -* The undocumented *endtime* argument to :meth:`subprocess.Popen.wait` should - not have been exposed and is hopefully not in use; it is deprecated and - will mostly likely be removed in Python 3.5. - -* The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated. - -* The :mod:`plistlib` :func:`~plistlib.readPlist`, - :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and - :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the - corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`, - :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data` - is deprecated in favor of just using the :class:`bytes` constructor. - -* The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by - ``EXT_SUFFIX``. - -* The ``U`` mode accepted by various ``open`` functions is deprecated. - In Python3 it does not do anything useful, and should be replaced by - appropriate uses of :class:`io.TextIOWrapper` (if needed) and its *newline* - argument. - -* The *parser* argument of :func:`xml.etree.ElementTree.iterparse` has - been deprecated, as has the *html* argument of - :func:`~xml.etree.ElementTree.XMLParser`. To prepare for the removal of the - latter, all arguments to ``XMLParser`` should be passed by keyword. - - -Deprecated Features -------------------- - -* Running :ref:`idle` with the ``-n`` flag (no subprocess) is deprecated. - However, the feature will not be removed until :issue:`18823` is resolved. - -* The site module adding a "site-python" directory to sys.path, if it - exists, is deprecated (:issue:`19375`). - - - -Removed -======= - - -Operating Systems No Longer Supported -------------------------------------- - -Support for the following operating systems has been removed from the source -and build tools: - -* OS/2 (:issue:`16135`). -* Windows 2000 (changeset e52df05b496a). -* Windows systems where ``COMSPEC`` points to ``command.com`` (:issue:`14470`). -* VMS (:issue:`16136`). - - -API and Feature Removals ------------------------- - -The following obsolete and previously deprecated APIs and features have been -removed: - -* The unmaintained ``Misc/TextMate`` and ``Misc/vim`` directories have been - removed (see the `devguide `_ - for suggestions on what to use instead). - -* The ``SO`` makefile macro is removed (it was replaced by the - ``SHLIB_SUFFIX`` and ``EXT_SUFFIX`` macros) (:issue:`16754`). - -* The ``PyThreadState.tick_counter`` field has been removed; its value has - been meaningless since Python 3.2, when the "new GIL" was introduced - (:issue:`19199`). - -* ``PyLoader`` and ``PyPycLoader`` have been removed from :mod:`importlib`. - (Contributed by Taras Lyapun in :issue:`15641`.) - -* The *strict* argument to :class:`~http.client.HTTPConnection` and - :class:`~http.client.HTTPSConnection` has been removed. HTTP 0.9-style - "Simple Responses" are no longer supported. - -* The deprecated :mod:`urllib.request.Request` getter and setter methods - ``add_data``, ``has_data``, ``get_data``, ``get_type``, ``get_host``, - ``get_selector``, ``set_proxy``, ``get_origin_req_host``, and - ``is_unverifiable`` have been removed (use direct attribute access instead). - -* Support for loading the deprecated ``TYPE_INT64`` has been removed from - :mod:`marshal`. (Contributed by Dan Riti in :issue:`15480`.) - -* :class:`inspect.Signature`: positional-only parameters are now required - to have a valid name. - -* :meth:`object.__format__` no longer accepts non-empty format strings, it now - raises a :exc:`TypeError` instead. Using a non-empty string has been - deprecated since Python 3.2. This change has been made to prevent a - situation where previously working (but incorrect) code would start failing - if an object gained a __format__ method, which means that your code may now - raise a :exc:`TypeError` if you are using an ``'s'`` format code with objects - that do not have a __format__ method that handles it. See :issue:`7994` for - background. - -* :meth:`difflib.SequenceMatcher.isbjunk` and - :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have - now been removed: use ``x in sm.bjunk`` and - ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object - (:issue:`13248`). - - -Code Cleanups -------------- - -* The unused and undocumented internal ``Scanner`` class has been removed from - the :mod:`pydoc` module. - -* The private and effectively unused ``_gestalt`` module has been removed, - along with the private :mod:`platform` functions ``_mac_ver_lookup``, - ``_mac_ver_gstalt``, and ``_bcd2str``, which would only have ever been called - on badly broken OSX systems (see :issue:`18393`). - -* The hardcoded copies of certain :mod:`stat` constants that were included in - the :mod:`tarfile` module namespace have been removed. - - - -Porting to Python 3.4 -===================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - - -Changes in 'python' Command Behavior ------------------------------------- - -* In a posix shell, setting the :envvar:`PATH` environment variable to - an empty value is equivalent to not setting it at all. However, setting - :envvar:`PYTHONPATH` to an empty value was *not* equivalent to not setting it - at all: setting :envvar:`PYTHONPATH` to an empty value was equivalent to - setting it to ``.``, which leads to confusion when reasoning by analogy to - how :envvar:`PATH` works. The behavior now conforms to the posix convention - for :envvar:`PATH`. - -* The [X refs, Y blocks] output of a debug (``--with-pydebug``) build of the - CPython interpreter is now off by default. It can be re-enabled using the - ``-X showrefcount`` option. (Contributed by Ezio Melotti in :issue:`17323`.) - -* The python command and most stdlib scripts (as well as :mod:`argparse`) now - output ``--version`` information to ``stdout`` instead of ``stderr`` (for - issue list see :ref:`other-improvements-3.4` above). - - -Changes in the Python API -------------------------- - -* The ABCs defined in :mod:`importlib.abc` now either raise the appropriate - exception or return a default value instead of raising - :exc:`NotImplementedError` blindly. This will only affect code calling - :func:`super` and falling through all the way to the ABCs. For compatibility, - catch both :exc:`NotImplementedError` or the appropriate exception as needed. - -* The module type now initializes the :attr:`__package__` and :attr:`__loader__` - attributes to ``None`` by default. To determine if these attributes were set - in a backwards-compatible fashion, use e.g. - ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) - -* :meth:`importlib.util.module_for_loader` now sets ``__loader__`` and - ``__package__`` unconditionally to properly support reloading. If this is not - desired then you will need to set these attributes manually. You can use - :func:`importlib.util.module_to_load` for module management. - -* Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, - ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. - Note that this restores a pre-3.3 behavior in that it means a module is - re-found when re-loaded (:issue:`19413`). - -* Frozen packages no longer set ``__path__`` to a list containing the package - name, they now set it to an empty list. The previous behavior could cause - the import system to do the wrong thing on submodule imports if there was - also a directory with the same name as the frozen package. The correct way - to determine if a module is a package or not is to use ``hasattr(module, - '__path__')`` (:issue:`18065`). - -* Frozen modules no longer define a ``__file__`` attribute. It's semantically - incorrect for frozen modules to set the attribute as they are not loaded from - any explicit location. If you must know that a module comes from frozen code - then you can see if the module's ``__spec__.location`` is set to ``'frozen'``, - check if the loader is a subclass of - :class:`importlib.machinery.FrozenImporter`, - or if Python 2 compatibility is necessary you can use :func:`imp.is_frozen`. - -* :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path - it would write to is a symlink or a non-regular file. This is to act as a - warning that import will overwrite those files with a regular file regardless - of what type of file path they were originally. - -* :meth:`importlib.abc.SourceLoader.get_source` no longer raises - :exc:`ImportError` when the source code being loaded triggers a - :exc:`SyntaxError` or :exc:`UnicodeDecodeError`. As :exc:`ImportError` is - meant to be raised only when source code cannot be found but it should, it was - felt to be over-reaching/overloading of that meaning when the source code is - found but improperly structured. If you were catching ImportError before and - wish to continue to ignore syntax or decoding issues, catch all three - exceptions now. - -* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly - set the ``__wrapped__`` attribute to the function being wrapped, even if - that function also had its ``__wrapped__`` attribute set. This means - ``__wrapped__`` attributes now correctly link a stack of decorated - functions rather than every ``__wrapped__`` attribute in the chain - referring to the innermost function. Introspection libraries that - assumed the previous behaviour was intentional can use - :func:`inspect.unwrap` to access the first function in the chain that has - no ``__wrapped__`` attribute. - -* :func:`inspect.getfullargspec` has been reimplemented on top of - :func:`inspect.signature` and hence handles a much wider variety of callable - objects than it did in the past. It is expected that additional builtin and - extension module callables will gain signature metadata over the course of - the Python 3.4 series. Code that assumes that - :func:`inspect.getfullargspec` will fail on non-Python callables may need - to be adjusted accordingly. - -* :class:`importlib.machinery.PathFinder` now passes on the current working - directory to objects in :data:`sys.path_hooks` for the empty string. This - results in :data:`sys.path_importer_cache` never containing ``''``, thus - iterating through :data:`sys.path_importer_cache` based on :data:`sys.path` - will not find all keys. A module's ``__file__`` when imported in the current - working directory will also now have an absolute path, including when using - ``-m`` with the interpreter (except for ``__main__.__file__`` when a script - has been executed directly using a relative path) (Contributed by Brett - Cannon in :issue:`18416`). is specified on the command-line) - (:issue:`18416`). - -* The removal of the *strict* argument to :class:`~http.client.HTTPConnection` - and :class:`~http.client.HTTPSConnection` changes the meaning of the - remaining arguments if you are specifying them positionally rather than by - keyword. If you've been paying attention to deprecation warnings your code - should already be specifying any additional arguments via keywords. - -* Strings between ``from __future__ import ...`` statements now *always* raise - a :exc:`SyntaxError`. Previously if there was no leading docstring, an - interstitial string would sometimes be ignored. This brings CPython into - compliance with the language spec; Jython and PyPy already were. - (:issue:`17434`). - -* :meth:`ssl.SSLSocket.getpeercert` and :meth:`ssl.SSLSocket.do_handshake` - now raise an :exc:`OSError` with ``ENOTCONN`` when the ``SSLSocket`` is not - connected, instead of the previous behavior of raising an - :exc:`AttributeError`. In addition, :meth:`~ssl.SSLSocket.getpeercert` - will raise a :exc:`ValueError` if the handshake has not yet been done. - -* :func:`base64.b32decode` now raises a :exc:`binascii.Error` when the - input string contains non-b32-alphabet characters, instead of a - :exc:`TypeError`. This particular :exc:`TypeError` was missed when the other - :exc:`TypeError`\ s were converted. (Contributed by Serhiy Storchaka in - :issue:`18011`.) Note: this change was also inadvertently applied in Python - 3.3.3. - -* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when - the creating :class:`cgi.FieldStorage` instance is garbage collected. If you - were pulling the file object out separately from the :class:`cgi.FieldStorage` - instance and not keeping the instance alive, then you should either store the - entire :class:`cgi.FieldStorage` instance or read the contents of the file - before the :class:`cgi.FieldStorage` instance is garbage collected. - -* Calling ``read`` or ``write`` on a closed SSL socket now raises an - informative :exc:`ValueError` rather than the previous more mysterious - :exc:`AttributeError` (:issue:`9177`). - -* :meth:`slice.indices` no longer produces an :exc:`OverflowError` for huge - values. As a consequence of this fix, :meth:`slice.indices` now raises a - :exc:`ValueError` if given a negative length; previously it returned nonsense - values (:issue:`14794`). - -* The :class:`complex` constructor, unlike the :mod:`cmath` functions, was - incorrectly accepting :class:`float` values if an object's ``__complex__`` - special method returned one. This now raises a :exc:`TypeError`. - (:issue:`16290`.) - -* The :class:`int` constructor in 3.2 and 3.3 erroneously accepts :class:`float` - values for the *base* parameter. It is unlikely anyone was doing this, but - if so, it will now raise a :exc:`TypeError` (:issue:`16772`). - -* Defaults for keyword-only arguments are now evaluated *after* defaults for - regular keyword arguments, instead of before. Hopefully no one wrote any - code that depends on the previous buggy behavior (:issue:`16967`). - -* Stale thread states are now cleared after :func:`~os.fork`. This may cause - some system resources to be released that previously were incorrectly kept - perpetually alive (for example, database connections kept in thread-local - storage). (:issue:`17094`.) - -* Parameter names in ``__annotations__`` dicts are now mangled properly, - similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in - :issue:`20625`.) - -* :attr:`hashlib.hash.name` now always returns the identifier in lower case. - Previously some builtin hashes had uppercase names, but now that it is a - formal public interface the naming has been made consistent (:issue:`18532`). - -* Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run - a set of tests may fail. Test suites should not be re-used in this fashion - since it means state is retained between test runs, breaking the test - isolation that :mod:`unittest` is designed to provide. However, if the lack - of isolation is considered acceptable, the old behavior can be restored by - creating a :mod:`~unittest.TestSuite` subclass that defines a - ``_removeTestAtIndex`` method that does nothing (see - :meth:`.TestSuite.__iter__`) (:issue:`11798`). - -* :mod:`unittest` now uses :mod:`argparse` for command line parsing. There are - certain invalid command forms that used to work that are no longer allowed; - in theory this should not cause backward compatibility issues since the - disallowed command forms didn't make any sense and are unlikely to be in use. - -* The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and - the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of - ``match`` objects now always return a *bytes* object when the string - to be matched is a :term:`bytes-like object`. Previously the return type - matched the input type, so if your code was depending on the return value - being, say, a ``bytearray``, you will need to change your code. - -* :mod:`audioop` functions now raise an error immediately if passed string - input, instead of failing randomly later on (:issue:`16685`). - -* The new *convert_charrefs* argument to :class:`~html.parser.HTMLParser` - currently defaults to ``False`` for backward compatibility, but will - eventually be changed to default to ``True``. It is recommended that you add - this keyword, with the appropriate value, to any - :class:`~html.parser.HTMLParser` calls in your code (:issue:`13633`). - -* Since the *digestmod* argument to the :func:`hmac.new` function will in the - future have no default, all calls to :func:`hmac.new` should be changed to - explicitly specify a *digestmod* (:issue:`17276`). - -* Calling :func:`sysconfig.get_config_var` with the ``SO`` key, or looking - ``SO`` up in the results of a call to :func:`sysconfig.get_config_vars` - is deprecated. This key should be replaced by ``EXT_SUFFIX`` or - ``SHLIB_SUFFIX``, depending on the context (:issue:`19555`). - -* Any calls to ``open`` functions that specify ``U`` should be modified. - ``U`` is ineffective in Python3 and will eventually raise an error if used. - Depending on the function, the equivalent of its old Python2 behavior can be - achieved using either a *newline* argument, or if necessary by wrapping the - stream in :mod:`~io.TextIOWrapper` to use its *newline* argument - (:issue:`15204`). - -* If you use :ref:`pyvenv ` in a script and desire that pip - *not* be installed, you must add ``--without-pip`` to your command - invocation. - -* The default behavior of :func:`json.dump` and :func:`json.dumps` when - an indent is specified has changed: it no longer produces trailing - spaces after the item separating commas at the ends of lines. This - will matter only if you have tests that are doing white-space-sensitive - comparisons of such output (:issue:`16333`). - -* :mod:`doctest` now looks for doctests in extension module ``__doc__`` - strings, so if your doctest test discovery includes extension modules that - have things that look like doctests in them you may see test failures you've - never seen before when running your tests (:issue:`3158`). - -* The :mod:`collections.abc` module has been slightly refactored as - part of the Python startup improvements. As a consequence of this, it is no - longer the case that importing :mod:`collections` automatically imports - :mod:`collections.abc`. If your program depended on the (undocumented) - implicit import, you will need to add an explicit ``import collections.abc`` - (:issue:`20784`). - - -Changes in the C API --------------------- - -* :c:func:`PyEval_EvalFrameEx`, :c:func:`PyObject_Repr`, and - :c:func:`PyObject_Str`, along with some other internal C APIs, now include - a debugging assertion that ensures they are not used in situations where - they may silently discard a currently active exception. In cases where - discarding the active exception is expected and desired (for example, - because it has already been saved locally with :c:func:`PyErr_Fetch` or - is being deliberately replaced with a different exception), an explicit - :c:func:`PyErr_Clear` call will be needed to avoid triggering the - assertion when invoking these operations (directly or indirectly) and - running against a version of Python that is compiled with assertions - enabled. - -* :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** - argument is not set. Previously only ``NULL`` was returned with no exception - set. - -* The result of the :c:data:`PyOS_ReadlineFunctionPointer` callback must - now be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred, instead of a - string allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc` - (:issue:`16742`) - -* :c:func:`PyThread_set_key_value` now always set the value. In Python - 3.3, the function did nothing if the key already exists (if the current - value is a non-NULL pointer). - -* The ``f_tstate`` (thread state) field of the :c:type:`PyFrameObject` - structure has been removed to fix a bug: see :issue:`14432` for the - rationale. - -Changed in 3.4.3 -================ - -.. _pep-476: - -PEP 476: Enabling certificate verification by default for stdlib http clients ------------------------------------------------------------------------------ - -:mod:`http.client` and modules which use it, such as :mod:`urllib.request` and -:mod:`xmlrpc.client`, will now verify that the server presents a certificate -which is signed by a CA in the platform trust store and whose hostname matches -the hostname being requested by default, significantly improving security for -many applications. - -For applications which require the old previous behavior, they can pass an -alternate context:: - - import urllib.request - import ssl - - # This disables all verification - context = ssl._create_unverified_context() - - # This allows using a specific certificate for the host, which doesn't need - # to be in the trust store - context = ssl.create_default_context(cafile="/path/to/file.crt") - - urllib.request.urlopen("https://invalid-cert", context=context) diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.5.rst --- a/Doc/whatsnew/3.5.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2495 +0,0 @@ -**************************** - What's New In Python 3.5 -**************************** - -:Editors: Elvis Pranskevichus , Yury Selivanov - -.. Rules for maintenance: - - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. - - * The maintainer will go through Misc/NEWS periodically and add - changes; it's therefore more important to add your changes to - Misc/NEWS than to this file. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small - or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to just add a fragmentary note about a change. For - example: "XXX Describe the transmogrify() function added to the - socket module." The maintainer will research the change and - write the necessary text. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - XXX Describe the transmogrify() function added to the socket - module. - (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - -This article explains the new features in Python 3.5, compared to 3.4. -Python 3.5 was released on September 13, 2015.  See the -`changelog `_ for a full -list of changes. - -.. seealso:: - - :pep:`478` - Python 3.5 Release Schedule - - -Summary -- Release highlights -============================= - -New syntax features: - -* :ref:`PEP 492 `, coroutines with async and await syntax. -* :ref:`PEP 465 `, a new matrix multiplication operator: ``a @ b``. -* :ref:`PEP 448 `, additional unpacking generalizations. - - -New library modules: - -* :mod:`typing`: :ref:`PEP 484 -- Type Hints `. -* :mod:`zipapp`: :ref:`PEP 441 Improving Python ZIP Application Support - `. - - -New built-in features: - -* ``bytes % args``, ``bytearray % args``: :ref:`PEP 461 ` -- - Adding ``%`` formatting to bytes and bytearray. - -* New :meth:`bytes.hex`, :meth:`bytearray.hex` and :meth:`memoryview.hex` - methods. (Contributed by Arnon Yaari in :issue:`9951`.) - -* :class:`memoryview` now supports tuple indexing (including multi-dimensional). - (Contributed by Antoine Pitrou in :issue:`23632`.) - -* Generators have a new ``gi_yieldfrom`` attribute, which returns the - object being iterated by ``yield from`` expressions. (Contributed - by Benno Leslie and Yury Selivanov in :issue:`24450`.) - -* A new :exc:`RecursionError` exception is now raised when maximum - recursion depth is reached. (Contributed by Georg Brandl - in :issue:`19235`.) - - -CPython implementation improvements: - -* When the ``LC_TYPE`` locale is the POSIX locale (``C`` locale), - :py:data:`sys.stdin` and :py:data:`sys.stdout` now use the - ``surrogateescape`` error handler, instead of the ``strict`` error handler. - (Contributed by Victor Stinner in :issue:`19977`.) - -* ``.pyo`` files are no longer used and have been replaced by a more flexible - scheme that includes the optimization level explicitly in ``.pyc`` name. - (See :ref:`PEP 488 overview `.) - -* Builtin and extension modules are now initialized in a multi-phase process, - which is similar to how Python modules are loaded. - (See :ref:`PEP 489 overview `.) - - -Significant improvements in the standard library: - -* :class:`collections.OrderedDict` is now - :ref:`implemented in C `, which makes it - 4 to 100 times faster. - -* The :mod:`ssl` module gained - :ref:`support for Memory BIO `, which decouples SSL - protocol handling from network IO. - -* The new :func:`os.scandir` function provides a - :ref:`better and significantly faster way ` - of directory traversal. - -* :func:`functools.lru_cache` has been mostly - :ref:`reimplemented in C `, yielding much better - performance. - -* The new :func:`subprocess.run` function provides a - :ref:`streamlined way to run subprocesses `. - -* The :mod:`traceback` module has been significantly - :ref:`enhanced ` for improved - performance and developer convenience. - - -Security improvements: - -* SSLv3 is now disabled throughout the standard library. - It can still be enabled by instantiating a :class:`ssl.SSLContext` - manually. (See :issue:`22638` for more details; this change was - backported to CPython 3.4 and 2.7.) - -* HTTP cookie parsing is now stricter, in order to protect - against potential injection attacks. (Contributed by Antoine Pitrou - in :issue:`22796`.) - - -Windows improvements: - -* A new installer for Windows has replaced the old MSI. - See :ref:`using-on-windows` for more information. - -* Windows builds now use Microsoft Visual C++ 14.0, and extension modules - should use the same. - - -Please read on for a comprehensive list of user-facing changes, including many -other smaller improvements, CPython optimizations, deprecations, and potential -porting issues. - - -New Features -============ - -.. _whatsnew-pep-492: - -PEP 492 - Coroutines with async and await syntax ------------------------------------------------- - -:pep:`492` greatly improves support for asynchronous programming in Python -by adding :term:`awaitable objects `, -:term:`coroutine functions `, -:term:`asynchronous iteration `, -and :term:`asynchronous context managers `. - -Coroutine functions are declared using the new :keyword:`async def` syntax:: - - >>> async def coro(): - ... return 'spam' - -Inside a coroutine function, the new :keyword:`await` expression can be used -to suspend coroutine execution until the result is available. Any object -can be *awaited*, as long as it implements the :term:`awaitable` protocol by -defining the :meth:`__await__` method. - -PEP 492 also adds :keyword:`async for` statement for convenient iteration -over asynchronous iterables. - -An example of a rudimentary HTTP client written using the new syntax:: - - import asyncio - - async def http_get(domain): - reader, writer = await asyncio.open_connection(domain, 80) - - writer.write(b'\r\n'.join([ - b'GET / HTTP/1.1', - b'Host: %b' % domain.encode('latin-1'), - b'Connection: close', - b'', b'' - ])) - - async for line in reader: - print('>>>', line) - - writer.close() - - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(http_get('example.com')) - finally: - loop.close() - - -Similarly to asynchronous iteration, there is a new syntax for asynchronous -context managers. The following script:: - - import asyncio - - async def coro(name, lock): - print('coro {}: waiting for lock'.format(name)) - async with lock: - print('coro {}: holding the lock'.format(name)) - await asyncio.sleep(1) - print('coro {}: releasing the lock'.format(name)) - - loop = asyncio.get_event_loop() - lock = asyncio.Lock() - coros = asyncio.gather(coro(1, lock), coro(2, lock)) - try: - loop.run_until_complete(coros) - finally: - loop.close() - -will output:: - - coro 2: waiting for lock - coro 2: holding the lock - coro 1: waiting for lock - coro 2: releasing the lock - coro 1: holding the lock - coro 1: releasing the lock - -Note that both :keyword:`async for` and :keyword:`async with` can only -be used inside a coroutine function declared with :keyword:`async def`. - -Coroutine functions are intended to be run inside a compatible event loop, -such as the :ref:`asyncio loop `. - -.. seealso:: - - :pep:`492` -- Coroutines with async and await syntax - PEP written and implemented by Yury Selivanov. - - -.. _whatsnew-pep-465: - -PEP 465 - A dedicated infix operator for matrix multiplication --------------------------------------------------------------- - -:pep:`465` adds the ``@`` infix operator for matrix multiplication. -Currently, no builtin Python types implement the new operator, however, it -can be implemented by defining :meth:`__matmul__`, :meth:`__rmatmul__`, -and :meth:`__imatmul__` for regular, reflected, and in-place matrix -multiplication. The semantics of these methods is similar to that of -methods defining other infix arithmetic operators. - -Matrix multiplication is a notably common operation in many fields of -mathematics, science, engineering, and the addition of ``@`` allows writing -cleaner code:: - - S = (H @ beta - r).T @ inv(H @ V @ H.T) @ (H @ beta - r) - -instead of:: - - S = dot((dot(H, beta) - r).T, - dot(inv(dot(dot(H, V), H.T)), dot(H, beta) - r)) - -NumPy 1.10 has support for the new operator:: - - >>> import numpy - - >>> x = numpy.ones(3) - >>> x - array([ 1., 1., 1.]) - - >>> m = numpy.eye(3) - >>> m - array([[ 1., 0., 0.], - [ 0., 1., 0.], - [ 0., 0., 1.]]) - - >>> x @ m - array([ 1., 1., 1.]) - - -.. seealso:: - - :pep:`465` -- A dedicated infix operator for matrix multiplication - PEP written by Nathaniel J. Smith; implemented by Benjamin Peterson. - - -.. _whatsnew-pep-448: - -PEP 448 - Additional Unpacking Generalizations ----------------------------------------------- - -:pep:`448` extends the allowed uses of the ``*`` iterable unpacking -operator and ``**`` dictionary unpacking operator. It is now possible -to use an arbitrary number of unpackings in function calls:: - - >>> print(*[1], *[2], 3, *[4, 5]) - 1 2 3 4 5 - - >>> def fn(a, b, c, d): - ... print(a, b, c, d) - ... - - >>> fn(**{'a': 1, 'c': 3}, **{'b': 2, 'd': 4}) - 1 2 3 4 - -Similarly, tuple, list, set, and dictionary displays allow multiple -unpackings:: - - >>> *range(4), 4 - (0, 1, 2, 3, 4) - - >>> [*range(4), 4] - [0, 1, 2, 3, 4] - - >>> {*range(4), 4, *(5, 6, 7)} - {0, 1, 2, 3, 4, 5, 6, 7} - - >>> {'x': 1, **{'y': 2}} - {'x': 1, 'y': 2} - -.. seealso:: - - :pep:`448` -- Additional Unpacking Generalizations - PEP written by Joshua Landau; implemented by Neil Girdhar, - Thomas Wouters, and Joshua Landau. - - -.. _whatsnew-pep-461: - -PEP 461 - % formatting support for bytes and bytearray ------------------------------------------------------- - -:pep:`461` adds support for the ``%`` -:ref:`interpolation operator ` to :class:`bytes` -and :class:`bytearray`. - -While interpolation is usually thought of as a string operation, there are -cases where interpolation on ``bytes`` or ``bytearrays`` makes sense, and the -work needed to make up for this missing functionality detracts from the -overall readability of the code. This issue is particularly important when -dealing with wire format protocols, which are often a mixture of binary -and ASCII compatible text. - -Examples:: - - >>> b'Hello %b!' % b'World' - b'Hello World!' - - >>> b'x=%i y=%f' % (1, 2.5) - b'x=1 y=2.500000' - -Unicode is not allowed for ``%b``, but it is accepted by ``%a`` (equivalent of -``repr(obj).encode('ascii', 'backslashreplace')``):: - - >>> b'Hello %b!' % 'World' - Traceback (most recent call last): - File "", line 1, in - TypeError: %b requires bytes, or an object that implements __bytes__, not 'str' - - >>> b'price: %a' % '10€' - b"price: '10\\u20ac'" - -Note that ``%s`` and ``%r`` conversion types, although supported, should -only be used in codebases that need compatibility with Python 2. - -.. seealso:: - - :pep:`461` -- Adding % formatting to bytes and bytearray - PEP written by Ethan Furman; implemented by Neil Schemenauer and - Ethan Furman. - - -.. _whatsnew-pep-484: - -PEP 484 - Type Hints --------------------- - -Function annotation syntax has been a Python feature since version 3.0 -(:pep:`3107`), however the semantics of annotations has been left undefined. - -Experience has shown that the majority of function annotation -uses were to provide type hints to function parameters and return values. It -became evident that it would be beneficial for Python users, if the -standard library included the base definitions and tools for type annotations. - -:pep:`484` introduces a :term:`provisional module ` to -provide these standard definitions and tools, along with some conventions -for situations where annotations are not available. - -For example, here is a simple function whose argument and return type -are declared in the annotations:: - - def greeting(name: str) -> str: - return 'Hello ' + name - -While these annotations are available at runtime through the usual -:attr:`__annotations__` attribute, *no automatic type checking happens at -runtime*. Instead, it is assumed that a separate off-line type checker -(e.g. `mypy `_) will be used for on-demand -source code analysis. - -The type system supports unions, generic types, and a special type -named :class:`~typing.Any` which is consistent with (i.e. assignable to -and from) all types. - -.. seealso:: - - * :mod:`typing` module documentation - * :pep:`484` -- Type Hints - PEP written by Guido van Rossum, Jukka Lehtosalo, and Łukasz Langa; - implemented by Guido van Rossum. - * :pep:`483` -- The Theory of Type Hints - PEP written by Guido van Rossum - - -.. _whatsnew-pep-471: - -PEP 471 - os.scandir() function -- a better and faster directory iterator -------------------------------------------------------------------------- - -:pep:`471` adds a new directory iteration function, :func:`os.scandir`, -to the standard library. Additionally, :func:`os.walk` is now -implemented using ``scandir``, which makes it 3 to 5 times faster -on POSIX systems and 7 to 20 times faster on Windows systems. This is -largely achieved by greatly reducing the number of calls to :func:`os.stat` -required to walk a directory tree. - -Additionally, ``scandir`` returns an iterator, as opposed to returning -a list of file names, which improves memory efficiency when iterating -over very large directories. - -The following example shows a simple use of :func:`os.scandir` to display all -the files (excluding directories) in the given *path* that don't start with -``'.'``. The :meth:`entry.is_file() ` call will generally -not make an additional system call:: - - for entry in os.scandir(path): - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) - -.. seealso:: - - :pep:`471` -- os.scandir() function -- a better and faster directory iterator - PEP written and implemented by Ben Hoyt with the help of Victor Stinner. - - -.. _whatsnew-pep-475: - -PEP 475: Retry system calls failing with EINTR ----------------------------------------------- - -An :py:data:`errno.EINTR` error code is returned whenever a system call, that -is waiting for I/O, is interrupted by a signal. Previously, Python would -raise :exc:`InterruptedError` in such cases. This meant that, when writing a -Python application, the developer had two choices: - -#. Ignore the ``InterruptedError``. -#. Handle the ``InterruptedError`` and attempt to restart the interrupted - system call at every call site. - -The first option makes an application fail intermittently. -The second option adds a large amount of boilerplate that makes the -code nearly unreadable. Compare:: - - print("Hello World") - -and:: - - while True: - try: - print("Hello World") - break - except InterruptedError: - continue - -:pep:`475` implements automatic retry of system calls on -``EINTR``. This removes the burden of dealing with ``EINTR`` -or :exc:`InterruptedError` in user code in most situations and makes -Python programs, including the standard library, more robust. Note that -the system call is only retried if the signal handler does not raise an -exception. - -Below is a list of functions which are now retried when interrupted -by a signal: - -* :func:`open` and :func:`io.open`; - -* functions of the :mod:`faulthandler` module; - -* :mod:`os` functions: :func:`~os.fchdir`, :func:`~os.fchmod`, - :func:`~os.fchown`, :func:`~os.fdatasync`, :func:`~os.fstat`, - :func:`~os.fstatvfs`, :func:`~os.fsync`, :func:`~os.ftruncate`, - :func:`~os.mkfifo`, :func:`~os.mknod`, :func:`~os.open`, - :func:`~os.posix_fadvise`, :func:`~os.posix_fallocate`, :func:`~os.pread`, - :func:`~os.pwrite`, :func:`~os.read`, :func:`~os.readv`, :func:`~os.sendfile`, - :func:`~os.wait3`, :func:`~os.wait4`, :func:`~os.wait`, - :func:`~os.waitid`, :func:`~os.waitpid`, :func:`~os.write`, - :func:`~os.writev`; - -* special cases: :func:`os.close` and :func:`os.dup2` now ignore - :py:data:`~errno.EINTR` errors; the syscall is not retried (see the PEP - for the rationale); - -* :mod:`select` functions: :func:`devpoll.poll() `, - :func:`epoll.poll() `, - :func:`kqueue.control() `, - :func:`poll.poll() `, :func:`~select.select`; - -* methods of the :class:`~socket.socket` class: :meth:`~socket.socket.accept`, - :meth:`~socket.socket.connect` (except for non-blocking sockets), - :meth:`~socket.socket.recv`, :meth:`~socket.socket.recvfrom`, - :meth:`~socket.socket.recvmsg`, :meth:`~socket.socket.send`, - :meth:`~socket.socket.sendall`, :meth:`~socket.socket.sendmsg`, - :meth:`~socket.socket.sendto`; - -* :func:`signal.sigtimedwait` and :func:`signal.sigwaitinfo`; - -* :func:`time.sleep`. - -.. seealso:: - - :pep:`475` -- Retry system calls failing with EINTR - PEP and implementation written by Charles-François Natali and - Victor Stinner, with the help of Antoine Pitrou (the French connection). - - -.. _whatsnew-pep-479: - -PEP 479: Change StopIteration handling inside generators --------------------------------------------------------- - -The interaction of generators and :exc:`StopIteration` in Python 3.4 and -earlier was sometimes surprising, and could conceal obscure bugs. Previously, -``StopIteration`` raised accidentally inside a generator function was -interpreted as the end of the iteration by the loop construct driving the -generator. - -:pep:`479` changes the behavior of generators: when a ``StopIteration`` -exception is raised inside a generator, it is replaced with a -:exc:`RuntimeError` before it exits the generator frame. The main goal of -this change is to ease debugging in the situation where an unguarded -:func:`next` call raises ``StopIteration`` and causes the iteration controlled -by the generator to terminate silently. This is particularly pernicious in -combination with the ``yield from`` construct. - -This is a backwards incompatible change, so to enable the new behavior, -a :term:`__future__` import is necessary:: - - >>> from __future__ import generator_stop - - >>> def gen(): - ... next(iter([])) - ... yield - ... - >>> next(gen()) - Traceback (most recent call last): - File "", line 2, in gen - StopIteration - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 1, in - RuntimeError: generator raised StopIteration - -Without a ``__future__`` import, a :exc:`PendingDeprecationWarning` will be -raised whenever a ``StopIteration`` exception is raised inside a generator. - -.. seealso:: - - :pep:`479` -- Change StopIteration handling inside generators - PEP written by Chris Angelico and Guido van Rossum. Implemented by - Chris Angelico, Yury Selivanov and Nick Coghlan. - - -.. _whatsnew-pep-485: - -PEP 485: A function for testing approximate equality ----------------------------------------------------- - -:pep:`485` adds the :func:`math.isclose` and :func:`cmath.isclose` -functions which tell whether two values are approximately equal or -"close" to each other. Whether or not two values are considered -close is determined according to given absolute and relative tolerances. -Relative tolerance is the maximum allowed difference between ``isclose`` -arguments, relative to the larger absolute value:: - - >>> import math - >>> a = 5.0 - >>> b = 4.99998 - >>> math.isclose(a, b, rel_tol=1e-5) - True - >>> math.isclose(a, b, rel_tol=1e-6) - False - -It is also possible to compare two values using absolute tolerance, which -must be a non-negative value:: - - >>> import math - >>> a = 5.0 - >>> b = 4.99998 - >>> math.isclose(a, b, abs_tol=0.00003) - True - >>> math.isclose(a, b, abs_tol=0.00001) - False - -.. seealso:: - - :pep:`485` -- A function for testing approximate equality - PEP written by Christopher Barker; implemented by Chris Barker and - Tal Einat. - - -.. _whatsnew-pep-486: - -PEP 486: Make the Python Launcher aware of virtual environments ---------------------------------------------------------------- - -:pep:`486` makes the Windows launcher (see :pep:`397`) aware of an active -virtual environment. When the default interpreter would be used and the -``VIRTUAL_ENV`` environment variable is set, the interpreter in the virtual -environment will be used. - -.. seealso:: - - :pep:`486` -- Make the Python Launcher aware of virtual environments - PEP written and implemented by Paul Moore. - - -.. _whatsnew-pep-488: - -PEP 488: Elimination of PYO files ---------------------------------- - -:pep:`488` does away with the concept of ``.pyo`` files. This means that -``.pyc`` files represent both unoptimized and optimized bytecode. To prevent the -need to constantly regenerate bytecode files, ``.pyc`` files now have an -optional ``opt-`` tag in their name when the bytecode is optimized. This has the -side-effect of no more bytecode file name clashes when running under either -:option:`-O` or :option:`-OO`. Consequently, bytecode files generated from -:option:`-O`, and :option:`-OO` may now exist simultaneously. -:func:`importlib.util.cache_from_source` has an updated API to help with -this change. - -.. seealso:: - - :pep:`488` -- Elimination of PYO files - PEP written and implemented by Brett Cannon. - - -.. _whatsnew-pep-489: - -PEP 489: Multi-phase extension module initialization ----------------------------------------------------- - -:pep:`489` updates extension module initialization to take advantage of the -two step module loading mechanism introduced by :pep:`451` in Python 3.4. - -This change brings the import semantics of extension modules that opt-in to -using the new mechanism much closer to those of Python source and bytecode -modules, including the ability to use any valid identifier as a module name, -rather than being restricted to ASCII. - -.. seealso:: - - :pep:`489` -- Multi-phase extension module initialization - PEP written by Petr Viktorin, Stefan Behnel, and Nick Coghlan; - implemented by Petr Viktorin. - - -Other Language Changes -====================== - -Some smaller changes made to the core Python language are: - -* Added the ``"namereplace"`` error handlers. The ``"backslashreplace"`` - error handlers now work with decoding and translating. - (Contributed by Serhiy Storchaka in :issue:`19676` and :issue:`22286`.) - -* The :option:`-b` option now affects comparisons of :class:`bytes` with - :class:`int`. (Contributed by Serhiy Storchaka in :issue:`23681`.) - -* New Kazakh ``kz1048`` and Tajik ``koi8_t`` :ref:`codecs `. - (Contributed by Serhiy Storchaka in :issue:`22682` and :issue:`22681`.) - -* Property docstrings are now writable. This is especially useful for - :func:`collections.namedtuple` docstrings. - (Contributed by Berker Peksag in :issue:`24064`.) - -* Circular imports involving relative imports are now supported. - (Contributed by Brett Cannon and Antoine Pitrou in :issue:`17636`.) - - -New Modules -=========== - -typing ------- - -The new :mod:`typing` :term:`provisional ` module -provides standard definitions and tools for function type annotations. -See :ref:`Type Hints ` for more information. - -.. _whatsnew-zipapp: - -zipapp ------- - -The new :mod:`zipapp` module (specified in :pep:`441`) provides an API and -command line tool for creating executable Python Zip Applications, which -were introduced in Python 2.6 in :issue:`1739468`, but which were not well -publicized, either at the time or since. - -With the new module, bundling your application is as simple as putting all -the files, including a ``__main__.py`` file, into a directory ``myapp`` -and running:: - - $ python -m zipapp myapp - $ python myapp.pyz - -The module implementation has been contributed by Paul Moore in -:issue:`23491`. - -.. seealso:: - - :pep:`441` -- Improving Python ZIP Application Support - - -Improved Modules -================ - -argparse --------- - -The :class:`~argparse.ArgumentParser` class now allows to disable -:ref:`abbreviated usage ` of long options by setting -:ref:`allow_abbrev` to ``False``. (Contributed by Jonathan Paugh, -Steven Bethard, paul j3 and Daniel Eriksson in :issue:`14910`.) - - -asyncio -------- - -Since the :mod:`asyncio` module is :term:`provisional `, -all changes introduced in Python 3.5 have also been backported to Python 3.4.x. - -Notable changes in the :mod:`asyncio` module since Python 3.4.0: - -* New debugging APIs: :meth:`loop.set_debug() ` - and :meth:`loop.get_debug() ` methods. - (Contributed by Victor Stinner.) - -* The proactor event loop now supports SSL. - (Contributed by Antoine Pitrou and Victor Stinner in :issue:`22560`.) - -* A new :meth:`loop.is_closed() ` method to - check if the event loop is closed. - (Contributed by Victor Stinner in :issue:`21326`.) - -* A new :meth:`loop.create_task() ` - to conveniently create and schedule a new :class:`~asyncio.Task` - for a coroutine. The ``create_task`` method is also used by all - asyncio functions that wrap coroutines into tasks, such as - :func:`asyncio.wait`, :func:`asyncio.gather`, etc. - (Contributed by Victor Stinner.) - -* A new :meth:`transport.get_write_buffer_limits() ` - method to inquire for *high-* and *low-* water limits of the flow - control. - (Contributed by Victor Stinner.) - -* The :func:`~asyncio.async` function is deprecated in favor of - :func:`~asyncio.ensure_future`. - (Contributed by Yury Selivanov.) - -* New :meth:`loop.set_task_factory() ` - and :meth:`loop.set_task_factory() ` - methods to customize the task factory that - :meth:`loop.create_task() ` method uses. - (Contributed by Yury Selivanov.) - -* New :meth:`Queue.join() ` and - :meth:`Queue.task_done() ` queue methods. - (Contributed by Victor Stinner.) - -* The ``JoinableQueue`` class was removed, in favor of the - :class:`asyncio.Queue` class. - (Contributed by Victor Stinner.) - -Updates in 3.5.1: - -* The :func:`~asyncio.ensure_future` function and all functions that - use it, such as :meth:`loop.run_until_complete() `, - now accept all kinds of :term:`awaitable objects `. - (Contributed by Yury Selivanov.) - -* New :func:`~asyncio.run_coroutine_threadsafe` function to submit - coroutines to event loops from other threads. - (Contributed by Vincent Michel.) - -* New :meth:`Transport.is_closing() ` - method to check if the transport is closing or closed. - (Contributed by Yury Selivanov.) - -* The :meth:`loop.create_server() ` - method can now accept a list of hosts. - (Contributed by Yann Sionneau.) - - -bz2 ---- - -The :meth:`BZ2Decompressor.decompress ` -method now accepts an optional *max_length* argument to limit the maximum -size of decompressed data. (Contributed by Nikolaus Rath in :issue:`15955`.) - - -cgi ---- - -The :class:`~cgi.FieldStorage` class now supports the :term:`context manager` -protocol. (Contributed by Berker Peksag in :issue:`20289`.) - - -cmath ------ - -A new function :func:`~cmath.isclose` provides a way to test for approximate -equality. (Contributed by Chris Barker and Tal Einat in :issue:`24270`.) - - -code ----- - -The :func:`InteractiveInterpreter.showtraceback() ` -method now prints the full chained traceback, just like the interactive -interpreter. (Contributed by Claudiu Popa in :issue:`17442`.) - - -collections ------------ - -.. _whatsnew-ordereddict: - -The :class:`~collections.OrderedDict` class is now implemented in C, which -makes it 4 to 100 times faster. (Contributed by Eric Snow in :issue:`16991`.) - -:meth:`OrderedDict.items() `, -:meth:`OrderedDict.keys() `, -:meth:`OrderedDict.values() ` views now support -:func:`reversed` iteration. -(Contributed by Serhiy Storchaka in :issue:`19505`.) - -The :class:`~collections.deque` class now defines -:meth:`~collections.deque.index`, :meth:`~collections.deque.insert`, and -:meth:`~collections.deque.copy`, and supports the ``+`` and ``*`` operators. -This allows deques to be recognized as a :class:`~collections.abc.MutableSequence` -and improves their substitutability for lists. -(Contributed by Raymond Hettinger in :issue:`23704`.) - -Docstrings produced by :func:`~collections.namedtuple` can now be updated:: - - Point = namedtuple('Point', ['x', 'y']) - Point.__doc__ += ': Cartesian coodinate' - Point.x.__doc__ = 'abscissa' - Point.y.__doc__ = 'ordinate' - -(Contributed by Berker Peksag in :issue:`24064`.) - -The :class:`~collections.UserString` class now implements the -:meth:`__getnewargs__`, :meth:`__rmod__`, :meth:`~str.casefold`, -:meth:`~str.format_map`, :meth:`~str.isprintable`, and :meth:`~str.maketrans` -methods to match the corresponding methods of :class:`str`. -(Contributed by Joe Jevnik in :issue:`22189`.) - - -collections.abc ---------------- - -The :meth:`Sequence.index() ` method now -accepts *start* and *stop* arguments to match the corresponding methods -of :class:`tuple`, :class:`list`, etc. -(Contributed by Devin Jeanpierre in :issue:`23086`.) - -A new :class:`~collections.abc.Generator` abstract base class. (Contributed -by Stefan Behnel in :issue:`24018`.) - -New :class:`~collections.abc.Awaitable`, :class:`~collections.abc.Coroutine`, -:class:`~collections.abc.AsyncIterator`, and -:class:`~collections.abc.AsyncIterable` abstract base classes. -(Contributed by Yury Selivanov in :issue:`24184`.) - -For earlier Python versions, a backport of the new ABCs is available in an -external `PyPI package `_. - - -compileall ----------- - -A new :mod:`compileall` option, :samp:`-j {N}`, allows to run *N* workers -sumultaneously to perform parallel bytecode compilation. -The :func:`~compileall.compile_dir` function has a corresponding ``workers`` -parameter. (Contributed by Claudiu Popa in :issue:`16104`.) - -Another new option, ``-r``, allows to control the maximum recursion -level for subdirectories. (Contributed by Claudiu Popa in :issue:`19628`.) - -The ``-q`` command line option can now be specified more than once, in -which case all output, including errors, will be suppressed. The corresponding -``quiet`` parameter in :func:`~compileall.compile_dir`, -:func:`~compileall.compile_file`, and :func:`~compileall.compile_path` can now -accept an integer value indicating the level of output suppression. -(Contributed by Thomas Kluyver in :issue:`21338`.) - - -concurrent.futures ------------------- - -The :meth:`Executor.map() ` method now accepts a -*chunksize* argument to allow batching of tasks to improve performance when -:meth:`~concurrent.futures.ProcessPoolExecutor` is used. -(Contributed by Dan O'Reilly in :issue:`11271`.) - -The number of workers in the :class:`~concurrent.futures.ThreadPoolExecutor` -constructor is optional now. The default value is 5 times the number of CPUs. -(Contributed by Claudiu Popa in :issue:`21527`.) - - -configparser ------------- - -:mod:`configparser` now provides a way to customize the conversion -of values by specifying a dictionary of converters in the -:class:`~configparser.ConfigParser` constructor, or by defining them -as methods in ``ConfigParser`` subclasses. Converters defined in -a parser instance are inherited by its section proxies. - -Example:: - - >>> import configparser - >>> conv = {} - >>> conv['list'] = lambda v: [e.strip() for e in v.split() if e.strip()] - >>> cfg = configparser.ConfigParser(converters=conv) - >>> cfg.read_string(""" - ... [s] - ... list = a b c d e f g - ... """) - >>> cfg.get('s', 'list') - 'a b c d e f g' - >>> cfg.getlist('s', 'list') - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> section = cfg['s'] - >>> section.getlist('list') - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - -(Contributed by Łukasz Langa in :issue:`18159`.) - - -contextlib ----------- - -The new :func:`~contextlib.redirect_stderr` :term:`context manager` (similar to -:func:`~contextlib.redirect_stdout`) makes it easier for utility scripts to -handle inflexible APIs that write their output to :data:`sys.stderr` and -don't provide any options to redirect it:: - - >>> import contextlib, io, logging - >>> f = io.StringIO() - >>> with contextlib.redirect_stderr(f): - ... logging.warning('warning') - ... - >>> f.getvalue() - 'WARNING:root:warning\n' - -(Contributed by Berker Peksag in :issue:`22389`.) - - -csv ---- - -The :meth:`~csv.csvwriter.writerow` method now supports arbitrary iterables, -not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) - - -curses ------- - -The new :func:`~curses.update_lines_cols` function updates the :envvar:`LINES` -and :envvar:`COLS` environment variables. This is useful for detecting -manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) - - -dbm ---- - -:func:`dumb.open ` always creates a new database when the flag -has the value ``"n"``. (Contributed by Claudiu Popa in :issue:`18039`.) - - -difflib -------- - -The charset of HTML documents generated by -:meth:`HtmlDiff.make_file() ` -can now be customized by using a new *charset* keyword-only argument. -The default charset of HTML document changed from ``"ISO-8859-1"`` -to ``"utf-8"``. -(Contributed by Berker Peksag in :issue:`2052`.) - -The :func:`~difflib.diff_bytes` function can now compare lists of byte -strings. This fixes a regression from Python 2. -(Contributed by Terry J. Reedy and Greg Ward in :issue:`17445`.) - - -distutils ---------- - -Both the ``build`` and ``build_ext`` commands now accept a ``-j`` option to -enable parallel building of extension modules. -(Contributed by Antoine Pitrou in :issue:`5309`.) - -The :mod:`distutils` module now supports ``xz`` compression, and can be -enabled by passing ``xztar`` as an argument to ``bdist --format``. -(Contributed by Serhiy Storchaka in :issue:`16314`.) - - -doctest -------- - -The :func:`~doctest.DocTestSuite` function returns an empty -:class:`unittest.TestSuite` if *module* contains no docstrings, instead of -raising :exc:`ValueError`. (Contributed by Glenn Jones in :issue:`15916`.) - - -email ------ - -A new policy option :attr:`Policy.mangle_from_ ` -controls whether or not lines that start with ``"From "`` in email bodies are -prefixed with a ``">"`` character by generators. The default is ``True`` for -:attr:`~email.policy.compat32` and ``False`` for all other policies. -(Contributed by Milan Oberkirch in :issue:`20098`.) - -A new -:meth:`Message.get_content_disposition() ` -method provides easy access to a canonical value for the -:mailheader:`Content-Disposition` header. -(Contributed by Abhilash Raj in :issue:`21083`.) - -A new policy option :attr:`EmailPolicy.utf8 ` -can be set to ``True`` to encode email headers using the UTF-8 charset instead -of using encoded words. This allows ``Messages`` to be formatted according to -:rfc:`6532` and used with an SMTP server that supports the :rfc:`6531` -``SMTPUTF8`` extension. (Contributed by R. David Murray in -:issue:`24211`.) - -The :class:`mime.text.MIMEText ` constructor now -accepts a :class:`charset.Charset ` instance. -(Contributed by Claude Paroz and Berker Peksag in :issue:`16324`.) - - -enum ----- - -The :class:`~enum.Enum` callable has a new parameter *start* to -specify the initial number of enum values if only *names* are provided:: - - >>> Animal = enum.Enum('Animal', 'cat dog', start=10) - >>> Animal.cat - - >>> Animal.dog - - -(Contributed by Ethan Furman in :issue:`21706`.) - - -faulthandler ------------- - -The :func:`~faulthandler.enable`, :func:`~faulthandler.register`, -:func:`~faulthandler.dump_traceback` and -:func:`~faulthandler.dump_traceback_later` functions now accept file -descriptors in addition to file-like objects. -(Contributed by Wei Wu in :issue:`23566`.) - - -functools ---------- - -.. _whatsnew-lrucache: - -Most of the :func:`~functools.lru_cache` machinery is now implemented in C, making -it significantly faster. (Contributed by Matt Joiner, Alexey Kachayev, and -Serhiy Storchaka in :issue:`14373`.) - - -glob ----- - -The :func:`~glob.iglob` and :func:`~glob.glob` functions now support recursive -search in subdirectories, using the ``"**"`` pattern. -(Contributed by Serhiy Storchaka in :issue:`13968`.) - - -gzip ----- - -The *mode* argument of the :class:`~gzip.GzipFile` constructor now -accepts ``"x"`` to request exclusive creation. -(Contributed by Tim Heaney in :issue:`19222`.) - - -heapq ------ - -Element comparison in :func:`~heapq.merge` can now be customized by -passing a :term:`key function` in a new optional *key* keyword argument, -and a new optional *reverse* keyword argument can be used to reverse element -comparison:: - - >>> import heapq - >>> a = ['9', '777', '55555'] - >>> b = ['88', '6666'] - >>> list(heapq.merge(a, b, key=len)) - ['9', '88', '777', '6666', '55555'] - >>> list(heapq.merge(reversed(a), reversed(b), key=len, reverse=True)) - ['55555', '6666', '777', '88', '9'] - -(Contributed by Raymond Hettinger in :issue:`13742`.) - - -http ----- - -A new :class:`HTTPStatus ` enum that defines a set of -HTTP status codes, reason phrases and long descriptions written in English. -(Contributed by Demian Brecht in :issue:`21793`.) - - -http.client ------------ - -:meth:`HTTPConnection.getresponse() ` -now raises a :exc:`~http.client.RemoteDisconnected` exception when a -remote server connection is closed unexpectedly. Additionally, if a -:exc:`ConnectionError` (of which ``RemoteDisconnected`` -is a subclass) is raised, the client socket is now closed automatically, -and will reconnect on the next request:: - - import http.client - conn = http.client.HTTPConnection('www.python.org') - for retries in range(3): - try: - conn.request('GET', '/') - resp = conn.getresponse() - except http.client.RemoteDisconnected: - pass - -(Contributed by Martin Panter in :issue:`3566`.) - - -idlelib and IDLE ----------------- - -Since idlelib implements the IDLE shell and editor and is not intended for -import by other programs, it gets improvements with every release. See -:file:`Lib/idlelib/NEWS.txt` for a cumulative list of changes since 3.4.0, -as well as changes made in future 3.5.x releases. This file is also available -from the IDLE :menuselection:`Help --> About IDLE` dialog. - - -imaplib -------- - -The :class:`~imaplib.IMAP4` class now supports the :term:`context manager` protocol. -When used in a :keyword:`with` statement, the IMAP4 ``LOGOUT`` -command will be called automatically at the end of the block. -(Contributed by Tarek Ziadé and Serhiy Storchaka in :issue:`4972`.) - -The :mod:`imaplib` module now supports :rfc:`5161` (ENABLE Extension) -and :rfc:`6855` (UTF-8 Support) via the :meth:`IMAP4.enable() ` -method. A new :attr:`IMAP4.utf8_enabled ` -attribute tracks whether or not :rfc:`6855` support is enabled. -(Contributed by Milan Oberkirch, R. David Murray, and Maciej Szulik in -:issue:`21800`.) - -The :mod:`imaplib` module now automatically encodes non-ASCII string usernames -and passwords using UTF-8, as recommended by the RFCs. (Contributed by Milan -Oberkirch in :issue:`21800`.) - - -imghdr ------- - -The :func:`~imghdr.what` function now recognizes the -`OpenEXR `_ format -(contributed by Martin Vignali and Claudiu Popa in :issue:`20295`), -and the `WebP `_ format -(contributed by Fabrice Aneche and Claudiu Popa in :issue:`20197`.) - - -importlib ---------- - -The :class:`util.LazyLoader ` class allows for -lazy loading of modules in applications where startup time is important. -(Contributed by Brett Cannon in :issue:`17621`.) - -The :func:`abc.InspectLoader.source_to_code() ` -method is now a static method. This makes it easier to initialize a module -object with code compiled from a string by running -``exec(code, module.__dict__)``. -(Contributed by Brett Cannon in :issue:`21156`.) - -The new :func:`util.module_from_spec() ` -function is now the preferred way to create a new module. As opposed to -creating a :class:`types.ModuleType` instance directly, this new function -will set the various import-controlled attributes based on the passed-in -spec object. (Contributed by Brett Cannon in :issue:`20383`.) - - -inspect -------- - -Both the :class:`~inspect.Signature` and :class:`~inspect.Parameter` classes are -now picklable and hashable. (Contributed by Yury Selivanov in :issue:`20726` -and :issue:`20334`.) - -A new -:meth:`BoundArguments.apply_defaults() ` -method provides a way to set default values for missing arguments:: - - >>> def foo(a, b='ham', *args): pass - >>> ba = inspect.signature(foo).bind('spam') - >>> ba.apply_defaults() - >>> ba.arguments - OrderedDict([('a', 'spam'), ('b', 'ham'), ('args', ())]) - -(Contributed by Yury Selivanov in :issue:`24190`.) - -A new class method -:meth:`Signature.from_callable() ` makes -subclassing of :class:`~inspect.Signature` easier. (Contributed -by Yury Selivanov and Eric Snow in :issue:`17373`.) - -The :func:`~inspect.signature` function now accepts a *follow_wrapped* -optional keyword argument, which, when set to ``False``, disables automatic -following of ``__wrapped__`` links. -(Contributed by Yury Selivanov in :issue:`20691`.) - -A set of new functions to inspect -:term:`coroutine functions ` and -:term:`coroutine objects ` has been added: -:func:`~inspect.iscoroutine`, :func:`~inspect.iscoroutinefunction`, -:func:`~inspect.isawaitable`, :func:`~inspect.getcoroutinelocals`, -and :func:`~inspect.getcoroutinestate`. -(Contributed by Yury Selivanov in :issue:`24017` and :issue:`24400`.) - -The :func:`~inspect.stack`, :func:`~inspect.trace`, -:func:`~inspect.getouterframes`, and :func:`~inspect.getinnerframes` -functions now return a list of named tuples. -(Contributed by Daniel Shahaf in :issue:`16808`.) - - -io --- - -A new :meth:`BufferedIOBase.readinto1() ` -method, that uses at most one call to the underlying raw stream's -:meth:`RawIOBase.read() ` or -:meth:`RawIOBase.readinto() ` methods. -(Contributed by Nikolaus Rath in :issue:`20578`.) - - -ipaddress ---------- - -Both the :class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` classes -now accept an ``(address, netmask)`` tuple argument, so as to easily construct -network objects from existing addresses:: - - >>> import ipaddress - >>> ipaddress.IPv4Network(('127.0.0.0', 8)) - IPv4Network('127.0.0.0/8') - >>> ipaddress.IPv4Network(('127.0.0.0', '255.0.0.0')) - IPv4Network('127.0.0.0/8') - -(Contributed by Peter Moody and Antoine Pitrou in :issue:`16531`.) - -A new :attr:`~ipaddress.IPv4Network.reverse_pointer` attribute for the -:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` classes -returns the name of the reverse DNS PTR record:: - - >>> import ipaddress - >>> addr = ipaddress.IPv4Address('127.0.0.1') - >>> addr.reverse_pointer - '1.0.0.127.in-addr.arpa' - >>> addr6 = ipaddress.IPv6Address('::1') - >>> addr6.reverse_pointer - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa' - -(Contributed by Leon Weber in :issue:`20480`.) - - -json ----- - -The :mod:`json.tool` command line interface now preserves the order of keys in -JSON objects passed in input. The new ``--sort-keys`` option can be used -to sort the keys alphabetically. (Contributed by Berker Peksag -in :issue:`21650`.) - -JSON decoder now raises :exc:`~json.JSONDecodeError` instead of -:exc:`ValueError` to provide better context information about the error. -(Contributed by Serhiy Storchaka in :issue:`19361`.) - - -linecache ---------- - -A new :func:`~linecache.lazycache` function can be used to capture information -about a non-file-based module to permit getting its lines later via -:func:`~linecache.getline`. This avoids doing I/O until a line is actually -needed, without having to carry the module globals around indefinitely. -(Contributed by Robert Collins in :issue:`17911`.) - - -locale ------- - -A new :func:`~locale.delocalize` function can be used to convert a string into -a normalized number string, taking the ``LC_NUMERIC`` settings into account:: - - >>> import locale - >>> locale.setlocale(locale.LC_NUMERIC, 'de_DE.UTF-8') - 'de_DE.UTF-8' - >>> locale.delocalize('1.234,56') - '1234.56' - >>> locale.setlocale(locale.LC_NUMERIC, 'en_US.UTF-8') - 'en_US.UTF-8' - >>> locale.delocalize('1,234.56') - '1234.56' - -(Contributed by Cédric Krier in :issue:`13918`.) - - -logging -------- - -All logging methods (:class:`~logging.Logger` :meth:`~logging.Logger.log`, -:meth:`~logging.Logger.exception`, :meth:`~logging.Logger.critical`, -:meth:`~logging.Logger.debug`, etc.), now accept exception instances -as an *exc_info* argument, in addition to boolean values and exception -tuples:: - - >>> import logging - >>> try: - ... 1/0 - ... except ZeroDivisionError as ex: - ... logging.error('exception', exc_info=ex) - ERROR:root:exception - -(Contributed by Yury Selivanov in :issue:`20537`.) - -The :class:`handlers.HTTPHandler ` class now -accepts an optional :class:`ssl.SSLContext` instance to configure SSL -settings used in an HTTP connection. -(Contributed by Alex Gaynor in :issue:`22788`.) - -The :class:`handlers.QueueListener ` class now -takes a *respect_handler_level* keyword argument which, if set to ``True``, -will pass messages to handlers taking handler levels into account. -(Contributed by Vinay Sajip.) - - -lzma ----- - -The :meth:`LZMADecompressor.decompress() ` -method now accepts an optional *max_length* argument to limit the maximum -size of decompressed data. -(Contributed by Martin Panter in :issue:`15955`.) - - -math ----- - -Two new constants have been added to the :mod:`math` module: :data:`~math.inf` -and :data:`~math.nan`. (Contributed by Mark Dickinson in :issue:`23185`.) - -A new function :func:`~math.isclose` provides a way to test for approximate -equality. (Contributed by Chris Barker and Tal Einat in :issue:`24270`.) - -A new :func:`~math.gcd` function has been added. The :func:`fractions.gcd` -function is now deprecated. (Contributed by Mark Dickinson and Serhiy -Storchaka in :issue:`22486`.) - - -multiprocessing ---------------- - -:func:`sharedctypes.synchronized() ` -objects now support the :term:`context manager` protocol. -(Contributed by Charles-François Natali in :issue:`21565`.) - - -operator --------- - -:func:`~operator.attrgetter`, :func:`~operator.itemgetter`, -and :func:`~operator.methodcaller` objects now support pickling. -(Contributed by Josh Rosenberg and Serhiy Storchaka in :issue:`22955`.) - -New :func:`~operator.matmul` and :func:`~operator.imatmul` functions -to perform matrix multiplication. -(Contributed by Benjamin Peterson in :issue:`21176`.) - - -os --- - -The new :func:`~os.scandir` function returning an iterator of -:class:`~os.DirEntry` objects has been added. If possible, :func:`~os.scandir` -extracts file attributes while scanning a directory, removing the need to -perform subsequent system calls to determine file type or attributes, which may -significantly improve performance. (Contributed by Ben Hoyt with the help -of Victor Stinner in :issue:`22524`.) - -On Windows, a new -:attr:`stat_result.st_file_attributes ` -attribute is now available. It corresponds to the ``dwFileAttributes`` member -of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by -``GetFileInformationByHandle()``. (Contributed by Ben Hoyt in :issue:`21719`.) - -The :func:`~os.urandom` function now uses the ``getrandom()`` syscall on Linux 3.17 -or newer, and ``getentropy()`` on OpenBSD 5.6 and newer, removing the need to -use ``/dev/urandom`` and avoiding failures due to potential file descriptor -exhaustion. (Contributed by Victor Stinner in :issue:`22181`.) - -New :func:`~os.get_blocking` and :func:`~os.set_blocking` functions allow to -get and set a file descriptor's blocking mode (:data:`~os.O_NONBLOCK`.) -(Contributed by Victor Stinner in :issue:`22054`.) - -The :func:`~os.truncate` and :func:`~os.ftruncate` functions are now supported -on Windows. (Contributed by Steve Dower in :issue:`23668`.) - -There is a new :func:`os.path.commonpath` function returning the longest -common sub-path of each passed pathname. Unlike the -:func:`os.path.commonprefix` function, it always returns a valid -path:: - - >>> os.path.commonprefix(['/usr/lib', '/usr/local/lib']) - '/usr/l' - - >>> os.path.commonpath(['/usr/lib', '/usr/local/lib']) - '/usr' - -(Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) - - -pathlib -------- - -The new :meth:`Path.samefile() ` method can be used -to check whether the path points to the same file as another path, which can -be either another :class:`~pathlib.Path` object, or a string:: - - >>> import pathlib - >>> p1 = pathlib.Path('/etc/hosts') - >>> p2 = pathlib.Path('/etc/../etc/hosts') - >>> p1.samefile(p2) - True - -(Contributed by Vajrasky Kok and Antoine Pitrou in :issue:`19775`.) - -The :meth:`Path.mkdir() ` method now accepts a new optional -*exist_ok* argument to match ``mkdir -p`` and :func:`os.makedirs` -functionality. (Contributed by Berker Peksag in :issue:`21539`.) - -There is a new :meth:`Path.expanduser() ` method to -expand ``~`` and ``~user`` prefixes. (Contributed by Serhiy Storchaka and -Claudiu Popa in :issue:`19776`.) - -A new :meth:`Path.home() ` class method can be used to get -a :class:`~pathlib.Path` instance representing the user’s home -directory. -(Contributed by Victor Salgado and Mayank Tripathi in :issue:`19777`.) - -New :meth:`Path.write_text() `, -:meth:`Path.read_text() `, -:meth:`Path.write_bytes() `, -:meth:`Path.read_bytes() ` methods to simplify -read/write operations on files. - -The following code snippet will create or rewrite existing file -``~/spam42``:: - - >>> import pathlib - >>> p = pathlib.Path('~/spam42') - >>> p.expanduser().write_text('ham') - 3 - -(Contributed by Christopher Welborn in :issue:`20218`.) - - -pickle ------- - -Nested objects, such as unbound methods or nested classes, can now be pickled -using :ref:`pickle protocols ` older than protocol version 4. -Protocol version 4 already supports these cases. (Contributed by Serhiy -Storchaka in :issue:`23611`.) - - -poplib ------- - -A new :meth:`POP3.utf8() ` command enables :rfc:`6856` -(Internationalized Email) support, if a POP server supports it. -(Contributed by Milan OberKirch in :issue:`21804`.) - - -re --- - -References and conditional references to groups with fixed length are now -allowed in lookbehind assertions:: - - >>> import re - >>> pat = re.compile(r'(a|b).(?<=\1)c') - >>> pat.match('aac') - <_sre.SRE_Match object; span=(0, 3), match='aac'> - >>> pat.match('bbc') - <_sre.SRE_Match object; span=(0, 3), match='bbc'> - -(Contributed by Serhiy Storchaka in :issue:`9179`.) - -The number of capturing groups in regular expressions is no longer limited to -100. (Contributed by Serhiy Storchaka in :issue:`22437`.) - -The :func:`~re.sub` and :func:`~re.subn` functions now replace unmatched -groups with empty strings instead of raising an exception. -(Contributed by Serhiy Storchaka in :issue:`1519638`.) - -The :class:`re.error` exceptions have new attributes, -:attr:`~re.error.msg`, :attr:`~re.error.pattern`, -:attr:`~re.error.pos`, :attr:`~re.error.lineno`, -and :attr:`~re.error.colno`, that provide better context -information about the error:: - - >>> re.compile(""" - ... (?x) - ... .++ - ... """) - Traceback (most recent call last): - ... - sre_constants.error: multiple repeat at position 16 (line 3, column 7) - -(Contributed by Serhiy Storchaka in :issue:`22578`.) - - -readline --------- - -A new :func:`~readline.append_history_file` function can be used to append -the specified number of trailing elements in history to the given file. -(Contributed by Bruno Cauet in :issue:`22940`.) - - -selectors ---------- - -The new :class:`~selectors.DevpollSelector` supports efficient -``/dev/poll`` polling on Solaris. -(Contributed by Giampaolo Rodola' in :issue:`18931`.) - - -shutil ------- - -The :func:`~shutil.move` function now accepts a *copy_function* argument, -allowing, for example, the :func:`~shutil.copy` function to be used instead of -the default :func:`~shutil.copy2` if there is a need to ignore file metadata -when moving. -(Contributed by Claudiu Popa in :issue:`19840`.) - -The :func:`~shutil.make_archive` function now supports the *xztar* format. -(Contributed by Serhiy Storchaka in :issue:`5411`.) - - -signal ------- - -On Windows, the :func:`~signal.set_wakeup_fd` function now also supports -socket handles. (Contributed by Victor Stinner in :issue:`22018`.) - -Various ``SIG*`` constants in the :mod:`signal` module have been converted into -:mod:`Enums `. This allows meaningful names to be printed -during debugging, instead of integer "magic numbers". -(Contributed by Giampaolo Rodola' in :issue:`21076`.) - - -smtpd ------ - -Both the :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now -accept a *decode_data* keyword argument to determine if the ``DATA`` portion of -the SMTP transaction is decoded using the ``"utf-8"`` codec or is instead -provided to the -:meth:`SMTPServer.process_message() ` -method as a byte string. The default is ``True`` for backward compatibility -reasons, but will change to ``False`` in Python 3.6. If *decode_data* is set -to ``False``, the ``process_message`` method must be prepared to accept keyword -arguments. -(Contributed by Maciej Szulik in :issue:`19662`.) - -The :class:`~smtpd.SMTPServer` class now advertises the ``8BITMIME`` extension -(:rfc:`6152`) if *decode_data* has been set ``True``. If the client -specifies ``BODY=8BITMIME`` on the ``MAIL`` command, it is passed to -:meth:`SMTPServer.process_message() ` -via the *mail_options* keyword. -(Contributed by Milan Oberkirch and R. David Murray in :issue:`21795`.) - -The :class:`~smtpd.SMTPServer` class now also supports the ``SMTPUTF8`` -extension (:rfc:`6531`: Internationalized Email). If the client specified -``SMTPUTF8 BODY=8BITMIME`` on the ``MAIL`` command, they are passed to -:meth:`SMTPServer.process_message() ` -via the *mail_options* keyword. It is the responsibility of the -``process_message`` method to correctly handle the ``SMTPUTF8`` data. -(Contributed by Milan Oberkirch in :issue:`21725`.) - -It is now possible to provide, directly or via name resolution, IPv6 -addresses in the :class:`~smtpd.SMTPServer` constructor, and have it -successfully connect. (Contributed by Milan Oberkirch in :issue:`14758`.) - - -smtplib -------- - -A new :meth:`SMTP.auth() ` method provides a convenient way to -implement custom authentication mechanisms. (Contributed by Milan -Oberkirch in :issue:`15014`.) - -The :meth:`SMTP.set_debuglevel() ` method now -accepts an additional debuglevel (2), which enables timestamps in debug -messages. (Contributed by Gavin Chappell and Maciej Szulik in :issue:`16914`.) - -Both the :meth:`SMTP.sendmail() ` and -:meth:`SMTP.send_message() ` methods now -support support :rfc:`6531` (SMTPUTF8). -(Contributed by Milan Oberkirch and R. David Murray in :issue:`22027`.) - - -sndhdr ------- - -The :func:`~sndhdr.what` and :func:`~sndhdr.whathdr` functions now return -a :func:`~collections.namedtuple`. (Contributed by Claudiu Popa in -:issue:`18615`.) - - -socket ------- - -Functions with timeouts now use a monotonic clock, instead of a system clock. -(Contributed by Victor Stinner in :issue:`22043`.) - -A new :meth:`socket.sendfile() ` method allows to -send a file over a socket by using the high-performance :func:`os.sendfile` -function on UNIX, resulting in uploads being from 2 to 3 times faster than when -using plain :meth:`socket.send() `. -(Contributed by Giampaolo Rodola' in :issue:`17552`.) - -The :meth:`socket.sendall() ` method no longer resets the -socket timeout every time bytes are received or sent. The socket timeout is -now the maximum total duration to send all data. -(Contributed by Victor Stinner in :issue:`23853`.) - -The *backlog* argument of the :meth:`socket.listen() ` -method is now optional. By default it is set to -:data:`SOMAXCONN ` or to ``128``, whichever is less. -(Contributed by Charles-François Natali in :issue:`21455`.) - - -ssl ---- - -.. _whatsnew-sslmemorybio: - -Memory BIO Support -~~~~~~~~~~~~~~~~~~ - -(Contributed by Geert Jansen in :issue:`21965`.) - -The new :class:`~ssl.SSLObject` class has been added to provide SSL protocol -support for cases when the network I/O capabilities of :class:`~ssl.SSLSocket` -are not necessary or are suboptimal. ``SSLObject`` represents -an SSL protocol instance, but does not implement any network I/O methods, and -instead provides a memory buffer interface. The new :class:`~ssl.MemoryBIO` -class can be used to pass data between Python and an SSL protocol instance. - -The memory BIO SSL support is primarily intended to be used in frameworks -implementing asynchronous I/O for which :class:`~ssl.SSLSocket`'s readiness -model ("select/poll") is inefficient. - -A new :meth:`SSLContext.wrap_bio() ` method can be used -to create a new ``SSLObject`` instance. - - -Application-Layer Protocol Negotiation Support -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(Contributed by Benjamin Peterson in :issue:`20188`.) - -Where OpenSSL support is present, the :mod:`ssl` module now implements -the *Application-Layer Protocol Negotiation* TLS extension as described -in :rfc:`7301`. - -The new :meth:`SSLContext.set_alpn_protocols() ` -can be used to specify which protocols a socket should advertise during -the TLS handshake. - -The new -:meth:`SSLSocket.selected_alpn_protocol() ` -returns the protocol that was selected during the TLS handshake. -The :data:`~ssl.HAS_ALPN` flag indicates whether ALPN support is present. - - -Other Changes -~~~~~~~~~~~~~ - -There is a new :meth:`SSLSocket.version() ` method to -query the actual protocol version in use. -(Contributed by Antoine Pitrou in :issue:`20421`.) - -The :class:`~ssl.SSLSocket` class now implements -a :meth:`SSLSocket.sendfile() ` method. -(Contributed by Giampaolo Rodola' in :issue:`17552`.) - -The :meth:`SSLSocket.send() ` method now raises either -the :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` exception on a -non-blocking socket if the operation would block. Previously, it would return -``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) - -The :func:`~ssl.cert_time_to_seconds` function now interprets the input time -as UTC and not as local time, per :rfc:`5280`. Additionally, the return -value is always an :class:`int`. (Contributed by Akira Li in :issue:`19940`.) - -New :meth:`SSLObject.shared_ciphers() ` and -:meth:`SSLSocket.shared_ciphers() ` methods return -the list of ciphers sent by the client during the handshake. -(Contributed by Benjamin Peterson in :issue:`23186`.) - -The :meth:`SSLSocket.do_handshake() `, -:meth:`SSLSocket.read() `, -:meth:`SSLSocket.shutdown() `, and -:meth:`SSLSocket.write() ` methods of the :class:`~ssl.SSLSocket` -class no longer reset the socket timeout every time bytes are received or sent. -The socket timeout is now the maximum total duration of the method. -(Contributed by Victor Stinner in :issue:`23853`.) - -The :func:`~ssl.match_hostname` function now supports matching of IP addresses. -(Contributed by Antoine Pitrou in :issue:`23239`.) - - -sqlite3 -------- - -The :class:`~sqlite3.Row` class now fully supports the sequence protocol, -in particular :func:`reversed` iteration and slice indexing. -(Contributed by Claudiu Popa in :issue:`10203`; by Lucas Sinclair, -Jessica McKellar, and Serhiy Storchaka in :issue:`13583`.) - - -.. _whatsnew-subprocess: - -subprocess ----------- - -The new :func:`~subprocess.run` function has been added. -It runs the specified command and returns a -:class:`~subprocess.CompletedProcess` object, which describes a finished -process. The new API is more consistent and is the recommended approach -to invoking subprocesses in Python code that does not need to maintain -compatibility with earlier Python versions. -(Contributed by Thomas Kluyver in :issue:`23342`.) - -Examples:: - - >>> subprocess.run(["ls", "-l"]) # doesn't capture output - CompletedProcess(args=['ls', '-l'], returncode=0) - - >>> subprocess.run("exit 1", shell=True, check=True) - Traceback (most recent call last): - ... - subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - - >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) - CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, - stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') - - -sys ---- - -A new :func:`~sys.set_coroutine_wrapper` function allows setting a global -hook that will be called whenever a :term:`coroutine object ` -is created by an :keyword:`async def` function. A corresponding -:func:`~sys.get_coroutine_wrapper` can be used to obtain a currently set -wrapper. Both functions are :term:`provisional `, -and are intended for debugging purposes only. (Contributed by Yury Selivanov -in :issue:`24017`.) - -A new :func:`~sys.is_finalizing` function can be used to check if the Python -interpreter is :term:`shutting down `. -(Contributed by Antoine Pitrou in :issue:`22696`.) - - -sysconfig ---------- - -The name of the user scripts directory on Windows now includes the first -two components of the Python version. (Contributed by Paul Moore -in :issue:`23437`.) - - -tarfile -------- - -The *mode* argument of the :func:`~tarfile.open` function now accepts ``"x"`` -to request exclusive creation. (Contributed by Berker Peksag in :issue:`21717`.) - -The :meth:`TarFile.extractall() ` and -:meth:`TarFile.extract() ` methods now take a keyword -argument *numeric_only*. If set to ``True``, the extracted files and -directories will be owned by the numeric ``uid`` and ``gid`` from the tarfile. -If set to ``False`` (the default, and the behavior in versions prior to 3.5), -they will be owned by the named user and group in the tarfile. -(Contributed by Michael Vogt and Eric Smith in :issue:`23193`.) - -The :meth:`TarFile.list() ` now accepts an optional -*members* keyword argument that can be set to a subset of the list returned -by :meth:`TarFile.getmembers() `. -(Contributed by Serhiy Storchaka in :issue:`21549`.) - - -threading ---------- - -Both the :meth:`Lock.acquire() ` and -:meth:`RLock.acquire() ` methods -now use a monotonic clock for timeout management. -(Contributed by Victor Stinner in :issue:`22043`.) - - -time ----- - -The :func:`~time.monotonic` function is now always available. -(Contributed by Victor Stinner in :issue:`22043`.) - - -timeit ------- - -A new command line option ``-u`` or :samp:`--unit={U}` can be used to specify the time -unit for the timer output. Supported options are ``usec``, ``msec``, -or ``sec``. (Contributed by Julian Gindi in :issue:`18983`.) - -The :func:`~timeit.timeit` function has a new *globals* parameter for -specifying the namespace in which the code will be running. -(Contributed by Ben Roberts in :issue:`2527`.) - - -tkinter -------- - -The :mod:`tkinter._fix` module used for setting up the Tcl/Tk environment -on Windows has been replaced by a private function in the :mod:`_tkinter` -module which makes no permanent changes to environment variables. -(Contributed by Zachary Ware in :issue:`20035`.) - - -.. _whatsnew-traceback: - -traceback ---------- - -New :func:`~traceback.walk_stack` and :func:`~traceback.walk_tb` -functions to conveniently traverse frame and traceback objects. -(Contributed by Robert Collins in :issue:`17911`.) - -New lightweight classes: :class:`~traceback.TracebackException`, -:class:`~traceback.StackSummary`, and :class:`~traceback.FrameSummary`. -(Contributed by Robert Collins in :issue:`17911`.) - -Both the :func:`~traceback.print_tb` and :func:`~traceback.print_stack` functions -now support negative values for the *limit* argument. -(Contributed by Dmitry Kazakov in :issue:`22619`.) - - -types ------ - -A new :func:`~types.coroutine` function to transform -:term:`generator ` and -:class:`generator-like ` objects into -:term:`awaitables `. -(Contributed by Yury Selivanov in :issue:`24017`.) - -A new type called :class:`~types.CoroutineType`, which is used for -:term:`coroutine` objects created by :keyword:`async def` functions. -(Contributed by Yury Selivanov in :issue:`24400`.) - - -unicodedata ------------ - -The :mod:`unicodedata` module now uses data from `Unicode 8.0.0 -`_. - - -unittest --------- - -The :meth:`TestLoader.loadTestsFromModule() ` -method now accepts a keyword-only argument *pattern* which is passed to -``load_tests`` as the third argument. Found packages are now checked for -``load_tests`` regardless of whether their path matches *pattern*, because it -is impossible for a package name to match the default pattern. -(Contributed by Robert Collins and Barry A. Warsaw in :issue:`16662`.) - -Unittest discovery errors now are exposed in the -:data:`TestLoader.errors ` attribute of the -:class:`~unittest.TestLoader` instance. -(Contributed by Robert Collins in :issue:`19746`.) - -A new command line option ``--locals`` to show local variables in -tracebacks. (Contributed by Robert Collins in :issue:`22936`.) - - -unittest.mock -------------- - -The :class:`~unittest.mock.Mock` class has the following improvements: - -* The class constructor has a new *unsafe* parameter, which causes mock - objects to raise :exc:`AttributeError` on attribute names starting - with ``"assert"``. - (Contributed by Kushal Das in :issue:`21238`.) - -* A new :meth:`Mock.assert_not_called() ` - method to check if the mock object was called. - (Contributed by Kushal Das in :issue:`21262`.) - -The :class:`~unittest.mock.MagicMock` class now supports :meth:`__truediv__`, -:meth:`__divmod__` and :meth:`__matmul__` operators. -(Contributed by Johannes Baiter in :issue:`20968`, and Håkan Lövdahl -in :issue:`23581` and :issue:`23568`.) - -It is no longer necessary to explicitly pass ``create=True`` to the -:func:`~unittest.mock.patch` function when patching builtin names. -(Contributed by Kushal Das in :issue:`17660`.) - - -urllib ------- - -A new -:class:`request.HTTPPasswordMgrWithPriorAuth ` -class allows HTTP Basic Authentication credentials to be managed so as to -eliminate unnecessary ``401`` response handling, or to unconditionally send -credentials on the first request in order to communicate with servers that -return a ``404`` response instead of a ``401`` if the ``Authorization`` header -is not sent. (Contributed by Matej Cepl in :issue:`19494` and Akshit Khurana in -:issue:`7159`.) - -A new *quote_via* argument for the -:func:`parse.urlencode() ` -function provides a way to control the encoding of query parts if needed. -(Contributed by Samwyse and Arnon Yaari in :issue:`13866`.) - -The :func:`request.urlopen() ` function accepts an -:class:`ssl.SSLContext` object as a *context* argument, which will be used for -the HTTPS connection. (Contributed by Alex Gaynor in :issue:`22366`.) - -The :func:`parse.urljoin() ` was updated to use the -:rfc:`3986` semantics for the resolution of relative URLs, rather than -:rfc:`1808` and :rfc:`2396`. -(Contributed by Demian Brecht and Senthil Kumaran in :issue:`22118`.) - - -wsgiref -------- - -The *headers* argument of the :class:`headers.Headers ` -class constructor is now optional. -(Contributed by Pablo Torres Navarrete and SilentGhost in :issue:`5800`.) - - -xmlrpc ------- - -The :class:`client.ServerProxy ` class now supports -the :term:`context manager` protocol. -(Contributed by Claudiu Popa in :issue:`20627`.) - -The :class:`client.ServerProxy ` constructor now accepts -an optional :class:`ssl.SSLContext` instance. -(Contributed by Alex Gaynor in :issue:`22960`.) - - -xml.sax -------- - -SAX parsers now support a character stream of the -:class:`xmlreader.InputSource ` object. -(Contributed by Serhiy Storchaka in :issue:`2175`.) - -:func:`~xml.sax.parseString` now accepts a :class:`str` instance. -(Contributed by Serhiy Storchaka in :issue:`10590`.) - - -zipfile -------- - -ZIP output can now be written to unseekable streams. -(Contributed by Serhiy Storchaka in :issue:`23252`.) - -The *mode* argument of :meth:`ZipFile.open() ` method now -accepts ``"x"`` to request exclusive creation. -(Contributed by Serhiy Storchaka in :issue:`21717`.) - - -Other module-level changes -========================== - -Many functions in the :mod:`mmap`, :mod:`ossaudiodev`, :mod:`socket`, -:mod:`ssl`, and :mod:`codecs` modules now accept writable -:term:`bytes-like objects `. -(Contributed by Serhiy Storchaka in :issue:`23001`.) - - -Optimizations -============= - -The :func:`os.walk` function has been sped up by 3 to 5 times on POSIX systems, -and by 7 to 20 times on Windows. This was done using the new :func:`os.scandir` -function, which exposes file information from the underlying ``readdir`` or -``FindFirstFile``/``FindNextFile`` system calls. (Contributed by -Ben Hoyt with help from Victor Stinner in :issue:`23605`.) - -Construction of ``bytes(int)`` (filled by zero bytes) is faster and uses less -memory for large objects. ``calloc()`` is used instead of ``malloc()`` to -allocate memory for these objects. -(Contributed by Victor Stinner in :issue:`21233`.) - -Some operations on :mod:`ipaddress` :class:`~ipaddress.IPv4Network` and -:class:`~ipaddress.IPv6Network` have been massively sped up, such as -:meth:`~ipaddress.IPv4Network.subnets`, :meth:`~ipaddress.IPv4Network.supernet`, -:func:`~ipaddress.summarize_address_range`, :func:`~ipaddress.collapse_addresses`. -The speed up can range from 3 to 15 times. -(Contributed by Antoine Pitrou, Michel Albert, and Markus in -:issue:`21486`, :issue:`21487`, :issue:`20826`, :issue:`23266`.) - -Pickling of :mod:`ipaddress` objects was optimized to produce significantly -smaller output. (Contributed by Serhiy Storchaka in :issue:`23133`.) - -Many operations on :class:`io.BytesIO` are now 50% to 100% faster. -(Contributed by Serhiy Storchaka in :issue:`15381` and David Wilson in -:issue:`22003`.) - -The :func:`marshal.dumps` function is now faster: 65-85% with versions 3 -and 4, 20-25% with versions 0 to 2 on typical data, and up to 5 times in -best cases. -(Contributed by Serhiy Storchaka in :issue:`20416` and :issue:`23344`.) - -The UTF-32 encoder is now 3 to 7 times faster. -(Contributed by Serhiy Storchaka in :issue:`15027`.) - -Regular expressions are now parsed up to 10% faster. -(Contributed by Serhiy Storchaka in :issue:`19380`.) - -The :func:`json.dumps` function was optimized to run with -``ensure_ascii=False`` as fast as with ``ensure_ascii=True``. -(Contributed by Naoki Inada in :issue:`23206`.) - -The :c:func:`PyObject_IsInstance` and :c:func:`PyObject_IsSubclass` -functions have been sped up in the common case that the second argument -has :class:`type` as its metaclass. -(Contributed Georg Brandl by in :issue:`22540`.) - -Method caching was slightly improved, yielding up to 5% performance -improvement in some benchmarks. -(Contributed by Antoine Pitrou in :issue:`22847`.) - -Objects from the :mod:`random` module now use 50% less memory on 64-bit -builds. (Contributed by Serhiy Storchaka in :issue:`23488`.) - -The :func:`property` getter calls are up to 25% faster. -(Contributed by Joe Jevnik in :issue:`23910`.) - -Instantiation of :class:`fractions.Fraction` is now up to 30% faster. -(Contributed by Stefan Behnel in :issue:`22464`.) - -String methods :meth:`~str.find`, :meth:`~str.rfind`, :meth:`~str.split`, -:meth:`~str.partition` and the :keyword:`in` string operator are now significantly -faster for searching 1-character substrings. -(Contributed by Serhiy Storchaka in :issue:`23573`.) - - -Build and C API Changes -======================= - -New ``calloc`` functions were added: - -* :c:func:`PyMem_RawCalloc`, -* :c:func:`PyMem_Calloc`, -* :c:func:`PyObject_Calloc`, -* :c:func:`_PyObject_GC_Calloc`. - -(Contributed by Victor Stinner in :issue:`21233`.) - -New encoding/decoding helper functions: - -* :c:func:`Py_DecodeLocale` (replaced ``_Py_char2wchar()``), -* :c:func:`Py_EncodeLocale` (replaced ``_Py_wchar2char()``). - -(Contributed by Victor Stinner in :issue:`18395`.) - -A new :c:func:`PyCodec_NameReplaceErrors` function to replace the unicode -encode error with ``\N{...}`` escapes. -(Contributed by Serhiy Storchaka in :issue:`19676`.) - -A new :c:func:`PyErr_FormatV` function similar to :c:func:`PyErr_Format`, -but accepts a ``va_list`` argument. -(Contributed by Antoine Pitrou in :issue:`18711`.) - -A new :c:data:`PyExc_RecursionError` exception. -(Contributed by Georg Brandl in :issue:`19235`.) - -New :c:func:`PyModule_FromDefAndSpec`, :c:func:`PyModule_FromDefAndSpec2`, -and :c:func:`PyModule_ExecDef` functions introduced by :pep:`489` -- -multi-phase extension module initialization. -(Contributed by Petr Viktorin in :issue:`24268`.) - -New :c:func:`PyNumber_MatrixMultiply` and -:c:func:`PyNumber_InPlaceMatrixMultiply` functions to perform matrix -multiplication. -(Contributed by Benjamin Peterson in :issue:`21176`. See also :pep:`465` -for details.) - -The :c:member:`PyTypeObject.tp_finalize` slot is now part of the stable ABI. - -Windows builds now require Microsoft Visual C++ 14.0, which -is available as part of `Visual Studio 2015 `_. - -Extension modules now include a platform information tag in their filename on -some platforms (the tag is optional, and CPython will import extensions without -it, although if the tag is present and mismatched, the extension won't be -loaded): - -* On Linux, extension module filenames end with - ``.cpython-m--.pyd``: - - * ```` is the major number of the Python version; - for Python 3.5 this is ``3``. - - * ```` is the minor number of the Python version; - for Python 3.5 this is ``5``. - - * ```` is the hardware architecture the extension module - was built to run on. It's most commonly either ``i386`` for 32-bit Intel - platforms or ``x86_64`` for 64-bit Intel (and AMD) platforms. - - * ```` is always ``linux-gnu``, except for extensions built to - talk to the 32-bit ABI on 64-bit platforms, in which case it is - ``linux-gnu32`` (and ```` will be ``x86_64``). - -* On Windows, extension module filenames end with - ``.cp-.pyd``: - - * ```` is the major number of the Python version; - for Python 3.5 this is ``3``. - - * ```` is the minor number of the Python version; - for Python 3.5 this is ``5``. - - * ```` is the platform the extension module was built for, - either ``win32`` for Win32, ``win_amd64`` for Win64, ``win_ia64`` for - Windows Itanium 64, and ``win_arm`` for Windows on ARM. - - * If built in debug mode, ```` will be ``_d``, - otherwise it will be blank. - -* On OS X platforms, extension module filenames now end with ``-darwin.so``. - -* On all other platforms, extension module filenames are the same as they were - with Python 3.4. - - -Deprecated -========== - -New Keywords ------------- - -``async`` and ``await`` are not recommended to be used as variable, class, -function or module names. Introduced by :pep:`492` in Python 3.5, they will -become proper keywords in Python 3.7. - - -Deprecated Python Behavior --------------------------- - -Raising the :exc:`StopIteration` exception inside a generator will now generate a silent -:exc:`PendingDeprecationWarning`, which will become a non-silent deprecation -warning in Python 3.6 and will trigger a :exc:`RuntimeError` in Python 3.7. -See :ref:`PEP 479: Change StopIteration handling inside generators ` -for details. - - -Unsupported Operating Systems ------------------------------ - -Windows XP is no longer supported by Microsoft, thus, per :PEP:`11`, CPython -3.5 is no longer officially supported on this OS. - - -Deprecated Python modules, functions and methods ------------------------------------------------- - -The :mod:`formatter` module has now graduated to full deprecation and is still -slated for removal in Python 3.6. - -The :func:`asyncio.async` function is deprecated in favor of -:func:`~asyncio.ensure_future`. - -The :mod:`smtpd` module has in the past always decoded the DATA portion of -email messages using the ``utf-8`` codec. This can now be controlled by the -new *decode_data* keyword to :class:`~smtpd.SMTPServer`. The default value is -``True``, but this default is deprecated. Specify the *decode_data* keyword -with an appropriate value to avoid the deprecation warning. - -Directly assigning values to the :attr:`~http.cookies.Morsel.key`, -:attr:`~http.cookies.Morsel.value` and -:attr:`~http.cookies.Morsel.coded_value` of :class:`http.cookies.Morsel` -objects is deprecated. Use the :meth:`~http.cookies.Morsel.set` method -instead. In addition, the undocumented *LegalChars* parameter of -:meth:`~http.cookies.Morsel.set` is deprecated, and is now ignored. - -Passing a format string as keyword argument *format_string* to the -:meth:`~string.Formatter.format` method of the :class:`string.Formatter` -class has been deprecated. -(Contributed by Serhiy Storchaka in :issue:`23671`.) - -The :func:`platform.dist` and :func:`platform.linux_distribution` functions -are now deprecated and will be removed in Python 3.7. Linux distributions use -too many different ways of describing themselves, so the functionality is -left to a package. -(Contributed by Vajrasky Kok and Berker Peksag in :issue:`1322`.) - -The previously undocumented ``from_function`` and ``from_builtin`` methods of -:class:`inspect.Signature` are deprecated. Use the new -:meth:`Signature.from_callable() ` -method instead. (Contributed by Yury Selivanov in :issue:`24248`.) - -The :func:`inspect.getargspec` function is deprecated and scheduled to be -removed in Python 3.6. (See :issue:`20438` for details.) - -The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getargvalues`, :func:`~inspect.getcallargs`, -:func:`~inspect.getargvalues`, :func:`~inspect.formatargspec`, and -:func:`~inspect.formatargvalues` functions are deprecated in favor of -the :func:`inspect.signature` API. -(Contributed by Yury Selivanov in :issue:`20438`.) - -Use of :const:`re.LOCALE` flag with str patterns or :const:`re.ASCII` is now -deprecated. (Contributed by Serhiy Storchaka in :issue:`22407`.) - -Use of unrecognized special sequences consisting of ``'\'`` and an ASCII letter -in regular expression patterns and replacement patterns now raises a -deprecation warning and will be forbidden in Python 3.6. -(Contributed by Serhiy Storchaka in :issue:`23622`.) - -The undocumented and unofficial *use_load_tests* default argument of the -:meth:`unittest.TestLoader.loadTestsFromModule` method now is -deprecated and ignored. -(Contributed by Robert Collins and Barry A. Warsaw in :issue:`16662`.) - - -Removed -======= - -API and Feature Removals ------------------------- - -The following obsolete and previously deprecated APIs and features have been -removed: - -* The ``__version__`` attribute has been dropped from the email package. The - email code hasn't been shipped separately from the stdlib for a long time, - and the ``__version__`` string was not updated in the last few releases. - -* The internal ``Netrc`` class in the :mod:`ftplib` module was deprecated in - 3.4, and has now been removed. - (Contributed by Matt Chaput in :issue:`6623`.) - -* The concept of ``.pyo`` files has been removed. - -* The JoinableQueue class in the provisional :mod:`asyncio` module was - deprecated in 3.4.4 and is now removed. - (Contributed by A. Jesse Jiryu Davis in :issue:`23464`.) - - -Porting to Python 3.5 -===================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - - -Changes in Python behavior --------------------------- - -* Due to an oversight, earlier Python versions erroneously accepted the - following syntax:: - - f(1 for x in [1], *args) - f(1 for x in [1], **kwargs) - - Python 3.5 now correctly raises a :exc:`SyntaxError`, as generator - expressions must be put in parentheses if not a sole argument to a function. - - -Changes in the Python API -------------------------- - -* :pep:`475`: System calls are now retried when interrupted by a signal instead - of raising :exc:`InterruptedError` if the Python signal handler does not - raise an exception. - -* Before Python 3.5, a :class:`datetime.time` object was considered to be false - if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. - -* The :meth:`ssl.SSLSocket.send()` method now raises either - :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` - on a non-blocking socket if the operation would block. Previously, - it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) - -* The ``__name__`` attribute of generators is now set from the function name, - instead of being set from the code name. Use ``gen.gi_code.co_name`` to - retrieve the code name. Generators also have a new ``__qualname__`` - attribute, the qualified name, which is now used for the representation - of a generator (``repr(gen)``). - (Contributed by Victor Stinner in :issue:`21205`.) - -* The deprecated "strict" mode and argument of :class:`~html.parser.HTMLParser`, - :meth:`HTMLParser.error`, and the :exc:`HTMLParserError` exception have been - removed. (Contributed by Ezio Melotti in :issue:`15114`.) - The *convert_charrefs* argument of :class:`~html.parser.HTMLParser` is - now ``True`` by default. (Contributed by Berker Peksag in :issue:`21047`.) - -* Although it is not formally part of the API, it is worth noting for porting - purposes (ie: fixing tests) that error messages that were previously of the - form "'sometype' does not support the buffer protocol" are now of the form "a - :term:`bytes-like object` is required, not 'sometype'". - (Contributed by Ezio Melotti in :issue:`16518`.) - -* If the current directory is set to a directory that no longer exists then - :exc:`FileNotFoundError` will no longer be raised and instead - :meth:`~importlib.machinery.FileFinder.find_spec` will return ``None`` - **without** caching ``None`` in :data:`sys.path_importer_cache`, which is - different than the typical case (:issue:`22834`). - -* HTTP status code and messages from :mod:`http.client` and :mod:`http.server` - were refactored into a common :class:`~http.HTTPStatus` enum. The values in - :mod:`http.client` and :mod:`http.server` remain available for backwards - compatibility. (Contributed by Demian Brecht in :issue:`21793`.) - -* When an import loader defines :meth:`importlib.machinery.Loader.exec_module` - it is now expected to also define - :meth:`~importlib.machinery.Loader.create_module` (raises a - :exc:`DeprecationWarning` now, will be an error in Python 3.6). If the loader - inherits from :class:`importlib.abc.Loader` then there is nothing to do, else - simply define :meth:`~importlib.machinery.Loader.create_module` to return - ``None``. (Contributed by Brett Cannon in :issue:`23014`.) - -* The :func:`re.split` function always ignored empty pattern matches, so the - ``"x*"`` pattern worked the same as ``"x+"``, and the ``"\b"`` pattern never - worked. Now :func:`re.split` raises a warning if the pattern could match - an empty string. For compatibility, use patterns that never match an empty - string (e.g. ``"x+"`` instead of ``"x*"``). Patterns that could only match - an empty string (such as ``"\b"``) now raise an error. - (Contributed by Serhiy Storchaka in :issue:`22818`.) - -* The :class:`http.cookies.Morsel` dict-like interface has been made self - consistent: morsel comparison now takes the :attr:`~http.cookies.Morsel.key` - and :attr:`~http.cookies.Morsel.value` into account, - :meth:`~http.cookies.Morsel.copy` now results in a - :class:`~http.cookies.Morsel` instance rather than a :class:`dict`, and - :meth:`~http.cookies.Morsel.update` will now raise an exception if any of the - keys in the update dictionary are invalid. In addition, the undocumented - *LegalChars* parameter of :func:`~http.cookies.Morsel.set` is deprecated and - is now ignored. (Contributed by Demian Brecht in :issue:`2211`.) - -* :pep:`488` has removed ``.pyo`` files from Python and introduced the optional - ``opt-`` tag in ``.pyc`` file names. The - :func:`importlib.util.cache_from_source` has gained an *optimization* - parameter to help control the ``opt-`` tag. Because of this, the - *debug_override* parameter of the function is now deprecated. `.pyo` files - are also no longer supported as a file argument to the Python interpreter and - thus serve no purpose when distributed on their own (i.e. sourcless code - distribution). Due to the fact that the magic number for bytecode has changed - in Python 3.5, all old `.pyo` files from previous versions of Python are - invalid regardless of this PEP. - -* The :mod:`socket` module now exports the :data:`~socket.CAN_RAW_FD_FRAMES` - constant on linux 3.6 and greater. - -* The :func:`ssl.cert_time_to_seconds` function now interprets the input time - as UTC and not as local time, per :rfc:`5280`. Additionally, the return - value is always an :class:`int`. (Contributed by Akira Li in :issue:`19940`.) - -* The ``pygettext.py`` Tool now uses the standard +NNNN format for timezones in - the POT-Creation-Date header. - -* The :mod:`smtplib` module now uses :data:`sys.stderr` instead of the previous - module-level :data:`stderr` variable for debug output. If your (test) - program depends on patching the module-level variable to capture the debug - output, you will need to update it to capture sys.stderr instead. - -* The :meth:`str.startswith` and :meth:`str.endswith` methods no longer return - ``True`` when finding the empty string and the indexes are completely out of - range. (Contributed by Serhiy Storchaka in :issue:`24284`.) - -* The :func:`inspect.getdoc` function now returns documentation strings - inherited from base classes. Documentation strings no longer need to be - duplicated if the inherited documentation is appropriate. To suppress an - inherited string, an empty string must be specified (or the documentation - may be filled in). This change affects the output of the :mod:`pydoc` - module and the :func:`help` function. - (Contributed by Serhiy Storchaka in :issue:`15582`.) - -* Nested :func:`functools.partial` calls are now flattened. If you were - relying on the previous behavior, you can now either add an attribute to a - :func:`functools.partial` object or you can create a subclass of - :func:`functools.partial`. - (Contributed by Alexander Belopolsky in :issue:`7830`.) - -Changes in the C API --------------------- - -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the - (non-public) :c:type:`PyMemoryViewObject` structure has been removed. - All extensions relying on the relevant parts in ``memoryobject.h`` - must be rebuilt. - -* The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. - Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions - to format the :func:`repr` of the object. - (Contributed by Serhiy Storchaka in :issue:`22453`.) - -* Because the lack of the :attr:`__module__` attribute breaks pickling and - introspection, a deprecation warning is now raised for builtin types without - the :attr:`__module__` attribute. This would be an AttributeError in - the future. - (Contributed by Serhiy Storchaka in :issue:`20204`.) - -* As part of the :pep:`492` implementation, the ``tp_reserved`` slot of - :c:type:`PyTypeObject` was replaced with a - :c:member:`tp_as_async` slot. Refer to :ref:`coro-objects` for - new types, structures and functions. - diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,290 +0,0 @@ -**************************** - What's New In Python 3.6 -**************************** - -:Release: |release| -:Date: |today| - -.. Rules for maintenance: - - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. - - * The maintainer will go through Misc/NEWS periodically and add - changes; it's therefore more important to add your changes to - Misc/NEWS than to this file. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small - or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to just add a fragmentary note about a change. For - example: "XXX Describe the transmogrify() function added to the - socket module." The maintainer will research the change and - write the necessary text. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - XXX Describe the transmogrify() function added to the socket - module. - (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - -This article explains the new features in Python 3.6, compared to 3.5. - -For full details, see the :source:`Misc/NEWS` file. - -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.6 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- Release highlights -============================= - -.. This section singles out the most important changes in Python 3.6. - Brevity is key. - -* None yet. - -.. PEP-sized items next. - -.. _pep-4XX: - -.. PEP 4XX: Virtual Environments -.. ============================= - - -.. (Implemented by Foo Bar.) - -.. .. seealso:: - - :pep:`4XX` - Python Virtual Environments - PEP written by Carl Meyer - - -Other Language Changes -====================== - -* None yet. - - -New Modules -=========== - -* None yet. - - -Improved Modules -================ - -datetime --------- - -The :meth:`datetime.strftime() ` and -:meth:`date.strftime() ` methods now support ISO 8601 date -directives ``%G``, ``%u`` and ``%V``. -(Contributed by Ashley Anderson in :issue:`12006`.) - - -pickle ------- - -Objects that need calling ``__new__`` with keyword arguments can now be pickled -using :ref:`pickle protocols ` older than protocol version 4. -Protocol version 4 already supports this case. (Contributed by Serhiy -Storchaka in :issue:`24164`.) - - -rlcompleter ------------ - -Private and special attribute names now are omitted unless the prefix starts -with underscores. A space or a colon is added after some completed keywords. -(Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) - -Names of most attributes listed by :func:`dir` are now completed. -Previously, names of properties and slots which were not yet created on -an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) - - -telnetlib ---------- - -:class:`~telnetlib.Telnet` is now a context manager (contributed by -Stéphane Wirtel in :issue:`25485`). - - -urllib.robotparser ------------------- - -:class:`~urllib.robotparser.RobotFileParser` now supports the ``Crawl-delay`` and -``Request-rate`` extensions. -(Contributed by Nikolay Bogoychev in :issue:`16099`.) - - -Optimizations -============= - -* The ASCII decoder is now up to 60 times as fast for error handlers - ``surrogateescape``, ``ignore`` and ``replace`` (Contributed - by Victor Stinner in :issue:`24870`). - -* The ASCII and the Latin1 encoders are now up to 3 times as fast for the - error handler ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). - -* The UTF-8 encoder is now up to 75 times as fast for error handlers - ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass`` (Contributed - by Victor Stinner in :issue:`25267`). - -* The UTF-8 decoder is now up to 15 times as fast for error handlers - ``ignore``, ``replace`` and ``surrogateescape`` (Contributed - by Victor Stinner in :issue:`25301`). - -* ``bytes % args`` is now up to 2 times faster. (Contributed by Victor Stinner - in :issue:`25349`). - -* ``bytearray % args`` is now between 2.5 and 5 times faster. (Contributed by - Victor Stinner in :issue:`25399`). - -* Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now - between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). - - -Build and C API Changes -======================= - -* New :c:func:`Py_FinalizeEx` API which indicates if flushing buffered data - failed (:issue:`5319`). - - -Deprecated -========== - -New Keywords ------------- - -``async`` and ``await`` are not recommended to be used as variable, class, -function or module names. Introduced by :pep:`492` in Python 3.5, they will -become proper keywords in Python 3.7. - - -Deprecated Python modules, functions and methods ------------------------------------------------- - -* :meth:`importlib.machinery.SourceFileLoader` and - :meth:`importlib.machinery.SourcelessFileLoader` are now deprecated. They - were the only remaining implementations of - :meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not - been deprecated in previous versions of Python in favour of - :meth:`importlib.abc.Loader.exec_module`. - - -Deprecated functions and types of the C API -------------------------------------------- - -* None yet. - - -Deprecated features -------------------- - -* The ``pyvenv`` script has been deprecated in favour of ``python3 -m venv``. - This prevents confusion as to what Python interpreter ``pyvenv`` is - connected to and thus what Python interpreter will be used by the virtual - environment. (Contributed by Brett Cannon in :issue:`25154`.) - -* When performing a relative import, falling back on ``__name__`` and - ``__path__`` from the calling module when ``__spec__`` or - ``__package__`` are not defined now raises an :exc:`ImportWarning`. - (Contributed by Rose Ames in :issue:`25791`.) - - -Removed -======= - -API and Feature Removals ------------------------- - -* ``inspect.getmoduleinfo()`` was removed (was deprecated since CPython 3.3). - :func:`inspect.getmodulename` should be used for obtaining the module - name for a given path. - -* ``traceback.Ignore`` class and ``traceback.usage``, ``traceback.modname``, - ``traceback.fullmodname``, ``traceback.find_lines_from_code``, - ``traceback.find_lines``, ``traceback.find_strings``, - ``traceback.find_executable_lines`` methods were removed from the - :mod:`traceback` module. They were undocumented methods deprecated since - Python 3.2 and equivalent functionality is available from private methods. - - -Porting to Python 3.6 -===================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - -Changes in the Python API -------------------------- - -* The format of the ``co_lnotab`` attribute of code objects changed to support - negative line number delta. By default, Python does not emit bytecode with - negative line number delta. Functions using ``frame.f_lineno``, - ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. - Functions decoding directly ``co_lnotab`` should be updated to use a signed - 8-bit integer type for the line number delta, but it's only required to - support applications using negative line number delta. See - ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode - it, and see the :pep:`511` for the rationale. - -* The functions in the :mod:`compileall` module now return booleans instead - of ``1`` or ``0`` to represent success or failure, respectively. Thanks to - booleans being a subclass of integers, this should only be an issue if you - were doing identity checks for ``1`` or ``0``. See :issue:`25768`. - -* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of - :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results - now raises :exc:`ValueError` for out-of-range values, rather than - returning :const:`None`. See :issue:`20059`. - -* The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of - :exc:`PendingDeprecationWarning`. - -* The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, - :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, - :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and - :mod:`wave`. This means they will export new symbols when ``import *`` - is used. See :issue:`23883`. - -* When performing a relative import, if ``__package__`` does not compare equal - to ``__spec__.parent`` then :exc:`ImportWarning` is raised. - (Contributed by Brett Cannon in :issue:`25791`.) - -* When a relative import is performed and no parent package is known, then - :exc:`ImportError` will be raised. Previously, :exc:`SystemError` could be - raised. (Contribute by Brett Cannon in :issue:`18018`.) - - -Changes in the C API --------------------- - -* :c:func:`Py_Exit` (and the main interpreter) now override the exit status - with 120 if flushing buffered data failed. See :issue:`5319`. diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/changelog.rst --- a/Doc/whatsnew/changelog.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,5 +0,0 @@ -+++++++++ -Changelog -+++++++++ - -.. miscnews:: ../../Misc/NEWS diff -r 6db40a9955dc -r 0d413f60cc23 Doc/whatsnew/index.rst --- a/Doc/whatsnew/index.rst Sun Jan 24 22:15:20 2016 -0800 +++ b/Doc/whatsnew/index.rst Mon Jan 25 17:05:13 2016 +0100 @@ -11,9 +11,6 @@ .. toctree:: :maxdepth: 2 - 3.6.rst - 3.5.rst - 3.4.rst 3.3.rst 3.2.rst 3.1.rst @@ -26,11 +23,3 @@ 2.2.rst 2.1.rst 2.0.rst - -The "Changelog" is a HTML version of the file :source:`Misc/NEWS` which -contains *all* nontrivial changes to Python for the current version. - -.. toctree:: - :maxdepth: 2 - - changelog.rst diff -r 6db40a9955dc -r 0d413f60cc23 Grammar/Grammar --- a/Grammar/Grammar Sun Jan 24 22:15:20 2016 -0800 +++ b/Grammar/Grammar Mon Jan 25 17:05:13 2016 +0100 @@ -7,8 +7,8 @@ # with someone who can; ask around on python-dev for help. Fred # Drake will probably be listening there. -# NOTE WELL: You should also follow all the steps listed at -# https://docs.python.org/devguide/grammar.html +# NOTE WELL: You should also follow all the steps listed in PEP 306, +# "How to Change Python's Grammar" # Start symbols for the grammar: # single_input is a single interactive statement; @@ -21,24 +21,16 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) - -async_funcdef: ASYNC funcdef +decorated: decorators (classdef | funcdef) funcdef: 'def' NAME parameters ['->' test] ':' suite - parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ - '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] - | '**' tfpdef [',']]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] - | '**' tfpdef [',']) +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' + ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) tfpdef: NAME [':' test] -varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ - '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] - | '**' vfpdef [',']]] - | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] - | '**' vfpdef [','] -) +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' + ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) vfpdef: NAME stmt: simple_stmt | compound_stmt @@ -48,7 +40,7 @@ expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | +augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter del_stmt: 'del' exprlist @@ -73,8 +65,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -98,7 +89,7 @@ not_test: 'not' not_test | comparison comparison: expr (comp_op expr)* # <> isn't actually a valid comparison operator in Python. It's here for the -# sake of a __future__ import described in PEP 401 (which really works :-) +# sake of a __future__ import described in PEP 401 comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' star_expr: '*' expr expr: xor_expr ('|' xor_expr)* @@ -106,10 +97,9 @@ and_expr: shift_expr ('&' shift_expr)* shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* +term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: atom_expr ['**' factor] -atom_expr: [AWAIT] atom trailer* +power: atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | @@ -121,29 +111,17 @@ sliceop: ':' [test] exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] testlist: test (',' test)* [','] -dictorsetmaker: ( ((test ':' test | '**' expr) - (comp_for | (',' (test ':' test | '**' expr))* [','])) | - ((test | star_expr) - (comp_for | (',' (test | star_expr))* [','])) ) +dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | + (test (comp_for | (',' test)* [','])) ) classdef: 'class' NAME ['(' [arglist] ')'] ':' suite -arglist: argument (',' argument)* [','] - +arglist: (argument ',')* (argument [','] + |'*' test (',' argument)* [',' '**' test] + |'**' test) # The reason that keywords are test nodes instead of NAME is that using NAME # results in an ambiguity. ast.c makes sure it's a NAME. -# "test '=' test" is really "keyword '=' test", but we have no such token. -# These need to be in a single rule to avoid grammar that is ambiguous -# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, -# we explicitly match '*' here, too, to give it proper precedence. -# Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguements are blocked; keyword unpackings -# that precede iterable unpackings are blocked; etc. -argument: ( test [comp_for] | - test '=' test | - '**' test | - '*' test ) - +argument: test [comp_for] | test '=' test # Really [keyword '='] test comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' or_test [comp_iter] comp_if: 'if' test_nocond [comp_iter] diff -r 6db40a9955dc -r 0d413f60cc23 Include/Python-ast.h --- a/Include/Python-ast.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/Python-ast.h Mon Jan 25 17:05:13 2016 +0100 @@ -15,9 +15,9 @@ typedef enum _boolop { And=1, Or=2 } boolop_ty; -typedef enum _operator { Add=1, Sub=2, Mult=3, MatMult=4, Div=5, Mod=6, Pow=7, - LShift=8, RShift=9, BitOr=10, BitXor=11, BitAnd=12, - FloorDiv=13 } operator_ty; +typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7, + RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 } + operator_ty; typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty; @@ -42,387 +42,350 @@ enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, Suite_kind=4}; struct _mod { - enum _mod_kind kind; - union { - struct { - asdl_seq *body; - } Module; - - struct { - asdl_seq *body; - } Interactive; - - struct { - expr_ty body; - } Expression; - - struct { - asdl_seq *body; - } Suite; - - } v; + enum _mod_kind kind; + union { + struct { + asdl_seq *body; + } Module; + + struct { + asdl_seq *body; + } Interactive; + + struct { + expr_ty body; + } Expression; + + struct { + asdl_seq *body; + } Suite; + + } v; }; -enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3, - Return_kind=4, Delete_kind=5, Assign_kind=6, - AugAssign_kind=7, For_kind=8, AsyncFor_kind=9, While_kind=10, - If_kind=11, With_kind=12, AsyncWith_kind=13, Raise_kind=14, - Try_kind=15, Assert_kind=16, Import_kind=17, - ImportFrom_kind=18, Global_kind=19, Nonlocal_kind=20, - Expr_kind=21, Pass_kind=22, Break_kind=23, Continue_kind=24}; +enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3, + Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7, + While_kind=8, If_kind=9, With_kind=10, Raise_kind=11, + Try_kind=12, Assert_kind=13, Import_kind=14, + ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17, + Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21}; struct _stmt { - enum _stmt_kind kind; - union { - struct { - identifier name; - arguments_ty args; - asdl_seq *body; - asdl_seq *decorator_list; - expr_ty returns; - } FunctionDef; - - struct { - identifier name; - arguments_ty args; - asdl_seq *body; - asdl_seq *decorator_list; - expr_ty returns; - } AsyncFunctionDef; - - struct { - identifier name; - asdl_seq *bases; - asdl_seq *keywords; - asdl_seq *body; - asdl_seq *decorator_list; - } ClassDef; - - struct { - expr_ty value; - } Return; - - struct { - asdl_seq *targets; - } Delete; - - struct { - asdl_seq *targets; - expr_ty value; - } Assign; - - struct { - expr_ty target; - operator_ty op; - expr_ty value; - } AugAssign; - - struct { - expr_ty target; - expr_ty iter; - asdl_seq *body; - asdl_seq *orelse; - } For; - - struct { - expr_ty target; - expr_ty iter; - asdl_seq *body; - asdl_seq *orelse; - } AsyncFor; - - struct { - expr_ty test; - asdl_seq *body; - asdl_seq *orelse; - } While; - - struct { - expr_ty test; - asdl_seq *body; - asdl_seq *orelse; - } If; - - struct { - asdl_seq *items; - asdl_seq *body; - } With; - - struct { - asdl_seq *items; - asdl_seq *body; - } AsyncWith; - - struct { - expr_ty exc; - expr_ty cause; - } Raise; - - struct { - asdl_seq *body; - asdl_seq *handlers; - asdl_seq *orelse; - asdl_seq *finalbody; - } Try; - - struct { - expr_ty test; - expr_ty msg; - } Assert; - - struct { - asdl_seq *names; - } Import; - - struct { - identifier module; - asdl_seq *names; - int level; - } ImportFrom; - - struct { - asdl_seq *names; - } Global; - - struct { - asdl_seq *names; - } Nonlocal; - - struct { - expr_ty value; - } Expr; - - } v; - int lineno; - int col_offset; + enum _stmt_kind kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + asdl_seq *decorator_list; + expr_ty returns; + } FunctionDef; + + struct { + identifier name; + asdl_seq *bases; + asdl_seq *keywords; + expr_ty starargs; + expr_ty kwargs; + asdl_seq *body; + asdl_seq *decorator_list; + } ClassDef; + + struct { + expr_ty value; + } Return; + + struct { + asdl_seq *targets; + } Delete; + + struct { + asdl_seq *targets; + expr_ty value; + } Assign; + + struct { + expr_ty target; + operator_ty op; + expr_ty value; + } AugAssign; + + struct { + expr_ty target; + expr_ty iter; + asdl_seq *body; + asdl_seq *orelse; + } For; + + struct { + expr_ty test; + asdl_seq *body; + asdl_seq *orelse; + } While; + + struct { + expr_ty test; + asdl_seq *body; + asdl_seq *orelse; + } If; + + struct { + asdl_seq *items; + asdl_seq *body; + } With; + + struct { + expr_ty exc; + expr_ty cause; + } Raise; + + struct { + asdl_seq *body; + asdl_seq *handlers; + asdl_seq *orelse; + asdl_seq *finalbody; + } Try; + + struct { + expr_ty test; + expr_ty msg; + } Assert; + + struct { + asdl_seq *names; + } Import; + + struct { + identifier module; + asdl_seq *names; + int level; + } ImportFrom; + + struct { + asdl_seq *names; + } Global; + + struct { + asdl_seq *names; + } Nonlocal; + + struct { + expr_ty value; + } Expr; + + } v; + int lineno; + int col_offset; }; enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, - Await_kind=12, Yield_kind=13, YieldFrom_kind=14, - Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, - FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, - NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, - Subscript_kind=25, Starred_kind=26, Name_kind=27, - List_kind=28, Tuple_kind=29}; + Yield_kind=12, YieldFrom_kind=13, Compare_kind=14, + Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18, + Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21, + Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25}; struct _expr { - enum _expr_kind kind; - union { - struct { - boolop_ty op; - asdl_seq *values; - } BoolOp; - - struct { - expr_ty left; - operator_ty op; - expr_ty right; - } BinOp; - - struct { - unaryop_ty op; - expr_ty operand; - } UnaryOp; - - struct { - arguments_ty args; - expr_ty body; - } Lambda; - - struct { - expr_ty test; - expr_ty body; - expr_ty orelse; - } IfExp; - - struct { - asdl_seq *keys; - asdl_seq *values; - } Dict; - - struct { - asdl_seq *elts; - } Set; - - struct { - expr_ty elt; - asdl_seq *generators; - } ListComp; - - struct { - expr_ty elt; - asdl_seq *generators; - } SetComp; - - struct { - expr_ty key; - expr_ty value; - asdl_seq *generators; - } DictComp; - - struct { - expr_ty elt; - asdl_seq *generators; - } GeneratorExp; - - struct { - expr_ty value; - } Await; - - struct { - expr_ty value; - } Yield; - - struct { - expr_ty value; - } YieldFrom; - - struct { - expr_ty left; - asdl_int_seq *ops; - asdl_seq *comparators; - } Compare; - - struct { - expr_ty func; - asdl_seq *args; - asdl_seq *keywords; - } Call; - - struct { - object n; - } Num; - - struct { - string s; - } Str; - - struct { - expr_ty value; - int conversion; - expr_ty format_spec; - } FormattedValue; - - struct { - asdl_seq *values; - } JoinedStr; - - struct { - bytes s; - } Bytes; - - struct { - singleton value; - } NameConstant; - - struct { - expr_ty value; - identifier attr; - expr_context_ty ctx; - } Attribute; - - struct { - expr_ty value; - slice_ty slice; - expr_context_ty ctx; - } Subscript; - - struct { - expr_ty value; - expr_context_ty ctx; - } Starred; - - struct { - identifier id; - expr_context_ty ctx; - } Name; - - struct { - asdl_seq *elts; - expr_context_ty ctx; - } List; - - struct { - asdl_seq *elts; - expr_context_ty ctx; - } Tuple; - - } v; - int lineno; - int col_offset; + enum _expr_kind kind; + union { + struct { + boolop_ty op; + asdl_seq *values; + } BoolOp; + + struct { + expr_ty left; + operator_ty op; + expr_ty right; + } BinOp; + + struct { + unaryop_ty op; + expr_ty operand; + } UnaryOp; + + struct { + arguments_ty args; + expr_ty body; + } Lambda; + + struct { + expr_ty test; + expr_ty body; + expr_ty orelse; + } IfExp; + + struct { + asdl_seq *keys; + asdl_seq *values; + } Dict; + + struct { + asdl_seq *elts; + } Set; + + struct { + expr_ty elt; + asdl_seq *generators; + } ListComp; + + struct { + expr_ty elt; + asdl_seq *generators; + } SetComp; + + struct { + expr_ty key; + expr_ty value; + asdl_seq *generators; + } DictComp; + + struct { + expr_ty elt; + asdl_seq *generators; + } GeneratorExp; + + struct { + expr_ty value; + } Yield; + + struct { + expr_ty value; + } YieldFrom; + + struct { + expr_ty left; + asdl_int_seq *ops; + asdl_seq *comparators; + } Compare; + + struct { + expr_ty func; + asdl_seq *args; + asdl_seq *keywords; + expr_ty starargs; + expr_ty kwargs; + } Call; + + struct { + object n; + } Num; + + struct { + string s; + } Str; + + struct { + bytes s; + } Bytes; + + struct { + expr_ty value; + identifier attr; + expr_context_ty ctx; + } Attribute; + + struct { + expr_ty value; + slice_ty slice; + expr_context_ty ctx; + } Subscript; + + struct { + expr_ty value; + expr_context_ty ctx; + } Starred; + + struct { + identifier id; + expr_context_ty ctx; + } Name; + + struct { + asdl_seq *elts; + expr_context_ty ctx; + } List; + + struct { + asdl_seq *elts; + expr_context_ty ctx; + } Tuple; + + } v; + int lineno; + int col_offset; }; enum _slice_kind {Slice_kind=1, ExtSlice_kind=2, Index_kind=3}; struct _slice { - enum _slice_kind kind; - union { - struct { - expr_ty lower; - expr_ty upper; - expr_ty step; - } Slice; - - struct { - asdl_seq *dims; - } ExtSlice; - - struct { - expr_ty value; - } Index; - - } v; + enum _slice_kind kind; + union { + struct { + expr_ty lower; + expr_ty upper; + expr_ty step; + } Slice; + + struct { + asdl_seq *dims; + } ExtSlice; + + struct { + expr_ty value; + } Index; + + } v; }; struct _comprehension { - expr_ty target; - expr_ty iter; - asdl_seq *ifs; + expr_ty target; + expr_ty iter; + asdl_seq *ifs; }; enum _excepthandler_kind {ExceptHandler_kind=1}; struct _excepthandler { - enum _excepthandler_kind kind; - union { - struct { - expr_ty type; - identifier name; - asdl_seq *body; - } ExceptHandler; - - } v; - int lineno; - int col_offset; + enum _excepthandler_kind kind; + union { + struct { + expr_ty type; + identifier name; + asdl_seq *body; + } ExceptHandler; + + } v; + int lineno; + int col_offset; }; struct _arguments { - asdl_seq *args; - arg_ty vararg; - asdl_seq *kwonlyargs; - asdl_seq *kw_defaults; - arg_ty kwarg; - asdl_seq *defaults; + asdl_seq *args; + identifier vararg; + expr_ty varargannotation; + asdl_seq *kwonlyargs; + identifier kwarg; + expr_ty kwargannotation; + asdl_seq *defaults; + asdl_seq *kw_defaults; }; struct _arg { - identifier arg; - expr_ty annotation; - int lineno; - int col_offset; + identifier arg; + expr_ty annotation; }; struct _keyword { - identifier arg; - expr_ty value; + identifier arg; + expr_ty value; }; struct _alias { - identifier name; - identifier asname; + identifier name; + identifier asname; }; struct _withitem { - expr_ty context_expr; - expr_ty optional_vars; + expr_ty context_expr; + expr_ty optional_vars; }; @@ -438,14 +401,11 @@ stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * decorator_list, expr_ty returns, int lineno, int col_offset, PyArena *arena); -#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq * - body, asdl_seq * decorator_list, expr_ty returns, - int lineno, int col_offset, PyArena *arena); -#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) +#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, - asdl_seq * body, asdl_seq * decorator_list, int lineno, - int col_offset, PyArena *arena); + expr_ty starargs, expr_ty kwargs, asdl_seq * body, + asdl_seq * decorator_list, int lineno, int col_offset, + PyArena *arena); #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3) stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena); #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3) @@ -460,9 +420,6 @@ #define For(a0, a1, a2, a3, a4, a5, a6) _Py_For(a0, a1, a2, a3, a4, a5, a6) stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, PyArena *arena); -#define AsyncFor(a0, a1, a2, a3, a4, a5, a6) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * - orelse, int lineno, int col_offset, PyArena *arena); #define While(a0, a1, a2, a3, a4, a5) _Py_While(a0, a1, a2, a3, a4, a5) stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, PyArena *arena); @@ -472,9 +429,6 @@ #define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4) stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena *arena); -#define AsyncWith(a0, a1, a2, a3, a4) _Py_AsyncWith(a0, a1, a2, a3, a4) -stmt_ty _Py_AsyncWith(asdl_seq * items, asdl_seq * body, int lineno, int - col_offset, PyArena *arena); #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4) stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena); @@ -537,8 +491,6 @@ #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); -#define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3) -expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena); #define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3) expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); #define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3) @@ -547,24 +499,16 @@ #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5) expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena); -#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5) -expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int - lineno, int col_offset, PyArena *arena); +#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7) +expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty + starargs, expr_ty kwargs, int lineno, int col_offset, PyArena + *arena); #define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3) expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena); #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3) expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena); -#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5) -expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, - int lineno, int col_offset, PyArena *arena); -#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3) -expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena - *arena); #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3) expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena); -#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3) -expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena - *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) @@ -598,13 +542,13 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq * body, int lineno, int col_offset, PyArena *arena); -#define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6) -arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * - kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg, - asdl_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4) -arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, - PyArena *arena); +#define arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8) +arguments_ty _Py_arguments(asdl_seq * args, identifier vararg, expr_ty + varargannotation, asdl_seq * kwonlyargs, identifier + kwarg, expr_ty kwargannotation, asdl_seq * defaults, + asdl_seq * kw_defaults, PyArena *arena); +#define arg(a0, a1, a2) _Py_arg(a0, a1, a2) +arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) diff -r 6db40a9955dc -r 0d413f60cc23 Include/Python.h --- a/Include/Python.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/Python.h Mon Jan 25 17:05:13 2016 +0100 @@ -68,7 +68,6 @@ #include "object.h" #include "objimpl.h" #include "typeslots.h" -#include "pyhash.h" #include "pydebug.h" @@ -85,7 +84,6 @@ #include "tupleobject.h" #include "listobject.h" #include "dictobject.h" -#include "odictobject.h" #include "enumobject.h" #include "setobject.h" #include "methodobject.h" @@ -103,7 +101,6 @@ #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" -#include "namespaceobject.h" #include "codecs.h" #include "pyerrors.h" @@ -113,7 +110,6 @@ #include "pyarena.h" #include "modsupport.h" #include "pythonrun.h" -#include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" #include "intrcheck.h" diff -r 6db40a9955dc -r 0d413f60cc23 Include/abstract.h --- a/Include/abstract.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/abstract.h Mon Jan 25 17:05:13 2016 +0100 @@ -95,7 +95,7 @@ numeric, sequence, and mapping. Each protocol consists of a collection of related operations. If an operation that is not provided by a particular type is invoked, then a standard exception, - NotImplementedError is raised with an operation name as an argument. + NotImplementedError is raised with a operation name as an argument. In addition, for convenience this interface defines a set of constructors for building objects of built-in types. This is needed so new objects can be returned from C functions that otherwise treat @@ -144,7 +144,7 @@ /* Implemented elsewhere: - int PyObject_HasAttrString(PyObject *o, const char *attr_name); + int PyObject_HasAttrString(PyObject *o, char *attr_name); Returns 1 if o has the attribute attr_name, and 0 otherwise. This is equivalent to the Python expression: @@ -156,7 +156,7 @@ /* Implemented elsewhere: - PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name); + PyObject* PyObject_GetAttrString(PyObject *o, char *attr_name); Retrieve an attributed named attr_name form object o. Returns the attribute value on success, or NULL on failure. @@ -189,11 +189,11 @@ /* Implemented elsewhere: - int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v); + int PyObject_SetAttrString(PyObject *o, char *attr_name, PyObject *v); Set the value of the attribute named attr_name, for object o, - to the value v. Raise an exception and return -1 on failure; return 0 on - success. This is the equivalent of the Python statement o.attr_name=v. + to the value, v. Returns -1 on failure. This is + the equivalent of the Python statement: o.attr_name=v. */ @@ -202,14 +202,14 @@ int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v); Set the value of the attribute named attr_name, for object o, - to the value v. Raise an exception and return -1 on failure; return 0 on - success. This is the equivalent of the Python statement o.attr_name=v. + to the value, v. Returns -1 on failure. This is + the equivalent of the Python statement: o.attr_name=v. */ /* implemented as a macro: - int PyObject_DelAttrString(PyObject *o, const char *attr_name); + int PyObject_DelAttrString(PyObject *o, char *attr_name); Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python @@ -266,12 +266,6 @@ PyAPI_FUNC(PyObject *) PyObject_Call(PyObject *callable_object, PyObject *args, PyObject *kw); -#ifndef Py_LIMITED_API - PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *func, - PyObject *result, - const char *where); -#endif - /* Call a callable Python object, callable_object, with arguments and keywords arguments. The 'args' argument can not be @@ -290,7 +284,7 @@ */ PyAPI_FUNC(PyObject *) PyObject_CallFunction(PyObject *callable_object, - const char *format, ...); + char *format, ...); /* Call a callable Python object, callable_object, with a @@ -302,9 +296,8 @@ */ - PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *o, - const char *method, - const char *format, ...); + PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *o, char *method, + char *format, ...); /* Call the method named m of object o with a variable number of @@ -315,9 +308,8 @@ Python expression: o.method(args). */ - PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o, - _Py_Identifier *method, - const char *format, ...); + PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o, _Py_Identifier *method, + char *format, ...); /* Like PyObject_CallMethod, but expect a _Py_Identifier* as the @@ -325,16 +317,13 @@ */ PyAPI_FUNC(PyObject *) _PyObject_CallFunction_SizeT(PyObject *callable, - const char *format, - ...); + char *format, ...); PyAPI_FUNC(PyObject *) _PyObject_CallMethod_SizeT(PyObject *o, - const char *name, - const char *format, - ...); + char *name, + char *format, ...); PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *o, _Py_Identifier *name, - const char *format, - ...); + char *format, ...); PyAPI_FUNC(PyObject *) PyObject_CallFunctionObjArgs(PyObject *callable, ...); @@ -350,9 +339,6 @@ PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o, PyObject *method, ...); - PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs(PyObject *o, - struct _Py_Identifier *method, - ...); /* Call the method named m of object o with a variable number of @@ -414,8 +400,7 @@ #define PyObject_Length PyObject_Size #ifndef Py_LIMITED_API - PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); - PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); + PyAPI_FUNC(Py_ssize_t) _PyObject_LengthHint(PyObject *o, Py_ssize_t); #endif /* @@ -435,12 +420,12 @@ PyAPI_FUNC(int) PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v); /* - Map the object key to the value v. Raise an exception and return -1 - on failure; return 0 on success. This is the equivalent of the Python - statement o[key]=v. + Map the object, key, to the value, v. Returns + -1 on failure. This is the equivalent of the Python + statement: o[key]=v. */ - PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, const char *key); + PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key); /* Remove the mapping for object, key, from the object *o. @@ -546,12 +531,11 @@ - /* Implementation in memoryobject.c */ PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, Py_buffer *view, - Py_ssize_t len, char order); + Py_ssize_t len, char fort); PyAPI_FUNC(int) PyBuffer_FromContiguous(Py_buffer *view, void *buf, - Py_ssize_t len, char order); + Py_ssize_t len, char fort); /* Copy len bytes of data from the contiguous chunk of memory @@ -664,12 +648,6 @@ o1*o2. */ - PyAPI_FUNC(PyObject *) PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2); - - /* - This is the equivalent of the Python expression: o1 @ o2. - */ - PyAPI_FUNC(PyObject *) PyNumber_FloorDivide(PyObject *o1, PyObject *o2); /* @@ -788,7 +766,7 @@ PyAPI_FUNC(PyObject *) PyNumber_Index(PyObject *o); /* - Returns the object converted to a Python int + Returns the object converted to a Python long or int or NULL with an error raised on failure. */ @@ -797,7 +775,7 @@ /* Returns the object converted to Py_ssize_t by going through PyNumber_Index first. If an overflow error occurs while - converting the int to Py_ssize_t, then the second argument + converting the int-or-long to Py_ssize_t, then the second argument is the error-type to return. If it is NULL, then the overflow error is cleared and the value is clipped. */ @@ -844,12 +822,6 @@ o1 *= o2. */ - PyAPI_FUNC(PyObject *) PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2); - - /* - This is the equivalent of the Python expression: o1 @= o2. - */ - PyAPI_FUNC(PyObject *) PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2); @@ -993,9 +965,9 @@ PyAPI_FUNC(int) PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v); /* - Assign object v to the ith element of o. Raise an exception and return - -1 on failure; return 0 on success. This is the equivalent of the - Python statement o[i]=v. + Assign object v to the ith element of o. Returns + -1 on failure. This is the equivalent of the Python + statement: o[i]=v. */ PyAPI_FUNC(int) PySequence_DelItem(PyObject *o, Py_ssize_t i); @@ -1039,7 +1011,7 @@ PyAPI_FUNC(PyObject *) PySequence_Fast(PyObject *o, const char* m); /* - Return the sequence, o, as a list, unless it's already a + Returns the sequence, o, as a list, unless it's already a tuple or list. Use PySequence_Fast_GET_ITEM to access the members of this list, and PySequence_Fast_GET_SIZE to get its length. @@ -1174,7 +1146,7 @@ /* implemented as a macro: - int PyMapping_DelItemString(PyObject *o, const char *key); + int PyMapping_DelItemString(PyObject *o, char *key); Remove the mapping for object, key, from the object *o. Returns -1 on failure. This is equivalent to @@ -1192,7 +1164,7 @@ */ #define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K)) - PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, const char *key); + PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, char *key); /* On success, return 1 if the mapping object has the key, key, @@ -1216,28 +1188,27 @@ PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o); /* - On success, return a list, a tuple or a dictionary view in case of a dict, - of the keys in object o. On failure, return NULL. + On success, return a list or tuple of the keys in object o. + On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o); /* - On success, return a list, a tuple or a dictionary view in case of a dict, - of the values in object o. On failure, return NULL. + On success, return a list or tuple of the values in object o. + On failure, return NULL. */ PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o); /* - On success, return a list, a tuple or a dictionary view in case of a dict, - of the items in object o, where each item is a tuple containing a key-value - pair. On failure, return NULL. + On success, return a list or tuple of the items in object o, + where each item is a tuple containing a key-value pair. + On failure, return NULL. */ - PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, - const char *key); + PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, char *key); /* Return element of o corresponding to the object, key, or NULL @@ -1245,7 +1216,7 @@ o[key]. */ - PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, const char *key, + PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, char *key, PyObject *value); /* diff -r 6db40a9955dc -r 0d413f60cc23 Include/asdl.h --- a/Include/asdl.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/asdl.h Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,6 @@ typedef PyObject * string; typedef PyObject * bytes; typedef PyObject * object; -typedef PyObject * singleton; /* It would be nice if the code generated by asdl_c.py was completely independent of Python, but it is a goal the requires too much work @@ -16,28 +15,26 @@ /* XXX A sequence should be typed so that its use can be typechecked. */ typedef struct { - Py_ssize_t size; + int size; void *elements[1]; } asdl_seq; typedef struct { - Py_ssize_t size; + int size; int elements[1]; } asdl_int_seq; -asdl_seq *_Py_asdl_seq_new(Py_ssize_t size, PyArena *arena); -asdl_int_seq *_Py_asdl_int_seq_new(Py_ssize_t size, PyArena *arena); +asdl_seq *asdl_seq_new(int size, PyArena *arena); +asdl_int_seq *asdl_int_seq_new(int size, PyArena *arena); #define asdl_seq_GET(S, I) (S)->elements[(I)] #define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size) #ifdef Py_DEBUG -#define asdl_seq_SET(S, I, V) \ - do { \ - Py_ssize_t _asdl_i = (I); \ - assert((S) != NULL); \ - assert(_asdl_i < (S)->size); \ +#define asdl_seq_SET(S, I, V) { \ + int _asdl_i = (I); \ + assert((S) && _asdl_i < (S)->size); \ (S)->elements[_asdl_i] = (V); \ - } while (0) +} #else #define asdl_seq_SET(S, I, V) (S)->elements[I] = (V) #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/ast.h --- a/Include/ast.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/ast.h Mon Jan 25 17:05:13 2016 +0100 @@ -10,11 +10,6 @@ PyCompilerFlags *flags, const char *filename, /* decoded from the filesystem encoding */ PyArena *arena); -PyAPI_FUNC(mod_ty) PyAST_FromNodeObject( - const node *n, - PyCompilerFlags *flags, - PyObject *filename, - PyArena *arena); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/bytearrayobject.h --- a/Include/bytearrayobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/bytearrayobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -22,11 +22,10 @@ #ifndef Py_LIMITED_API typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ - char *ob_bytes; /* Physical backing buffer */ - char *ob_start; /* Logical start inside ob_bytes */ /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ - int ob_exports; /* How many buffer exports */ + int ob_exports; /* how many buffer exports */ + Py_ssize_t ob_alloc; /* How many bytes allocated */ + char *ob_bytes; } PyByteArrayObject; #endif @@ -50,8 +49,8 @@ #ifndef Py_LIMITED_API #define PyByteArray_AS_STRING(self) \ (assert(PyByteArray_Check(self)), \ - Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string) -#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self)) + Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_bytes : _PyByteArray_empty_string) +#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)),Py_SIZE(self)) PyAPI_DATA(char) _PyByteArray_empty_string[]; #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/bytes_methods.h --- a/Include/bytes_methods.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/bytes_methods.h Mon Jan 25 17:05:13 2016 +0100 @@ -17,12 +17,12 @@ /* These store their len sized answer in the given preallocated *result arg. */ extern void _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len); extern void _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len); -extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); -extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); -extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); +extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len); -/* The maketrans() static method. */ -extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); +/* This one gets the raw argument list. */ +extern PyObject* _Py_bytes_maketrans(PyObject *args); /* Shared __doc__ strings. */ extern const char _Py_isspace__doc__[]; diff -r 6db40a9955dc -r 0d413f60cc23 Include/bytesobject.h --- a/Include/bytesobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/bytesobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -62,14 +62,8 @@ PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( - const char *format, - Py_ssize_t format_len, - PyObject *args, - int use_bytearray); -PyAPI_FUNC(PyObject*) _PyBytes_FromHex( - PyObject *string, - int use_bytearray); +PyAPI_FUNC(PyObject *) _PyBytes_FormatLong(PyObject*, int, int, + int, char**, int*); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, @@ -94,11 +88,11 @@ 0-terminated (passing a string with embedded NULL characters will cause an exception). */ PyAPI_FUNC(int) PyBytes_AsStringAndSize( - PyObject *obj, /* string or Unicode object */ - char **s, /* pointer to buffer variable */ - Py_ssize_t *len /* pointer to length variable or NULL - (only possible for 0-terminated - strings) */ + register PyObject *obj, /* string or Unicode object */ + register char **s, /* pointer to buffer variable */ + register Py_ssize_t *len /* pointer to length variable or NULL + (only possible for 0-terminated + strings) */ ); /* Using the current locale, insert the thousands grouping @@ -130,87 +124,6 @@ #define F_ALT (1<<3) #define F_ZERO (1<<4) -#ifndef Py_LIMITED_API -/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; - - /* Number of allocated size. */ - Py_ssize_t allocated; - - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ - int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ - int overallocate; - - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; - -/* Initialize a bytes writer - - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); -#endif /* Py_LIMITED_API */ - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/ceval.h --- a/Include/ceval.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/ceval.h Mon Jan 25 17:05:13 2016 +0100 @@ -23,8 +23,6 @@ #ifndef Py_LIMITED_API PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); -PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void); #endif struct _frame; /* Avoid including frameobject.h */ @@ -48,16 +46,16 @@ In Python 3.0, this protection has two levels: * normal anti-recursion protection is triggered when the recursion level - exceeds the current recursion limit. It raises a RecursionError, and sets + exceeds the current recursion limit. It raises a RuntimeError, and sets the "overflowed" flag in the thread state structure. This flag temporarily *disables* the normal protection; this allows cleanup code to potentially outgrow the recursion limit while processing the - RecursionError. + RuntimeError. * "last chance" anti-recursion protection is triggered when the recursion level exceeds "current recursion limit + 50". By construction, this protection can only be triggered when the "overflowed" flag is set. It means the cleanup code has itself gone into an infinite loop, or the - RecursionError has been mistakingly ignored. When this protection is + RuntimeError has been mistakingly ignored. When this protection is triggered, the interpreter aborts with a Fatal Error. In addition, the "overflowed" flag is automatically reset when the @@ -79,7 +77,7 @@ do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ PyThreadState_GET()->overflowed = 0; \ } while(0) -PyAPI_FUNC(int) _Py_CheckRecursiveCall(const char *where); +PyAPI_FUNC(int) _Py_CheckRecursiveCall(char *where); PyAPI_DATA(int) _Py_CheckRecursionLimit; #ifdef USE_STACKCHECK @@ -94,16 +92,10 @@ # define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit) #endif -/* Compute the "lower-water mark" for a recursion limit. When - * Py_LeaveRecursiveCall() is called with a recursion depth below this mark, - * the overflowed flag is reset to 0. */ -#define _Py_RecursionLimitLowerWaterMark(limit) \ - (((limit) > 200) \ - ? ((limit) - 50) \ - : (3 * ((limit) >> 2))) - #define _Py_MakeEndRecCheck(x) \ - (--(x) < _Py_RecursionLimitLowerWaterMark(_Py_CheckRecursionLimit)) + (--(x) < ((_Py_CheckRecursionLimit > 100) \ + ? (_Py_CheckRecursionLimit - 50) \ + : (3 * (_Py_CheckRecursionLimit >> 2)))) #define Py_ALLOW_RECURSION \ do { unsigned char _old = PyThreadState_GET()->recursion_critical;\ @@ -206,14 +198,6 @@ PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif -/* Masks and values used by FORMAT_VALUE opcode. */ -#define FVC_MASK 0x3 -#define FVC_NONE 0x0 -#define FVC_STR 0x1 -#define FVC_REPR 0x2 -#define FVC_ASCII 0x3 -#define FVS_MASK 0x4 -#define FVS_HAVE_SPEC 0x4 #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/code.h --- a/Include/code.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/code.h Mon Jan 25 17:05:13 2016 +0100 @@ -21,12 +21,7 @@ PyObject *co_varnames; /* tuple of strings (local variable names) */ PyObject *co_freevars; /* tuple of strings (free variable names) */ PyObject *co_cellvars; /* tuple of strings (cell variable names) */ - /* The rest aren't used in either hash or comparisons, except for - co_name (used in both) and co_firstlineno (used only in - comparisons). This is done to preserve the name and line number - for tracebacks and debuggers; otherwise, constant de-duplication - would collapse identical functions/lambdas defined on different lines. - */ + /* The rest doesn't count for hash or comparisons */ unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */ PyObject *co_filename; /* unicode (where it was loaded from) */ PyObject *co_name; /* unicode (name, for reference) */ @@ -51,11 +46,6 @@ */ #define CO_NOFREE 0x0040 -/* The CO_COROUTINE flag is set for coroutine functions (defined with - ``async def`` keywords) */ -#define CO_COROUTINE 0x0080 -#define CO_ITERABLE_COROUTINE 0x0100 - /* These are no longer used. */ #if 0 #define CO_GENERATOR_ALLOWED 0x1000 @@ -67,7 +57,6 @@ #define CO_FUTURE_UNICODE_LITERALS 0x20000 #define CO_FUTURE_BARRY_AS_BDFL 0x40000 -#define CO_FUTURE_GENERATOR_STOP 0x80000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. The maximum number of @@ -108,25 +97,16 @@ int ap_upper; } PyAddrPair; -#ifndef Py_LIMITED_API /* Update *bounds to describe the first and one-past-the-last instructions in the same line as lasti. Return the number of that line. */ +#ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds); - -/* Create a comparable key used to compare constants taking in account the - * object type. It is used to make sure types are not coerced (e.g., float and - * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms - * - * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) - * depending on the type and the value. The type is the first item to not - * compare bytes and str which can raise a BytesWarning exception. */ -PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); #endif PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, - PyObject *names, PyObject *lnotab); + PyObject *names, PyObject *lineno_obj); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/codecs.h --- a/Include/codecs.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/codecs.h Mon Jan 25 17:05:13 2016 +0100 @@ -49,10 +49,6 @@ PyAPI_FUNC(PyObject *) _PyCodec_Lookup( const char *encoding ); - -PyAPI_FUNC(int) _PyCodec_Forget( - const char *encoding - ); #endif /* Codec registry encoding check API. @@ -71,7 +67,7 @@ object is passed through the encoder function found for the given encoding using the error handling method defined by errors. errors may be NULL to use the default method defined for the codec. - + Raises a LookupError in case no encoder can be found. */ @@ -87,7 +83,7 @@ object is passed through the decoder function found for the given encoding using the error handling method defined by errors. errors may be NULL to use the default method defined for the codec. - + Raises a LookupError in case no encoder can be found. */ @@ -98,54 +94,7 @@ const char *errors ); -#ifndef Py_LIMITED_API -/* Text codec specific encoding and decoding API. - - Checks the encoding against a list of codecs which do not - implement a str<->bytes encoding before attempting the - operation. - - Please note that these APIs are internal and should not - be used in Python C extensions. - - XXX (ncoghlan): should we make these, or something like them, public - in Python 3.5+? - - */ -PyAPI_FUNC(PyObject *) _PyCodec_LookupTextEncoding( - const char *encoding, - const char *alternate_command - ); - -PyAPI_FUNC(PyObject *) _PyCodec_EncodeText( - PyObject *object, - const char *encoding, - const char *errors - ); - -PyAPI_FUNC(PyObject *) _PyCodec_DecodeText( - PyObject *object, - const char *encoding, - const char *errors - ); - -/* These two aren't actually text encoding specific, but _io.TextIOWrapper - * is the only current API consumer. - */ -PyAPI_FUNC(PyObject *) _PyCodecInfo_GetIncrementalDecoder( - PyObject *codec_info, - const char *errors - ); - -PyAPI_FUNC(PyObject *) _PyCodecInfo_GetIncrementalEncoder( - PyObject *codec_info, - const char *errors - ); -#endif - - - -/* --- Codec Lookup APIs -------------------------------------------------- +/* --- Codec Lookup APIs -------------------------------------------------- All APIs return a codec object with incremented refcount and are based on _PyCodec_Lookup(). The same comments w/r to the encoding @@ -165,14 +114,14 @@ const char *encoding ); -/* Get an IncrementalEncoder object for the given encoding. */ +/* Get a IncrementalEncoder object for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_IncrementalEncoder( const char *encoding, const char *errors ); -/* Get an IncrementalDecoder object function for the given encoding. */ +/* Get a IncrementalDecoder object function for the given encoding. */ PyAPI_FUNC(PyObject *) PyCodec_IncrementalDecoder( const char *encoding, @@ -225,9 +174,6 @@ /* replace the unicode encode error with backslash escapes (\x, \u and \U) */ PyAPI_FUNC(PyObject *) PyCodec_BackslashReplaceErrors(PyObject *exc); -/* replace the unicode encode error with backslash escapes (\N, \x, \u and \U) */ -PyAPI_FUNC(PyObject *) PyCodec_NameReplaceErrors(PyObject *exc); - PyAPI_DATA(const char *) Py_hexdigits; #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/compile.h --- a/Include/compile.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/compile.h Mon Jan 25 17:05:13 2016 +0100 @@ -27,7 +27,6 @@ #define FUTURE_PRINT_FUNCTION "print_function" #define FUTURE_UNICODE_LITERALS "unicode_literals" #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" -#define FUTURE_GENERATOR_STOP "generator_stop" struct _mod; /* Declare the existence of this type */ #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) @@ -37,27 +36,11 @@ PyCompilerFlags *flags, int optimize, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject( - struct _mod *mod, - PyObject *filename, - PyCompilerFlags *flags, - int optimize, - PyArena *arena); -PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST( - struct _mod * mod, - const char *filename /* decoded from the filesystem encoding */ - ); -PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject( - struct _mod * mod, - PyObject *filename - ); +PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); /* _Py_Mangle is defined in compile.c */ PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); -#define PY_INVALID_STACK_EFFECT INT_MAX -PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/complexobject.h --- a/Include/complexobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/complexobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -14,13 +14,21 @@ /* Operations on complex numbers from complexmodule.c */ -PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); -PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); -PyAPI_FUNC(double) _Py_c_abs(Py_complex); +#define c_sum _Py_c_sum +#define c_diff _Py_c_diff +#define c_neg _Py_c_neg +#define c_prod _Py_c_prod +#define c_quot _Py_c_quot +#define c_pow _Py_c_pow +#define c_abs _Py_c_abs + +PyAPI_FUNC(Py_complex) c_sum(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) c_diff(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) c_neg(Py_complex); +PyAPI_FUNC(Py_complex) c_prod(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) c_quot(Py_complex, Py_complex); +PyAPI_FUNC(Py_complex) c_pow(Py_complex, Py_complex); +PyAPI_FUNC(double) c_abs(Py_complex); #endif /* Complex object interface */ @@ -55,12 +63,10 @@ /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ #ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyComplex_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/datetime.h --- a/Include/datetime.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/datetime.h Mon Jan 25 17:05:13 2016 +0100 @@ -42,7 +42,7 @@ typedef struct { - PyObject_HEAD /* a pure abstract base class */ + PyObject_HEAD /* a pure abstract base clase */ } PyDateTime_TZInfo; diff -r 6db40a9955dc -r 0d413f60cc23 Include/dictobject.h --- a/Include/dictobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/dictobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -13,25 +13,78 @@ tuning dictionaries, and several ideas for possible optimizations. */ +/* +There are three kinds of slots in the table: + +1. Unused. me_key == me_value == NULL + Does not hold an active (key, value) pair now and never did. Unused can + transition to Active upon key insertion. This is the only case in which + me_key is NULL, and is each slot's initial state. + +2. Active. me_key != NULL and me_key != dummy and me_value != NULL + Holds an active (key, value) pair. Active can transition to Dummy upon + key deletion. This is the only case in which me_value != NULL. + +3. Dummy. me_key == dummy and me_value == NULL + Previously held an active (key, value) pair, but that was deleted and an + active pair has not yet overwritten the slot. Dummy can transition to + Active upon key insertion. Dummy slots cannot be made Unused again + (cannot have me_key set to NULL), else the probe sequence in case of + collision would have no way to know they were once active. + +Note: .popitem() abuses the me_hash field of an Unused or Dummy slot to +hold a search finger. The me_hash field of Unused or Dummy slots has no +meaning otherwise. +*/ + +/* PyDict_MINSIZE is the minimum size of a dictionary. This many slots are + * allocated directly in the dict object (in the ma_smalltable member). + * It must be a power of 2, and at least 4. 8 allows dicts with no more + * than 5 active entries to live in ma_smalltable (and so avoid an + * additional malloc); instrumentation suggested this suffices for the + * majority of dicts (consisting mostly of usually-small instance dicts and + * usually-small dicts created to pass keyword arguments). + */ #ifndef Py_LIMITED_API - -typedef struct _dictkeysobject PyDictKeysObject; - -/* The ma_values pointer is NULL for a combined table - * or points to an array of PyObject* for a split table - */ -typedef struct { - PyObject_HEAD - Py_ssize_t ma_used; - PyDictKeysObject *ma_keys; - PyObject **ma_values; -} PyDictObject; +#define PyDict_MINSIZE 8 typedef struct { + /* Cached hash code of me_key. */ + Py_hash_t me_hash; + PyObject *me_key; + PyObject *me_value; +} PyDictEntry; + +/* +To ensure the lookup algorithm terminates, there must be at least one Unused +slot (NULL key) in the table. +The value ma_fill is the number of non-NULL keys (sum of Active and Dummy); +ma_used is the number of non-NULL, non-dummy keys (== the number of non-NULL +values == the number of Active items). +To avoid slowing down lookups on a near-full table, we resize the table when +it's two-thirds full. +*/ +typedef struct _dictobject PyDictObject; +struct _dictobject { PyObject_HEAD - PyDictObject *dv_dict; -} _PyDictViewObject; + Py_ssize_t ma_fill; /* # Active + # Dummy */ + Py_ssize_t ma_used; /* # Active */ + /* The table contains ma_mask + 1 slots, and that's a power of 2. + * We store the mask instead of the size because the mask is more + * frequently needed. + */ + Py_ssize_t ma_mask; + + /* ma_table points to ma_smalltable for small tables, else to + * additional malloc'ed memory. ma_table is never NULL! This rule + * saves repeated runtime null-tests in the workhorse getitem and + * setitem calls. + */ + PyDictEntry *ma_table; + PyDictEntry *(*ma_lookup)(PyDictObject *mp, PyObject *key, Py_hash_t hash); + PyDictEntry ma_smalltable[PyDict_MINSIZE]; +}; #endif /* Py_LIMITED_API */ PyAPI_DATA(PyTypeObject) PyDict_Type; @@ -45,9 +98,9 @@ #define PyDict_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS) #define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) -#define PyDictKeys_Check(op) PyObject_TypeCheck(op, &PyDictKeys_Type) -#define PyDictItems_Check(op) PyObject_TypeCheck(op, &PyDictItems_Type) -#define PyDictValues_Check(op) PyObject_TypeCheck(op, &PyDictValues_Type) +#define PyDictKeys_Check(op) (Py_TYPE(op) == &PyDictKeys_Type) +#define PyDictItems_Check(op) (Py_TYPE(op) == &PyDictItems_Type) +#define PyDictValues_Check(op) (Py_TYPE(op) == &PyDictValues_Type) /* This excludes Values, since they are not sets. */ # define PyDictViewSet_Check(op) \ (PyDictKeys_Check(op) || PyDictItems_Check(op)) @@ -55,36 +108,15 @@ PyAPI_FUNC(PyObject *) PyDict_New(void); PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -#endif PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, - struct _Py_Identifier *key); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) PyDict_SetDefault( - PyObject *mp, PyObject *key, PyObject *defaultobj); -#endif PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, - PyObject *item, Py_hash_t hash); -#endif PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -#endif PyAPI_FUNC(void) PyDict_Clear(PyObject *mp); PyAPI_FUNC(int) PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value); #ifndef Py_LIMITED_API -PyDictKeysObject *_PyDict_NewKeysForClass(void); -PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); PyAPI_FUNC(int) _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); -PyObject *_PyDictView_New(PyObject *, PyTypeObject *); #endif PyAPI_FUNC(PyObject *) PyDict_Keys(PyObject *mp); PyAPI_FUNC(PyObject *) PyDict_Values(PyObject *mp); @@ -97,11 +129,6 @@ PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); -Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); -Py_ssize_t _PyDict_SizeOf(PyDictObject *); -PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *); -PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); -#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) PyAPI_FUNC(int) PyDict_ClearFreeList(void); #endif @@ -118,10 +145,6 @@ PyObject *other, int override); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); -#endif - /* PyDict_MergeFromSeq2 updates/merges from an iterable object producing iterable objects of length 2. If override is true, the last occurrence of a key wins, else the first. The Python dict constructor dict(seq2) @@ -132,19 +155,9 @@ int override); PyAPI_FUNC(PyObject *) PyDict_GetItemString(PyObject *dp, const char *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemId(PyObject *dp, struct _Py_Identifier *key); PyAPI_FUNC(int) PyDict_SetItemString(PyObject *dp, const char *key, PyObject *item); -PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, struct _Py_Identifier *key, PyObject *item); PyAPI_FUNC(int) PyDict_DelItemString(PyObject *dp, const char *key); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, struct _Py_Identifier *key); -PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); - -int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); -PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); -#endif - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/dtoa.h --- a/Include/dtoa.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/dtoa.h Mon Jan 25 17:05:13 2016 +0100 @@ -8,8 +8,6 @@ PyAPI_FUNC(char *) _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); PyAPI_FUNC(void) _Py_dg_freedtoa(char *s); -PyAPI_FUNC(double) _Py_dg_stdnan(int sign); -PyAPI_FUNC(double) _Py_dg_infinity(int sign); #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/dynamic_annotations.h --- a/Include/dynamic_annotations.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/dynamic_annotations.h Mon Jan 25 17:05:13 2016 +0100 @@ -150,7 +150,7 @@ /* Report that a new memory at "address" of size "size" has been allocated. This might be used when the memory has been retrieved from a free list and - is about to be reused, or when the locking discipline for a variable + is about to be reused, or when a the locking discipline for a variable changes. */ #define _Py_ANNOTATE_NEW_MEMORY(address, size) \ AnnotateNewMemory(__FILE__, __LINE__, address, size) diff -r 6db40a9955dc -r 0d413f60cc23 Include/fileobject.h --- a/Include/fileobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/fileobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -8,9 +8,8 @@ #define PY_STDIOTEXTMODE "b" -PyAPI_FUNC(PyObject *) PyFile_FromFd(int, const char *, const char *, int, - const char *, const char *, - const char *, int); +PyAPI_FUNC(PyObject *) PyFile_FromFd(int, char *, char *, int, char *, char *, + char *, int); PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int); PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int); PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *); @@ -32,6 +31,17 @@ #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int); PyAPI_DATA(PyTypeObject) PyStdPrinter_Type; + +#if defined _MSC_VER && _MSC_VER >= 1400 +/* A routine to check if a file descriptor is valid on Windows. Returns 0 + * and sets errno to EBADF if it isn't. This is to avoid Assertions + * from various functions in the Windows CRT beginning with + * Visual Studio 2005 + */ +int _PyVerify_fd(int fd); +#else +#define _PyVerify_fd(A) (1) /* dummy */ +#endif #endif /* Py_LIMITED_API */ /* A routine to check if a file descriptor can be select()-ed. */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/fileutils.h --- a/Include/fileutils.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/fileutils.h Mon Jan 25 17:05:13 2016 +0100 @@ -7,59 +7,24 @@ PyAPI_FUNC(PyObject *) _Py_device_encoding(int); -PyAPI_FUNC(wchar_t *) Py_DecodeLocale( +PyAPI_FUNC(wchar_t *) _Py_char2wchar( const char *arg, size_t *size); -PyAPI_FUNC(char*) Py_EncodeLocale( +PyAPI_FUNC(char*) _Py_wchar2char( const wchar_t *text, size_t *error_pos); -#ifndef Py_LIMITED_API - -#ifdef MS_WINDOWS -struct _Py_stat_struct { - unsigned long st_dev; - __int64 st_ino; - unsigned short st_mode; - int st_nlink; - int st_uid; - int st_gid; - unsigned long st_rdev; - __int64 st_size; - time_t st_atime; - int st_atime_nsec; - time_t st_mtime; - int st_mtime_nsec; - time_t st_ctime; - int st_ctime_nsec; - unsigned long st_file_attributes; -}; -#else -# define _Py_stat_struct stat +#if defined(HAVE_STAT) && !defined(MS_WINDOWS) +PyAPI_FUNC(int) _Py_wstat( + const wchar_t* path, + struct stat *buf); #endif -PyAPI_FUNC(int) _Py_fstat( - int fd, - struct _Py_stat_struct *status); - -PyAPI_FUNC(int) _Py_fstat_noraise( - int fd, - struct _Py_stat_struct *status); -#endif /* Py_LIMITED_API */ - +#ifdef HAVE_STAT PyAPI_FUNC(int) _Py_stat( PyObject *path, - struct stat *status); - -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _Py_open( - const char *pathname, - int flags); - -PyAPI_FUNC(int) _Py_open_noraise( - const char *pathname, - int flags); + struct stat *statbuf); #endif PyAPI_FUNC(FILE *) _Py_wfopen( @@ -67,28 +32,9 @@ const wchar_t *mode); PyAPI_FUNC(FILE*) _Py_fopen( - const char *pathname, - const char *mode); - -PyAPI_FUNC(FILE*) _Py_fopen_obj( PyObject *path, const char *mode); -PyAPI_FUNC(Py_ssize_t) _Py_read( - int fd, - void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write( - int fd, - const void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( - int fd, - const void *buf, - size_t count); - #ifdef HAVE_READLINK PyAPI_FUNC(int) _Py_wreadlink( const wchar_t *path, @@ -107,34 +53,6 @@ wchar_t *buf, size_t size); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _Py_get_inheritable(int fd); - -PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, - int *atomic_flag_works); - -PyAPI_FUNC(int) _Py_dup(int fd); - -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_get_blocking(int fd); - -PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#endif /* !MS_WINDOWS */ - -#if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 -/* A routine to check if a file descriptor is valid on Windows. Returns 0 - * and sets errno to EBADF if it isn't. This is to avoid Assertions - * from various functions in the Windows CRT beginning with - * Visual Studio 2005 - */ -int _PyVerify_fd(int fd); - -#else -#define _PyVerify_fd(A) (1) /* dummy */ -#endif - -#endif /* Py_LIMITED_API */ - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/floatobject.h --- a/Include/floatobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/floatobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -110,16 +110,12 @@ /* free list api */ PyAPI_FUNC(int) PyFloat_ClearFreeList(void); -PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); - /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/frameobject.h --- a/Include/frameobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/frameobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -33,12 +33,11 @@ frame (which shouldn't be impacted when the generator "yields" from an except handler). These three fields exist exactly for that, and are unused for - non-generator frames. See the save_exc_state and swap_exc_state - functions in ceval.c for details of their use. */ + non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE + macros in ceval.c for details of their use. */ PyObject *f_exc_type, *f_exc_value, *f_exc_traceback; - /* Borrowed reference to a generator, or NULL */ - PyObject *f_gen; + PyThreadState *f_tstate; int f_lasti; /* Last instruction if called */ /* Call PyFrame_GetLineNumber() instead of reading this field directly. As of 2.3 f_lineno is only valid when tracing is @@ -47,7 +46,6 @@ bytecode index. */ int f_lineno; /* Current line number */ int f_iblock; /* index in f_blockstack */ - char f_executing; /* whether the frame is still executing */ PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ } PyFrameObject; @@ -77,14 +75,10 @@ /* Conversions between "fast locals" and locals in dictionary */ PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); - -PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); PyAPI_FUNC(int) PyFrame_ClearFreeList(void); -PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); - /* Return the line of code the frame is currently executing. */ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); diff -r 6db40a9955dc -r 0d413f60cc23 Include/genobject.h --- a/Include/genobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/genobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -10,26 +10,21 @@ struct _frame; /* Avoid including frameobject.h */ -/* _PyGenObject_HEAD defines the initial segment of generator - and coroutine objects. */ -#define _PyGenObject_HEAD(prefix) \ - PyObject_HEAD \ - /* Note: gi_frame can be NULL if the generator is "finished" */ \ - struct _frame *prefix##_frame; \ - /* True if generator is being executed. */ \ - char prefix##_running; \ - /* The code object backing the generator */ \ - PyObject *prefix##_code; \ - /* List of weak reference. */ \ - PyObject *prefix##_weakreflist; \ - /* Name of the generator. */ \ - PyObject *prefix##_name; \ - /* Qualified name of the generator. */ \ - PyObject *prefix##_qualname; +typedef struct { + PyObject_HEAD + /* The gi_ prefix is intended to remind of generator-iterator. */ -typedef struct { - /* The gi_ prefix is intended to remind of generator-iterator. */ - _PyGenObject_HEAD(gi) + /* Note: gi_frame can be NULL if the generator is "finished" */ + struct _frame *gi_frame; + + /* True if generator is being executed. */ + char gi_running; + + /* The code object backing the generator */ + PyObject *gi_code; + + /* List of weak reference. */ + PyObject *gi_weakreflist; } PyGenObject; PyAPI_DATA(PyTypeObject) PyGen_Type; @@ -38,28 +33,9 @@ #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type) PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); -PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *, - PyObject *name, PyObject *qualname); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); -PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); +PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); PyObject *_PyGen_Send(PyGenObject *, PyObject *); -PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); - -#ifndef Py_LIMITED_API -typedef struct { - _PyGenObject_HEAD(cr) -} PyCoroObject; - -PyAPI_DATA(PyTypeObject) PyCoro_Type; -PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; - -#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type) -PyObject *_PyCoro_GetAwaitableIter(PyObject *o); -PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *, - PyObject *name, PyObject *qualname); -#endif - -#undef _PyGenObject_HEAD #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/graminit.h --- a/Include/graminit.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/graminit.h Mon Jan 25 17:05:13 2016 +0100 @@ -6,82 +6,79 @@ #define decorator 259 #define decorators 260 #define decorated 261 -#define async_funcdef 262 -#define funcdef 263 -#define parameters 264 -#define typedargslist 265 -#define tfpdef 266 -#define varargslist 267 -#define vfpdef 268 -#define stmt 269 -#define simple_stmt 270 -#define small_stmt 271 -#define expr_stmt 272 -#define testlist_star_expr 273 -#define augassign 274 -#define del_stmt 275 -#define pass_stmt 276 -#define flow_stmt 277 -#define break_stmt 278 -#define continue_stmt 279 -#define return_stmt 280 -#define yield_stmt 281 -#define raise_stmt 282 -#define import_stmt 283 -#define import_name 284 -#define import_from 285 -#define import_as_name 286 -#define dotted_as_name 287 -#define import_as_names 288 -#define dotted_as_names 289 -#define dotted_name 290 -#define global_stmt 291 -#define nonlocal_stmt 292 -#define assert_stmt 293 -#define compound_stmt 294 -#define async_stmt 295 -#define if_stmt 296 -#define while_stmt 297 -#define for_stmt 298 -#define try_stmt 299 -#define with_stmt 300 -#define with_item 301 -#define except_clause 302 -#define suite 303 -#define test 304 -#define test_nocond 305 -#define lambdef 306 -#define lambdef_nocond 307 -#define or_test 308 -#define and_test 309 -#define not_test 310 -#define comparison 311 -#define comp_op 312 -#define star_expr 313 -#define expr 314 -#define xor_expr 315 -#define and_expr 316 -#define shift_expr 317 -#define arith_expr 318 -#define term 319 -#define factor 320 -#define power 321 -#define atom_expr 322 -#define atom 323 -#define testlist_comp 324 -#define trailer 325 -#define subscriptlist 326 -#define subscript 327 -#define sliceop 328 -#define exprlist 329 -#define testlist 330 -#define dictorsetmaker 331 -#define classdef 332 -#define arglist 333 -#define argument 334 -#define comp_iter 335 -#define comp_for 336 -#define comp_if 337 -#define encoding_decl 338 -#define yield_expr 339 -#define yield_arg 340 +#define funcdef 262 +#define parameters 263 +#define typedargslist 264 +#define tfpdef 265 +#define varargslist 266 +#define vfpdef 267 +#define stmt 268 +#define simple_stmt 269 +#define small_stmt 270 +#define expr_stmt 271 +#define testlist_star_expr 272 +#define augassign 273 +#define del_stmt 274 +#define pass_stmt 275 +#define flow_stmt 276 +#define break_stmt 277 +#define continue_stmt 278 +#define return_stmt 279 +#define yield_stmt 280 +#define raise_stmt 281 +#define import_stmt 282 +#define import_name 283 +#define import_from 284 +#define import_as_name 285 +#define dotted_as_name 286 +#define import_as_names 287 +#define dotted_as_names 288 +#define dotted_name 289 +#define global_stmt 290 +#define nonlocal_stmt 291 +#define assert_stmt 292 +#define compound_stmt 293 +#define if_stmt 294 +#define while_stmt 295 +#define for_stmt 296 +#define try_stmt 297 +#define with_stmt 298 +#define with_item 299 +#define except_clause 300 +#define suite 301 +#define test 302 +#define test_nocond 303 +#define lambdef 304 +#define lambdef_nocond 305 +#define or_test 306 +#define and_test 307 +#define not_test 308 +#define comparison 309 +#define comp_op 310 +#define star_expr 311 +#define expr 312 +#define xor_expr 313 +#define and_expr 314 +#define shift_expr 315 +#define arith_expr 316 +#define term 317 +#define factor 318 +#define power 319 +#define atom 320 +#define testlist_comp 321 +#define trailer 322 +#define subscriptlist 323 +#define subscript 324 +#define sliceop 325 +#define exprlist 326 +#define testlist 327 +#define dictorsetmaker 328 +#define classdef 329 +#define arglist 330 +#define argument 331 +#define comp_iter 332 +#define comp_for 333 +#define comp_if 334 +#define encoding_decl 335 +#define yield_expr 336 +#define yield_arg 337 diff -r 6db40a9955dc -r 0d413f60cc23 Include/grammar.h --- a/Include/grammar.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/grammar.h Mon Jan 25 17:05:13 2016 +0100 @@ -37,7 +37,7 @@ typedef struct { int s_narcs; arc *s_arc; /* Array of arcs */ - + /* Optional accelerators */ int s_lower; /* Lowest label index */ int s_upper; /* Highest label index */ @@ -69,14 +69,14 @@ /* FUNCTIONS */ grammar *newgrammar(int start); -dfa *adddfa(grammar *g, int type, const char *name); +dfa *adddfa(grammar *g, int type, char *name); int addstate(dfa *d); void addarc(dfa *d, int from, int to, int lbl); dfa *PyGrammar_FindDFA(grammar *g, int type); -int addlabel(labellist *ll, int type, const char *str); -int findlabel(labellist *ll, int type, const char *str); -const char *PyGrammar_LabelRepr(label *lb); +int addlabel(labellist *ll, int type, char *str); +int findlabel(labellist *ll, int type, char *str); +char *PyGrammar_LabelRepr(label *lb); void translatelabels(grammar *g); void addfirstsets(grammar *g); diff -r 6db40a9955dc -r 0d413f60cc23 Include/import.h --- a/Include/import.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/import.h Mon Jan 25 17:05:13 2016 +0100 @@ -7,25 +7,22 @@ extern "C" { #endif -PyAPI_FUNC(void) _PyImportZip_Init(void); - -PyMODINIT_FUNC PyInit_imp(void); PyAPI_FUNC(long) PyImport_GetMagicNumber(void); PyAPI_FUNC(const char *) PyImport_GetMagicTag(void); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule( - const char *name, /* UTF-8 encoded string */ + char *name, /* UTF-8 encoded string */ PyObject *co ); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx( - const char *name, /* UTF-8 encoded string */ + char *name, /* UTF-8 encoded string */ PyObject *co, - const char *pathname /* decoded from the filesystem encoding */ + char *pathname /* decoded from the filesystem encoding */ ); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames( - const char *name, /* UTF-8 encoded string */ + char *name, /* UTF-8 encoded string */ PyObject *co, - const char *pathname, /* decoded from the filesystem encoding */ - const char *cpathname /* decoded from the filesystem encoding */ + char *pathname, /* decoded from the filesystem encoding */ + char *cpathname /* decoded from the filesystem encoding */ ); PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( PyObject *name, @@ -62,7 +59,7 @@ ); #define PyImport_ImportModuleEx(n, g, l, f) \ - PyImport_ImportModuleLevel(n, g, l, f, 0) + PyImport_ImportModuleLevel(n, g, l, f, -1) PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path); PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name); @@ -72,7 +69,7 @@ PyObject *name ); PyAPI_FUNC(int) PyImport_ImportFrozenModule( - const char *name /* UTF-8 encoded string */ + char *name /* UTF-8 encoded string */ ); #ifndef Py_LIMITED_API @@ -86,18 +83,18 @@ PyAPI_FUNC(void) _PyImport_ReInitLock(void); -PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin( +PyAPI_FUNC(PyObject *)_PyImport_FindBuiltin( const char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *); -PyAPI_FUNC(int) _PyImport_FixupBuiltin( +PyAPI_FUNC(PyObject *)_PyImport_FindExtensionObject(PyObject *, PyObject *); +PyAPI_FUNC(int)_PyImport_FixupBuiltin( PyObject *mod, - const char *name /* UTF-8 encoded string */ + char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); +PyAPI_FUNC(int)_PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); struct _inittab { - const char *name; /* ASCII encoded string */ + char *name; /* ASCII encoded string */ PyObject* (*initfunc)(void); }; PyAPI_DATA(struct _inittab *) PyImport_Inittab; @@ -113,15 +110,15 @@ #ifndef Py_LIMITED_API struct _frozen { - const char *name; /* ASCII encoded string */ - const unsigned char *code; + char *name; /* ASCII encoded string */ + unsigned char *code; int size; }; /* Embedding apps may change this pointer to point to their favorite collection of frozen modules: */ -PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules; +PyAPI_DATA(struct _frozen *) PyImport_FrozenModules; #endif #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/listobject.h --- a/Include/listobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/listobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -46,7 +46,7 @@ PyAPI_DATA(PyTypeObject) PySortWrapper_Type; #define PyList_Check(op) \ - PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LIST_SUBCLASS) + PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LIST_SUBCLASS) #define PyList_CheckExact(op) (Py_TYPE(op) == &PyList_Type) PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); @@ -64,7 +64,6 @@ PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(int) PyList_ClearFreeList(void); -PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #endif /* Macro, trading safety for speed */ @@ -72,7 +71,6 @@ #define PyList_GET_ITEM(op, i) (((PyListObject *)(op))->ob_item[i]) #define PyList_SET_ITEM(op, i, v) (((PyListObject *)(op))->ob_item[i] = (v)) #define PyList_GET_SIZE(op) Py_SIZE(op) -#define _PyList_ITEMS(op) (((PyListObject *)(op))->ob_item) #endif #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/longintrepr.h --- a/Include/longintrepr.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/longintrepr.h Mon Jan 25 17:05:13 2016 +0100 @@ -8,7 +8,7 @@ /* This is published for the benefit of "friends" marshal.c and _decimal.c. */ -/* Parameters of the integer representation. There are two different +/* Parameters of the long integer representation. There are two different sets of parameters: one set for 30-bit digits, stored in an unsigned 32-bit integer type, and one set for 15-bit digits with each digit stored in an unsigned short. The value of PYLONG_BITS_IN_DIGIT, defined either at @@ -29,7 +29,7 @@ of bits in an unsigned long, as do the PyLong <-> long (or unsigned long) conversion functions - - the Python int <-> size_t/Py_ssize_t conversion functions expect that + - the long <-> size_t/Py_ssize_t conversion functions expect that PyLong_SHIFT is strictly less than the number of bits in a size_t - the marshal code currently expects that PyLong_SHIFT is a multiple of 15 @@ -83,7 +83,7 @@ so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. + aware that longs abuse ob_size's sign bit. */ struct _longobject { diff -r 6db40a9955dc -r 0d413f60cc23 Include/longobject.h --- a/Include/longobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/longobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -26,9 +26,6 @@ PyAPI_FUNC(size_t) PyLong_AsSize_t(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLong(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); -#endif PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong @@ -52,21 +49,7 @@ #error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)" #endif /* SIZEOF_PID_T */ -#if SIZEOF_VOID_P == SIZEOF_INT -# define _Py_PARSE_INTPTR "i" -# define _Py_PARSE_UINTPTR "I" -#elif SIZEOF_VOID_P == SIZEOF_LONG -# define _Py_PARSE_INTPTR "l" -# define _Py_PARSE_UINTPTR "k" -#elif defined(SIZEOF_LONG_LONG) && SIZEOF_VOID_P == SIZEOF_LONG_LONG -# define _Py_PARSE_INTPTR "L" -# define _Py_PARSE_UINTPTR "K" -#else -# error "void* different in size from int, long and long long" -#endif /* SIZEOF_VOID_P */ - -/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), - _PyBytes_DecodeEscapeRecode(), etc. */ +/* Used by Python/mystrtoul.c. */ #ifndef Py_LIMITED_API PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; #endif @@ -94,11 +77,10 @@ PyAPI_FUNC(PY_LONG_LONG) PyLong_AsLongLongAndOverflow(PyObject *, int *); #endif /* HAVE_LONG_LONG */ -PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); +PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int); PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); #endif #ifndef Py_LIMITED_API @@ -127,7 +109,7 @@ PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in - base 256, and return a Python int with the same numeric value. + base 256, and return a Python long with the same numeric value. If n is 0, the integer is 0. Else: If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the @@ -137,7 +119,7 @@ non-negative if bit 0x80 of the MSB is clear, negative if set. Error returns: + Return NULL with the appropriate exception set if there's not - enough memory to create the Python int. + enough memory to create the Python long. */ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( const unsigned char* bytes, size_t n, @@ -166,48 +148,24 @@ unsigned char* bytes, size_t n, int little_endian, int is_signed); -/* _PyLong_FromNbInt: Convert the given object to a PyLongObject - using the nb_int slot, if available. Raise TypeError if either the - nb_int slot is not available or the result of the call to nb_int - returns something not of type int. -*/ -PyAPI_FUNC(PyLongObject *)_PyLong_FromNbInt(PyObject *); /* _PyLong_Format: Convert the long to a string object with given base, appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); - -PyAPI_FUNC(int) _PyLong_FormatWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - int base, - int alternate); - -PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, - char *str, - PyObject *obj, - int base, - int alternate); +PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *aa, int base); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ -/* These aren't really part of the int object, but they're handy. The +/* These aren't really part of the long object, but they're handy. The functions are in Python/mystrtoul.c. */ -PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); -PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); - -/* For use by the gcd function in mathmodule.c */ -PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); +PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int); +PyAPI_FUNC(long) PyOS_strtol(char *, char **, int); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/marshal.h --- a/Include/marshal.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/marshal.h Mon Jan 25 17:05:13 2016 +0100 @@ -7,7 +7,7 @@ extern "C" { #endif -#define Py_MARSHAL_VERSION 4 +#define Py_MARSHAL_VERSION 2 PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); @@ -19,8 +19,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); #endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(char *, Py_ssize_t); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/memoryobject.h --- a/Include/memoryobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/memoryobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -45,6 +45,9 @@ } _PyManagedBufferObject; +/* static storage used for casting between formats */ +#define _Py_MEMORYVIEW_MAX_FORMAT 3 /* must be >= 3 */ + /* memoryview state flags */ #define _Py_MEMORYVIEW_RELEASED 0x001 /* access to master buffer blocked */ #define _Py_MEMORYVIEW_C 0x002 /* C-contiguous layout */ @@ -59,7 +62,7 @@ int flags; /* state flags */ Py_ssize_t exports; /* number of buffer re-exports */ Py_buffer view; /* private copy of the exporter's view */ - PyObject *weakreflist; + char format[_Py_MEMORYVIEW_MAX_FORMAT]; /* used for casting */ Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */ } PyMemoryViewObject; #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/methodobject.h --- a/Include/methodobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/methodobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, - PyObject *); + PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); PyAPI_FUNC(PyCFunction) PyCFunction_GetFunction(PyObject *); @@ -33,22 +33,22 @@ (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \ NULL : ((PyCFunctionObject *)func) -> m_self) #define PyCFunction_GET_FLAGS(func) \ - (((PyCFunctionObject *)func) -> m_ml -> ml_flags) + (((PyCFunctionObject *)func) -> m_ml -> ml_flags) #endif PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); struct PyMethodDef { - const char *ml_name; /* The name of the built-in function/method */ - PyCFunction ml_meth; /* The C function that implements it */ - int ml_flags; /* Combination of METH_xxx flags, which mostly - describe the args expected by the C func */ - const char *ml_doc; /* The __doc__ attribute, or NULL */ + const char *ml_name; /* The name of the built-in function/method */ + PyCFunction ml_meth; /* The C function that implements it */ + int ml_flags; /* Combination of METH_xxx flags, which mostly + describe the args expected by the C func */ + const char *ml_doc; /* The __doc__ attribute, or NULL */ }; typedef struct PyMethodDef PyMethodDef; #define PyCFunction_New(ML, SELF) PyCFunction_NewEx((ML), (SELF), NULL) -PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *, - PyObject *); +PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *, + PyObject *); /* Flag passed to newmethodobject */ /* #define METH_OLDARGS 0x0000 -- unsupported now */ @@ -66,7 +66,7 @@ /* METH_COEXIST allows a method to be entered even though a slot has already filled the entry. When defined, the flag allows a separate - method, "__contains__" for example, to coexist with a defined + method, "__contains__" for example, to coexist with a defined slot like sq_contains. */ #define METH_COEXIST 0x0040 @@ -77,17 +77,11 @@ PyMethodDef *m_ml; /* Description of the C function to call */ PyObject *m_self; /* Passed as 'self' arg to the C func, can be NULL */ PyObject *m_module; /* The __module__ attribute, can be anything */ - PyObject *m_weakreflist; /* List of weak references */ } PyCFunctionObject; #endif PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyCFunction_DebugMallocStats(FILE *out); -PyAPI_FUNC(void) _PyMethod_DebugMallocStats(FILE *out); -#endif - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/modsupport.h --- a/Include/modsupport.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/modsupport.h Mon Jan 25 17:05:13 2016 +0100 @@ -12,36 +12,32 @@ /* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier to mean Py_ssize_t */ #ifdef PY_SSIZE_T_CLEAN -#define PyArg_Parse _PyArg_Parse_SizeT -#define PyArg_ParseTuple _PyArg_ParseTuple_SizeT -#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT -#define PyArg_VaParse _PyArg_VaParse_SizeT -#define PyArg_VaParseTupleAndKeywords _PyArg_VaParseTupleAndKeywords_SizeT -#define Py_BuildValue _Py_BuildValue_SizeT -#define Py_VaBuildValue _Py_VaBuildValue_SizeT +#define PyArg_Parse _PyArg_Parse_SizeT +#define PyArg_ParseTuple _PyArg_ParseTuple_SizeT +#define PyArg_ParseTupleAndKeywords _PyArg_ParseTupleAndKeywords_SizeT +#define PyArg_VaParse _PyArg_VaParse_SizeT +#define PyArg_VaParseTupleAndKeywords _PyArg_VaParseTupleAndKeywords_SizeT +#define Py_BuildValue _Py_BuildValue_SizeT +#define Py_VaBuildValue _Py_VaBuildValue_SizeT #else PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list); #endif -/* Due to a glitch in 3.2, the _SizeT versions weren't exported from the DLL. */ -#if !defined(PY_SSIZE_T_CLEAN) || !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...); -PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...); +PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3); PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, ...); PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *); PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...); PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...); PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); -#endif #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kw); -PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); +#endif PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list); PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, const char *, char **, va_list); -#endif PyAPI_FUNC(PyObject *) Py_VaBuildValue(const char *, va_list); PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *); @@ -50,13 +46,6 @@ #define PyModule_AddIntMacro(m, c) PyModule_AddIntConstant(m, #c, c) #define PyModule_AddStringMacro(m, c) PyModule_AddStringConstant(m, #c, c) -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -PyAPI_FUNC(int) PyModule_SetDocString(PyObject *, const char *); -PyAPI_FUNC(int) PyModule_AddFunctions(PyObject *, PyMethodDef *); -PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def); -#endif - #define Py_CLEANUP_SUPPORTED 0x20000 #define PYTHON_API_VERSION 1013 @@ -74,35 +63,35 @@ Please add a line or two to the top of this log for each API version change: - 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths + 22-Feb-2006 MvL 1013 PEP 353 - long indices for sequence lengths - 19-Aug-2002 GvR 1012 Changes to string object struct for - interning changes, saving 3 bytes. + 19-Aug-2002 GvR 1012 Changes to string object struct for + interning changes, saving 3 bytes. - 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side + 17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side 25-Jan-2001 FLD 1010 Parameters added to PyCode_New() and PyFrame_New(); Python 2.1a2 14-Mar-2000 GvR 1009 Unicode API added - 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!) + 3-Jan-1999 GvR 1007 Decided to change back! (Don't reuse 1008!) - 3-Dec-1998 GvR 1008 Python 1.5.2b1 + 3-Dec-1998 GvR 1008 Python 1.5.2b1 - 18-Jan-1997 GvR 1007 string interning and other speedups + 18-Jan-1997 GvR 1007 string interning and other speedups - 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-( + 11-Oct-1996 GvR renamed Py_Ellipses to Py_Ellipsis :-( - 30-Jul-1996 GvR Slice and ellipses syntax added + 30-Jul-1996 GvR Slice and ellipses syntax added - 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-) + 23-Jul-1996 GvR For 1.4 -- better safe than sorry this time :-) - 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( ) + 7-Nov-1995 GvR Keyword arguments (should've been done at 1.3 :-( ) - 10-Jan-1995 GvR Renamed globals to new naming scheme + 10-Jan-1995 GvR Renamed globals to new naming scheme - 9-Jan-1995 GvR Initial version (incompatible with older API) + 9-Jan-1995 GvR Initial version (incompatible with older API) */ /* The PYTHON_ABI_VERSION is introduced in PEP 384. For the lifetime of @@ -112,11 +101,10 @@ #define PYTHON_ABI_STRING "3" #ifdef Py_TRACE_REFS - /* When we are tracing reference counts, rename module creation functions so + /* When we are tracing reference counts, rename PyModule_Create2 so modules compiled with incompatible settings will generate a link-time error. */ #define PyModule_Create2 PyModule_Create2TraceRefs - #define PyModule_FromDefAndSpec2 PyModule_FromDefAndSpec2TraceRefs #endif PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*, @@ -124,27 +112,12 @@ #ifdef Py_LIMITED_API #define PyModule_Create(module) \ - PyModule_Create2(module, PYTHON_ABI_VERSION) + PyModule_Create2(module, PYTHON_ABI_VERSION) #else #define PyModule_Create(module) \ - PyModule_Create2(module, PYTHON_API_VERSION) + PyModule_Create2(module, PYTHON_API_VERSION) #endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -PyAPI_FUNC(PyObject *) PyModule_FromDefAndSpec2(PyModuleDef *def, - PyObject *spec, - int module_api_version); - -#ifdef Py_LIMITED_API -#define PyModule_FromDefAndSpec(module, spec) \ - PyModule_FromDefAndSpec2(module, spec, PYTHON_ABI_VERSION) -#else -#define PyModule_FromDefAndSpec(module, spec) \ - PyModule_FromDefAndSpec2(module, spec, PYTHON_API_VERSION) -#endif /* Py_LIMITED_API */ -#endif /* New in 3.5 */ - #ifndef Py_LIMITED_API PyAPI_DATA(char *) _Py_PackageContext; #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/moduleobject.h --- a/Include/moduleobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/moduleobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -25,17 +25,10 @@ PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(void) _PyModule_Clear(PyObject *); -PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *); #endif PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*); PyAPI_FUNC(void*) PyModule_GetState(PyObject*); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -PyAPI_FUNC(PyObject *) PyModuleDef_Init(struct PyModuleDef*); -PyAPI_DATA(PyTypeObject) PyModuleDef_Type; -#endif - typedef struct PyModuleDef_Base { PyObject_HEAD PyObject* (*m_init)(void); @@ -50,35 +43,19 @@ NULL, /* m_copy */ \ } -struct PyModuleDef_Slot; -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -typedef struct PyModuleDef_Slot{ - int slot; - void *value; -} PyModuleDef_Slot; - -#define Py_mod_create 1 -#define Py_mod_exec 2 - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 2 -#endif - -#endif /* New in 3.5 */ - typedef struct PyModuleDef{ PyModuleDef_Base m_base; const char* m_name; const char* m_doc; Py_ssize_t m_size; PyMethodDef *m_methods; - struct PyModuleDef_Slot* m_slots; + inquiry m_reload; traverseproc m_traverse; inquiry m_clear; freefunc m_free; }PyModuleDef; + #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/namespaceobject.h --- a/Include/namespaceobject.h Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ - -/* simple namespace object interface */ - -#ifndef NAMESPACEOBJECT_H -#define NAMESPACEOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_DATA(PyTypeObject) _PyNamespace_Type; - -PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); - -#ifdef __cplusplus -} -#endif -#endif /* !NAMESPACEOBJECT_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/node.h --- a/Include/node.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/node.h Mon Jan 25 17:05:13 2016 +0100 @@ -20,13 +20,10 @@ PyAPI_FUNC(int) PyNode_AddChild(node *n, int type, char *str, int lineno, int col_offset); PyAPI_FUNC(void) PyNode_Free(node *n); -#ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); -#endif /* Node access functions */ #define NCH(n) ((n)->n_nchildren) - + #define CHILD(n, i) (&(n)->n_child[i]) #define RCHILD(n, i) (CHILD(n, NCH(n) + i)) #define TYPE(n) ((n)->n_type) diff -r 6db40a9955dc -r 0d413f60cc23 Include/object.h --- a/Include/object.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/object.h Mon Jan 25 17:05:13 2016 +0100 @@ -65,7 +65,6 @@ #error Py_LIMITED_API is incompatible with Py_DEBUG, Py_TRACE_REFS, and Py_REF_DEBUG #endif - #ifdef Py_TRACE_REFS /* Define pointers to support a doubly-linked list of all live heap objects. */ #define _PyObject_HEAD_EXTRA \ @@ -144,8 +143,7 @@ PyObject *object; } _Py_Identifier; -#define _Py_static_string_init(value) { 0, value, 0 } -#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) +#define _Py_static_string(varname, value) static _Py_Identifier varname = { 0, value, 0 } #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) /* @@ -276,9 +274,6 @@ binaryfunc nb_inplace_true_divide; unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; } PyNumberMethods; typedef struct { @@ -301,11 +296,6 @@ objobjargproc mp_ass_subscript; } PyMappingMethods; -typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; -} PyAsyncMethods; typedef struct { getbufferproc bf_getbuffer; @@ -351,8 +341,7 @@ printfunc tp_print; getattrfunc tp_getattr; setattrfunc tp_setattr; - PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) - or tp_reserved (Python 3) */ + void *tp_reserved; /* formerly known as tp_compare */ reprfunc tp_repr; /* Method suites for standard classes */ @@ -373,7 +362,7 @@ PyBufferProcs *tp_as_buffer; /* Flags to define presence of optional/expanded features */ - unsigned long tp_flags; + long tp_flags; const char *tp_doc; /* Documentation string */ @@ -419,8 +408,6 @@ /* Type attribute cache version tag. Added in version 2.6 */ unsigned int tp_version_tag; - destructor tp_finalize; - #ifdef COUNT_ALLOCS /* these must be last and never explicitly initialized */ Py_ssize_t tp_allocs; @@ -441,17 +428,11 @@ const char* name; int basicsize; int itemsize; - unsigned int flags; + int flags; PyType_Slot *slots; /* terminated by slot==0. */ } PyType_Spec; PyAPI_FUNC(PyObject*) PyType_FromSpec(PyType_Spec*); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 -PyAPI_FUNC(PyObject*) PyType_FromSpecWithBases(PyType_Spec*, PyObject*); -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000 -PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int); -#endif #ifndef Py_LIMITED_API /* The *real* layout of a type object when allocated on the heap */ @@ -459,7 +440,6 @@ /* Note: there's a dependency on the order of these members in slotptr() in typeobject.c . */ PyTypeObject ht_type; - PyAsyncMethods as_async; PyNumberMethods as_number; PyMappingMethods as_mapping; PySequenceMethods as_sequence; /* as_sequence comes after as_mapping, @@ -469,7 +449,6 @@ see add_operators() in typeobject.c . */ PyBufferProcs as_buffer; PyObject *ht_name, *ht_slots, *ht_qualname; - struct _dictkeysobject *ht_cached_keys; /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; @@ -487,7 +466,7 @@ PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */ PyAPI_DATA(PyTypeObject) PySuper_Type; /* built-in 'super' */ -PyAPI_FUNC(unsigned long) PyType_GetFlags(PyTypeObject*); +PyAPI_FUNC(long) PyType_GetFlags(PyTypeObject*); #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) @@ -499,18 +478,12 @@ PyObject *, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); #endif PyAPI_FUNC(unsigned int) PyType_ClearCache(void); PyAPI_FUNC(void) PyType_Modified(PyTypeObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); -PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); -#endif - /* Generic operations on objects */ struct _Py_Identifier; #ifndef Py_LIMITED_API @@ -544,6 +517,7 @@ PyAPI_FUNC(PyObject *) PyObject_GenericGetAttr(PyObject *, PyObject *); PyAPI_FUNC(int) PyObject_GenericSetAttr(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); PyAPI_FUNC(int) PyObject_GenericSetDict(PyObject *, PyObject *, void *); PyAPI_FUNC(Py_hash_t) PyObject_Hash(PyObject *); PyAPI_FUNC(Py_hash_t) PyObject_HashNotImplemented(PyObject *); @@ -552,10 +526,6 @@ PyAPI_FUNC(int) PyCallable_Check(PyObject *); PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); -PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); -#endif /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ @@ -565,11 +535,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, PyObject *, PyObject *); -/* Helper to look up a builtin object */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) -_PyObject_GetBuiltin(const char *name); -#endif /* PyObject_Dir(obj) acts like Python builtins.dir(obj), returning a list of strings. PyObject_Dir(NULL) is like builtins.dir(), @@ -583,6 +548,26 @@ PyAPI_FUNC(int) Py_ReprEnter(PyObject *); PyAPI_FUNC(void) Py_ReprLeave(PyObject *); +/* Helpers for hash functions */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double); +PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*); +PyAPI_FUNC(Py_hash_t) _Py_HashBytes(unsigned char*, Py_ssize_t); +#endif + +typedef struct { + Py_hash_t prefix; + Py_hash_t suffix; +} _Py_HashSecret_t; +PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; + +#ifdef Py_DEBUG +PyAPI_DATA(int) _Py_HashSecret_Initialized; +#endif + +/* Helper for passing objects to printf and the like */ +#define PyObject_REPR(obj) _PyUnicode_AsString(PyObject_Repr(obj)) + /* Flag bits for printing: */ #define Py_PRINT_RAW 1 /* No string quotes etc. */ @@ -610,55 +595,50 @@ */ /* Set if the type object is dynamically allocated */ -#define Py_TPFLAGS_HEAPTYPE (1UL << 9) +#define Py_TPFLAGS_HEAPTYPE (1L<<9) /* Set if the type allows subclassing */ -#define Py_TPFLAGS_BASETYPE (1UL << 10) +#define Py_TPFLAGS_BASETYPE (1L<<10) /* Set if the type is 'ready' -- fully initialized */ -#define Py_TPFLAGS_READY (1UL << 12) +#define Py_TPFLAGS_READY (1L<<12) /* Set while the type is being 'readied', to prevent recursive ready calls */ -#define Py_TPFLAGS_READYING (1UL << 13) +#define Py_TPFLAGS_READYING (1L<<13) /* Objects support garbage collection (see objimp.h) */ -#define Py_TPFLAGS_HAVE_GC (1UL << 14) +#define Py_TPFLAGS_HAVE_GC (1L<<14) /* These two bits are preserved for Stackless Python, next after this is 17 */ #ifdef STACKLESS -#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15) +#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3L<<15) #else #define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0 #endif /* Objects support type attribute cache */ -#define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) -#define Py_TPFLAGS_VALID_VERSION_TAG (1UL << 19) +#define Py_TPFLAGS_HAVE_VERSION_TAG (1L<<18) +#define Py_TPFLAGS_VALID_VERSION_TAG (1L<<19) /* Type is abstract and cannot be instantiated */ -#define Py_TPFLAGS_IS_ABSTRACT (1UL << 20) +#define Py_TPFLAGS_IS_ABSTRACT (1L<<20) /* These flags are used to determine if a type is a subclass. */ -#define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24) -#define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25) -#define Py_TPFLAGS_TUPLE_SUBCLASS (1UL << 26) -#define Py_TPFLAGS_BYTES_SUBCLASS (1UL << 27) -#define Py_TPFLAGS_UNICODE_SUBCLASS (1UL << 28) -#define Py_TPFLAGS_DICT_SUBCLASS (1UL << 29) -#define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) -#define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) +#define Py_TPFLAGS_INT_SUBCLASS (1L<<23) +#define Py_TPFLAGS_LONG_SUBCLASS (1L<<24) +#define Py_TPFLAGS_LIST_SUBCLASS (1L<<25) +#define Py_TPFLAGS_TUPLE_SUBCLASS (1L<<26) +#define Py_TPFLAGS_BYTES_SUBCLASS (1L<<27) +#define Py_TPFLAGS_UNICODE_SUBCLASS (1L<<28) +#define Py_TPFLAGS_DICT_SUBCLASS (1L<<29) +#define Py_TPFLAGS_BASE_EXC_SUBCLASS (1L<<30) +#define Py_TPFLAGS_TYPE_SUBCLASS (1L<<31) #define Py_TPFLAGS_DEFAULT ( \ Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ Py_TPFLAGS_HAVE_VERSION_TAG | \ 0) -/* NOTE: The following flags reuse lower bits (removed as part of the - * Python 3.0 transition). */ - -/* Type structure has tp_finalize member (3.4) */ -#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0) - #ifdef Py_LIMITED_API #define PyType_HasFeature(t,f) ((PyType_GetFlags(t) & (f)) != 0) #else @@ -693,6 +673,12 @@ complications in the deallocation function. (This is actually a decision that's up to the implementer of each new type so if you want, you can count such references to the type object.) + +*** WARNING*** The Py_DECREF macro must have a side-effect-free argument +since it may evaluate its argument multiple times. (The alternative +would be to mace it a proper function or assign it to a global temporary +variable first, both of which are slower; and in a multi-threaded +environment the global variable trick is not safe.) */ /* First define a pile of simple helper macros, one set per special @@ -709,6 +695,7 @@ PyAPI_FUNC(void) _Py_NegativeRefcount(const char *fname, int lineno, PyObject *op); PyAPI_FUNC(PyObject *) _PyDict_Dummy(void); +PyAPI_FUNC(PyObject *) _PySet_Dummy(void); PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); #define _Py_INC_REFTOTAL _Py_RefTotal++ #define _Py_DEC_REFTOTAL _Py_RefTotal-- @@ -718,17 +705,11 @@ _Py_NegativeRefcount(__FILE__, __LINE__, \ (PyObject *)(OP)); \ } -/* Py_REF_DEBUG also controls the display of refcounts and memory block - * allocations at the interactive prompt and at interpreter shutdown - */ -PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void); -#define _PY_DEBUG_PRINT_TOTAL_REFS() _PyDebug_PrintTotalRefs() #else #define _Py_INC_REFTOTAL #define _Py_DEC_REFTOTAL #define _Py_REF_DEBUG_COMMA #define _Py_CHECK_REFCNT(OP) /* a semicolon */; -#define _PY_DEBUG_PRINT_TOTAL_REFS() #endif /* Py_REF_DEBUG */ #ifdef COUNT_ALLOCS @@ -776,20 +757,19 @@ #define Py_INCREF(op) ( \ _Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA \ - ((PyObject *)(op))->ob_refcnt++) + ((PyObject*)(op))->ob_refcnt++) #define Py_DECREF(op) \ do { \ - PyObject *_py_decref_tmp = (PyObject *)(op); \ if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA \ - --(_py_decref_tmp)->ob_refcnt != 0) \ - _Py_CHECK_REFCNT(_py_decref_tmp) \ + --((PyObject*)(op))->ob_refcnt != 0) \ + _Py_CHECK_REFCNT(op) \ else \ - _Py_Dealloc(_py_decref_tmp); \ + _Py_Dealloc((PyObject *)(op)); \ } while (0) /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear - * and tp_dealloc implementations. + * and tp_dealloc implementatons. * * Note that "the obvious" code can be deadly: * @@ -824,53 +804,16 @@ */ #define Py_CLEAR(op) \ do { \ - PyObject *_py_tmp = (PyObject *)(op); \ - if (_py_tmp != NULL) { \ + if (op) { \ + PyObject *_py_tmp = (PyObject *)(op); \ (op) = NULL; \ Py_DECREF(_py_tmp); \ } \ } while (0) /* Macros to use in case the object pointer may be NULL: */ -#define Py_XINCREF(op) \ - do { \ - PyObject *_py_xincref_tmp = (PyObject *)(op); \ - if (_py_xincref_tmp != NULL) \ - Py_INCREF(_py_xincref_tmp); \ - } while (0) - -#define Py_XDECREF(op) \ - do { \ - PyObject *_py_xdecref_tmp = (PyObject *)(op); \ - if (_py_xdecref_tmp != NULL) \ - Py_DECREF(_py_xdecref_tmp); \ - } while (0) - -#ifndef Py_LIMITED_API -/* Safely decref `op` and set `op` to `op2`. - * - * As in case of Py_CLEAR "the obvious" code can be deadly: - * - * Py_XDECREF(op); - * op = op2; - * - * The safe way is: - * - * Py_SETREF(op, op2); - * - * That arranges to set `op` to `op2` _before_ decref'ing, so that any code - * triggered as a side-effect of `op` getting torn down no longer believes - * `op` points to a valid object. - */ - -#define Py_SETREF(op, op2) \ - do { \ - PyObject *_py_tmp = (PyObject *)(op); \ - (op) = (op2); \ - Py_XDECREF(_py_tmp); \ - } while (0) - -#endif /* ifndef Py_LIMITED_API */ +#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) +#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0) /* These are provided as conveniences to Python runtime embedders, so that @@ -879,9 +822,6 @@ PyAPI_FUNC(void) Py_IncRef(PyObject *); PyAPI_FUNC(void) Py_DecRef(PyObject *); -PyAPI_DATA(PyTypeObject) _PyNone_Type; -PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; - /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). @@ -1013,41 +953,24 @@ with the call stack never exceeding a depth of PyTrash_UNWIND_LEVEL. */ -/* This is the old private API, invoked by the macros before 3.2.4. - Kept for binary compatibility of extensions using the stable ABI. */ PyAPI_FUNC(void) _PyTrash_deposit_object(PyObject*); PyAPI_FUNC(void) _PyTrash_destroy_chain(void); PyAPI_DATA(int) _PyTrash_delete_nesting; PyAPI_DATA(PyObject *) _PyTrash_delete_later; -/* The new thread-safe private API, invoked by the macros below. */ -PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyObject*); -PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void); - #define PyTrash_UNWIND_LEVEL 50 #define Py_TRASHCAN_SAFE_BEGIN(op) \ - do { \ - PyThreadState *_tstate = PyThreadState_GET(); \ - if (_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ - ++_tstate->trash_delete_nesting; - /* The body of the deallocator is here. */ + if (_PyTrash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ + ++_PyTrash_delete_nesting; + /* The body of the deallocator is here. */ #define Py_TRASHCAN_SAFE_END(op) \ - --_tstate->trash_delete_nesting; \ - if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ - _PyTrash_thread_destroy_chain(); \ - } \ - else \ - _PyTrash_thread_deposit_object((PyObject*)op); \ - } while (0); - -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) -_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, - size_t sizeof_block); -PyAPI_FUNC(void) -_PyObject_DebugTypeStats(FILE *out); -#endif /* ifndef Py_LIMITED_API */ + --_PyTrash_delete_nesting; \ + if (_PyTrash_delete_later && _PyTrash_delete_nesting <= 0) \ + _PyTrash_destroy_chain(); \ + } \ + else \ + _PyTrash_deposit_object((PyObject*)op); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/objimpl.h --- a/Include/objimpl.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/objimpl.h Mon Jan 25 17:05:13 2016 +0100 @@ -94,28 +94,49 @@ the object gets initialized via PyObject_{Init, InitVar} after obtaining the raw memory. */ -PyAPI_FUNC(void *) PyObject_Malloc(size_t size); -PyAPI_FUNC(void *) PyObject_Calloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyObject_Free(void *ptr); +PyAPI_FUNC(void *) PyObject_Malloc(size_t); +PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t); +PyAPI_FUNC(void) PyObject_Free(void *); -/* This function returns the number of allocated memory blocks, regardless of size */ -PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); /* Macros */ #ifdef WITH_PYMALLOC -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); -#endif /* #ifndef Py_LIMITED_API */ -#endif +#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); +PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); +PyAPI_FUNC(void) _PyObject_DebugFree(void *p); +PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); +PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); +PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); +PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); +PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); +PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); +PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p); +PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes); +PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes); +PyAPI_FUNC(void) _PyMem_DebugFree(void *p); +#define PyObject_MALLOC _PyObject_DebugMalloc +#define PyObject_Malloc _PyObject_DebugMalloc +#define PyObject_REALLOC _PyObject_DebugRealloc +#define PyObject_Realloc _PyObject_DebugRealloc +#define PyObject_FREE _PyObject_DebugFree +#define PyObject_Free _PyObject_DebugFree -/* Macros */ +#else /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */ #define PyObject_MALLOC PyObject_Malloc #define PyObject_REALLOC PyObject_Realloc #define PyObject_FREE PyObject_Free +#endif + +#else /* ! WITH_PYMALLOC */ +#define PyObject_MALLOC PyMem_MALLOC +#define PyObject_REALLOC PyMem_REALLOC +#define PyObject_FREE PyMem_FREE + +#endif /* WITH_PYMALLOC */ + #define PyObject_Del PyObject_Free -#define PyObject_DEL PyObject_Free - +#define PyObject_DEL PyObject_FREE /* * Generic object allocator interface @@ -148,7 +169,7 @@ value is rounded up to the closest multiple of sizeof(void *), in order to ensure that pointer fields at the end of the object are correctly aligned for the platform (this is of special importance for subclasses of, e.g., - str or int, so that pointers can be stored after the embedded data). + str or long, so that pointers can be stored after the embedded data). Note that there's no memory wastage in doing this, as malloc has to return (at worst) pointer-aligned memory anyway. @@ -158,9 +179,12 @@ #endif #define _PyObject_VAR_SIZE(typeobj, nitems) \ - _Py_SIZE_ROUND_UP((typeobj)->tp_basicsize + \ - (nitems)*(typeobj)->tp_itemsize, \ - SIZEOF_VOID_P) + (size_t) \ + ( ( (typeobj)->tp_basicsize + \ + (nitems)*(typeobj)->tp_itemsize + \ + (SIZEOF_VOID_P - 1) \ + ) & ~(SIZEOF_VOID_P - 1) \ + ) #define PyObject_NEW(type, typeobj) \ ( (type *) PyObject_Init( \ @@ -199,26 +223,6 @@ constructor you would start directly with PyObject_Init/InitVar */ -#ifndef Py_LIMITED_API -typedef struct { - /* user context passed as the first argument to the 2 functions */ - void *ctx; - - /* allocate an arena of size bytes */ - void* (*alloc) (void *ctx, size_t size); - - /* free an arena */ - void (*free) (void *ctx, void *ptr, size_t size); -} PyObjectArenaAllocator; - -/* Get the arena allocator. */ -PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator); - -/* Set the arena allocator. */ -PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); -#endif - - /* * Garbage Collection Support * ========================== @@ -227,10 +231,6 @@ /* C equivalent of gc.collect(). */ PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyGC_CollectNoFail(void); -#endif - /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) @@ -250,37 +250,13 @@ union _gc_head *gc_prev; Py_ssize_t gc_refs; } gc; - double dummy; /* force worst-case alignment */ + long double dummy; /* force worst-case alignment */ } PyGC_Head; extern PyGC_Head *_PyGC_generation0; #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1) -/* Bit 0 is set when tp_finalize is called */ -#define _PyGC_REFS_MASK_FINALIZED (1 << 0) -/* The (N-1) most significant bits contain the gc state / refcount */ -#define _PyGC_REFS_SHIFT (1) -#define _PyGC_REFS_MASK (((size_t) -1) << _PyGC_REFS_SHIFT) - -#define _PyGCHead_REFS(g) ((g)->gc.gc_refs >> _PyGC_REFS_SHIFT) -#define _PyGCHead_SET_REFS(g, v) do { \ - (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK) \ - | (((size_t)(v)) << _PyGC_REFS_SHIFT); \ - } while (0) -#define _PyGCHead_DECREF(g) ((g)->gc.gc_refs -= 1 << _PyGC_REFS_SHIFT) - -#define _PyGCHead_FINALIZED(g) (((g)->gc.gc_refs & _PyGC_REFS_MASK_FINALIZED) != 0) -#define _PyGCHead_SET_FINALIZED(g, v) do { \ - (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \ - | (v != 0); \ - } while (0) - -#define _PyGC_FINALIZED(o) _PyGCHead_FINALIZED(_Py_AS_GC(o)) -#define _PyGC_SET_FINALIZED(o, v) _PyGCHead_SET_FINALIZED(_Py_AS_GC(o), v) - -#define _PyGC_REFS(o) _PyGCHead_REFS(_Py_AS_GC(o)) - #define _PyGC_REFS_UNTRACKED (-2) #define _PyGC_REFS_REACHABLE (-3) #define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4) @@ -289,9 +265,9 @@ * collector it must be safe to call the ob_traverse method. */ #define _PyObject_GC_TRACK(o) do { \ PyGC_Head *g = _Py_AS_GC(o); \ - if (_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED) \ + if (g->gc.gc_refs != _PyGC_REFS_UNTRACKED) \ Py_FatalError("GC object already tracked"); \ - _PyGCHead_SET_REFS(g, _PyGC_REFS_REACHABLE); \ + g->gc.gc_refs = _PyGC_REFS_REACHABLE; \ g->gc.gc_next = _PyGC_generation0; \ g->gc.gc_prev = _PyGC_generation0->gc.gc_prev; \ g->gc.gc_prev->gc.gc_next = g; \ @@ -304,8 +280,8 @@ */ #define _PyObject_GC_UNTRACK(o) do { \ PyGC_Head *g = _Py_AS_GC(o); \ - assert(_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED); \ - _PyGCHead_SET_REFS(g, _PyGC_REFS_UNTRACKED); \ + assert(g->gc.gc_refs != _PyGC_REFS_UNTRACKED); \ + g->gc.gc_refs = _PyGC_REFS_UNTRACKED; \ g->gc.gc_prev->gc.gc_next = g->gc.gc_next; \ g->gc.gc_next->gc.gc_prev = g->gc.gc_prev; \ g->gc.gc_next = NULL; \ @@ -313,7 +289,7 @@ /* True if the object is currently tracked by the GC. */ #define _PyObject_GC_IS_TRACKED(o) \ - (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED) + ((_Py_AS_GC(o))->gc.gc_refs != _PyGC_REFS_UNTRACKED) /* True if the object may be tracked by the GC in the future, or already is. This can be useful to implement some optimizations. */ @@ -322,8 +298,7 @@ (!PyTuple_CheckExact(obj) || _PyObject_GC_IS_TRACKED(obj))) #endif /* Py_LIMITED_API */ -PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size); -PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size); +PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t); PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); PyAPI_FUNC(void) PyObject_GC_Track(void *); diff -r 6db40a9955dc -r 0d413f60cc23 Include/odictobject.h --- a/Include/odictobject.h Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,43 +0,0 @@ -#ifndef Py_ODICTOBJECT_H -#define Py_ODICTOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - - -/* OrderedDict */ - -#ifndef Py_LIMITED_API - -typedef struct _odictobject PyODictObject; - -PyAPI_DATA(PyTypeObject) PyODict_Type; -PyAPI_DATA(PyTypeObject) PyODictIter_Type; -PyAPI_DATA(PyTypeObject) PyODictKeys_Type; -PyAPI_DATA(PyTypeObject) PyODictItems_Type; -PyAPI_DATA(PyTypeObject) PyODictValues_Type; - -#endif /* Py_LIMITED_API */ - -#define PyODict_Check(op) PyObject_TypeCheck(op, &PyODict_Type) -#define PyODict_CheckExact(op) (Py_TYPE(op) == &PyODict_Type) -#define PyODict_SIZE(op) ((PyDictObject *)op)->ma_used -#define PyODict_HasKey(od, key) (PyMapping_HasKey(PyObject *)od, key) - -PyAPI_FUNC(PyObject *) PyODict_New(void); -PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item); -PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key); - -/* wrappers around PyDict* functions */ -#define PyODict_GetItem(od, key) PyDict_GetItem((PyObject *)od, key) -#define PyODict_GetItemWithError(od, key) \ - PyDict_GetItemWithError((PyObject *)od, key) -#define PyODict_Contains(od, key) PyDict_Contains((PyObject *)od, key) -#define PyODict_Size(od) PyDict_Size((PyObject *)od) -#define PyODict_GetItemString(od, key) \ - PyDict_GetItemString((PyObject *)od, key) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_ODICTOBJECT_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/opcode.h --- a/Include/opcode.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/opcode.h Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,3 @@ -/* Auto-generated by Tools/scripts/generate_opcode_h.py */ #ifndef Py_OPCODE_H #define Py_OPCODE_H #ifdef __cplusplus @@ -6,123 +5,141 @@ #endif - /* Instruction opcodes for compiled code */ -#define POP_TOP 1 -#define ROT_TWO 2 -#define ROT_THREE 3 -#define DUP_TOP 4 -#define DUP_TOP_TWO 5 -#define NOP 9 -#define UNARY_POSITIVE 10 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define BINARY_MATRIX_MULTIPLY 16 -#define INPLACE_MATRIX_MULTIPLY 17 -#define BINARY_POWER 19 -#define BINARY_MULTIPLY 20 -#define BINARY_MODULO 22 -#define BINARY_ADD 23 -#define BINARY_SUBTRACT 24 -#define BINARY_SUBSCR 25 -#define BINARY_FLOOR_DIVIDE 26 -#define BINARY_TRUE_DIVIDE 27 -#define INPLACE_FLOOR_DIVIDE 28 -#define INPLACE_TRUE_DIVIDE 29 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define INPLACE_ADD 55 -#define INPLACE_SUBTRACT 56 -#define INPLACE_MULTIPLY 57 -#define INPLACE_MODULO 59 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define BINARY_LSHIFT 62 -#define BINARY_RSHIFT 63 -#define BINARY_AND 64 -#define BINARY_XOR 65 -#define BINARY_OR 66 -#define INPLACE_POWER 67 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define PRINT_EXPR 70 -#define LOAD_BUILD_CLASS 71 -#define YIELD_FROM 72 -#define GET_AWAITABLE 73 -#define INPLACE_LSHIFT 75 -#define INPLACE_RSHIFT 76 -#define INPLACE_AND 77 -#define INPLACE_XOR 78 -#define INPLACE_OR 79 -#define BREAK_LOOP 80 -#define WITH_CLEANUP_START 81 -#define WITH_CLEANUP_FINISH 82 -#define RETURN_VALUE 83 -#define IMPORT_STAR 84 -#define YIELD_VALUE 86 -#define POP_BLOCK 87 -#define END_FINALLY 88 -#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define JUMP_IF_FALSE_OR_POP 111 -#define JUMP_IF_TRUE_OR_POP 112 -#define JUMP_ABSOLUTE 113 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define CONTINUE_LOOP 119 -#define SETUP_LOOP 120 -#define SETUP_EXCEPT 121 -#define SETUP_FINALLY 122 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define RAISE_VARARGS 130 -#define CALL_FUNCTION 131 -#define MAKE_FUNCTION 132 -#define BUILD_SLICE 133 -#define MAKE_CLOSURE 134 -#define LOAD_CLOSURE 135 -#define LOAD_DEREF 136 -#define STORE_DEREF 137 -#define DELETE_DEREF 138 -#define CALL_FUNCTION_VAR 140 -#define CALL_FUNCTION_KW 141 -#define CALL_FUNCTION_VAR_KW 142 -#define SETUP_WITH 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define LOAD_CLASSDEREF 148 -#define BUILD_LIST_UNPACK 149 -#define BUILD_MAP_UNPACK 150 -#define BUILD_MAP_UNPACK_WITH_CALL 151 -#define BUILD_TUPLE_UNPACK 152 -#define BUILD_SET_UNPACK 153 -#define SETUP_ASYNC_WITH 154 -#define FORMAT_VALUE 155 +/* Instruction opcodes for compiled code */ + +#define POP_TOP 1 +#define ROT_TWO 2 +#define ROT_THREE 3 +#define DUP_TOP 4 +#define DUP_TOP_TWO 5 +#define NOP 9 + +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 + +#define UNARY_INVERT 15 + +#define BINARY_POWER 19 + +#define BINARY_MULTIPLY 20 + +#define BINARY_MODULO 22 +#define BINARY_ADD 23 +#define BINARY_SUBTRACT 24 +#define BINARY_SUBSCR 25 +#define BINARY_FLOOR_DIVIDE 26 +#define BINARY_TRUE_DIVIDE 27 +#define INPLACE_FLOOR_DIVIDE 28 +#define INPLACE_TRUE_DIVIDE 29 + +#define STORE_MAP 54 +#define INPLACE_ADD 55 +#define INPLACE_SUBTRACT 56 +#define INPLACE_MULTIPLY 57 + +#define INPLACE_MODULO 59 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 + +#define BINARY_LSHIFT 62 +#define BINARY_RSHIFT 63 +#define BINARY_AND 64 +#define BINARY_XOR 65 +#define BINARY_OR 66 +#define INPLACE_POWER 67 +#define GET_ITER 68 +#define STORE_LOCALS 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define YIELD_FROM 72 + +#define INPLACE_LSHIFT 75 +#define INPLACE_RSHIFT 76 +#define INPLACE_AND 77 +#define INPLACE_XOR 78 +#define INPLACE_OR 79 +#define BREAK_LOOP 80 +#define WITH_CLEANUP 81 + +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 + +#define YIELD_VALUE 86 +#define POP_BLOCK 87 +#define END_FINALLY 88 +#define POP_EXCEPT 89 + +#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */ + +#define STORE_NAME 90 /* Index in name list */ +#define DELETE_NAME 91 /* "" */ +#define UNPACK_SEQUENCE 92 /* Number of sequence items */ +#define FOR_ITER 93 +#define UNPACK_EX 94 /* Num items before variable part + + (Num items after variable part << 8) */ + +#define STORE_ATTR 95 /* Index in name list */ +#define DELETE_ATTR 96 /* "" */ +#define STORE_GLOBAL 97 /* "" */ +#define DELETE_GLOBAL 98 /* "" */ + +#define LOAD_CONST 100 /* Index in const list */ +#define LOAD_NAME 101 /* Index in name list */ +#define BUILD_TUPLE 102 /* Number of tuple items */ +#define BUILD_LIST 103 /* Number of list items */ +#define BUILD_SET 104 /* Number of set items */ +#define BUILD_MAP 105 /* Always zero for now */ +#define LOAD_ATTR 106 /* Index in name list */ +#define COMPARE_OP 107 /* Comparison operator */ +#define IMPORT_NAME 108 /* Index in name list */ +#define IMPORT_FROM 109 /* Index in name list */ + +#define JUMP_FORWARD 110 /* Number of bytes to skip */ +#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */ +#define JUMP_IF_TRUE_OR_POP 112 /* "" */ +#define JUMP_ABSOLUTE 113 /* "" */ +#define POP_JUMP_IF_FALSE 114 /* "" */ +#define POP_JUMP_IF_TRUE 115 /* "" */ + +#define LOAD_GLOBAL 116 /* Index in name list */ + +#define CONTINUE_LOOP 119 /* Start of loop (absolute) */ +#define SETUP_LOOP 120 /* Target address (relative) */ +#define SETUP_EXCEPT 121 /* "" */ +#define SETUP_FINALLY 122 /* "" */ + +#define LOAD_FAST 124 /* Local variable number */ +#define STORE_FAST 125 /* Local variable number */ +#define DELETE_FAST 126 /* Local variable number */ + +#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */ +/* CALL_FUNCTION_XXX opcodes defined below depend on this definition */ +#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */ +#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */ +#define BUILD_SLICE 133 /* Number of items */ + +#define MAKE_CLOSURE 134 /* same as MAKE_FUNCTION */ +#define LOAD_CLOSURE 135 /* Load free variable from closure */ +#define LOAD_DEREF 136 /* Load and dereference from closure cell */ +#define STORE_DEREF 137 /* Store into cell */ +#define DELETE_DEREF 138 /* Delete closure cell */ + +/* The next 3 opcodes must be contiguous and satisfy + (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */ +#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */ +#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */ +#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */ + +#define SETUP_WITH 143 + +/* Support for opargs more than 16 bits long */ +#define EXTENDED_ARG 144 + +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 + /* EXCEPT_HANDLER is a special, implicit block type which is created when entering an except handler. It is not an opcode but we define it here @@ -131,9 +148,8 @@ #define EXCEPT_HANDLER 257 -enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, - PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, PyCmp_IN, PyCmp_NOT_IN, - PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD}; +enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, + PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD}; #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) diff -r 6db40a9955dc -r 0d413f60cc23 Include/osdefs.h --- a/Include/osdefs.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/osdefs.h Mon Jan 25 17:05:13 2016 +0100 @@ -7,12 +7,21 @@ /* Operating system dependencies */ -#ifdef MS_WINDOWS +/* Mod by chrish: QNX has WATCOM, but isn't DOS */ +#if !defined(__QNX__) +#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__) || defined(PYOS_OS2) +#if defined(PYOS_OS2) && defined(PYCC_GCC) +#define MAXPATHLEN 260 +#define SEP L'/' +#define ALTSEP L'\\' +#else #define SEP L'\\' #define ALTSEP L'/' #define MAXPATHLEN 256 +#endif #define DELIM L';' #endif +#endif /* Filename separator */ #ifndef SEP @@ -20,14 +29,6 @@ #endif /* Max pathname length */ -#ifdef __hpux -#include -#include -#ifndef PATH_MAX -#define PATH_MAX MAXPATHLEN -#endif -#endif - #ifndef MAXPATHLEN #if defined(PATH_MAX) && PATH_MAX > 1024 #define MAXPATHLEN PATH_MAX diff -r 6db40a9955dc -r 0d413f60cc23 Include/parsetok.h --- a/Include/parsetok.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/parsetok.h Mon Jan 25 17:05:13 2016 +0100 @@ -38,49 +38,29 @@ PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int, perrdetail *); PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int, - const char *, const char *, - perrdetail *); + char *, char *, perrdetail *); PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int, perrdetail *, int); -PyAPI_FUNC(node *) PyParser_ParseFileFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - const char *enc, - grammar *g, - int start, - const char *ps1, - const char *ps2, - perrdetail *err_ret, - int flags); +PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, + const char*, grammar *, + int, char *, char *, + perrdetail *, int); PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ const char *enc, grammar *g, int start, - const char *ps1, - const char *ps2, - perrdetail *err_ret, - int *flags); -PyAPI_FUNC(node *) PyParser_ParseFileObject( - FILE *fp, - PyObject *filename, - const char *enc, - grammar *g, - int start, - const char *ps1, - const char *ps2, + char *ps1, + char *ps2, perrdetail *err_ret, int *flags); -PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename( - const char *s, - const char *filename, /* decoded from the filesystem encoding */ - grammar *g, - int start, - perrdetail *err_ret, - int flags); +PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *, + const char *, + grammar *, int, + perrdetail *, int); PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx( const char *s, const char *filename, /* decoded from the filesystem encoding */ @@ -88,13 +68,6 @@ int start, perrdetail *err_ret, int *flags); -PyAPI_FUNC(node *) PyParser_ParseStringObject( - const char *s, - PyObject *filename, - grammar *g, - int start, - perrdetail *err_ret, - int *flags); /* Note that the following functions are defined in pythonrun.c, not in parsetok.c */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/patchlevel.h --- a/Include/patchlevel.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/patchlevel.h Mon Jan 25 17:05:13 2016 +0100 @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 6 +#define PY_MINOR_VERSION 3 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 0 +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0a0" +#define PY_VERSION "3.3.0a1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff -r 6db40a9955dc -r 0d413f60cc23 Include/py_curses.h --- a/Include/py_curses.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/py_curses.h Mon Jan 25 17:05:13 2016 +0100 @@ -103,8 +103,8 @@ #endif /* general error messages */ -static const char catchall_ERR[] = "curses function returned ERR"; -static const char catchall_NULL[] = "curses function returned NULL"; +static char *catchall_ERR = "curses function returned ERR"; +static char *catchall_NULL = "curses function returned NULL"; /* Function Prototype Macros - They are ugly but very, very useful. ;-) diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyatomic.h --- a/Include/pyatomic.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pyatomic.h Mon Jan 25 17:05:13 2016 +0100 @@ -1,13 +1,14 @@ +#ifndef Py_LIMITED_API #ifndef Py_ATOMIC_H #define Py_ATOMIC_H -#ifdef Py_BUILD_CORE +/* XXX: When compilers start offering a stdatomic.h with lock-free + atomic_int and atomic_address types, include that here and rewrite + the atomic operations in terms of it. */ #include "dynamic_annotations.h" -#include "pyconfig.h" - -#if defined(HAVE_STD_ATOMIC) -#include +#ifdef __cplusplus +extern "C" { #endif /* This is modeled after the atomics interface from C1x, according to @@ -19,76 +20,6 @@ * Beware, the implementations here are deep magic. */ -#if defined(HAVE_STD_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = memory_order_relaxed, - _Py_memory_order_acquire = memory_order_acquire, - _Py_memory_order_release = memory_order_release, - _Py_memory_order_acq_rel = memory_order_acq_rel, - _Py_memory_order_seq_cst = memory_order_seq_cst -} _Py_memory_order; - -typedef struct _Py_atomic_address { - atomic_uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - atomic_int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER) - -/* Use builtin atomic operations in GCC >= 4.7 */ -#elif defined(HAVE_BUILTIN_ATOMIC) - -typedef enum _Py_memory_order { - _Py_memory_order_relaxed = __ATOMIC_RELAXED, - _Py_memory_order_acquire = __ATOMIC_ACQUIRE, - _Py_memory_order_release = __ATOMIC_RELEASE, - _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, - _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST -} _Py_memory_order; - -typedef struct _Py_atomic_address { - Py_uintptr_t _value; -} _Py_atomic_address; - -typedef struct _Py_atomic_int { - int _value; -} _Py_atomic_int; - -#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ - __atomic_signal_fence(ORDER) - -#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ - __atomic_thread_fence(ORDER) - -#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_RELEASE), \ - __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)) - -#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ - (assert((ORDER) == __ATOMIC_RELAXED \ - || (ORDER) == __ATOMIC_SEQ_CST \ - || (ORDER) == __ATOMIC_ACQUIRE \ - || (ORDER) == __ATOMIC_CONSUME), \ - __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER)) - -#else - typedef enum _Py_memory_order { _Py_memory_order_relaxed, _Py_memory_order_acquire, @@ -98,7 +29,7 @@ } _Py_memory_order; typedef struct _Py_atomic_address { - Py_uintptr_t _value; + void *_value; } _Py_atomic_address; typedef struct _Py_atomic_int { @@ -231,7 +162,6 @@ ((ATOMIC_VAL)->_value) #endif /* !gcc x86 */ -#endif /* Standardized shortcuts. */ #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ @@ -246,5 +176,9 @@ #define _Py_atomic_load_relaxed(ATOMIC_VAL) \ _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed) -#endif /* Py_BUILD_CORE */ +#ifdef __cplusplus +} +#endif + #endif /* Py_ATOMIC_H */ +#endif /* Py_LIMITED_API */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pydebug.h --- a/Include/pydebug.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pydebug.h Mon Jan 25 17:05:13 2016 +0100 @@ -5,8 +5,6 @@ extern "C" { #endif -/* These global variable are defined in pylifecycle.c */ -/* XXX (ncoghlan): move these declarations to pylifecycle.h? */ PyAPI_DATA(int) Py_DebugFlag; PyAPI_DATA(int) Py_VerboseFlag; PyAPI_DATA(int) Py_QuietFlag; @@ -22,7 +20,6 @@ PyAPI_DATA(int) Py_NoUserSiteDirectory; PyAPI_DATA(int) Py_UnbufferedStdioFlag; PyAPI_DATA(int) Py_HashRandomizationFlag; -PyAPI_DATA(int) Py_IsolatedFlag; /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyerrors.h --- a/Include/pyerrors.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pyerrors.h Mon Jan 25 17:05:13 2016 +0100 @@ -10,8 +10,7 @@ /* PyException_HEAD defines the initial segment of every exception class. */ #define PyException_HEAD PyObject_HEAD PyObject *dict;\ PyObject *args; PyObject *traceback;\ - PyObject *context; PyObject *cause;\ - char suppress_context; + PyObject *context; PyObject *cause; typedef struct { PyException_HEAD @@ -29,13 +28,6 @@ typedef struct { PyException_HEAD - PyObject *msg; - PyObject *name; - PyObject *path; -} PyImportErrorObject; - -typedef struct { - PyException_HEAD PyObject *encoding; PyObject *object; Py_ssize_t start; @@ -53,7 +45,6 @@ PyObject *myerrno; PyObject *strerror; PyObject *filename; - PyObject *filename2; #ifdef MS_WINDOWS PyObject *winerror; #endif @@ -76,9 +67,6 @@ PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); -#endif PyAPI_FUNC(void) PyErr_SetString( PyObject *exception, const char *string /* decoded from utf-8 */ @@ -87,11 +75,9 @@ PyAPI_FUNC(void) PyErr_Clear(void); PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **); PyAPI_FUNC(void) PyErr_Restore(PyObject *, PyObject *, PyObject *); -PyAPI_FUNC(void) PyErr_GetExcInfo(PyObject **, PyObject **, PyObject **); -PyAPI_FUNC(void) PyErr_SetExcInfo(PyObject *, PyObject *, PyObject *); #if defined(__clang__) || \ - (defined(__GNUC_MAJOR__) && \ + (defined(__GNUC__) && \ ((__GNUC_MAJOR__ >= 3) || \ (__GNUC_MAJOR__ == 2) && (__GNUC_MINOR__ >= 5))) #define _Py_NO_RETURN __attribute__((__noreturn__)) @@ -99,13 +85,12 @@ #define _Py_NO_RETURN #endif -/* Defined in Python/pylifecycle.c */ PyAPI_FUNC(void) Py_FatalError(const char *message) _Py_NO_RETURN; #if defined(Py_DEBUG) || defined(Py_LIMITED_API) #define _PyErr_OCCURRED() PyErr_Occurred() #else -#define _PyErr_OCCURRED() (PyThreadState_GET()->curexc_type) +#define _PyErr_OCCURRED() (_PyThreadState_Current->curexc_type) #endif /* Error testing and normalization */ @@ -120,13 +105,12 @@ /* Cause manipulation (PEP 3134) */ PyAPI_FUNC(PyObject *) PyException_GetCause(PyObject *); PyAPI_FUNC(void) PyException_SetCause(PyObject *, PyObject *); +PyAPI_FUNC(int) _PyException_SetCauseChecked(PyObject *, PyObject *); /* Context manipulation (PEP 3134) */ PyAPI_FUNC(PyObject *) PyException_GetContext(PyObject *); PyAPI_FUNC(void) PyException_SetContext(PyObject *, PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -#endif + /* */ @@ -147,7 +131,6 @@ PyAPI_DATA(PyObject *) PyExc_BaseException; PyAPI_DATA(PyObject *) PyExc_Exception; -PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration; PyAPI_DATA(PyObject *) PyExc_StopIteration; PyAPI_DATA(PyObject *) PyExc_GeneratorExit; PyAPI_DATA(PyObject *) PyExc_ArithmeticError; @@ -167,7 +150,6 @@ PyAPI_DATA(PyObject *) PyExc_NameError; PyAPI_DATA(PyObject *) PyExc_OverflowError; PyAPI_DATA(PyObject *) PyExc_RuntimeError; -PyAPI_DATA(PyObject *) PyExc_RecursionError; PyAPI_DATA(PyObject *) PyExc_NotImplementedError; PyAPI_DATA(PyObject *) PyExc_SyntaxError; PyAPI_DATA(PyObject *) PyExc_IndentationError; @@ -207,6 +189,9 @@ #ifdef MS_WINDOWS PyAPI_DATA(PyObject *) PyExc_WindowsError; #endif +#ifdef __VMS +PyAPI_DATA(PyObject *) PyExc_VMSError; +#endif PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst; @@ -231,8 +216,6 @@ PyAPI_FUNC(PyObject *) PyErr_SetFromErrno(PyObject *); PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilenameObject( PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilenameObjects( - PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilename( PyObject *exc, const char *filename /* decoded from the filesystem encoding */ @@ -247,12 +230,6 @@ const char *format, /* ASCII-encoded string */ ... ); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -PyAPI_FUNC(PyObject *) PyErr_FormatV( - PyObject *exception, - const char *format, - va_list vargs); -#endif #ifdef MS_WINDOWS PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilename( @@ -267,8 +244,6 @@ PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErr(int); PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilenameObject( PyObject *,int, PyObject *); -PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilenameObjects( - PyObject *,int, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilename( PyObject *exc, int ierr, @@ -281,11 +256,6 @@ PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErr(PyObject *, int); #endif /* MS_WINDOWS */ -PyAPI_FUNC(PyObject *) PyErr_SetExcWithArgsKwargs(PyObject *, PyObject *, - PyObject *); -PyAPI_FUNC(PyObject *) PyErr_SetImportError(PyObject *, PyObject *, - PyObject *); - /* Export the old function so that the existing API remains available: */ PyAPI_FUNC(void) PyErr_BadInternalCall(void); PyAPI_FUNC(void) _PyErr_BadInternalCall(const char *filename, int lineno); @@ -300,28 +270,6 @@ const char *name, const char *doc, PyObject *base, PyObject *dict); PyAPI_FUNC(void) PyErr_WriteUnraisable(PyObject *); -/* In exceptions.c */ -#ifndef Py_LIMITED_API -/* Helper that attempts to replace the current exception with one of the - * same type but with a prefix added to the exception text. The resulting - * exception description looks like: - * - * prefix (exc_type: original_exc_str) - * - * Only some exceptions can be safely replaced. If the function determines - * it isn't safe to perform the replacement, it will leave the original - * unmodified exception in place. - * - * Returns a borrowed reference to the new exception (if any), NULL if the - * existing exception was left in place. - */ -PyAPI_FUNC(PyObject *) _PyErr_TrySetFromCause( - const char *prefix_format, /* ASCII-encoded string */ - ... - ); -#endif - - /* In sigcheck.c or signalmodule.c */ PyAPI_FUNC(int) PyErr_CheckSignals(void); PyAPI_FUNC(void) PyErr_SetInterrupt(void); @@ -339,20 +287,9 @@ const char *filename, /* decoded from the filesystem encoding */ int lineno, int col_offset); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) PyErr_SyntaxLocationObject( - PyObject *filename, - int lineno, - int col_offset); -#endif PyAPI_FUNC(PyObject *) PyErr_ProgramText( const char *filename, /* decoded from the filesystem encoding */ int lineno); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( - PyObject *filename, - int lineno); -#endif /* The following functions are used to create and modify unicode exceptions from C */ @@ -449,6 +386,9 @@ const char *reason /* UTF-8 encoded string */ ); +/* create a StopIteration exception with the given value */ +PyAPI_FUNC(PyObject *) PyStopIteration_Create(PyObject *); + /* These APIs aren't really part of the error implementation, but often needed to format error messages; the native C lib APIs are not available on all platforms, which is why we provide emulations diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyexpat.h --- a/Include/pyexpat.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pyexpat.h Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,7 @@ #define PyExpat_CAPI_MAGIC "pyexpat.expat_CAPI 1.0" #define PyExpat_CAPSULE_NAME "pyexpat.expat_CAPI" -struct PyExpat_CAPI +struct PyExpat_CAPI { char* magic; /* set to PyExpat_CAPI_MAGIC */ int size; /* set to sizeof(struct PyExpat_CAPI) */ @@ -43,11 +43,6 @@ XML_Parser parser, XML_UnknownEncodingHandler handler, void *encodingHandlerData); void (*SetUserData)(XML_Parser parser, void *userData); - void (*SetStartDoctypeDeclHandler)(XML_Parser parser, - XML_StartDoctypeDeclHandler start); - enum XML_Status (*SetEncoding)(XML_Parser parser, const XML_Char *encoding); - int (*DefaultUnknownEncodingHandler)( - void *encodingHandlerData, const XML_Char *name, XML_Encoding *info); /* always add new stuff to the end! */ }; diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyfpe.h --- a/Include/pyfpe.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pyfpe.h Mon Jan 25 17:05:13 2016 +0100 @@ -4,8 +4,8 @@ extern "C" { #endif /* - --------------------------------------------------------------------- - / Copyright (c) 1996. \ + --------------------------------------------------------------------- + / Copyright (c) 1996. \ | The Regents of the University of California. | | All rights reserved. | | | @@ -37,8 +37,8 @@ | opinions of authors expressed herein do not necessarily state or | | reflect those of the United States Government or the University | | of California, and shall not be used for advertising or product | - \ endorsement purposes. / - --------------------------------------------------------------------- + \ endorsement purposes. / + --------------------------------------------------------------------- */ /* diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyhash.h --- a/Include/pyhash.h Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,149 +0,0 @@ -#ifndef Py_HASH_H - -#define Py_HASH_H -#ifdef __cplusplus -extern "C" { -#endif - -/* Helpers for hash functions */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double); -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*); -PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); -#endif - -/* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ - -/* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Objects/object.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ - -#if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 -#else -# define _PyHASH_BITS 31 -#endif - -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_NAN 0 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER - - -/* hash secret - * - * memory layout on 64 bit systems - * cccccccc cccccccc cccccccc uc -- unsigned char[24] - * pppppppp ssssssss ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two PY_UINT64_T - * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt - * - * memory layout on 32 bit systems - * cccccccc cccccccc cccccccc uc - * ppppssss ........ ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two PY_UINT64_T (*) - * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt - * - * (*) The siphash member may not be available on 32 bit platforms without - * an unsigned int64 data type. - */ -#ifndef Py_LIMITED_API -typedef union { - /* ensure 24 bytes */ - unsigned char uc[24]; - /* two Py_hash_t for FNV */ - struct { - Py_hash_t prefix; - Py_hash_t suffix; - } fnv; -#ifdef PY_UINT64_T - /* two uint64 for SipHash24 */ - struct { - PY_UINT64_T k0; - PY_UINT64_T k1; - } siphash; -#endif - /* a different (!) Py_hash_t for small string optimization */ - struct { - unsigned char padding[16]; - Py_hash_t suffix; - } djbx33a; - struct { - unsigned char padding[16]; - Py_hash_t hashsalt; - } expat; -} _Py_HashSecret_t; -PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; -#endif - -#ifdef Py_DEBUG -PyAPI_DATA(int) _Py_HashSecret_Initialized; -#endif - - -/* hash function definition */ -#ifndef Py_LIMITED_API -typedef struct { - Py_hash_t (*const hash)(const void *, Py_ssize_t); - const char *name; - const int hash_bits; - const int seed_bits; -} PyHash_FuncDef; - -PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); -#endif - - -/* cutoff for small string DJBX33A optimization in range [1, cutoff). - * - * About 50% of the strings in a typical Python application are smaller than - * 6 to 7 chars. However DJBX33A is vulnerable to hash collision attacks. - * NEVER use DJBX33A for long strings! - * - * A Py_HASH_CUTOFF of 0 disables small string optimization. 32 bit platforms - * should use a smaller cutoff because it is easier to create colliding - * strings. A cutoff of 7 on 64bit platforms and 5 on 32bit platforms should - * provide a decent safety margin. - */ -#ifndef Py_HASH_CUTOFF -# define Py_HASH_CUTOFF 0 -#elif (Py_HASH_CUTOFF > 7 || Py_HASH_CUTOFF < 0) -# error Py_HASH_CUTOFF must in range 0...7. -#endif /* Py_HASH_CUTOFF */ - - -/* hash algorithm selection - * - * The values for Py_HASH_SIPHASH24 and Py_HASH_FNV are hard-coded in the - * configure script. - * - * - FNV is available on all platforms and architectures. - * - SIPHASH24 only works on plaforms that provide PY_UINT64_T and doesn't - * require aligned memory for integers. - * - With EXTERNAL embedders can provide an alternative implementation with:: - * - * PyHash_FuncDef PyHash_Func = {...}; - * - * XXX: Figure out __declspec() for extern PyHash_FuncDef. - */ -#define Py_HASH_EXTERNAL 0 -#define Py_HASH_SIPHASH24 1 -#define Py_HASH_FNV 2 - -#ifndef Py_HASH_ALGORITHM -# if (defined(PY_UINT64_T) && defined(PY_UINT32_T) \ - && !defined(HAVE_ALIGNED_REQUIRED)) -# define Py_HASH_ALGORITHM Py_HASH_SIPHASH24 -# else -# define Py_HASH_ALGORITHM Py_HASH_FNV -# endif /* uint64_t && uint32_t && aligned */ -#endif /* Py_HASH_ALGORITHM */ - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_HASH_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pylifecycle.h --- a/Include/pylifecycle.h Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,125 +0,0 @@ - -/* Interfaces to configure, query, create & destroy the Python runtime */ - -#ifndef Py_PYLIFECYCLE_H -#define Py_PYLIFECYCLE_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); - -#ifndef Py_LIMITED_API -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, - const char *errors); -#endif - -PyAPI_FUNC(void) Py_Initialize(void); -PyAPI_FUNC(void) Py_InitializeEx(int); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int); -#endif -PyAPI_FUNC(void) Py_Finalize(void); -PyAPI_FUNC(int) Py_FinalizeEx(void); -PyAPI_FUNC(int) Py_IsInitialized(void); -PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); -PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); - - -/* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level - * exit functions. - */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void)); -#endif -PyAPI_FUNC(int) Py_AtExit(void (*func)(void)); - -PyAPI_FUNC(void) Py_Exit(int); - -/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_RestoreSignals(void); - -PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); -#endif - -/* Bootstrap __main__ (defined in Modules/main.c) */ -PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); - -/* In getpath.c */ -PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); -PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); -PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); -PyAPI_FUNC(wchar_t *) Py_GetPath(void); -PyAPI_FUNC(void) Py_SetPath(const wchar_t *); -#ifdef MS_WINDOWS -int _Py_CheckPython3(); -#endif - -/* In their own files */ -PyAPI_FUNC(const char *) Py_GetVersion(void); -PyAPI_FUNC(const char *) Py_GetPlatform(void); -PyAPI_FUNC(const char *) Py_GetCopyright(void); -PyAPI_FUNC(const char *) Py_GetCompiler(void); -PyAPI_FUNC(const char *) Py_GetBuildInfo(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_hgidentifier(void); -PyAPI_FUNC(const char *) _Py_hgversion(void); -#endif - -/* Internal -- various one-time initializations */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); -PyAPI_FUNC(PyObject *) _PySys_Init(void); -PyAPI_FUNC(void) _PyImport_Init(void); -PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); -PyAPI_FUNC(void) _PyImportHooks_Init(void); -PyAPI_FUNC(int) _PyFrame_Init(void); -PyAPI_FUNC(int) _PyFloat_Init(void); -PyAPI_FUNC(int) PyByteArray_Init(void); -PyAPI_FUNC(void) _PyRandom_Init(void); -#endif - -/* Various internal finalizers */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyExc_Fini(void); -PyAPI_FUNC(void) _PyImport_Fini(void); -PyAPI_FUNC(void) PyMethod_Fini(void); -PyAPI_FUNC(void) PyFrame_Fini(void); -PyAPI_FUNC(void) PyCFunction_Fini(void); -PyAPI_FUNC(void) PyDict_Fini(void); -PyAPI_FUNC(void) PyTuple_Fini(void); -PyAPI_FUNC(void) PyList_Fini(void); -PyAPI_FUNC(void) PySet_Fini(void); -PyAPI_FUNC(void) PyBytes_Fini(void); -PyAPI_FUNC(void) PyByteArray_Fini(void); -PyAPI_FUNC(void) PyFloat_Fini(void); -PyAPI_FUNC(void) PyOS_FiniInterrupts(void); -PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); -PyAPI_FUNC(void) _PyGC_Fini(void); -PyAPI_FUNC(void) PySlice_Fini(void); -PyAPI_FUNC(void) _PyType_Fini(void); -PyAPI_FUNC(void) _PyRandom_Fini(void); - -PyAPI_DATA(PyThreadState *) _Py_Finalizing; -#endif - -/* Signals */ -typedef void (*PyOS_sighandler_t)(int); -PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int); -PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); - -/* Random */ -PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_PYLIFECYCLE_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pymacro.h --- a/Include/pymacro.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pymacro.h Mon Jan 25 17:05:13 2016 +0100 @@ -1,26 +1,13 @@ #ifndef Py_PYMACRO_H #define Py_PYMACRO_H -/* Minimum value between x and y */ #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) - -/* Maximum value between x and y */ #define Py_MAX(x, y) (((x) > (y)) ? (x) : (y)) -/* Absolute value of the number x */ -#define Py_ABS(x) ((x) < 0 ? -(x) : (x)) - -#define _Py_XSTRINGIFY(x) #x - -/* Convert the argument to a string. For example, Py_STRINGIFY(123) is replaced - with "123" by the preprocessor. Defines are also replaced by their value. - For example Py_STRINGIFY(__LINE__) is replaced by the line number, not - by "__LINE__". */ -#define Py_STRINGIFY(x) _Py_XSTRINGIFY(x) - /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) + /* Assert a build-time dependency, as an expression. Your compile will fail if the condition isn't true, or can't be evaluated @@ -36,21 +23,14 @@ #define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) - /* Get the number of elements in a visible array This does not work on pointers, or arrays declared as [], or function parameters. With correct compiler support, such usage will cause a build error (see Py_BUILD_ASSERT_EXPR). - Written by Rusty Russell, public domain, http://ccodearchive.net/ - - Requires at GCC 3.1+ */ -#if (defined(__GNUC__) && !defined(__STRICT_ANSI__) && \ - (((__GNUC__ == 3) && (__GNU_MINOR__ >= 1)) || (__GNUC__ >= 4))) + Written by Rusty Russell, public domain, http://ccodearchive.net/ */ +#if defined(__GNUC__) /* Two gcc extensions. &a[0] degrades to a pointer: a different type from an array */ #define Py_ARRAY_LENGTH(array) \ @@ -72,24 +52,4 @@ #define PyDoc_STR(str) "" #endif -/* Below "a" is a power of 2. */ -/* Round down size "n" to be a multiple of "a". */ -#define _Py_SIZE_ROUND_DOWN(n, a) ((size_t)(n) & ~(size_t)((a) - 1)) -/* Round up size "n" to be a multiple of "a". */ -#define _Py_SIZE_ROUND_UP(n, a) (((size_t)(n) + \ - (size_t)((a) - 1)) & ~(size_t)((a) - 1)) -/* Round pointer "p" down to the closest "a"-aligned address <= "p". */ -#define _Py_ALIGN_DOWN(p, a) ((void *)((Py_uintptr_t)(p) & ~(Py_uintptr_t)((a) - 1))) -/* Round pointer "p" up to the closest "a"-aligned address >= "p". */ -#define _Py_ALIGN_UP(p, a) ((void *)(((Py_uintptr_t)(p) + \ - (Py_uintptr_t)((a) - 1)) & ~(Py_uintptr_t)((a) - 1))) -/* Check if pointer "p" is aligned to "a"-bytes boundary. */ -#define _Py_IS_ALIGNED(p, a) (!((Py_uintptr_t)(p) & (Py_uintptr_t)((a) - 1))) - -#ifdef __GNUC__ -#define Py_UNUSED(name) _unused_ ## name __attribute__((unused)) -#else -#define Py_UNUSED(name) _unused_ ## name -#endif - #endif /* Py_PYMACRO_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pymath.h --- a/Include/pymath.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pymath.h Mon Jan 25 17:05:13 2016 +0100 @@ -150,29 +150,7 @@ * doesn't support NaNs. */ #if !defined(Py_NAN) && !defined(Py_NO_NAN) -#if !defined(__INTEL_COMPILER) - #define Py_NAN (Py_HUGE_VAL * 0.) -#else /* __INTEL_COMPILER */ - #if defined(ICC_NAN_STRICT) - #pragma float_control(push) - #pragma float_control(precise, on) - #pragma float_control(except, on) - #if defined(_MSC_VER) - __declspec(noinline) - #else /* Linux */ - __attribute__((noinline)) - #endif /* _MSC_VER */ - static double __icc_nan() - { - return sqrt(-1.0); - } - #pragma float_control (pop) - #define Py_NAN __icc_nan() - #else /* ICC_NAN_RELAXED as default for Intel Compiler */ - static const union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; - #define Py_NAN (__nan_store.__icc_nan) - #endif /* ICC_NAN_STRICT */ -#endif /* __INTEL_COMPILER */ +#define Py_NAN (Py_HUGE_VAL * 0.) #endif /* Py_OVERFLOWED(X) diff -r 6db40a9955dc -r 0d413f60cc23 Include/pymem.h --- a/Include/pymem.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pymem.h Mon Jan 25 17:05:13 2016 +0100 @@ -11,14 +11,6 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); -PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyMem_RawFree(void *ptr); -#endif - - /* BEWARE: Each interface exports both functions and macros. Extension modules should @@ -57,17 +49,21 @@ performed on failure (no exception is set, no warning is printed, etc). */ -PyAPI_FUNC(void *) PyMem_Malloc(size_t size); -PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyMem_Free(void *ptr); +PyAPI_FUNC(void *) PyMem_Malloc(size_t); +PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t); +PyAPI_FUNC(void) PyMem_Free(void *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); -PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); -#endif +/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are + no longer supported. They used to call PyErr_NoMemory() on failure. */ /* Macros. */ +#ifdef PYMALLOC_DEBUG +/* Redirect all memory operations to Python's debugging allocator. */ +#define PyMem_MALLOC _PyMem_DebugMalloc +#define PyMem_REALLOC _PyMem_DebugRealloc +#define PyMem_FREE _PyMem_DebugFree + +#else /* ! PYMALLOC_DEBUG */ /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL for malloc(0), which would be treated as an error. Some platforms @@ -75,9 +71,13 @@ pymalloc. To solve these problems, allocate an extra byte. */ /* Returns NULL to indicate error if a negative size or size larger than Py_ssize_t can represent is supplied. Helps prevents security holes. */ -#define PyMem_MALLOC(n) PyMem_Malloc(n) -#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n) -#define PyMem_FREE(p) PyMem_Free(p) +#define PyMem_MALLOC(n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \ + : malloc((n) ? (n) : 1)) +#define PyMem_REALLOC(p, n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \ + : realloc((p), (n) ? (n) : 1)) +#define PyMem_FREE free + +#endif /* PYMALLOC_DEBUG */ /* * Type-oriented memory interface @@ -115,72 +115,6 @@ #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE -#ifndef Py_LIMITED_API -typedef enum { - /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ - PYMEM_DOMAIN_RAW, - - /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */ - PYMEM_DOMAIN_MEM, - - /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */ - PYMEM_DOMAIN_OBJ -} PyMemAllocatorDomain; - -typedef struct { - /* user context passed as the first argument to the 4 functions */ - void *ctx; - - /* allocate a memory block */ - void* (*malloc) (void *ctx, size_t size); - - /* allocate a memory block initialized by zeros */ - void* (*calloc) (void *ctx, size_t nelem, size_t elsize); - - /* allocate or resize a memory block */ - void* (*realloc) (void *ctx, void *ptr, size_t new_size); - - /* release a memory block */ - void (*free) (void *ctx, void *ptr); -} PyMemAllocatorEx; - -/* Get the memory block allocator of the specified domain. */ -PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain, - PyMemAllocatorEx *allocator); - -/* Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-NULL pointer when requesting - zero bytes. - - For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL - is not held when the allocator is called. - - If the new allocator is not a hook (don't call the previous allocator), the - PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks - on top on the new allocator. */ -PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain, - PyMemAllocatorEx *allocator); - -/* Setup hooks to detect bugs in the following Python memory allocator - functions: - - - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree() - - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free() - - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() - - Newly allocated memory is filled with the byte 0xCB, freed memory is filled - with the byte 0xDB. Additionnal checks: - - - detect API violations, ex: PyObject_Free() called on a buffer allocated - by PyMem_Malloc() - - detect write before the start of the buffer (buffer underflow) - - detect write after the end of the buffer (buffer overflow) - - The function does nothing if Python is not compiled is debug mode. */ -PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); -#endif - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/pyport.h --- a/Include/pyport.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pyport.h Mon Jan 25 17:05:13 2016 +0100 @@ -80,53 +80,40 @@ #endif #endif /* HAVE_LONG_LONG */ -/* a build with 30-bit digits for Python integers needs an exact-width +/* a build with 30-bit digits for Python long integers needs an exact-width * 32-bit unsigned integer type to store those digits. (We could just use * type 'unsigned long', but that would be wasteful on a system where longs * are 64-bits.) On Unix systems, the autoconf macro AC_TYPE_UINT32_T defines * uint32_t to be such a type unless stdint.h or inttypes.h defines uint32_t. * However, it doesn't set HAVE_UINT32_T, so we do that here. */ -#ifdef uint32_t +#if (defined UINT32_MAX || defined uint32_t) +#ifndef PY_UINT32_T #define HAVE_UINT32_T 1 -#endif - -#ifdef HAVE_UINT32_T -#ifndef PY_UINT32_T #define PY_UINT32_T uint32_t #endif #endif /* Macros for a 64-bit unsigned integer type; used for type 'twodigits' in the - * integer implementation, when 30-bit digits are enabled. + * long integer implementation, when 30-bit digits are enabled. */ -#ifdef uint64_t +#if (defined UINT64_MAX || defined uint64_t) +#ifndef PY_UINT64_T #define HAVE_UINT64_T 1 -#endif - -#ifdef HAVE_UINT64_T -#ifndef PY_UINT64_T #define PY_UINT64_T uint64_t #endif #endif /* Signed variants of the above */ -#ifdef int32_t +#if (defined INT32_MAX || defined int32_t) +#ifndef PY_INT32_T #define HAVE_INT32_T 1 -#endif - -#ifdef HAVE_INT32_T -#ifndef PY_INT32_T #define PY_INT32_T int32_t #endif #endif - -#ifdef int64_t +#if (defined INT64_MAX || defined int64_t) +#ifndef PY_INT64_T #define HAVE_INT64_T 1 -#endif - -#ifdef HAVE_INT64_T -#ifndef PY_INT64_T #define PY_INT64_T int64_t #endif #endif @@ -144,6 +131,23 @@ #endif #endif +/* Prime multiplier used in string and various other hashes. */ +#define _PyHASH_MULTIPLIER 1000003 /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Objects/object.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +#define _PyHASH_BITS 61 +#else +#define _PyHASH_BITS 31 +#endif +#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define _PyHASH_INF 314159 +#define _PyHASH_NAN 0 +#define _PyHASH_IMAG _PyHASH_MULTIPLIER + /* uintptr_t is the C9X name for an unsigned integral type such that a * legitimate void* can be cast to uintptr_t and then back to void* again * without loss of information. Similarly for intptr_t, wrt a signed @@ -182,19 +186,10 @@ #endif /* Py_hash_t is the same size as a pointer. */ -#define SIZEOF_PY_HASH_T SIZEOF_SIZE_T typedef Py_ssize_t Py_hash_t; /* Py_uhash_t is the unsigned equivalent needed to calculate numeric hash. */ -#define SIZEOF_PY_UHASH_T SIZEOF_SIZE_T typedef size_t Py_uhash_t; -/* Only used for compatibility with code that may not be PY_SSIZE_T_CLEAN. */ -#ifdef PY_SSIZE_T_CLEAN -typedef Py_ssize_t Py_ssize_clean_t; -#else -typedef int Py_ssize_clean_t; -#endif - /* Largest possible value of size_t. SIZE_MAX is part of C99, so it might be defined on some platforms. If it is not defined, (size_t)-1 is a portable @@ -211,6 +206,10 @@ /* Smallest negative value of type Py_ssize_t. */ #define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1) +#if SIZEOF_PID_T > SIZEOF_LONG +# error "Python doesn't support sizeof(pid_t) > sizeof(long)" +#endif + /* PY_FORMAT_SIZE_T is a platform-specific modifier for use in a printf * format to convert an argument with the width of a size_t or Py_ssize_t. * C99 introduced "z" for this purpose, but not all platforms support that; @@ -255,7 +254,7 @@ */ #ifdef HAVE_LONG_LONG # ifndef PY_FORMAT_LONG_LONG -# ifdef MS_WINDOWS +# if defined(MS_WIN64) || defined(MS_WINDOWS) # define PY_FORMAT_LONG_LONG "I64" # else # error "This platform's pyconfig.h needs to define PY_FORMAT_LONG_LONG" @@ -270,7 +269,7 @@ * for platforms that support that. * * If PY_LOCAL_AGGRESSIVE is defined before python.h is included, more - * "aggressive" inlining/optimization is enabled for the entire module. This + * "aggressive" inlining/optimizaion is enabled for the entire module. This * may lead to code bloat, and may slow things down for those reasons. It may * also lead to errors, if the code relies on pointer aliasing. Use with * care. @@ -357,21 +356,40 @@ * stat() and fstat() fiddling * *******************************/ +/* We expect that stat and fstat exist on most systems. + * It's confirmed on Unix, Mac and Windows. + * If you don't have them, add + * #define DONT_HAVE_STAT + * and/or + * #define DONT_HAVE_FSTAT + * to your pyconfig.h. Python code beyond this should check HAVE_STAT and + * HAVE_FSTAT instead. + * Also + * #define HAVE_SYS_STAT_H + * if exists on your platform, and + * #define HAVE_STAT_H + * if does. + */ +#ifndef DONT_HAVE_STAT +#define HAVE_STAT +#endif + +#ifndef DONT_HAVE_FSTAT +#define HAVE_FSTAT +#endif + #ifdef HAVE_SYS_STAT_H +#if defined(PYOS_OS2) && defined(PYCC_GCC) +#include +#endif #include #elif defined(HAVE_STAT_H) #include #endif -#ifndef S_IFMT +#if defined(PYCC_VACPP) /* VisualAge C/C++ Failed to Define MountType Field in sys/stat.h */ -#define S_IFMT 0170000 -#endif - -#ifndef S_IFLNK -/* Windows doesn't define S_IFLNK but posixmodule.c maps - * IO_REPARSE_TAG_SYMLINK to S_IFLNK */ -# define S_IFLNK 0120000 +#define S_IFMT (S_IFDIR|S_IFCHR|S_IFREG) #endif #ifndef S_ISREG @@ -382,9 +400,6 @@ #define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) #endif -#ifndef S_ISCHR -#define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR) -#endif #ifdef __cplusplus /* Move this down here since some C++ #include's don't like to be included @@ -542,49 +557,6 @@ _Py_set_387controlword(old_387controlword) #endif -/* get and set x87 control word for VisualStudio/x86 */ -#if defined(_MSC_VER) && !defined(_WIN64) /* x87 not supported in 64-bit */ -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_387controlword, new_387controlword, out_387controlword -/* We use the __control87_2 function to set only the x87 control word. - The SSE control word is unaffected. */ -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __control87_2(0, 0, &old_387controlword, NULL); \ - new_387controlword = \ - (old_387controlword & ~(_MCW_PC | _MCW_RC)) | (_PC_53 | _RC_NEAR); \ - if (new_387controlword != old_387controlword) \ - __control87_2(new_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_387controlword != old_387controlword) \ - __control87_2(old_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) -#endif - -#ifdef HAVE_GCC_ASM_FOR_MC68881 -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_fpcr, new_fpcr -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ - /* Set double precision / round to nearest. */ \ - new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr)); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ - } while (0) -#endif - /* default definitions are empty */ #ifndef HAVE_PY_SET_53BIT_PRECISION #define _Py_SET_53BIT_PRECISION_HEADER @@ -653,7 +625,7 @@ /* On QNX 6, struct termio must be declared by including sys/termio.h if TCGETA, TCSETA, TCSETAW, or TCSETAF are used. sys/termio.h must be included before termios.h or it will generate an error. */ -#if defined(HAVE_SYS_TERMIO_H) && !defined(__hpux) +#ifdef HAVE_SYS_TERMIO_H #include #endif @@ -826,6 +798,15 @@ #endif /* + * Add PyArg_ParseTuple format where available. + */ +#ifdef HAVE_ATTRIBUTE_FORMAT_PARSETUPLE +#define Py_FORMAT_PARSETUPLE(func,p1,p2) __attribute__((format(func,p1,p2))) +#else +#define Py_FORMAT_PARSETUPLE(func,p1,p2) +#endif + +/* * Specify alignment on compilers that support it. */ #if defined(__GNUC__) && __GNUC__ >= 3 @@ -863,38 +844,4 @@ #endif #endif -/* - * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is - * detected by configure and defined in pyconfig.h. The code in pyconfig.h - * also takes care of Apple's universal builds. - */ - -#ifdef WORDS_BIGENDIAN -#define PY_BIG_ENDIAN 1 -#define PY_LITTLE_ENDIAN 0 -#else -#define PY_BIG_ENDIAN 0 -#define PY_LITTLE_ENDIAN 1 -#endif - -#ifdef Py_BUILD_CORE -/* - * Macros to protect CRT calls against instant termination when passed an - * invalid parameter (issue23524). - */ -#if defined _MSC_VER && _MSC_VER >= 1900 - -extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; -#define _Py_BEGIN_SUPPRESS_IPH { _invalid_parameter_handler _Py_old_handler = \ - _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); -#define _Py_END_SUPPRESS_IPH _set_thread_local_invalid_parameter_handler(_Py_old_handler); } - -#else - -#define _Py_BEGIN_SUPPRESS_IPH -#define _Py_END_SUPPRESS_IPH - -#endif /* _MSC_VER >= 1900 */ -#endif /* Py_BUILD_CORE */ - #endif /* Py_PYPORT_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pystate.h --- a/Include/pystate.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pystate.h Mon Jan 25 17:05:13 2016 +0100 @@ -25,7 +25,7 @@ PyObject *modules_by_index; PyObject *sysdict; PyObject *builtins; - PyObject *importlib; + PyObject *modules_reloading; PyObject *codec_search_path; PyObject *codec_search_cache; @@ -40,7 +40,6 @@ int tscdump; #endif - PyObject *builtins_copy; } PyInterpreterState; #endif @@ -69,7 +68,6 @@ typedef struct _ts { /* See Python/ceval.c for comments explaining most fields */ - struct _ts *prev; struct _ts *next; PyInterpreterState *interp; @@ -100,43 +98,21 @@ PyObject *dict; /* Stores per-thread state */ + /* XXX doesn't mean anything anymore (the comment below is obsolete) + => deprecate or remove? */ + /* tick_counter is incremented whenever the check_interval ticker + * reaches zero. The purpose is to give a useful measure of the number + * of interpreted bytecode instructions in a given thread. This + * extremely lightweight statistic collector may be of interest to + * profilers (like psyco.jit()), although nothing in the core uses it. + */ + int tick_counter; + int gilstate_counter; PyObject *async_exc; /* Asynchronous exception to raise */ long thread_id; /* Thread id where this tstate was created */ - int trash_delete_nesting; - PyObject *trash_delete_later; - - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyThreadState_Current is already NULL, so only the simplest of C code - * can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. - */ - void (*on_delete)(void *); - void *on_delete_data; - - PyObject *coroutine_wrapper; - int in_coroutine_wrapper; - /* XXX signal handlers should also be here */ } PyThreadState; @@ -147,38 +123,19 @@ PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 -/* New in 3.3 */ -PyAPI_FUNC(int) PyState_AddModule(PyObject*, struct PyModuleDef*); -PyAPI_FUNC(int) PyState_RemoveModule(struct PyModuleDef*); -#endif PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyState_ClearModules(void); -#endif PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *); PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *); PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *); PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); -PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); #ifdef WITH_THREAD PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); PyAPI_FUNC(void) _PyGILState_Reinit(void); #endif -/* Return the current thread state. The global interpreter lock must be held. - * When the current thread state is NULL, this issues a fatal error (so that - * the caller needn't check for NULL). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); - -#ifdef WITH_THREAD -/* Similar to PyThreadState_Get(), but don't issue a fatal error - * if it is NULL. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); -#endif - PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); PyAPI_FUNC(PyObject *) PyThreadState_GetDict(void); PyAPI_FUNC(int) PyThreadState_SetAsyncExc(long, PyObject *); @@ -188,12 +145,15 @@ /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ -#ifdef Py_BUILD_CORE +#ifndef Py_LIMITED_API PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current; -# define PyThreadState_GET() \ - ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)) +#endif + +#if defined(Py_DEBUG) || defined(Py_LIMITED_API) +#define PyThreadState_GET() PyThreadState_Get() #else -# define PyThreadState_GET() PyThreadState_Get() +#define PyThreadState_GET() \ + ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)) #endif typedef @@ -243,13 +203,6 @@ */ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); -/* Helper/diagnostic function - return 1 if the current thread - * currently holds the GIL, 0 otherwise - */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyGILState_Check(void); -#endif - #endif /* #ifdef WITH_THREAD */ /* The implementation of sys._current_frames() Returns a dict mapping diff -r 6db40a9955dc -r 0d413f60cc23 Include/pystrhex.h --- a/Include/pystrhex.h Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -#ifndef Py_STRHEX_H -#define Py_STRHEX_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* Returns a str() containing the hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); -/* Returns a bytes() containing the ASCII hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen); - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_STRHEX_H */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/pythonrun.h --- a/Include/pythonrun.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pythonrun.h Mon Jan 25 17:05:13 2016 +0100 @@ -9,8 +9,7 @@ #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ - CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ - CO_FUTURE_GENERATOR_STOP) + CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL) #define PyCF_MASK_OBSOLETE (CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 @@ -23,6 +22,19 @@ } PyCompilerFlags; #endif +PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + +PyAPI_FUNC(void) Py_Initialize(void); +PyAPI_FUNC(void) Py_InitializeEx(int); +PyAPI_FUNC(void) Py_Finalize(void); +PyAPI_FUNC(int) Py_IsInitialized(void); +PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); +PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); + #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); @@ -40,10 +52,6 @@ FILE *fp, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); -PyAPI_FUNC(int) PyRun_InteractiveOneObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ @@ -55,29 +63,13 @@ int start, PyCompilerFlags *flags, PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject( - const char *s, - PyObject *filename, - int start, - PyCompilerFlags *flags, - PyArena *arena); PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ const char* enc, int start, - const char *ps1, - const char *ps2, - PyCompilerFlags *flags, - int *errcode, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( - FILE *fp, - PyObject *filename, - const char* enc, - int start, - const char *ps1, - const char *ps2, + char *ps1, + char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); @@ -90,12 +82,9 @@ PyParser_SimpleParseFileFlags(FP, S, B, 0) #endif PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, - int); -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *, - const char *, - int, int); + int); PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, - int, int); + int, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, @@ -122,27 +111,36 @@ int start, PyCompilerFlags *flags, int optimize); -PyAPI_FUNC(PyObject *) Py_CompileStringObject( - const char *str, - PyObject *filename, int start, - PyCompilerFlags *flags, - int optimize); #endif PyAPI_FUNC(struct symtable *) Py_SymtableString( const char *str, const char *filename, /* decoded from the filesystem encoding */ int start); -#ifndef Py_LIMITED_API -PyAPI_FUNC(struct symtable *) Py_SymtableStringObject( - const char *str, - PyObject *filename, - int start); -#endif PyAPI_FUNC(void) PyErr_Print(void); PyAPI_FUNC(void) PyErr_PrintEx(int); PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *); +/* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level + * exit functions. + */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void)); +#endif +PyAPI_FUNC(int) Py_AtExit(void (*func)(void)); + +PyAPI_FUNC(void) Py_Exit(int); + +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _Py_RestoreSignals(void); + +PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); +#endif + +/* Bootstrap */ +PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); + #ifndef Py_LIMITED_API /* Use macros for a bunch of old variants */ #define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL) @@ -164,12 +162,67 @@ PyRun_FileExFlags(fp, p, s, g, l, 0, flags) #endif +/* In getpath.c */ +PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); +PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); +PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); +PyAPI_FUNC(wchar_t *) Py_GetPath(void); +PyAPI_FUNC(void) Py_SetPath(const wchar_t *); +#ifdef MS_WINDOWS +int _Py_CheckPython3(); +#endif + +/* In their own files */ +PyAPI_FUNC(const char *) Py_GetVersion(void); +PyAPI_FUNC(const char *) Py_GetPlatform(void); +PyAPI_FUNC(const char *) Py_GetCopyright(void); +PyAPI_FUNC(const char *) Py_GetCompiler(void); +PyAPI_FUNC(const char *) Py_GetBuildInfo(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(const char *) _Py_hgidentifier(void); +PyAPI_FUNC(const char *) _Py_hgversion(void); +#endif + +/* Internal -- various one-time initializations */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); +PyAPI_FUNC(PyObject *) _PySys_Init(void); +PyAPI_FUNC(void) _PyImport_Init(void); +PyAPI_FUNC(void) _PyExc_Init(void); +PyAPI_FUNC(void) _PyImportHooks_Init(void); +PyAPI_FUNC(int) _PyFrame_Init(void); +PyAPI_FUNC(void) _PyFloat_Init(void); +PyAPI_FUNC(int) PyByteArray_Init(void); +PyAPI_FUNC(void) _PyRandom_Init(void); +#endif + +/* Various internal finalizers */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyExc_Fini(void); +PyAPI_FUNC(void) _PyImport_Fini(void); +PyAPI_FUNC(void) PyMethod_Fini(void); +PyAPI_FUNC(void) PyFrame_Fini(void); +PyAPI_FUNC(void) PyCFunction_Fini(void); +PyAPI_FUNC(void) PyDict_Fini(void); +PyAPI_FUNC(void) PyTuple_Fini(void); +PyAPI_FUNC(void) PyList_Fini(void); +PyAPI_FUNC(void) PySet_Fini(void); +PyAPI_FUNC(void) PyBytes_Fini(void); +PyAPI_FUNC(void) PyByteArray_Fini(void); +PyAPI_FUNC(void) PyFloat_Fini(void); +PyAPI_FUNC(void) PyOS_FiniInterrupts(void); +PyAPI_FUNC(void) _PyGC_Fini(void); +PyAPI_FUNC(void) PySlice_Fini(void); + +PyAPI_DATA(PyThreadState *) _Py_Finalizing; +#endif + /* Stuff with no proper home (yet) */ #ifndef Py_LIMITED_API -PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); +PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, char *); #endif PyAPI_DATA(int) (*PyOS_InputHook)(void); -PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); +PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, char *); #ifndef Py_LIMITED_API PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; #endif @@ -189,6 +242,14 @@ PyAPI_FUNC(int) PyOS_CheckStack(void); #endif +/* Signals */ +typedef void (*PyOS_sighandler_t)(int); +PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int); +PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); + +/* Random */ +PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size); + #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/pytime.h --- a/Include/pytime.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/pytime.h Mon Jan 25 17:05:13 2016 +0100 @@ -13,42 +13,36 @@ extern "C" { #endif -#ifdef PY_INT64_T -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef PY_INT64_T _PyTime_t; -#define _PyTime_MIN PY_LLONG_MIN -#define _PyTime_MAX PY_LLONG_MAX +#ifdef HAVE_GETTIMEOFDAY +typedef struct timeval _PyTime_timeval; #else -# error "_PyTime_t need signed 64-bit integer type" +typedef struct { + time_t tv_sec; /* seconds since Jan. 1, 1970 */ + long tv_usec; /* and microseconds */ +} _PyTime_timeval; #endif -typedef enum { - /* Round towards minus infinity (-inf). - For example, used to read a clock. */ - _PyTime_ROUND_FLOOR=0, - /* Round towards infinity (+inf). - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going to nearest even integer. - For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_EVEN -} _PyTime_round_t; +/* Similar to POSIX gettimeofday but cannot fail. If system gettimeofday + * fails or is not available, fall back to lower resolution clocks. + */ +PyAPI_FUNC(void) _PyTime_gettimeofday(_PyTime_timeval *tp); -/* Convert a time_t to a PyLong. */ -PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( - time_t sec); +#define _PyTime_ADD_SECONDS(tv, interval) \ +do { \ + tv.tv_usec += (long) (((long) interval - interval) * 1000000); \ + tv.tv_sec += (time_t) interval + (time_t) (tv.tv_usec / 1000000); \ + tv.tv_usec %= 1000000; \ +} while (0) -/* Convert a PyLong to a time_t. */ -PyAPI_FUNC(time_t) _PyLong_AsTime_t( - PyObject *obj); +#define _PyTime_INTERVAL(tv_start, tv_end) \ + ((tv_end.tv_sec - tv_start.tv_sec) + \ + (tv_end.tv_usec - tv_start.tv_usec) * 0.000001) +#ifndef Py_LIMITED_API /* Convert a number of seconds, int or float, to time_t. */ PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, - time_t *sec, - _PyTime_round_t); + time_t *sec); /* Convert a number of seconds, int or float, to a timeval structure. usec is in the range [0; 999999] and rounded towards zero. @@ -56,8 +50,7 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimeval( PyObject *obj, time_t *sec, - long *usec, - _PyTime_round_t); + long *usec); /* Convert a number of seconds, int or float, to a timespec structure. nsec is in the range [0; 999999999] and rounded towards zero. @@ -65,128 +58,11 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimespec( PyObject *obj, time_t *sec, - long *nsec, - _PyTime_round_t); - - -/* Create a timestamp from a number of seconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); - -/* Macro to create a timestamp from a number of seconds, no integer overflow. - Only use the macro for small values, prefer _PyTime_FromSeconds(). */ -#define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -/* Create a timestamp from a number of nanoseconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(PY_LONG_LONG ns); - -/* Convert a number of seconds (Python float or int) to a timetamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timetamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds as a C double. */ -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - -/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of microseconds (10^-6 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int - object. */ -PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); - -/* Convert a timestamp to a timeval structure (microsecond resolution). - tv_usec is always positive. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */ -PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds (secs) and microseconds (us). - us is always positive. This function is similar to _PyTime_AsTimeval() - except that secs is always a time_t type, whereas the timeval structure - uses a C long for tv_sec on Windows. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, - time_t *secs, - int *us, - _PyTime_round_t round); - -#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -/* Convert a timestamp to a timespec structure (nanosecond resolution). - tv_nsec is always positive. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); + long *nsec); #endif -/* Get the current time from the system clock. - - The function cannot fail. _PyTime_Init() ensures that the system clock - works. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - The function cannot fail. _PyTime_Init() ensures that a monotonic clock - is available and works. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - - -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; - -/* Get the current time from the system clock. - * Fill clock information if info is not NULL. - * Raise an exception and return -1 on error, return 0 on success. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -/* Initialize time. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_Init(void); +/* Dummy to force linking. */ +PyAPI_FUNC(void) _PyTime_Init(void); #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/setobject.h --- a/Include/setobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/setobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -6,44 +6,38 @@ extern "C" { #endif + +/* +There are three kinds of slots in the table: + +1. Unused: key == NULL +2. Active: key != NULL and key != dummy +3. Dummy: key == dummy + +Note: .pop() abuses the hash field of an Unused or Dummy slot to +hold a search finger. The hash field of Unused or Dummy slots has +no meaning otherwise. +*/ #ifndef Py_LIMITED_API - -/* There are three kinds of entries in the table: - -1. Unused: key == NULL and hash == 0 -2. Dummy: key == dummy and hash == -1 -3. Active: key != NULL and key != dummy and hash != -1 - -The hash field of Unused slots is always zero. - -The hash field of Dummy slots are set to -1 -meaning that dummy entries can be detected by -either entry->key==dummy or by entry->hash==-1. -*/ - #define PySet_MINSIZE 8 typedef struct { + /* Cached hash code of the key. */ + Py_hash_t hash; PyObject *key; - Py_hash_t hash; /* Cached hash code of the key */ } setentry; -/* The SetObject data structure is shared by set and frozenset objects. -Invariant for sets: - - hash is -1 - -Invariants for frozensets: - - data is immutable. - - hash is the hash of the frozenset or -1 if not computed yet. - +/* +This data structure is shared by set and frozenset objects. */ -typedef struct { +typedef struct _setobject PySetObject; +struct _setobject { PyObject_HEAD - Py_ssize_t fill; /* Number active and dummy entries*/ - Py_ssize_t used; /* Number active entries */ + Py_ssize_t fill; /* # Active + # Dummy */ + Py_ssize_t used; /* # Active */ /* The table contains mask + 1 slots, and that's a power of 2. * We store the mask instead of the size because the mask is more @@ -51,42 +45,29 @@ */ Py_ssize_t mask; - /* The table points to a fixed-size smalltable for small tables - * or to additional malloc'ed memory for bigger tables. - * The table pointer is never NULL which saves us from repeated - * runtime null-tests. + /* table points to smalltable for small tables, else to + * additional malloc'ed memory. table is never NULL! This rule + * saves repeated runtime null-tests. */ setentry *table; - Py_hash_t hash; /* Only used by frozenset objects */ - Py_ssize_t finger; /* Search finger for pop() */ + setentry *(*lookup)(PySetObject *so, PyObject *key, Py_hash_t hash); + setentry smalltable[PySet_MINSIZE]; - setentry smalltable[PySet_MINSIZE]; + Py_hash_t hash; /* only used by frozenset objects */ PyObject *weakreflist; /* List of weak references */ -} PySetObject; - -#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used) - -PyAPI_DATA(PyObject *) _PySet_Dummy; - -PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); -PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); -PyAPI_FUNC(int) PySet_ClearFreeList(void); - -#endif /* Section excluded by Py_LIMITED_API */ +}; +#endif /* Py_LIMITED_API */ PyAPI_DATA(PyTypeObject) PySet_Type; PyAPI_DATA(PyTypeObject) PyFrozenSet_Type; PyAPI_DATA(PyTypeObject) PySetIter_Type; -PyAPI_FUNC(PyObject *) PySet_New(PyObject *); -PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *); - -PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key); -PyAPI_FUNC(int) PySet_Clear(PyObject *set); -PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key); -PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key); -PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); -PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset); +/* Invariants for frozensets: + * data is immutable. + * hash is the hash of the frozenset or -1 if not computed yet. + * Invariants for sets: + * hash is -1 + */ #define PyFrozenSet_CheckExact(ob) (Py_TYPE(ob) == &PyFrozenSet_Type) #define PyAnySet_CheckExact(ob) \ @@ -102,6 +83,26 @@ (Py_TYPE(ob) == &PyFrozenSet_Type || \ PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) +PyAPI_FUNC(PyObject *) PySet_New(PyObject *); +PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *); +PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset); +#ifndef Py_LIMITED_API +#define PySet_GET_SIZE(so) (((PySetObject *)(so))->used) +#endif +PyAPI_FUNC(int) PySet_Clear(PyObject *set); +PyAPI_FUNC(int) PySet_Contains(PyObject *anyset, PyObject *key); +PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key); +PyAPI_FUNC(int) PySet_Add(PyObject *set, PyObject *key); +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); +#endif +PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); + +PyAPI_FUNC(int) PySet_ClearFreeList(void); +#endif + #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/sliceobject.h --- a/Include/sliceobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/sliceobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -34,14 +34,11 @@ PyObject* step); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop); -PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, - PyObject **start_ptr, PyObject **stop_ptr, - PyObject **step_ptr); #endif PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step); PyAPI_FUNC(int) PySlice_GetIndicesEx(PyObject *r, Py_ssize_t length, - Py_ssize_t *start, Py_ssize_t *stop, + Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step, Py_ssize_t *slicelength); #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 Include/structmember.h --- a/Include/structmember.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/structmember.h Mon Jan 25 17:05:13 2016 +0100 @@ -16,41 +16,42 @@ pointer is NULL. */ typedef struct PyMemberDef { - char *name; - int type; - Py_ssize_t offset; - int flags; - char *doc; + /* Current version, use this */ + char *name; + int type; + Py_ssize_t offset; + int flags; + char *doc; } PyMemberDef; /* Types */ -#define T_SHORT 0 -#define T_INT 1 -#define T_LONG 2 -#define T_FLOAT 3 -#define T_DOUBLE 4 -#define T_STRING 5 -#define T_OBJECT 6 +#define T_SHORT 0 +#define T_INT 1 +#define T_LONG 2 +#define T_FLOAT 3 +#define T_DOUBLE 4 +#define T_STRING 5 +#define T_OBJECT 6 /* XXX the ordering here is weird for binary compatibility */ -#define T_CHAR 7 /* 1-character string */ -#define T_BYTE 8 /* 8-bit signed int */ +#define T_CHAR 7 /* 1-character string */ +#define T_BYTE 8 /* 8-bit signed int */ /* unsigned variants: */ -#define T_UBYTE 9 -#define T_USHORT 10 -#define T_UINT 11 -#define T_ULONG 12 +#define T_UBYTE 9 +#define T_USHORT 10 +#define T_UINT 11 +#define T_ULONG 12 /* Added by Jack: strings contained in the structure */ -#define T_STRING_INPLACE 13 +#define T_STRING_INPLACE 13 /* Added by Lillo: bools contained in the structure (assumed char) */ -#define T_BOOL 14 +#define T_BOOL 14 -#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError - when the value is NULL, instead of - converting to None. */ +#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError + when the value is NULL, instead of + converting to None. */ #ifdef HAVE_LONG_LONG -#define T_LONGLONG 17 +#define T_LONGLONG 17 #define T_ULONGLONG 18 #endif /* HAVE_LONG_LONG */ @@ -59,10 +60,10 @@ /* Flags */ -#define READONLY 1 -#define READ_RESTRICTED 2 +#define READONLY 1 +#define READ_RESTRICTED 2 #define PY_WRITE_RESTRICTED 4 -#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) +#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) /* Current API, use this */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/structseq.h --- a/Include/structseq.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/structseq.h Mon Jan 25 17:05:13 2016 +0100 @@ -24,8 +24,6 @@ #ifndef Py_LIMITED_API PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc); -PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type, - PyStructSequence_Desc *desc); #endif PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc); diff -r 6db40a9955dc -r 0d413f60cc23 Include/symtable.h --- a/Include/symtable.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/symtable.h Mon Jan 25 17:05:13 2016 +0100 @@ -16,7 +16,7 @@ struct _symtable_entry; struct symtable { - PyObject *st_filename; /* name of file being compiled, + const char *st_filename; /* name of file being compiled, decoded from the filesystem encoding */ struct _symtable_entry *st_cur; /* current symbol table entry */ struct _symtable_entry *st_top; /* symbol table entry for module */ @@ -30,8 +30,6 @@ PyObject *st_private; /* name of current class or NULL */ PyFutureFeatures *st_future; /* module's future features that affect the symbol table */ - int recursion_depth; /* current recursion depth */ - int recursion_limit; /* recursion limit */ }; typedef struct _symtable_entry { @@ -41,8 +39,8 @@ PyObject *ste_name; /* string: name of current block */ PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ - PyObject *ste_directives;/* locations of global and nonlocal statements */ _Py_block_ty ste_type; /* module, class, or function */ + int ste_unoptimized; /* false if namespace is optimized */ int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ unsigned ste_child_free : 1; /* true if a child block has free vars, @@ -52,9 +50,6 @@ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ unsigned ste_returns_value : 1; /* true if namespace uses return with an argument */ - unsigned ste_needs_class_closure : 1; /* for class scopes, true if a - closure over __class__ - should be created */ int ste_lineno; /* first line of block */ int ste_col_offset; /* offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ @@ -73,10 +68,6 @@ mod_ty mod, const char *filename, /* decoded from the filesystem encoding */ PyFutureFeatures *future); -PyAPI_FUNC(struct symtable *) PySymtable_BuildObject( - mod_ty mod, - PyObject *filename, - PyFutureFeatures *future); PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); PyAPI_FUNC(void) PySymtable_Free(struct symtable *); @@ -107,6 +98,10 @@ #define FREE 4 #define CELL 5 +/* The following two names are used for the ste_unoptimized bit field */ +#define OPT_IMPORT_STAR 1 +#define OPT_TOPLEVEL 2 /* top-level names, including eval and exec */ + #define GENERATOR 1 #define GENERATOR_EXPRESSION 2 diff -r 6db40a9955dc -r 0d413f60cc23 Include/sysmodule.h --- a/Include/sysmodule.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/sysmodule.h Mon Jan 25 17:05:13 2016 +0100 @@ -8,12 +8,7 @@ #endif PyAPI_FUNC(PyObject *) PySys_GetObject(const char *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key); -#endif PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *); -PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *); - PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **); PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); @@ -25,6 +20,10 @@ PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); +#ifndef Py_LIMITED_API +PyAPI_DATA(PyObject *) _PySys_TraceFunc, *_PySys_ProfileFunc; +#endif + PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(void) PySys_AddWarnOption(const wchar_t *); PyAPI_FUNC(void) PySys_AddWarnOptionUnicode(PyObject *); @@ -33,10 +32,6 @@ PyAPI_FUNC(void) PySys_AddXOption(const wchar_t *); PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); -#endif - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/token.h --- a/Include/token.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/token.h Mon Jan 25 17:05:13 2016 +0100 @@ -58,16 +58,13 @@ #define DOUBLESTAREQUAL 46 #define DOUBLESLASH 47 #define DOUBLESLASHEQUAL 48 -#define AT 49 -#define ATEQUAL 50 -#define RARROW 51 -#define ELLIPSIS 52 +#define AT 49 +#define RARROW 50 +#define ELLIPSIS 51 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */ -#define OP 53 -#define AWAIT 54 -#define ASYNC 55 -#define ERRORTOKEN 56 -#define N_TOKENS 57 +#define OP 52 +#define ERRORTOKEN 53 +#define N_TOKENS 54 /* Special definitions for cooperation with parser */ @@ -78,7 +75,7 @@ #define ISEOF(x) ((x) == ENDMARKER) -PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */ +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) PyToken_OneChar(int); PyAPI_FUNC(int) PyToken_TwoChars(int, int); PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); diff -r 6db40a9955dc -r 0d413f60cc23 Include/traceback.h --- a/Include/traceback.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/traceback.h Mon Jan 25 17:05:13 2016 +0100 @@ -24,7 +24,6 @@ PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int); -PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); #endif /* Reveal traceback type so we can typecheck traceback objects */ diff -r 6db40a9955dc -r 0d413f60cc23 Include/tupleobject.h --- a/Include/tupleobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/tupleobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -63,9 +63,6 @@ #endif PyAPI_FUNC(int) PyTuple_ClearFreeList(void); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); -#endif /* Py_LIMITED_API */ #ifdef __cplusplus } diff -r 6db40a9955dc -r 0d413f60cc23 Include/typeslots.h --- a/Include/typeslots.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/typeslots.h Mon Jan 25 17:05:13 2016 +0100 @@ -74,12 +74,3 @@ #define Py_tp_members 72 #define Py_tp_getset 73 #define Py_tp_free 74 -#define Py_nb_matrix_multiply 75 -#define Py_nb_inplace_matrix_multiply 76 -#define Py_am_await 77 -#define Py_am_aiter 78 -#define Py_am_anext 79 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -#define Py_tp_finalize 80 -#endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/ucnhash.h --- a/Include/ucnhash.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/ucnhash.h Mon Jan 25 17:05:13 2016 +0100 @@ -16,7 +16,7 @@ int size; /* Get name for a given character code. Returns non-zero if - success, zero if not. Does not set Python exceptions. + success, zero if not. Does not set Python exceptions. If self is NULL, data come from the default version of the database. If it is not NULL, it should be a unicodedata.ucd_X_Y_Z object */ int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen, diff -r 6db40a9955dc -r 0d413f60cc23 Include/unicodeobject.h --- a/Include/unicodeobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/unicodeobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -180,17 +180,17 @@ } while (0) /* macros to work with surrogates */ -#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF) -#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF) -#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF) +#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDFFF) +#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) +#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) /* Join two surrogate characters and return a single Py_UCS4 value. */ #define Py_UNICODE_JOIN_SURROGATES(high, low) \ (((((Py_UCS4)(high) & 0x03FF) << 10) | \ ((Py_UCS4)(low) & 0x03FF)) + 0x10000) /* high surrogate = top 10 bits added to D800 */ -#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 - (0x10000 >> 10) + ((ch) >> 10)) +#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 | (((ch) - 0x10000) >> 10)) /* low surrogate = bottom 10 bits added to DC00 */ -#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF)) +#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 | (((ch) - 0x10000) & 0x3FF)) /* Check if substring matches at given offset. The offset must be valid, and the substring must not be empty. */ @@ -343,9 +343,6 @@ the data pointer is filled out. The bit is redundant, and helps to minimize the test in PyUnicode_IS_READY(). */ unsigned int ready:1; - /* Padding to ensure that PyUnicode_DATA() is always aligned to - 4 bytes (see issue #19537 on m68k). */ - unsigned int :24; } state; wchar_t *wstr; /* wchar_t representation (null-terminated) */ } PyASCIIObject; @@ -605,7 +602,7 @@ ); #endif -/* Initializes the canonical string representation from the deprecated +/* Initializes the canonical string representation from a the deprecated wstr/Py_UNICODE representation. This function is used to convert Unicode objects which were created using the old API to the new flexible format introduced with PEP 393. @@ -651,20 +648,8 @@ Py_ssize_t from_start, Py_ssize_t how_many ); - -/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so - may crash if parameters are invalid (e.g. if the output string - is too short). */ -PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( - PyObject *to, - Py_ssize_t to_start, - PyObject *from, - Py_ssize_t from_start, - Py_ssize_t how_many - ); #endif -#ifndef Py_LIMITED_API /* Fill a string with a character: write fill_char into unicode[start:start+length]. @@ -673,21 +658,13 @@ Return the number of written character, or return -1 and raise an exception on error. */ +#ifndef Py_LIMITED_API PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( PyObject *unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char ); - -/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash - if parameters are invalid (e.g. if length is longer than the string). */ -PyAPI_FUNC(void) _PyUnicode_FastFill( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t length, - Py_UCS4 fill_char - ); #endif /* Create a Unicode Object from the Py_UNICODE buffer u of the given @@ -719,19 +696,13 @@ const char *u /* UTF-8 encoded string */ ); -#ifndef Py_LIMITED_API /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. Scan the string to find the maximum character. */ +#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( int kind, const void *buffer, Py_ssize_t size); - -/* Create a new string from a buffer of ASCII characters. - WARNING: Don't check if the string contains any non-ASCII character. */ -PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( - const char *buffer, - Py_ssize_t size); #endif PyAPI_FUNC(PyObject*) PyUnicode_Substring( @@ -739,15 +710,6 @@ Py_ssize_t start, Py_ssize_t end); -#ifndef Py_LIMITED_API -/* Compute the maximum character of the substring unicode[start:end]. - Return 127 for an empty string. */ -PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t end); -#endif - /* Copy the string into a UCS4 buffer including the null character if copy_null is set. Return NULL and raise an exception on error. Raise a ValueError if the buffer is smaller than the string. Return buffer on success. @@ -849,7 +811,7 @@ Coercion is done in the following way: - 1. bytes, bytearray and other bytes-like objects are decoded + 1. bytes, bytearray and other char buffer compatible objects are decoded under the assumptions that they contain data using the UTF-8 encoding. Decoding is done in "strict" mode. @@ -862,7 +824,7 @@ */ PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject( - PyObject *obj, /* Object */ + register PyObject *obj, /* Object */ const char *encoding, /* encoding */ const char *errors /* error handling */ ); @@ -881,7 +843,7 @@ */ PyAPI_FUNC(PyObject*) PyUnicode_FromObject( - PyObject *obj /* Object */ + register PyObject *obj /* Object */ ); PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV( @@ -894,130 +856,12 @@ ); #ifndef Py_LIMITED_API -typedef struct { - PyObject *buffer; - void *data; - enum PyUnicode_Kind kind; - Py_UCS4 maxchar; - Py_ssize_t size; - Py_ssize_t pos; - - /* minimum number of allocated characters (default: 0) */ - Py_ssize_t min_length; - - /* minimum character (default: 127, ASCII) */ - Py_UCS4 min_char; - - /* If non-zero, overallocate the buffer (default: 0). */ - unsigned char overallocate; - - /* If readonly is 1, buffer is a shared string (cannot be modified) - and size is set to 0. */ - unsigned char readonly; -} _PyUnicodeWriter ; - -/* Initialize a Unicode writer. - * - * By default, the minimum buffer size is 0 character and overallocation is - * disabled. Set min_length, min_char and overallocate attributes to control - * the allocation of the buffer. */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Init(_PyUnicodeWriter *writer); - -/* Prepare the buffer to write 'length' characters - with the specified maximum character. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_Prepare(WRITER, LENGTH, MAXCHAR) \ - (((MAXCHAR) <= (WRITER)->maxchar \ - && (LENGTH) <= (WRITER)->size - (WRITER)->pos) \ - ? 0 \ - : (((LENGTH) == 0) \ - ? 0 \ - : _PyUnicodeWriter_PrepareInternal((WRITER), (LENGTH), (MAXCHAR)))) - -/* Don't call this function directly, use the _PyUnicodeWriter_Prepare() macro - instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, - Py_ssize_t length, Py_UCS4 maxchar); - -/* Prepare the buffer to have at least the kind KIND. - For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will - support characters in range U+000-U+FFFF. - - Return 0 on success, raise an exception and return -1 on error. */ -#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ - (assert((KIND) != PyUnicode_WCHAR_KIND), \ - (KIND) <= (WRITER)->kind \ - ? 0 \ - : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) - -/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() - macro instead. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, - enum PyUnicode_Kind kind); - -/* Append a Unicode character. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, - Py_UCS4 ch - ); - -/* Append a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, - PyObject *str /* Unicode string */ - ); - -/* Append a substring of a Unicode string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, - PyObject *str, /* Unicode string */ - Py_ssize_t start, - Py_ssize_t end - ); - -/* Append an ASCII-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, - const char *str, /* ASCII-encoded byte string */ - Py_ssize_t len /* number of bytes, or -1 if unknown */ - ); - -/* Append a latin1-encoded byte string. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) -_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, - const char *str, /* latin1-encoded byte string */ - Py_ssize_t len /* length in bytes */ - ); - -/* Get the value of the writer as an Unicode string. Clear the - buffer of the writer. Raise an exception and return NULL - on error. */ -PyAPI_FUNC(PyObject *) -_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) -_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); -#endif - -#ifndef Py_LIMITED_API /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); +PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); @@ -1043,7 +887,7 @@ The buffer is copied into the new object. */ PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar( - const wchar_t *w, /* wchar_t buffer */ + register const wchar_t *w, /* wchar_t buffer */ Py_ssize_t size /* size of buffer */ ); @@ -1061,7 +905,7 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar( PyObject *unicode, /* Unicode object */ - wchar_t *w, /* wchar_t buffer */ + register wchar_t *w, /* wchar_t buffer */ Py_ssize_t size /* size of buffer */ ); @@ -1069,7 +913,7 @@ always ends with a nul character. If size is not NULL, write the number of wide characters (excluding the null character) into *size. - Returns a buffer allocated by PyMem_Malloc() (use PyMem_Free() to free it) + Returns a buffer allocated by PyMem_Alloc() (use PyMem_Free() to free it) on success. On error, returns NULL, *size is undefined and raises a MemoryError. */ @@ -1088,7 +932,8 @@ /* Create a Unicode Object from the given Unicode code point ordinal. - The ordinal must be in range(0x110000). A ValueError is + The ordinal must be in range(0x10000) on narrow Python builds + (UCS2), and range(0x110000) on wide builds (UCS4). A ValueError is raised in case it is not. */ @@ -1792,7 +1637,7 @@ /* Encode a Unicode object to the current locale encoding. The encoder is strict is *surrogateescape* is equal to zero, otherwise the "surrogateescape" error handler is used. Return a bytes object. The string - cannot contain embedded null characters. */ + cannot contain embedded null characters.. */ PyAPI_FUNC(PyObject*) PyUnicode_EncodeLocale( PyObject *unicode, @@ -2016,21 +1861,13 @@ ); /* Compare two strings and return -1, 0, 1 for less than, equal, - greater than resp. - Raise an exception and return -1 on error. */ + greater than resp. */ PyAPI_FUNC(int) PyUnicode_Compare( PyObject *left, /* Left string */ PyObject *right /* Right string */ ); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyUnicode_CompareWithId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); -#endif - PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString( PyObject *left, const char *right /* ASCII-encoded string */ @@ -2058,7 +1895,7 @@ int op /* Operation: Py_EQ, Py_NE, Py_GT, etc. */ ); -/* Apply an argument tuple or dictionary to a format string and return +/* Apply a argument tuple or dictionary to a format string and return the resulting Unicode string. */ PyAPI_FUNC(PyObject *) PyUnicode_Format( @@ -2077,6 +1914,12 @@ PyObject *element /* Element string */ ); +/* Checks whether the string contains any NUL characters. */ + +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PyUnicode_HasNULChars(PyObject *); +#endif + /* Checks whether argument is a valid identifier. */ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s); @@ -2256,8 +2099,6 @@ Py_UNICODE c ); -PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int); - /* Create a copy of a unicode string ending with a nul character. Return NULL and raise a MemoryError exception on memory allocation failure, otherwise return a new allocated buffer (use PyMem_Free() to free the buffer). */ @@ -2278,10 +2119,6 @@ /* Clear all static strings. */ PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void); -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *); - #ifdef __cplusplus } #endif diff -r 6db40a9955dc -r 0d413f60cc23 Include/warnings.h --- a/Include/warnings.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/warnings.h Mon Jan 25 17:05:13 2016 +0100 @@ -17,15 +17,6 @@ Py_ssize_t stack_level, const char *format, /* ASCII-encoded string */ ...); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyErr_WarnExplicitObject( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno, - PyObject *module, - PyObject *registry); -#endif PyAPI_FUNC(int) PyErr_WarnExplicit( PyObject *category, const char *message, /* UTF-8 encoded string */ @@ -34,14 +25,6 @@ const char *module, /* UTF-8 encoded string */ PyObject *registry); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) -PyErr_WarnExplicitFormat(PyObject *category, - const char *filename, int lineno, - const char *module, PyObject *registry, - const char *format, ...); -#endif - /* DEPRECATED: Use PyErr_WarnEx() instead. */ #ifndef Py_LIMITED_API #define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) diff -r 6db40a9955dc -r 0d413f60cc23 Include/weakrefobject.h --- a/Include/weakrefobject.h Sun Jan 24 22:15:20 2016 -0800 +++ b/Include/weakrefobject.h Mon Jan 25 17:05:13 2016 +0100 @@ -51,6 +51,9 @@ ((Py_TYPE(op) == &_PyWeakref_ProxyType) || \ (Py_TYPE(op) == &_PyWeakref_CallableProxyType)) +/* This macro calls PyWeakref_CheckRef() last since that can involve a + function call; this makes it more likely that the function call + will be avoided. */ #define PyWeakref_Check(op) \ (PyWeakref_CheckRef(op) || PyWeakref_CheckProxy(op)) @@ -67,17 +70,7 @@ PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); #endif -/* Explanation for the Py_REFCNT() check: when a weakref's target is part - of a long chain of deallocations which triggers the trashcan mechanism, - clearing the weakrefs can be delayed long after the target's refcount - has dropped to zero. In the meantime, code accessing the weakref will - be able to "see" the target object even though it is supposed to be - unreachable. See issue #16602. */ - -#define PyWeakref_GET_OBJECT(ref) \ - (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ - ? ((PyWeakReference *)(ref))->wr_object \ - : Py_None) +#define PyWeakref_GET_OBJECT(ref) (((PyWeakReference *)(ref))->wr_object) #ifdef __cplusplus diff -r 6db40a9955dc -r 0d413f60cc23 LICENSE --- a/LICENSE Sun Jan 24 22:15:20 2016 -0800 +++ b/LICENSE Mon Jan 25 17:05:13 2016 +0100 @@ -36,9 +36,44 @@ 2.1 2.0+1.6.1 2001 PSF no 2.0.1 2.0+1.6.1 2001 PSF yes 2.1.1 2.1+2.0.1 2001 PSF yes + 2.2 2.1.1 2001 PSF yes 2.1.2 2.1.1 2002 PSF yes 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes + 2.2.1 2.2 2002 PSF yes + 2.2.2 2.2.1 2002 PSF yes + 2.2.3 2.2.2 2003 PSF yes + 2.3 2.2.2 2002-2003 PSF yes + 2.3.1 2.3 2002-2003 PSF yes + 2.3.2 2.3.1 2002-2003 PSF yes + 2.3.3 2.3.2 2002-2003 PSF yes + 2.3.4 2.3.3 2004 PSF yes + 2.3.5 2.3.4 2005 PSF yes + 2.4 2.3 2004 PSF yes + 2.4.1 2.4 2005 PSF yes + 2.4.2 2.4.1 2005 PSF yes + 2.4.3 2.4.2 2006 PSF yes + 2.4.4 2.4.3 2006 PSF yes + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 3.0 2.6 2008 PSF yes + 3.0.1 3.0 2009 PSF yes + 3.1 3.0.1 2009 PSF yes + 3.1.1 3.1 2009 PSF yes + 3.1.2 3.1.1 2010 PSF yes + 3.1.3 3.1.2 2010 PSF yes + 3.1.4 3.1.3 2011 PSF yes + 3.2 3.1 2011 PSF yes + 3.2.1 3.2 2011 PSF yes + 3.2.2 3.2.1 2011 PSF yes + 3.3.0 3.2 2012 PSF yes Footnotes: @@ -74,9 +109,8 @@ distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016 Python Software Foundation; All Rights -Reserved" are retained in Python alone or in any derivative version prepared by -Licensee. +2011, 2012 Python Software Foundation; All Rights Reserved" are retained in Python +alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make diff -r 6db40a9955dc -r 0d413f60cc23 Lib/__future__.py --- a/Lib/__future__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/__future__.py Mon Jan 25 17:05:13 2016 +0100 @@ -56,7 +56,6 @@ "print_function", "unicode_literals", "barry_as_FLUFL", - "generator_stop", ] __all__ = ["all_feature_names"] + all_feature_names @@ -73,7 +72,6 @@ CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals CO_FUTURE_BARRY_AS_BDFL = 0x40000 -CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): @@ -116,7 +114,7 @@ CO_FUTURE_DIVISION) absolute_import = _Feature((2, 5, 0, "alpha", 1), - (3, 0, 0, "alpha", 0), + (2, 7, 0, "alpha", 0), CO_FUTURE_ABSOLUTE_IMPORT) with_statement = _Feature((2, 5, 0, "alpha", 1), @@ -134,7 +132,3 @@ barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2), (3, 9, 0, "alpha", 0), CO_FUTURE_BARRY_AS_BDFL) - -generator_stop = _Feature((3, 5, 0, "beta", 1), - (3, 7, 0, "alpha", 0), - CO_FUTURE_GENERATOR_STOP) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_bootlocale.py --- a/Lib/_bootlocale.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -"""A minimal subset of the locale module used at interpreter startup -(imported by the _io module), in order to reduce startup time. - -Don't import directly from third-party code; use the `locale` module instead! -""" - -import sys -import _locale - -if sys.platform.startswith("win"): - def getpreferredencoding(do_setlocale=True): - return _locale._getdefaultlocale()[1] -else: - try: - _locale.CODESET - except AttributeError: - def getpreferredencoding(do_setlocale=True): - # This path for legacy systems needs the more complex - # getdefaultlocale() function, import the full locale module. - import locale - return locale.getpreferredencoding(do_setlocale) - else: - def getpreferredencoding(do_setlocale=True): - assert not do_setlocale - result = _locale.nl_langinfo(_locale.CODESET) - if not result and sys.platform == 'darwin': - # nl_langinfo can return an empty string - # when the setting has an invalid value. - # Default to UTF-8 in that case because - # UTF-8 is the default charset on OSX and - # returning nothing will crash the - # interpreter. - result = 'UTF-8' - return result diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_collections_abc.py --- a/Lib/_collections_abc.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,939 +0,0 @@ -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. - -Unit tests are in test_collections. -""" - -from abc import ABCMeta, abstractmethod -import sys - -__all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", - "Hashable", "Iterable", "Iterator", "Generator", - "Sized", "Container", "Callable", - "Set", "MutableSet", - "Mapping", "MutableMapping", - "MappingView", "KeysView", "ItemsView", "ValuesView", - "Sequence", "MutableSequence", - "ByteString", - ] - -# This module has been renamed from collections.abc to _collections_abc to -# speed up interpreter startup. Some of the types such as MutableMapping are -# required early but collections module imports a lot of other modules. -# See issue #19218 -__name__ = "collections.abc" - -# Private list of types that we want to register with the various ABCs -# so that they will pass tests like: -# it = iter(somebytearray) -# assert isinstance(it, Iterable) -# Note: in other implementations, these types many not be distinct -# and they make have their own implementation specific types that -# are not included on this list. -bytes_iterator = type(iter(b'')) -bytearray_iterator = type(iter(bytearray())) -#callable_iterator = ??? -dict_keyiterator = type(iter({}.keys())) -dict_valueiterator = type(iter({}.values())) -dict_itemiterator = type(iter({}.items())) -list_iterator = type(iter([])) -list_reverseiterator = type(iter(reversed([]))) -range_iterator = type(iter(range(0))) -set_iterator = type(iter(set())) -str_iterator = type(iter("")) -tuple_iterator = type(iter(())) -zip_iterator = type(iter(zip())) -## views ## -dict_keys = type({}.keys()) -dict_values = type({}.values()) -dict_items = type({}.items()) -## misc ## -mappingproxy = type(type.__dict__) -generator = type((lambda: (yield))()) -## coroutine ## -async def _coro(): pass -_coro = _coro() -coroutine = type(_coro) -_coro.close() # Prevent ResourceWarning -del _coro - - -### ONE-TRICK PONIES ### - -class Hashable(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __hash__(self): - return 0 - - @classmethod - def __subclasshook__(cls, C): - if cls is Hashable: - for B in C.__mro__: - if "__hash__" in B.__dict__: - if B.__dict__["__hash__"]: - return True - break - return NotImplemented - - -class Awaitable(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __await__(self): - yield - - @classmethod - def __subclasshook__(cls, C): - if cls is Awaitable: - for B in C.__mro__: - if "__await__" in B.__dict__: - if B.__dict__["__await__"]: - return True - break - return NotImplemented - - -class Coroutine(Awaitable): - - __slots__ = () - - @abstractmethod - def send(self, value): - """Send a value into the coroutine. - Return next yielded value or raise StopIteration. - """ - raise StopIteration - - @abstractmethod - def throw(self, typ, val=None, tb=None): - """Raise an exception in the coroutine. - Return next yielded value or raise StopIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val - - def close(self): - """Raise GeneratorExit inside coroutine. - """ - try: - self.throw(GeneratorExit) - except (GeneratorExit, StopIteration): - pass - else: - raise RuntimeError("coroutine ignored GeneratorExit") - - @classmethod - def __subclasshook__(cls, C): - if cls is Coroutine: - mro = C.__mro__ - for method in ('__await__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True - return NotImplemented - - -Coroutine.register(coroutine) - - -class AsyncIterable(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - async def __aiter__(self): - return AsyncIterator() - - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterable: - if any("__aiter__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - -class AsyncIterator(AsyncIterable): - - __slots__ = () - - @abstractmethod - async def __anext__(self): - """Return the next item or raise StopAsyncIteration when exhausted.""" - raise StopAsyncIteration - - async def __aiter__(self): - return self - - @classmethod - def __subclasshook__(cls, C): - if cls is AsyncIterator: - if (any("__anext__" in B.__dict__ for B in C.__mro__) and - any("__aiter__" in B.__dict__ for B in C.__mro__)): - return True - return NotImplemented - - -class Iterable(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __iter__(self): - while False: - yield None - - @classmethod - def __subclasshook__(cls, C): - if cls is Iterable: - if any("__iter__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - -class Iterator(Iterable): - - __slots__ = () - - @abstractmethod - def __next__(self): - 'Return the next item from the iterator. When exhausted, raise StopIteration' - raise StopIteration - - def __iter__(self): - return self - - @classmethod - def __subclasshook__(cls, C): - if cls is Iterator: - if (any("__next__" in B.__dict__ for B in C.__mro__) and - any("__iter__" in B.__dict__ for B in C.__mro__)): - return True - return NotImplemented - -Iterator.register(bytes_iterator) -Iterator.register(bytearray_iterator) -#Iterator.register(callable_iterator) -Iterator.register(dict_keyiterator) -Iterator.register(dict_valueiterator) -Iterator.register(dict_itemiterator) -Iterator.register(list_iterator) -Iterator.register(list_reverseiterator) -Iterator.register(range_iterator) -Iterator.register(set_iterator) -Iterator.register(str_iterator) -Iterator.register(tuple_iterator) -Iterator.register(zip_iterator) - - -class Generator(Iterator): - - __slots__ = () - - def __next__(self): - """Return the next item from the generator. - When exhausted, raise StopIteration. - """ - return self.send(None) - - @abstractmethod - def send(self, value): - """Send a value into the generator. - Return next yielded value or raise StopIteration. - """ - raise StopIteration - - @abstractmethod - def throw(self, typ, val=None, tb=None): - """Raise an exception in the generator. - Return next yielded value or raise StopIteration. - """ - if val is None: - if tb is None: - raise typ - val = typ() - if tb is not None: - val = val.with_traceback(tb) - raise val - - def close(self): - """Raise GeneratorExit inside generator. - """ - try: - self.throw(GeneratorExit) - except (GeneratorExit, StopIteration): - pass - else: - raise RuntimeError("generator ignored GeneratorExit") - - @classmethod - def __subclasshook__(cls, C): - if cls is Generator: - mro = C.__mro__ - for method in ('__iter__', '__next__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True - return NotImplemented - - -Generator.register(generator) - - -class Sized(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __len__(self): - return 0 - - @classmethod - def __subclasshook__(cls, C): - if cls is Sized: - if any("__len__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - -class Container(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __contains__(self, x): - return False - - @classmethod - def __subclasshook__(cls, C): - if cls is Container: - if any("__contains__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - -class Callable(metaclass=ABCMeta): - - __slots__ = () - - @abstractmethod - def __call__(self, *args, **kwds): - return False - - @classmethod - def __subclasshook__(cls, C): - if cls is Callable: - if any("__call__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - -### SETS ### - - -class Set(Sized, Iterable, Container): - - """A set is a finite, iterable container. - - This class provides concrete generic implementations of all - methods except for __contains__, __iter__ and __len__. - - To override the comparisons (presumably for speed, as the - semantics are fixed), redefine __le__ and __ge__, - then the other operations will automatically follow suit. - """ - - __slots__ = () - - def __le__(self, other): - if not isinstance(other, Set): - return NotImplemented - if len(self) > len(other): - return False - for elem in self: - if elem not in other: - return False - return True - - def __lt__(self, other): - if not isinstance(other, Set): - return NotImplemented - return len(self) < len(other) and self.__le__(other) - - def __gt__(self, other): - if not isinstance(other, Set): - return NotImplemented - return len(self) > len(other) and self.__ge__(other) - - def __ge__(self, other): - if not isinstance(other, Set): - return NotImplemented - if len(self) < len(other): - return False - for elem in other: - if elem not in self: - return False - return True - - def __eq__(self, other): - if not isinstance(other, Set): - return NotImplemented - return len(self) == len(other) and self.__le__(other) - - @classmethod - def _from_iterable(cls, it): - '''Construct an instance of the class from any iterable input. - - Must override this method if the class constructor signature - does not accept an iterable for an input. - ''' - return cls(it) - - def __and__(self, other): - if not isinstance(other, Iterable): - return NotImplemented - return self._from_iterable(value for value in other if value in self) - - __rand__ = __and__ - - def isdisjoint(self, other): - 'Return True if two sets have a null intersection.' - for value in other: - if value in self: - return False - return True - - def __or__(self, other): - if not isinstance(other, Iterable): - return NotImplemented - chain = (e for s in (self, other) for e in s) - return self._from_iterable(chain) - - __ror__ = __or__ - - def __sub__(self, other): - if not isinstance(other, Set): - if not isinstance(other, Iterable): - return NotImplemented - other = self._from_iterable(other) - return self._from_iterable(value for value in self - if value not in other) - - def __rsub__(self, other): - if not isinstance(other, Set): - if not isinstance(other, Iterable): - return NotImplemented - other = self._from_iterable(other) - return self._from_iterable(value for value in other - if value not in self) - - def __xor__(self, other): - if not isinstance(other, Set): - if not isinstance(other, Iterable): - return NotImplemented - other = self._from_iterable(other) - return (self - other) | (other - self) - - __rxor__ = __xor__ - - def _hash(self): - """Compute the hash value of a set. - - Note that we don't define __hash__: not all sets are hashable. - But if you define a hashable set type, its __hash__ should - call this function. - - This must be compatible __eq__. - - All sets ought to compare equal if they contain the same - elements, regardless of how they are implemented, and - regardless of the order of the elements; so there's not much - freedom for __eq__ or __hash__. We match the algorithm used - by the built-in frozenset type. - """ - MAX = sys.maxsize - MASK = 2 * MAX + 1 - n = len(self) - h = 1927868237 * (n + 1) - h &= MASK - for x in self: - hx = hash(x) - h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 - h &= MASK - h = h * 69069 + 907133923 - h &= MASK - if h > MAX: - h -= MASK + 1 - if h == -1: - h = 590923713 - return h - -Set.register(frozenset) - - -class MutableSet(Set): - """A mutable set is a finite, iterable container. - - This class provides concrete generic implementations of all - methods except for __contains__, __iter__, __len__, - add(), and discard(). - - To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and - then the other operations will automatically follow suit. - """ - - __slots__ = () - - @abstractmethod - def add(self, value): - """Add an element.""" - raise NotImplementedError - - @abstractmethod - def discard(self, value): - """Remove an element. Do not raise an exception if absent.""" - raise NotImplementedError - - def remove(self, value): - """Remove an element. If not a member, raise a KeyError.""" - if value not in self: - raise KeyError(value) - self.discard(value) - - def pop(self): - """Return the popped value. Raise KeyError if empty.""" - it = iter(self) - try: - value = next(it) - except StopIteration: - raise KeyError - self.discard(value) - return value - - def clear(self): - """This is slow (creates N new iterators!) but effective.""" - try: - while True: - self.pop() - except KeyError: - pass - - def __ior__(self, it): - for value in it: - self.add(value) - return self - - def __iand__(self, it): - for value in (self - it): - self.discard(value) - return self - - def __ixor__(self, it): - if it is self: - self.clear() - else: - if not isinstance(it, Set): - it = self._from_iterable(it) - for value in it: - if value in self: - self.discard(value) - else: - self.add(value) - return self - - def __isub__(self, it): - if it is self: - self.clear() - else: - for value in it: - self.discard(value) - return self - -MutableSet.register(set) - - -### MAPPINGS ### - - -class Mapping(Sized, Iterable, Container): - - __slots__ = () - - """A Mapping is a generic container for associating key/value - pairs. - - This class provides concrete generic implementations of all - methods except for __getitem__, __iter__, and __len__. - - """ - - @abstractmethod - def __getitem__(self, key): - raise KeyError - - def get(self, key, default=None): - 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' - try: - return self[key] - except KeyError: - return default - - def __contains__(self, key): - try: - self[key] - except KeyError: - return False - else: - return True - - def keys(self): - "D.keys() -> a set-like object providing a view on D's keys" - return KeysView(self) - - def items(self): - "D.items() -> a set-like object providing a view on D's items" - return ItemsView(self) - - def values(self): - "D.values() -> an object providing a view on D's values" - return ValuesView(self) - - def __eq__(self, other): - if not isinstance(other, Mapping): - return NotImplemented - return dict(self.items()) == dict(other.items()) - -Mapping.register(mappingproxy) - - -class MappingView(Sized): - - __slots__ = '_mapping', - - def __init__(self, mapping): - self._mapping = mapping - - def __len__(self): - return len(self._mapping) - - def __repr__(self): - return '{0.__class__.__name__}({0._mapping!r})'.format(self) - - -class KeysView(MappingView, Set): - - __slots__ = () - - @classmethod - def _from_iterable(self, it): - return set(it) - - def __contains__(self, key): - return key in self._mapping - - def __iter__(self): - yield from self._mapping - -KeysView.register(dict_keys) - - -class ItemsView(MappingView, Set): - - __slots__ = () - - @classmethod - def _from_iterable(self, it): - return set(it) - - def __contains__(self, item): - key, value = item - try: - v = self._mapping[key] - except KeyError: - return False - else: - return v == value - - def __iter__(self): - for key in self._mapping: - yield (key, self._mapping[key]) - -ItemsView.register(dict_items) - - -class ValuesView(MappingView): - - __slots__ = () - - def __contains__(self, value): - for key in self._mapping: - if value == self._mapping[key]: - return True - return False - - def __iter__(self): - for key in self._mapping: - yield self._mapping[key] - -ValuesView.register(dict_values) - - -class MutableMapping(Mapping): - - __slots__ = () - - """A MutableMapping is a generic container for associating - key/value pairs. - - This class provides concrete generic implementations of all - methods except for __getitem__, __setitem__, __delitem__, - __iter__, and __len__. - - """ - - @abstractmethod - def __setitem__(self, key, value): - raise KeyError - - @abstractmethod - def __delitem__(self, key): - raise KeyError - - __marker = object() - - def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - ''' - try: - value = self[key] - except KeyError: - if default is self.__marker: - raise - return default - else: - del self[key] - return value - - def popitem(self): - '''D.popitem() -> (k, v), remove and return some (key, value) pair - as a 2-tuple; but raise KeyError if D is empty. - ''' - try: - key = next(iter(self)) - except StopIteration: - raise KeyError - value = self[key] - del self[key] - return key, value - - def clear(self): - 'D.clear() -> None. Remove all items from D.' - try: - while True: - self.popitem() - except KeyError: - pass - - def update(*args, **kwds): - ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E: D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v - ''' - if not args: - raise TypeError("descriptor 'update' of 'MutableMapping' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('update expected at most 1 arguments, got %d' % - len(args)) - if args: - other = args[0] - if isinstance(other, Mapping): - for key in other: - self[key] = other[key] - elif hasattr(other, "keys"): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - def setdefault(self, key, default=None): - 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' - try: - return self[key] - except KeyError: - self[key] = default - return default - -MutableMapping.register(dict) - - -### SEQUENCES ### - - -class Sequence(Sized, Iterable, Container): - - """All the operations on a read-only sequence. - - Concrete subclasses must override __new__ or __init__, - __getitem__, and __len__. - """ - - __slots__ = () - - @abstractmethod - def __getitem__(self, index): - raise IndexError - - def __iter__(self): - i = 0 - try: - while True: - v = self[i] - yield v - i += 1 - except IndexError: - return - - def __contains__(self, value): - for v in self: - if v == value: - return True - return False - - def __reversed__(self): - for i in reversed(range(len(self))): - yield self[i] - - def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. - ''' - if start is not None and start < 0: - start = max(len(self) + start, 0) - if stop is not None and stop < 0: - stop += len(self) - - i = start - while stop is None or i < stop: - try: - if self[i] == value: - return i - except IndexError: - break - i += 1 - raise ValueError - - def count(self, value): - 'S.count(value) -> integer -- return number of occurrences of value' - return sum(1 for v in self if v == value) - -Sequence.register(tuple) -Sequence.register(str) -Sequence.register(range) -Sequence.register(memoryview) - - -class ByteString(Sequence): - - """This unifies bytes and bytearray. - - XXX Should add all their methods. - """ - - __slots__ = () - -ByteString.register(bytes) -ByteString.register(bytearray) - - -class MutableSequence(Sequence): - - __slots__ = () - - """All the operations on a read-write sequence. - - Concrete subclasses must provide __new__ or __init__, - __getitem__, __setitem__, __delitem__, __len__, and insert(). - - """ - - @abstractmethod - def __setitem__(self, index, value): - raise IndexError - - @abstractmethod - def __delitem__(self, index): - raise IndexError - - @abstractmethod - def insert(self, index, value): - 'S.insert(index, value) -- insert value before index' - raise IndexError - - def append(self, value): - 'S.append(value) -- append value to the end of the sequence' - self.insert(len(self), value) - - def clear(self): - 'S.clear() -> None -- remove all items from S' - try: - while True: - self.pop() - except IndexError: - pass - - def reverse(self): - 'S.reverse() -- reverse *IN PLACE*' - n = len(self) - for i in range(n//2): - self[i], self[n-i-1] = self[n-i-1], self[i] - - def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' - for v in values: - self.append(v) - - def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. - ''' - v = self[index] - del self[index] - return v - - def remove(self, value): - '''S.remove(value) -- remove first occurrence of value. - Raise ValueError if the value is not present. - ''' - del self[self.index(value)] - - def __iadd__(self, values): - self.extend(values) - return self - -MutableSequence.register(list) -MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_compat_pickle.py --- a/Lib/_compat_pickle.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/_compat_pickle.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,13 +6,18 @@ # lib2to3 and use the mapping defined there, because lib2to3 uses pickle. # Thus, this could cause the module to be imported recursively. IMPORT_MAPPING = { + 'StringIO': 'io', + 'cStringIO': 'io', + 'cPickle': 'pickle', '__builtin__' : 'builtins', 'copy_reg': 'copyreg', 'Queue': 'queue', 'SocketServer': 'socketserver', 'ConfigParser': 'configparser', 'repr': 'reprlib', + 'FileDialog': 'tkinter.filedialog', 'tkFileDialog': 'tkinter.filedialog', + 'SimpleDialog': 'tkinter.simpledialog', 'tkSimpleDialog': 'tkinter.simpledialog', 'tkColorChooser': 'tkinter.colorchooser', 'tkCommonDialog': 'tkinter.commondialog', @@ -34,6 +39,7 @@ 'dbm': 'dbm.ndbm', 'gdbm': 'dbm.gnu', 'xmlrpclib': 'xmlrpc.client', + 'DocXMLRPCServer': 'xmlrpc.server', 'SimpleXMLRPCServer': 'xmlrpc.server', 'httplib': 'http.client', 'htmlentitydefs' : 'html.entities', @@ -41,13 +47,16 @@ 'Cookie': 'http.cookies', 'cookielib': 'http.cookiejar', 'BaseHTTPServer': 'http.server', + 'SimpleHTTPServer': 'http.server', + 'CGIHTTPServer': 'http.server', 'test.test_support': 'test.support', 'commands': 'subprocess', + 'UserString' : 'collections', + 'UserList' : 'collections', 'urlparse' : 'urllib.parse', 'robotparser' : 'urllib.robotparser', - 'urllib2': 'urllib.request', - 'anydbm': 'dbm', - '_abcoll' : 'collections.abc', + 'whichdb': 'dbm', + 'anydbm': 'dbm' } @@ -59,186 +68,14 @@ ('__builtin__', 'reduce'): ('functools', 'reduce'), ('__builtin__', 'intern'): ('sys', 'intern'), ('__builtin__', 'unichr'): ('builtins', 'chr'), - ('__builtin__', 'unicode'): ('builtins', 'str'), + ('__builtin__', 'basestring'): ('builtins', 'str'), ('__builtin__', 'long'): ('builtins', 'int'), ('itertools', 'izip'): ('builtins', 'zip'), ('itertools', 'imap'): ('builtins', 'map'), ('itertools', 'ifilter'): ('builtins', 'filter'), ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'), - ('itertools', 'izip_longest'): ('itertools', 'zip_longest'), - ('UserDict', 'IterableUserDict'): ('collections', 'UserDict'), - ('UserList', 'UserList'): ('collections', 'UserList'), - ('UserString', 'UserString'): ('collections', 'UserString'), - ('whichdb', 'whichdb'): ('dbm', 'whichdb'), - ('_socket', 'fromfd'): ('socket', 'fromfd'), - ('_multiprocessing', 'Connection'): ('multiprocessing.connection', 'Connection'), - ('multiprocessing.process', 'Process'): ('multiprocessing.context', 'Process'), - ('multiprocessing.forking', 'Popen'): ('multiprocessing.popen_fork', 'Popen'), - ('urllib', 'ContentTooShortError'): ('urllib.error', 'ContentTooShortError'), - ('urllib', 'getproxies'): ('urllib.request', 'getproxies'), - ('urllib', 'pathname2url'): ('urllib.request', 'pathname2url'), - ('urllib', 'quote_plus'): ('urllib.parse', 'quote_plus'), - ('urllib', 'quote'): ('urllib.parse', 'quote'), - ('urllib', 'unquote_plus'): ('urllib.parse', 'unquote_plus'), - ('urllib', 'unquote'): ('urllib.parse', 'unquote'), - ('urllib', 'url2pathname'): ('urllib.request', 'url2pathname'), - ('urllib', 'urlcleanup'): ('urllib.request', 'urlcleanup'), - ('urllib', 'urlencode'): ('urllib.parse', 'urlencode'), - ('urllib', 'urlopen'): ('urllib.request', 'urlopen'), - ('urllib', 'urlretrieve'): ('urllib.request', 'urlretrieve'), - ('urllib2', 'HTTPError'): ('urllib.error', 'HTTPError'), - ('urllib2', 'URLError'): ('urllib.error', 'URLError'), } -PYTHON2_EXCEPTIONS = ( - "ArithmeticError", - "AssertionError", - "AttributeError", - "BaseException", - "BufferError", - "BytesWarning", - "DeprecationWarning", - "EOFError", - "EnvironmentError", - "Exception", - "FloatingPointError", - "FutureWarning", - "GeneratorExit", - "IOError", - "ImportError", - "ImportWarning", - "IndentationError", - "IndexError", - "KeyError", - "KeyboardInterrupt", - "LookupError", - "MemoryError", - "NameError", - "NotImplementedError", - "OSError", - "OverflowError", - "PendingDeprecationWarning", - "ReferenceError", - "RuntimeError", - "RuntimeWarning", - # StandardError is gone in Python 3, so we map it to Exception - "StopIteration", - "SyntaxError", - "SyntaxWarning", - "SystemError", - "SystemExit", - "TabError", - "TypeError", - "UnboundLocalError", - "UnicodeDecodeError", - "UnicodeEncodeError", - "UnicodeError", - "UnicodeTranslateError", - "UnicodeWarning", - "UserWarning", - "ValueError", - "Warning", - "ZeroDivisionError", -) - -try: - WindowsError -except NameError: - pass -else: - PYTHON2_EXCEPTIONS += ("WindowsError",) - -for excname in PYTHON2_EXCEPTIONS: - NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) - -MULTIPROCESSING_EXCEPTIONS = ( - 'AuthenticationError', - 'BufferTooShort', - 'ProcessError', - 'TimeoutError', -) - -for excname in MULTIPROCESSING_EXCEPTIONS: - NAME_MAPPING[("multiprocessing", excname)] = ("multiprocessing.context", excname) - # Same, but for 3.x to 2.x REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items()) -assert len(REVERSE_IMPORT_MAPPING) == len(IMPORT_MAPPING) REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items()) -assert len(REVERSE_NAME_MAPPING) == len(NAME_MAPPING) - -# Non-mutual mappings. - -IMPORT_MAPPING.update({ - 'cPickle': 'pickle', - '_elementtree': 'xml.etree.ElementTree', - 'FileDialog': 'tkinter.filedialog', - 'SimpleDialog': 'tkinter.simpledialog', - 'DocXMLRPCServer': 'xmlrpc.server', - 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', - # For compatibility with broken pickles saved in old Python 3 versions - 'UserDict': 'collections', - 'UserList': 'collections', - 'UserString': 'collections', - 'whichdb': 'dbm', - 'StringIO': 'io', - 'cStringIO': 'io', -}) - -REVERSE_IMPORT_MAPPING.update({ - '_bz2': 'bz2', - '_dbm': 'dbm', - '_functools': 'functools', - '_gdbm': 'gdbm', - '_pickle': 'pickle', -}) - -NAME_MAPPING.update({ - ('__builtin__', 'basestring'): ('builtins', 'str'), - ('exceptions', 'StandardError'): ('builtins', 'Exception'), - ('UserDict', 'UserDict'): ('collections', 'UserDict'), - ('socket', '_socketobject'): ('socket', 'SocketType'), -}) - -REVERSE_NAME_MAPPING.update({ - ('_functools', 'reduce'): ('__builtin__', 'reduce'), - ('tkinter.filedialog', 'FileDialog'): ('FileDialog', 'FileDialog'), - ('tkinter.filedialog', 'LoadFileDialog'): ('FileDialog', 'LoadFileDialog'), - ('tkinter.filedialog', 'SaveFileDialog'): ('FileDialog', 'SaveFileDialog'), - ('tkinter.simpledialog', 'SimpleDialog'): ('SimpleDialog', 'SimpleDialog'), - ('xmlrpc.server', 'ServerHTMLDoc'): ('DocXMLRPCServer', 'ServerHTMLDoc'), - ('xmlrpc.server', 'XMLRPCDocGenerator'): - ('DocXMLRPCServer', 'XMLRPCDocGenerator'), - ('xmlrpc.server', 'DocXMLRPCRequestHandler'): - ('DocXMLRPCServer', 'DocXMLRPCRequestHandler'), - ('xmlrpc.server', 'DocXMLRPCServer'): - ('DocXMLRPCServer', 'DocXMLRPCServer'), - ('xmlrpc.server', 'DocCGIXMLRPCRequestHandler'): - ('DocXMLRPCServer', 'DocCGIXMLRPCRequestHandler'), - ('http.server', 'SimpleHTTPRequestHandler'): - ('SimpleHTTPServer', 'SimpleHTTPRequestHandler'), - ('http.server', 'CGIHTTPRequestHandler'): - ('CGIHTTPServer', 'CGIHTTPRequestHandler'), - ('_socket', 'socket'): ('socket', '_socketobject'), -}) - -PYTHON3_OSERROR_EXCEPTIONS = ( - 'BrokenPipeError', - 'ChildProcessError', - 'ConnectionAbortedError', - 'ConnectionError', - 'ConnectionRefusedError', - 'ConnectionResetError', - 'FileExistsError', - 'FileNotFoundError', - 'InterruptedError', - 'IsADirectoryError', - 'NotADirectoryError', - 'PermissionError', - 'ProcessLookupError', - 'TimeoutError', -) - -for excname in PYTHON3_OSERROR_EXCEPTIONS: - REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_compression.py --- a/Lib/_compression.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,152 +0,0 @@ -"""Internal classes used by the gzip, lzma and bz2 modules""" - -import io - - -BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size - - -class BaseStream(io.BufferedIOBase): - """Mode-checking helper functions.""" - - def _check_not_closed(self): - if self.closed: - raise ValueError("I/O operation on closed file") - - def _check_can_read(self): - if not self.readable(): - raise io.UnsupportedOperation("File not open for reading") - - def _check_can_write(self): - if not self.writable(): - raise io.UnsupportedOperation("File not open for writing") - - def _check_can_seek(self): - if not self.readable(): - raise io.UnsupportedOperation("Seeking is only supported " - "on files open for reading") - if not self.seekable(): - raise io.UnsupportedOperation("The underlying file object " - "does not support seeking") - - -class DecompressReader(io.RawIOBase): - """Adapts the decompressor API to a RawIOBase reader API""" - - def readable(self): - return True - - def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args): - self._fp = fp - self._eof = False - self._pos = 0 # Current offset in decompressed stream - - # Set to size of decompressed stream once it is known, for SEEK_END - self._size = -1 - - # Save the decompressor factory and arguments. - # If the file contains multiple compressed streams, each - # stream will need a separate decompressor object. A new decompressor - # object is also needed when implementing a backwards seek(). - self._decomp_factory = decomp_factory - self._decomp_args = decomp_args - self._decompressor = self._decomp_factory(**self._decomp_args) - - # Exception class to catch from decompressor signifying invalid - # trailing data to ignore - self._trailing_error = trailing_error - - def close(self): - self._decompressor = None - return super().close() - - def seekable(self): - return self._fp.seekable() - - def readinto(self, b): - with memoryview(b) as view, view.cast("B") as byte_view: - data = self.read(len(byte_view)) - byte_view[:len(data)] = data - return len(data) - - def read(self, size=-1): - if size < 0: - return self.readall() - - if not size or self._eof: - return b"" - data = None # Default if EOF is encountered - # Depending on the input data, our call to the decompressor may not - # return any data. In this case, try again after reading another block. - while True: - if self._decompressor.eof: - rawblock = (self._decompressor.unused_data or - self._fp.read(BUFFER_SIZE)) - if not rawblock: - break - # Continue to next stream. - self._decompressor = self._decomp_factory( - **self._decomp_args) - try: - data = self._decompressor.decompress(rawblock, size) - except self._trailing_error: - # Trailing data isn't a valid compressed stream; ignore it. - break - else: - if self._decompressor.needs_input: - rawblock = self._fp.read(BUFFER_SIZE) - if not rawblock: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") - else: - rawblock = b"" - data = self._decompressor.decompress(rawblock, size) - if data: - break - if not data: - self._eof = True - self._size = self._pos - return b"" - self._pos += len(data) - return data - - # Rewind the file to the beginning of the data stream. - def _rewind(self): - self._fp.seek(0) - self._eof = False - self._pos = 0 - self._decompressor = self._decomp_factory(**self._decomp_args) - - def seek(self, offset, whence=io.SEEK_SET): - # Recalculate offset as an absolute file position. - if whence == io.SEEK_SET: - pass - elif whence == io.SEEK_CUR: - offset = self._pos + offset - elif whence == io.SEEK_END: - # Seeking relative to EOF - we need to know the file's size. - if self._size < 0: - while self.read(io.DEFAULT_BUFFER_SIZE): - pass - offset = self._size + offset - else: - raise ValueError("Invalid value for whence: {}".format(whence)) - - # Make it so that offset is the number of bytes to skip forward. - if offset < self._pos: - self._rewind() - else: - offset -= self._pos - - # Read and discard data until we reach the desired position. - while offset > 0: - data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset)) - if not data: - break - offset -= len(data) - - return self._pos - - def tell(self): - """Return the current file position.""" - return self._pos diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_dummy_thread.py --- a/Lib/_dummy_thread.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/_dummy_thread.py Mon Jan 25 17:05:13 2016 +0100 @@ -81,10 +81,6 @@ raise error("setting thread stack size not supported") return 0 -def _set_sentinel(): - """Dummy implementation of _thread._set_sentinel().""" - return LockType() - class LockType(object): """Class implementing dummy implementation of _thread.LockType. @@ -140,14 +136,6 @@ def locked(self): return self.locked_status - def __repr__(self): - return "<%s %s.%s object at %s>" % ( - "locked" if self.locked_status else "unlocked", - self.__class__.__module__, - self.__class__.__qualname__, - hex(id(self)) - ) - # Used to signal that interrupt_main was called in a "thread" _interrupt = False # True when not executing in a "thread" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_osx_support.py --- a/Lib/_osx_support.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,502 +0,0 @@ -"""Shared OS X support functions.""" - -import os -import re -import sys - -__all__ = [ - 'compiler_fixup', - 'customize_config_vars', - 'customize_compiler', - 'get_platform_osx', -] - -# configuration variables that may contain universal build flags, -# like "-arch" or "-isdkroot", that may need customization for -# the user environment -_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', - 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', - 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS') - -# configuration variables that may contain compiler calls -_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') - -# prefix added to original configuration variable names -_INITPRE = '_OSX_SUPPORT_INITIAL_' - - -def _find_executable(executable, path=None): - """Tries to find 'executable' in the directories listed in 'path'. - - A string listing directories separated by 'os.pathsep'; defaults to - os.environ['PATH']. Returns the complete filename or None if not found. - """ - if path is None: - path = os.environ['PATH'] - - paths = path.split(os.pathsep) - base, ext = os.path.splitext(executable) - - if (sys.platform == 'win32') and (ext != '.exe'): - executable = executable + '.exe' - - if not os.path.isfile(executable): - for p in paths: - f = os.path.join(p, executable) - if os.path.isfile(f): - # the file exists, we have a shot at spawn working - return f - return None - else: - return executable - - -def _read_output(commandstring): - """Output from successful command execution or None""" - # Similar to os.popen(commandstring, "r").read(), - # but without actually using os.popen because that - # function is not usable during python bootstrap. - # tempfile is also not available then. - import contextlib - try: - import tempfile - fp = tempfile.NamedTemporaryFile() - except ImportError: - fp = open("/tmp/_osx_support.%s"%( - os.getpid(),), "w+b") - - with contextlib.closing(fp) as fp: - cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) - return fp.read().decode('utf-8').strip() if not os.system(cmd) else None - - -def _find_build_tool(toolname): - """Find a build tool on current path or using xcrun""" - return (_find_executable(toolname) - or _read_output("/usr/bin/xcrun -find %s" % (toolname,)) - or '' - ) - -_SYSTEM_VERSION = None - -def _get_system_version(): - """Return the OS X system version as a string""" - # Reading this plist is a documented way to get the system - # version (see the documentation for the Gestalt Manager) - # We avoid using platform.mac_ver to avoid possible bootstrap issues during - # the build of Python itself (distutils is used to build standard library - # extensions). - - global _SYSTEM_VERSION - - if _SYSTEM_VERSION is None: - _SYSTEM_VERSION = '' - try: - f = open('/System/Library/CoreServices/SystemVersion.plist') - except OSError: - # We're on a plain darwin box, fall back to the default - # behaviour. - pass - else: - try: - m = re.search(r'ProductUserVisibleVersion\s*' - r'(.*?)', f.read()) - finally: - f.close() - if m is not None: - _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) - # else: fall back to the default behaviour - - return _SYSTEM_VERSION - -def _remove_original_values(_config_vars): - """Remove original unmodified values for testing""" - # This is needed for higher-level cross-platform tests of get_platform. - for k in list(_config_vars): - if k.startswith(_INITPRE): - del _config_vars[k] - -def _save_modified_value(_config_vars, cv, newvalue): - """Save modified and original unmodified value of configuration var""" - - oldvalue = _config_vars.get(cv, '') - if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars): - _config_vars[_INITPRE + cv] = oldvalue - _config_vars[cv] = newvalue - -def _supports_universal_builds(): - """Returns True if universal builds are supported on this system""" - # As an approximation, we assume that if we are running on 10.4 or above, - # then we are running with an Xcode environment that supports universal - # builds, in particular -isysroot and -arch arguments to the compiler. This - # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' - return bool(osx_version >= (10, 4)) if osx_version else False - - -def _find_appropriate_compiler(_config_vars): - """Find appropriate C compiler for extension module builds""" - - # Issue #13590: - # The OSX location for the compiler varies between OSX - # (or rather Xcode) releases. With older releases (up-to 10.5) - # the compiler is in /usr/bin, with newer releases the compiler - # can only be found inside Xcode.app if the "Command Line Tools" - # are not installed. - # - # Futhermore, the compiler that can be used varies between - # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' - # as the compiler, after that 'clang' should be used because - # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that - # miscompiles Python. - - # skip checks if the compiler was overriden with a CC env variable - if 'CC' in os.environ: - return _config_vars - - # The CC config var might contain additional arguments. - # Ignore them while searching. - cc = oldcc = _config_vars['CC'].split()[0] - if not _find_executable(cc): - # Compiler is not found on the shell search PATH. - # Now search for clang, first on PATH (if the Command LIne - # Tools have been installed in / or if the user has provided - # another location via CC). If not found, try using xcrun - # to find an uninstalled clang (within a selected Xcode). - - # NOTE: Cannot use subprocess here because of bootstrap - # issues when building Python itself (and os.popen is - # implemented on top of subprocess and is therefore not - # usable as well) - - cc = _find_build_tool('clang') - - elif os.path.basename(cc).startswith('gcc'): - # Compiler is GCC, check if it is LLVM-GCC - data = _read_output("'%s' --version" - % (cc.replace("'", "'\"'\"'"),)) - if data and 'llvm-gcc' in data: - # Found LLVM-GCC, fall back to clang - cc = _find_build_tool('clang') - - if not cc: - raise SystemError( - "Cannot locate working compiler") - - if cc != oldcc: - # Found a replacement compiler. - # Modify config vars using new compiler, if not already explicitly - # overriden by an env variable, preserving additional arguments. - for cv in _COMPILER_CONFIG_VARS: - if cv in _config_vars and cv not in os.environ: - cv_split = _config_vars[cv].split() - cv_split[0] = cc if cv != 'CXX' else cc + '++' - _save_modified_value(_config_vars, cv, ' '.join(cv_split)) - - return _config_vars - - -def _remove_universal_flags(_config_vars): - """Remove all universal build arguments from config vars""" - - for cv in _UNIVERSAL_CONFIG_VARS: - # Do not alter a config var explicitly overriden by env var - if cv in _config_vars and cv not in os.environ: - flags = _config_vars[cv] - flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII) - flags = re.sub('-isysroot [^ \t]*', ' ', flags) - _save_modified_value(_config_vars, cv, flags) - - return _config_vars - - -def _remove_unsupported_archs(_config_vars): - """Remove any unsupported archs from config vars""" - # Different Xcode releases support different sets for '-arch' - # flags. In particular, Xcode 4.x no longer supports the - # PPC architectures. - # - # This code automatically removes '-arch ppc' and '-arch ppc64' - # when these are not supported. That makes it possible to - # build extensions on OSX 10.7 and later with the prebuilt - # 32-bit installer on the python.org website. - - # skip checks if the compiler was overriden with a CC env variable - if 'CC' in os.environ: - return _config_vars - - if re.search('-arch\s+ppc', _config_vars['CFLAGS']) is not None: - # NOTE: Cannot use subprocess here because of bootstrap - # issues when building Python itself - status = os.system( - """echo 'int main{};' | """ - """'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null""" - %(_config_vars['CC'].replace("'", "'\"'\"'"),)) - if status: - # The compile failed for some reason. Because of differences - # across Xcode and compiler versions, there is no reliable way - # to be sure why it failed. Assume here it was due to lack of - # PPC support and remove the related '-arch' flags from each - # config variables not explicitly overriden by an environment - # variable. If the error was for some other reason, we hope the - # failure will show up again when trying to compile an extension - # module. - for cv in _UNIVERSAL_CONFIG_VARS: - if cv in _config_vars and cv not in os.environ: - flags = _config_vars[cv] - flags = re.sub('-arch\s+ppc\w*\s', ' ', flags) - _save_modified_value(_config_vars, cv, flags) - - return _config_vars - - -def _override_all_archs(_config_vars): - """Allow override of all archs with ARCHFLAGS env var""" - # NOTE: This name was introduced by Apple in OSX 10.5 and - # is used by several scripting languages distributed with - # that OS release. - if 'ARCHFLAGS' in os.environ: - arch = os.environ['ARCHFLAGS'] - for cv in _UNIVERSAL_CONFIG_VARS: - if cv in _config_vars and '-arch' in _config_vars[cv]: - flags = _config_vars[cv] - flags = re.sub('-arch\s+\w+\s', ' ', flags) - flags = flags + ' ' + arch - _save_modified_value(_config_vars, cv, flags) - - return _config_vars - - -def _check_for_unavailable_sdk(_config_vars): - """Remove references to any SDKs not available""" - # If we're on OSX 10.5 or later and the user tries to - # compile an extension using an SDK that is not present - # on the current machine it is better to not use an SDK - # than to fail. This is particularly important with - # the standalone Command Line Tools alternative to a - # full-blown Xcode install since the CLT packages do not - # provide SDKs. If the SDK is not present, it is assumed - # that the header files and dev libs have been installed - # to /usr and /System/Library by either a standalone CLT - # package or the CLT component within Xcode. - cflags = _config_vars.get('CFLAGS', '') - m = re.search(r'-isysroot\s+(\S+)', cflags) - if m is not None: - sdk = m.group(1) - if not os.path.exists(sdk): - for cv in _UNIVERSAL_CONFIG_VARS: - # Do not alter a config var explicitly overriden by env var - if cv in _config_vars and cv not in os.environ: - flags = _config_vars[cv] - flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) - _save_modified_value(_config_vars, cv, flags) - - return _config_vars - - -def compiler_fixup(compiler_so, cc_args): - """ - This function will strip '-isysroot PATH' and '-arch ARCH' from the - compile flags if the user has specified one them in extra_compile_flags. - - This is needed because '-arch ARCH' adds another architecture to the - build, without a way to remove an architecture. Furthermore GCC will - barf if multiple '-isysroot' arguments are present. - """ - stripArch = stripSysroot = False - - compiler_so = list(compiler_so) - - if not _supports_universal_builds(): - # OSX before 10.4.0, these don't support -arch and -isysroot at - # all. - stripArch = stripSysroot = True - else: - stripArch = '-arch' in cc_args - stripSysroot = '-isysroot' in cc_args - - if stripArch or 'ARCHFLAGS' in os.environ: - while True: - try: - index = compiler_so.index('-arch') - # Strip this argument and the next one: - del compiler_so[index:index+2] - except ValueError: - break - - if 'ARCHFLAGS' in os.environ and not stripArch: - # User specified different -arch flags in the environ, - # see also distutils.sysconfig - compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() - - if stripSysroot: - while True: - try: - index = compiler_so.index('-isysroot') - # Strip this argument and the next one: - del compiler_so[index:index+2] - except ValueError: - break - - # Check if the SDK that is used during compilation actually exists, - # the universal build requires the usage of a universal SDK and not all - # users have that installed by default. - sysroot = None - if '-isysroot' in cc_args: - idx = cc_args.index('-isysroot') - sysroot = cc_args[idx+1] - elif '-isysroot' in compiler_so: - idx = compiler_so.index('-isysroot') - sysroot = compiler_so[idx+1] - - if sysroot and not os.path.isdir(sysroot): - from distutils import log - log.warn("Compiling with an SDK that doesn't seem to exist: %s", - sysroot) - log.warn("Please check your Xcode installation") - - return compiler_so - - -def customize_config_vars(_config_vars): - """Customize Python build configuration variables. - - Called internally from sysconfig with a mutable mapping - containing name/value pairs parsed from the configured - makefile used to build this interpreter. Returns - the mapping updated as needed to reflect the environment - in which the interpreter is running; in the case of - a Python from a binary installer, the installed - environment may be very different from the build - environment, i.e. different OS levels, different - built tools, different available CPU architectures. - - This customization is performed whenever - distutils.sysconfig.get_config_vars() is first - called. It may be used in environments where no - compilers are present, i.e. when installing pure - Python dists. Customization of compiler paths - and detection of unavailable archs is deferred - until the first extension module build is - requested (in distutils.sysconfig.customize_compiler). - - Currently called from distutils.sysconfig - """ - - if not _supports_universal_builds(): - # On Mac OS X before 10.4, check if -arch and -isysroot - # are in CFLAGS or LDFLAGS and remove them if they are. - # This is needed when building extensions on a 10.3 system - # using a universal build of python. - _remove_universal_flags(_config_vars) - - # Allow user to override all archs with ARCHFLAGS env var - _override_all_archs(_config_vars) - - # Remove references to sdks that are not found - _check_for_unavailable_sdk(_config_vars) - - return _config_vars - - -def customize_compiler(_config_vars): - """Customize compiler path and configuration variables. - - This customization is performed when the first - extension module build is requested - in distutils.sysconfig.customize_compiler). - """ - - # Find a compiler to use for extension module builds - _find_appropriate_compiler(_config_vars) - - # Remove ppc arch flags if not supported here - _remove_unsupported_archs(_config_vars) - - # Allow user to override all archs with ARCHFLAGS env var - _override_all_archs(_config_vars) - - return _config_vars - - -def get_platform_osx(_config_vars, osname, release, machine): - """Filter values for get_platform()""" - # called from get_platform() in sysconfig and distutils.util - # - # For our purposes, we'll assume that the system version from - # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set - # to. This makes the compatibility story a bit more sane because the - # machine is going to compile and link as if it were - # MACOSX_DEPLOYMENT_TARGET. - - macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') - macrelease = _get_system_version() or macver - macver = macver or macrelease - - if macver: - release = macver - osname = "macosx" - - # Use the original CFLAGS value, if available, so that we - # return the same machine type for the platform string. - # Otherwise, distutils may consider this a cross-compiling - # case and disallow installs. - cflags = _config_vars.get(_INITPRE+'CFLAGS', - _config_vars.get('CFLAGS', '')) - if macrelease: - try: - macrelease = tuple(int(i) for i in macrelease.split('.')[0:2]) - except ValueError: - macrelease = (10, 0) - else: - # assume no universal support - macrelease = (10, 0) - - if (macrelease >= (10, 4)) and '-arch' in cflags.strip(): - # The universal build will build fat binaries, but not on - # systems before 10.4 - - machine = 'fat' - - archs = re.findall('-arch\s+(\S+)', cflags) - archs = tuple(sorted(set(archs))) - - if len(archs) == 1: - machine = archs[0] - elif archs == ('i386', 'ppc'): - machine = 'fat' - elif archs == ('i386', 'x86_64'): - machine = 'intel' - elif archs == ('i386', 'ppc', 'x86_64'): - machine = 'fat3' - elif archs == ('ppc64', 'x86_64'): - machine = 'fat64' - elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): - machine = 'universal' - else: - raise ValueError( - "Don't know machine value for archs=%r" % (archs,)) - - elif machine == 'i386': - # On OSX the machine type returned by uname is always the - # 32-bit variant, even if the executable architecture is - # the 64-bit variant - if sys.maxsize >= 2**32: - machine = 'x86_64' - - elif machine in ('PowerPC', 'Power_Macintosh'): - # Pick a sane name for the PPC architecture. - # See 'i386' case - if sys.maxsize >= 2**32: - machine = 'ppc64' - else: - machine = 'ppc' - - return (osname, release, machine) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_pydecimal.py --- a/Lib/_pydecimal.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,6430 +0,0 @@ -# Copyright (c) 2004 Python Software Foundation. -# All rights reserved. - -# Written by Eric Price -# and Facundo Batista -# and Raymond Hettinger -# and Aahz -# and Tim Peters - -# This module should be kept in sync with the latest updates of the -# IBM specification as it evolves. Those updates will be treated -# as bug fixes (deviation from the spec is a compatibility, usability -# bug) and will be backported. At this point the spec is stabilizing -# and the updates are becoming fewer, smaller, and less significant. - -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" - -__all__ = [ - # Two major classes - 'Decimal', 'Context', - - # Named tuple representation - 'DecimalTuple', - - # Contexts - 'DefaultContext', 'BasicContext', 'ExtendedContext', - - # Exceptions - 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', - 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', - 'FloatOperation', - - # Exceptional conditions that trigger InvalidOperation - 'DivisionImpossible', 'InvalidContext', 'ConversionSyntax', 'DivisionUndefined', - - # Constants for use in setting up contexts - 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', - 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', - - # Functions for manipulating contexts - 'setcontext', 'getcontext', 'localcontext', - - # Limits for the C version for compatibility - 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', - - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' -] - -__xname__ = __name__ # sys.modules lookup (--without-threads) -__name__ = 'decimal' # For pickling -__version__ = '1.70' # Highest version of the spec this complies with - # See http://speleotrove.com/decimal/ -__libmpdec_version__ = "2.4.1" # compatible libmpdec version - -import math as _math -import numbers as _numbers -import sys - -try: - from collections import namedtuple as _namedtuple - DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') -except ImportError: - DecimalTuple = lambda *args: args - -# Rounding -ROUND_DOWN = 'ROUND_DOWN' -ROUND_HALF_UP = 'ROUND_HALF_UP' -ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' -ROUND_CEILING = 'ROUND_CEILING' -ROUND_FLOOR = 'ROUND_FLOOR' -ROUND_UP = 'ROUND_UP' -ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' -ROUND_05UP = 'ROUND_05UP' - -# Compatibility with the C version -HAVE_THREADS = True -if sys.maxsize == 2**63-1: - MAX_PREC = 999999999999999999 - MAX_EMAX = 999999999999999999 - MIN_EMIN = -999999999999999999 -else: - MAX_PREC = 425000000 - MAX_EMAX = 425000000 - MIN_EMIN = -425000000 - -MIN_ETINY = MIN_EMIN - (MAX_PREC-1) - -# Errors - -class DecimalException(ArithmeticError): - """Base exception class. - - Used exceptions derive from this. - If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only - called if the others are present. This isn't actually used for - anything, though. - - handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after - the explanation in _raise_error (For example, - context._raise_error(NewError, '(-x)!', self._sign) would - call NewError().handle(context, self._sign).) - - To define a new exception, it should be sufficient to have it derive - from DecimalException. - """ - def handle(self, context, *args): - pass - - -class Clamped(DecimalException): - """Exponent of a 0 changed to fit bounds. - - This occurs and signals clamped if the exponent of a result has been - altered in order to fit the constraints of a specific concrete - representation. This may occur when the exponent of a zero result would - be outside the bounds of a representation, or when a large normal - number would have an encoded exponent that cannot be represented. In - this latter case, the exponent is reduced to fit and the corresponding - number of zero digits are appended to the coefficient ("fold-down"). - """ - -class InvalidOperation(DecimalException): - """An invalid operation was performed. - - Various bad things cause this: - - Something creates a signaling NaN - -INF + INF - 0 * (+-)INF - (+-)INF / (+-)INF - x % 0 - (+-)INF % x - x._rescale( non-integer ) - sqrt(-x) , x > 0 - 0 ** 0 - x ** (non-integer) - x ** (+-)INF - An operand is invalid - - The result of the operation after these is a quiet positive NaN, - except when the cause is a signaling NaN, in which case the result is - also a quiet NaN, but with the original sign, and an optional - diagnostic information. - """ - def handle(self, context, *args): - if args: - ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) - return ans._fix_nan(context) - return _NaN - -class ConversionSyntax(InvalidOperation): - """Trying to convert badly formed string. - - This occurs and signals invalid-operation if an string is being - converted to a number and it does not conform to the numeric string - syntax. The result is [0,qNaN]. - """ - def handle(self, context, *args): - return _NaN - -class DivisionByZero(DecimalException, ZeroDivisionError): - """Division by 0. - - This occurs and signals division-by-zero if division of a finite number - by zero was attempted (during a divide-integer or divide operation, or a - power operation with negative right-hand operand), and the dividend was - not zero. - - The result of the operation is [sign,inf], where sign is the exclusive - or of the signs of the operands for divide, or is 1 for an odd power of - -0, for power. - """ - - def handle(self, context, sign, *args): - return _SignedInfinity[sign] - -class DivisionImpossible(InvalidOperation): - """Cannot perform the division adequately. - - This occurs and signals invalid-operation if the integer result of a - divide-integer or remainder operation had too many digits (would be - longer than precision). The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class DivisionUndefined(InvalidOperation, ZeroDivisionError): - """Undefined result of division. - - This occurs and signals invalid-operation if division by zero was - attempted (during a divide-integer, divide, or remainder operation), and - the dividend is also zero. The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class Inexact(DecimalException): - """Had to round, losing information. - - This occurs and signals inexact whenever the result of an operation is - not exact (that is, it needed to be rounded and any discarded digits - were non-zero), or if an overflow or underflow condition occurs. The - result in all cases is unchanged. - - The inexact signal may be tested (or trapped) to determine if a given - operation (or sequence of operations) was inexact. - """ - -class InvalidContext(InvalidOperation): - """Invalid context. Unknown rounding, for example. - - This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. - """ - - def handle(self, context, *args): - return _NaN - -class Rounded(DecimalException): - """Number got rounded (not necessarily changed during rounding). - - This occurs and signals rounded whenever the result of an operation is - rounded (that is, some zero or non-zero digits were discarded from the - coefficient), or if an overflow or underflow condition occurs. The - result in all cases is unchanged. - - The rounded signal may be tested (or trapped) to determine if a given - operation (or sequence of operations) caused a loss of precision. - """ - -class Subnormal(DecimalException): - """Exponent < Emin before rounding. - - This occurs and signals subnormal whenever the result of a conversion or - operation is subnormal (that is, its adjusted exponent is less than - Emin, before any rounding). The result in all cases is unchanged. - - The subnormal signal may be tested (or trapped) to determine if a given - or operation (or sequence of operations) yielded a subnormal result. - """ - -class Overflow(Inexact, Rounded): - """Numerical overflow. - - This occurs and signals overflow if the adjusted exponent of a result - (from a conversion or from an operation that is not an attempt to divide - by zero), after rounding, would be greater than the largest value that - can be handled by the implementation (the value Emax). - - The result depends on the rounding mode: - - For round-half-up and round-half-even (and for round-half-down and - round-up, if implemented), the result of the operation is [sign,inf], - where sign is the sign of the intermediate result. For round-down, the - result is the largest finite number that can be represented in the - current precision, with the sign of the intermediate result. For - round-ceiling, the result is the same as for round-down if the sign of - the intermediate result is 1, or is [0,inf] otherwise. For round-floor, - the result is the same as for round-down if the sign of the intermediate - result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded - will also be raised. - """ - - def handle(self, context, sign, *args): - if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, - ROUND_HALF_DOWN, ROUND_UP): - return _SignedInfinity[sign] - if sign == 0: - if context.rounding == ROUND_CEILING: - return _SignedInfinity[sign] - return _dec_from_triple(sign, '9'*context.prec, - context.Emax-context.prec+1) - if sign == 1: - if context.rounding == ROUND_FLOOR: - return _SignedInfinity[sign] - return _dec_from_triple(sign, '9'*context.prec, - context.Emax-context.prec+1) - - -class Underflow(Inexact, Rounded, Subnormal): - """Numerical underflow with result rounded to 0. - - This occurs and signals underflow if a result is inexact and the - adjusted exponent of the result would be smaller (more negative) than - the smallest value that can be handled by the implementation (the value - Emin). That is, the result is both inexact and subnormal. - - The result after an underflow will be a subnormal number rounded, if - necessary, so that its exponent is not less than Etiny. This may result - in 0 with the sign of the intermediate result and an exponent of Etiny. - - In all cases, Inexact, Rounded, and Subnormal will also be raised. - """ - -class FloatOperation(DecimalException, TypeError): - """Enable stricter semantics for mixing floats and Decimals. - - If the signal is not trapped (default), mixing floats and Decimals is - permitted in the Decimal() constructor, context.create_decimal() and - all comparison operators. Both conversion and comparisons are exact. - Any occurrence of a mixed operation is silently recorded by setting - FloatOperation in the context flags. Explicit conversions with - Decimal.from_float() or context.create_decimal_from_float() do not - set the flag. - - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. - """ - -# List of public traps and flags -_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, - Underflow, InvalidOperation, Subnormal, FloatOperation] - -# Map conditions (per the spec) to signals -_condition_map = {ConversionSyntax:InvalidOperation, - DivisionImpossible:InvalidOperation, - DivisionUndefined:InvalidOperation, - InvalidContext:InvalidOperation} - -# Valid rounding modes -_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, - ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) - -##### Context Functions ################################################## - -# The getcontext() and setcontext() function manage access to a thread-local -# current context. Py2.4 offers direct support for thread locals. If that -# is not available, use threading.current_thread() which is slower but will -# work for older Pythons. If threads are not part of the build, create a -# mock threading object with threading.local() returning the module namespace. - -try: - import threading -except ImportError: - # Python was compiled without threads; create a mock object instead - class MockThreading(object): - def local(self, sys=sys): - return sys.modules[__xname__] - threading = MockThreading() - del MockThreading - -try: - threading.local - -except AttributeError: - - # To fix reloading, force it to create a new context - # Old contexts have different exceptions in their dicts, making problems. - if hasattr(threading.current_thread(), '__decimal_context__'): - del threading.current_thread().__decimal_context__ - - def setcontext(context): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - threading.current_thread().__decimal_context__ = context - - def getcontext(): - """Returns this thread's context. - - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return threading.current_thread().__decimal_context__ - except AttributeError: - context = Context() - threading.current_thread().__decimal_context__ = context - return context - -else: - - local = threading.local() - if hasattr(local, '__decimal_context__'): - del local.__decimal_context__ - - def getcontext(_local=local): - """Returns this thread's context. - - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return _local.__decimal_context__ - except AttributeError: - context = Context() - _local.__decimal_context__ = context - return context - - def setcontext(context, _local=local): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - _local.__decimal_context__ = context - - del threading, local # Don't contaminate the namespace - -def localcontext(ctx=None): - """Return a context manager for a copy of the supplied context - - Uses a copy of the current context if no context is specified - The returned context manager creates a local decimal context - in a with statement: - def sin(x): - with localcontext() as ctx: - ctx.prec += 2 - # Rest of sin calculation algorithm - # uses a precision 2 greater than normal - return +s # Convert result to normal precision - - def sin(x): - with localcontext(ExtendedContext): - # Rest of sin calculation algorithm - # uses the Extended Context from the - # General Decimal Arithmetic Specification - return +s # Convert result to normal context - - >>> setcontext(DefaultContext) - >>> print(getcontext().prec) - 28 - >>> with localcontext(): - ... ctx = getcontext() - ... ctx.prec += 2 - ... print(ctx.prec) - ... - 30 - >>> with localcontext(ExtendedContext): - ... print(getcontext().prec) - ... - 9 - >>> print(getcontext().prec) - 28 - """ - if ctx is None: ctx = getcontext() - return _ContextManager(ctx) - - -##### Decimal class ####################################################### - -# Do not subclass Decimal from numbers.Real and do not register it as such -# (because Decimals are not interoperable with floats). See the notes in -# numbers.py for more detail. - -class Decimal(object): - """Floating point class for decimal arithmetic.""" - - __slots__ = ('_exp','_int','_sign', '_is_special') - # Generally, the value of the Decimal instance is given by - # (-1)**_sign * _int * 10**_exp - # Special values are signified by _is_special == True - - # We're immutable, so use __new__ not __init__ - def __new__(cls, value="0", context=None): - """Create a decimal point instance. - - >>> Decimal('3.14') # string input - Decimal('3.14') - >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) - Decimal('3.14') - >>> Decimal(314) # int - Decimal('314') - >>> Decimal(Decimal(314)) # another decimal instance - Decimal('314') - >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay - Decimal('3.14') - """ - - # Note that the coefficient, self._int, is actually stored as - # a string rather than as a tuple of digits. This speeds up - # the "digits to integer" and "integer to digits" conversions - # that are used in almost every arithmetic operation on - # Decimals. This is an internal detail: the as_tuple function - # and the Decimal constructor still deal with tuples of - # digits. - - self = object.__new__(cls) - - # From a string - # REs insist on real strings, so we can too. - if isinstance(value, str): - m = _parser(value.strip()) - if m is None: - if context is None: - context = getcontext() - return context._raise_error(ConversionSyntax, - "Invalid literal for Decimal: %r" % value) - - if m.group('sign') == "-": - self._sign = 1 - else: - self._sign = 0 - intpart = m.group('int') - if intpart is not None: - # finite number - fracpart = m.group('frac') or '' - exp = int(m.group('exp') or '0') - self._int = str(int(intpart+fracpart)) - self._exp = exp - len(fracpart) - self._is_special = False - else: - diag = m.group('diag') - if diag is not None: - # NaN - self._int = str(int(diag or '0')).lstrip('0') - if m.group('signal'): - self._exp = 'N' - else: - self._exp = 'n' - else: - # infinity - self._int = '0' - self._exp = 'F' - self._is_special = True - return self - - # From an integer - if isinstance(value, int): - if value >= 0: - self._sign = 0 - else: - self._sign = 1 - self._exp = 0 - self._int = str(abs(value)) - self._is_special = False - return self - - # From another decimal - if isinstance(value, Decimal): - self._exp = value._exp - self._sign = value._sign - self._int = value._int - self._is_special = value._is_special - return self - - # From an internal working value - if isinstance(value, _WorkRep): - self._sign = value.sign - self._int = str(value.int) - self._exp = int(value.exp) - self._is_special = False - return self - - # tuple/list conversion (possibly from as_tuple()) - if isinstance(value, (list,tuple)): - if len(value) != 3: - raise ValueError('Invalid tuple size in creation of Decimal ' - 'from list or tuple. The list or tuple ' - 'should have exactly three elements.') - # process sign. The isinstance test rejects floats - if not (isinstance(value[0], int) and value[0] in (0,1)): - raise ValueError("Invalid sign. The first value in the tuple " - "should be an integer; either 0 for a " - "positive number or 1 for a negative number.") - self._sign = value[0] - if value[2] == 'F': - # infinity: value[1] is ignored - self._int = '0' - self._exp = value[2] - self._is_special = True - else: - # process and validate the digits in value[1] - digits = [] - for digit in value[1]: - if isinstance(digit, int) and 0 <= digit <= 9: - # skip leading zeros - if digits or digit != 0: - digits.append(digit) - else: - raise ValueError("The second value in the tuple must " - "be composed of integers in the range " - "0 through 9.") - if value[2] in ('n', 'N'): - # NaN: digits form the diagnostic - self._int = ''.join(map(str, digits)) - self._exp = value[2] - self._is_special = True - elif isinstance(value[2], int): - # finite number: digits give the coefficient - self._int = ''.join(map(str, digits or [0])) - self._exp = value[2] - self._is_special = False - else: - raise ValueError("The third value in the tuple must " - "be an integer, or one of the " - "strings 'F', 'n', 'N'.") - return self - - if isinstance(value, float): - if context is None: - context = getcontext() - context._raise_error(FloatOperation, - "strict semantics for mixing floats and Decimals are " - "enabled") - value = Decimal.from_float(value) - self._exp = value._exp - self._sign = value._sign - self._int = value._int - self._is_special = value._is_special - return self - - raise TypeError("Cannot convert %r to Decimal" % value) - - @classmethod - def from_float(cls, f): - """Converts a float to a decimal number, exactly. - - Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). - Since 0.1 is not exactly representable in binary floating point, the - value is stored as the nearest representable value which is - 0x1.999999999999ap-4. The exact equivalent of the value in decimal - is 0.1000000000000000055511151231257827021181583404541015625. - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - >>> Decimal.from_float(float('nan')) - Decimal('NaN') - >>> Decimal.from_float(float('inf')) - Decimal('Infinity') - >>> Decimal.from_float(-float('inf')) - Decimal('-Infinity') - >>> Decimal.from_float(-0.0) - Decimal('-0') - - """ - if isinstance(f, int): # handle integer inputs - return cls(f) - if not isinstance(f, float): - raise TypeError("argument must be int or float.") - if _math.isinf(f) or _math.isnan(f): - return cls(repr(f)) - if _math.copysign(1.0, f) == 1.0: - sign = 0 - else: - sign = 1 - n, d = abs(f).as_integer_ratio() - k = d.bit_length() - 1 - result = _dec_from_triple(sign, str(n*5**k), -k) - if cls is Decimal: - return result - else: - return cls(result) - - def _isnan(self): - """Returns whether the number is not actually one. - - 0 if a number - 1 if NaN - 2 if sNaN - """ - if self._is_special: - exp = self._exp - if exp == 'n': - return 1 - elif exp == 'N': - return 2 - return 0 - - def _isinfinity(self): - """Returns whether the number is infinite - - 0 if finite or not a number - 1 if +INF - -1 if -INF - """ - if self._exp == 'F': - if self._sign: - return -1 - return 1 - return 0 - - def _check_nans(self, other=None, context=None): - """Returns whether the number is not actually one. - - if self, other are sNaN, signal - if self, other are NaN return nan - return 0 - - Done before operations. - """ - - self_is_nan = self._isnan() - if other is None: - other_is_nan = False - else: - other_is_nan = other._isnan() - - if self_is_nan or other_is_nan: - if context is None: - context = getcontext() - - if self_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - self) - if other_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - other) - if self_is_nan: - return self._fix_nan(context) - - return other._fix_nan(context) - return 0 - - def _compare_check_nans(self, other, context): - """Version of _check_nans used for the signaling comparisons - compare_signal, __le__, __lt__, __ge__, __gt__. - - Signal InvalidOperation if either self or other is a (quiet - or signaling) NaN. Signaling NaNs take precedence over quiet - NaNs. - - Return 0 if neither operand is a NaN. - - """ - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - if self.is_snan(): - return context._raise_error(InvalidOperation, - 'comparison involving sNaN', - self) - elif other.is_snan(): - return context._raise_error(InvalidOperation, - 'comparison involving sNaN', - other) - elif self.is_qnan(): - return context._raise_error(InvalidOperation, - 'comparison involving NaN', - self) - elif other.is_qnan(): - return context._raise_error(InvalidOperation, - 'comparison involving NaN', - other) - return 0 - - def __bool__(self): - """Return True if self is nonzero; otherwise return False. - - NaNs and infinities are considered nonzero. - """ - return self._is_special or self._int != '0' - - def _cmp(self, other): - """Compare the two non-NaN decimal instances self and other. - - Returns -1 if self < other, 0 if self == other and 1 - if self > other. This routine is for internal use only.""" - - if self._is_special or other._is_special: - self_inf = self._isinfinity() - other_inf = other._isinfinity() - if self_inf == other_inf: - return 0 - elif self_inf < other_inf: - return -1 - else: - return 1 - - # check for zeros; Decimal('0') == Decimal('-0') - if not self: - if not other: - return 0 - else: - return -((-1)**other._sign) - if not other: - return (-1)**self._sign - - # If different signs, neg one is less - if other._sign < self._sign: - return -1 - if self._sign < other._sign: - return 1 - - self_adjusted = self.adjusted() - other_adjusted = other.adjusted() - if self_adjusted == other_adjusted: - self_padded = self._int + '0'*(self._exp - other._exp) - other_padded = other._int + '0'*(other._exp - self._exp) - if self_padded == other_padded: - return 0 - elif self_padded < other_padded: - return -(-1)**self._sign - else: - return (-1)**self._sign - elif self_adjusted > other_adjusted: - return (-1)**self._sign - else: # self_adjusted < other_adjusted - return -((-1)**self._sign) - - # Note: The Decimal standard doesn't cover rich comparisons for - # Decimals. In particular, the specification is silent on the - # subject of what should happen for a comparison involving a NaN. - # We take the following approach: - # - # == comparisons involving a quiet NaN always return False - # != comparisons involving a quiet NaN always return True - # == or != comparisons involving a signaling NaN signal - # InvalidOperation, and return False or True as above if the - # InvalidOperation is not trapped. - # <, >, <= and >= comparisons involving a (quiet or signaling) - # NaN signal InvalidOperation, and return False if the - # InvalidOperation is not trapped. - # - # This behavior is designed to conform as closely as possible to - # that specified by IEEE 754. - - def __eq__(self, other, context=None): - self, other = _convert_for_comparison(self, other, equality_op=True) - if other is NotImplemented: - return other - if self._check_nans(other, context): - return False - return self._cmp(other) == 0 - - def __lt__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) < 0 - - def __le__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) <= 0 - - def __gt__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) > 0 - - def __ge__(self, other, context=None): - self, other = _convert_for_comparison(self, other) - if other is NotImplemented: - return other - ans = self._compare_check_nans(other, context) - if ans: - return False - return self._cmp(other) >= 0 - - def compare(self, other, context=None): - """Compare self to other. Return a decimal value: - - a or b is a NaN ==> Decimal('NaN') - a < b ==> Decimal('-1') - a == b ==> Decimal('0') - a > b ==> Decimal('1') - """ - other = _convert_other(other, raiseit=True) - - # Compare(NaN, NaN) = NaN - if (self._is_special or other and other._is_special): - ans = self._check_nans(other, context) - if ans: - return ans - - return Decimal(self._cmp(other)) - - def __hash__(self): - """x.__hash__() <==> hash(x)""" - - # In order to make sure that the hash of a Decimal instance - # agrees with the hash of a numerically equal integer, float - # or Fraction, we follow the rules for numeric hashes outlined - # in the documentation. (See library docs, 'Built-in Types'). - if self._is_special: - if self.is_snan(): - raise TypeError('Cannot hash a signaling NaN value.') - elif self.is_nan(): - return _PyHASH_NAN - else: - if self._sign: - return -_PyHASH_INF - else: - return _PyHASH_INF - - if self._exp >= 0: - exp_hash = pow(10, self._exp, _PyHASH_MODULUS) - else: - exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) - hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS - ans = hash_ if self >= 0 else -hash_ - return -2 if ans == -1 else ans - - def as_tuple(self): - """Represents the number as a triple tuple. - - To show the internals exactly as they are. - """ - return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) - - def as_integer_ratio(self): - """Express a finite Decimal instance in the form n / d. - - Returns a pair (n, d) of integers. When called on an infinity - or NaN, raises OverflowError or ValueError respectively. - - >>> Decimal('3.14').as_integer_ratio() - (157, 50) - >>> Decimal('-123e5').as_integer_ratio() - (-12300000, 1) - >>> Decimal('0.00').as_integer_ratio() - (0, 1) - - """ - if self._is_special: - if self.is_nan(): - raise ValueError("cannot convert NaN to integer ratio") - else: - raise OverflowError("cannot convert Infinity to integer ratio") - - if not self: - return 0, 1 - - # Find n, d in lowest terms such that abs(self) == n / d; - # we'll deal with the sign later. - n = int(self._int) - if self._exp >= 0: - # self is an integer. - n, d = n * 10**self._exp, 1 - else: - # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). - d5 = -self._exp - while d5 > 0 and n % 5 == 0: - n //= 5 - d5 -= 1 - - # (n & -n).bit_length() - 1 counts trailing zeros in binary - # representation of n (provided n is nonzero). - d2 = -self._exp - shift2 = min((n & -n).bit_length() - 1, d2) - if shift2: - n >>= shift2 - d2 -= shift2 - - d = 5**d5 << d2 - - if self._sign: - n = -n - return n, d - - def __repr__(self): - """Represents the number as an instance of Decimal.""" - # Invariant: eval(repr(d)) == d - return "Decimal('%s')" % str(self) - - def __str__(self, eng=False, context=None): - """Return string representation of the number in scientific notation. - - Captures all of the information in the underlying representation. - """ - - sign = ['', '-'][self._sign] - if self._is_special: - if self._exp == 'F': - return sign + 'Infinity' - elif self._exp == 'n': - return sign + 'NaN' + self._int - else: # self._exp == 'N' - return sign + 'sNaN' + self._int - - # number of digits of self._int to left of decimal point - leftdigits = self._exp + len(self._int) - - # dotplace is number of digits of self._int to the left of the - # decimal point in the mantissa of the output string (that is, - # after adjusting the exponent) - if self._exp <= 0 and leftdigits > -6: - # no exponent required - dotplace = leftdigits - elif not eng: - # usual scientific notation: 1 digit on left of the point - dotplace = 1 - elif self._int == '0': - # engineering notation, zero - dotplace = (leftdigits + 1) % 3 - 1 - else: - # engineering notation, nonzero - dotplace = (leftdigits - 1) % 3 + 1 - - if dotplace <= 0: - intpart = '0' - fracpart = '.' + '0'*(-dotplace) + self._int - elif dotplace >= len(self._int): - intpart = self._int+'0'*(dotplace-len(self._int)) - fracpart = '' - else: - intpart = self._int[:dotplace] - fracpart = '.' + self._int[dotplace:] - if leftdigits == dotplace: - exp = '' - else: - if context is None: - context = getcontext() - exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) - - return sign + intpart + fracpart + exp - - def to_eng_string(self, context=None): - """Convert to engineering-type string. - - Engineering notation has an exponent which is a multiple of 3, so there - are up to 3 digits left of the decimal place. - - Same rules for when in exponential and when as a value as in __str__. - """ - return self.__str__(eng=True, context=context) - - def __neg__(self, context=None): - """Returns a copy with the sign switched. - - Rounds, if it has reason. - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - if not self and context.rounding != ROUND_FLOOR: - # -Decimal('0') is Decimal('0'), not Decimal('-0'), except - # in ROUND_FLOOR rounding mode. - ans = self.copy_abs() - else: - ans = self.copy_negate() - - return ans._fix(context) - - def __pos__(self, context=None): - """Returns a copy, unless it is a sNaN. - - Rounds the number (if more then precision digits) - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - if not self and context.rounding != ROUND_FLOOR: - # + (-0) = 0, except in ROUND_FLOOR rounding mode. - ans = self.copy_abs() - else: - ans = Decimal(self) - - return ans._fix(context) - - def __abs__(self, round=True, context=None): - """Returns the absolute value of self. - - If the keyword argument 'round' is false, do not round. The - expression self.__abs__(round=False) is equivalent to - self.copy_abs(). - """ - if not round: - return self.copy_abs() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if self._sign: - ans = self.__neg__(context=context) - else: - ans = self.__pos__(context=context) - - return ans - - def __add__(self, other, context=None): - """Returns self + other. - - -INF + INF (or the reverse) cause InvalidOperation errors. - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - # If both INF, same sign => same as both, opposite => error. - if self._sign != other._sign and other._isinfinity(): - return context._raise_error(InvalidOperation, '-INF + INF') - return Decimal(self) - if other._isinfinity(): - return Decimal(other) # Can't both be infinity here - - exp = min(self._exp, other._exp) - negativezero = 0 - if context.rounding == ROUND_FLOOR and self._sign != other._sign: - # If the answer is 0, the sign should be negative, in this case. - negativezero = 1 - - if not self and not other: - sign = min(self._sign, other._sign) - if negativezero: - sign = 1 - ans = _dec_from_triple(sign, '0', exp) - ans = ans._fix(context) - return ans - if not self: - exp = max(exp, other._exp - context.prec-1) - ans = other._rescale(exp, context.rounding) - ans = ans._fix(context) - return ans - if not other: - exp = max(exp, self._exp - context.prec-1) - ans = self._rescale(exp, context.rounding) - ans = ans._fix(context) - return ans - - op1 = _WorkRep(self) - op2 = _WorkRep(other) - op1, op2 = _normalize(op1, op2, context.prec) - - result = _WorkRep() - if op1.sign != op2.sign: - # Equal and opposite - if op1.int == op2.int: - ans = _dec_from_triple(negativezero, '0', exp) - ans = ans._fix(context) - return ans - if op1.int < op2.int: - op1, op2 = op2, op1 - # OK, now abs(op1) > abs(op2) - if op1.sign == 1: - result.sign = 1 - op1.sign, op2.sign = op2.sign, op1.sign - else: - result.sign = 0 - # So we know the sign, and op1 > 0. - elif op1.sign == 1: - result.sign = 1 - op1.sign, op2.sign = (0, 0) - else: - result.sign = 0 - # Now, op1 > abs(op2) > 0 - - if op2.sign == 0: - result.int = op1.int + op2.int - else: - result.int = op1.int - op2.int - - result.exp = op1.exp - ans = Decimal(result) - ans = ans._fix(context) - return ans - - __radd__ = __add__ - - def __sub__(self, other, context=None): - """Return self - other""" - other = _convert_other(other) - if other is NotImplemented: - return other - - if self._is_special or other._is_special: - ans = self._check_nans(other, context=context) - if ans: - return ans - - # self - other is computed as self + other.copy_negate() - return self.__add__(other.copy_negate(), context=context) - - def __rsub__(self, other, context=None): - """Return other - self""" - other = _convert_other(other) - if other is NotImplemented: - return other - - return other.__sub__(self, context=context) - - def __mul__(self, other, context=None): - """Return self * other. - - (+-) INF * 0 (or its reverse) raise InvalidOperation. - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - resultsign = self._sign ^ other._sign - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - if not other: - return context._raise_error(InvalidOperation, '(+-)INF * 0') - return _SignedInfinity[resultsign] - - if other._isinfinity(): - if not self: - return context._raise_error(InvalidOperation, '0 * (+-)INF') - return _SignedInfinity[resultsign] - - resultexp = self._exp + other._exp - - # Special case for multiplying by zero - if not self or not other: - ans = _dec_from_triple(resultsign, '0', resultexp) - # Fixing in case the exponent is out of bounds - ans = ans._fix(context) - return ans - - # Special case for multiplying by power of 10 - if self._int == '1': - ans = _dec_from_triple(resultsign, other._int, resultexp) - ans = ans._fix(context) - return ans - if other._int == '1': - ans = _dec_from_triple(resultsign, self._int, resultexp) - ans = ans._fix(context) - return ans - - op1 = _WorkRep(self) - op2 = _WorkRep(other) - - ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) - ans = ans._fix(context) - - return ans - __rmul__ = __mul__ - - def __truediv__(self, other, context=None): - """Return self / other.""" - other = _convert_other(other) - if other is NotImplemented: - return NotImplemented - - if context is None: - context = getcontext() - - sign = self._sign ^ other._sign - - if self._is_special or other._is_special: - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity() and other._isinfinity(): - return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') - - if self._isinfinity(): - return _SignedInfinity[sign] - - if other._isinfinity(): - context._raise_error(Clamped, 'Division by infinity') - return _dec_from_triple(sign, '0', context.Etiny()) - - # Special cases for zeroes - if not other: - if not self: - return context._raise_error(DivisionUndefined, '0 / 0') - return context._raise_error(DivisionByZero, 'x / 0', sign) - - if not self: - exp = self._exp - other._exp - coeff = 0 - else: - # OK, so neither = 0, INF or NaN - shift = len(other._int) - len(self._int) + context.prec + 1 - exp = self._exp - other._exp - shift - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if shift >= 0: - coeff, remainder = divmod(op1.int * 10**shift, op2.int) - else: - coeff, remainder = divmod(op1.int, op2.int * 10**-shift) - if remainder: - # result is not exact; adjust to ensure correct rounding - if coeff % 5 == 0: - coeff += 1 - else: - # result is exact; get as close to ideal exponent as possible - ideal_exp = self._exp - other._exp - while exp < ideal_exp and coeff % 10 == 0: - coeff //= 10 - exp += 1 - - ans = _dec_from_triple(sign, str(coeff), exp) - return ans._fix(context) - - def _divide(self, other, context): - """Return (self // other, self % other), to context.prec precision. - - Assumes that neither self nor other is a NaN, that self is not - infinite and that other is nonzero. - """ - sign = self._sign ^ other._sign - if other._isinfinity(): - ideal_exp = self._exp - else: - ideal_exp = min(self._exp, other._exp) - - expdiff = self.adjusted() - other.adjusted() - if not self or other._isinfinity() or expdiff <= -2: - return (_dec_from_triple(sign, '0', 0), - self._rescale(ideal_exp, context.rounding)) - if expdiff <= context.prec: - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if op1.exp >= op2.exp: - op1.int *= 10**(op1.exp - op2.exp) - else: - op2.int *= 10**(op2.exp - op1.exp) - q, r = divmod(op1.int, op2.int) - if q < 10**context.prec: - return (_dec_from_triple(sign, str(q), 0), - _dec_from_triple(self._sign, str(r), ideal_exp)) - - # Here the quotient is too large to be representable - ans = context._raise_error(DivisionImpossible, - 'quotient too large in //, % or divmod') - return ans, ans - - def __rtruediv__(self, other, context=None): - """Swaps self/other and returns __truediv__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__truediv__(self, context=context) - - def __divmod__(self, other, context=None): - """ - Return (self // other, self % other) - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return (ans, ans) - - sign = self._sign ^ other._sign - if self._isinfinity(): - if other._isinfinity(): - ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') - return ans, ans - else: - return (_SignedInfinity[sign], - context._raise_error(InvalidOperation, 'INF % x')) - - if not other: - if not self: - ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') - return ans, ans - else: - return (context._raise_error(DivisionByZero, 'x // 0', sign), - context._raise_error(InvalidOperation, 'x % 0')) - - quotient, remainder = self._divide(other, context) - remainder = remainder._fix(context) - return quotient, remainder - - def __rdivmod__(self, other, context=None): - """Swaps self/other and returns __divmod__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__divmod__(self, context=context) - - def __mod__(self, other, context=None): - """ - self % other - """ - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - return context._raise_error(InvalidOperation, 'INF % x') - elif not other: - if self: - return context._raise_error(InvalidOperation, 'x % 0') - else: - return context._raise_error(DivisionUndefined, '0 % 0') - - remainder = self._divide(other, context)[1] - remainder = remainder._fix(context) - return remainder - - def __rmod__(self, other, context=None): - """Swaps self/other and returns __mod__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__mod__(self, context=context) - - def remainder_near(self, other, context=None): - """ - Remainder nearest to 0- abs(remainder-near) <= other/2 - """ - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - # self == +/-infinity -> InvalidOperation - if self._isinfinity(): - return context._raise_error(InvalidOperation, - 'remainder_near(infinity, x)') - - # other == 0 -> either InvalidOperation or DivisionUndefined - if not other: - if self: - return context._raise_error(InvalidOperation, - 'remainder_near(x, 0)') - else: - return context._raise_error(DivisionUndefined, - 'remainder_near(0, 0)') - - # other = +/-infinity -> remainder = self - if other._isinfinity(): - ans = Decimal(self) - return ans._fix(context) - - # self = 0 -> remainder = self, with ideal exponent - ideal_exponent = min(self._exp, other._exp) - if not self: - ans = _dec_from_triple(self._sign, '0', ideal_exponent) - return ans._fix(context) - - # catch most cases of large or small quotient - expdiff = self.adjusted() - other.adjusted() - if expdiff >= context.prec + 1: - # expdiff >= prec+1 => abs(self/other) > 10**prec - return context._raise_error(DivisionImpossible) - if expdiff <= -2: - # expdiff <= -2 => abs(self/other) < 0.1 - ans = self._rescale(ideal_exponent, context.rounding) - return ans._fix(context) - - # adjust both arguments to have the same exponent, then divide - op1 = _WorkRep(self) - op2 = _WorkRep(other) - if op1.exp >= op2.exp: - op1.int *= 10**(op1.exp - op2.exp) - else: - op2.int *= 10**(op2.exp - op1.exp) - q, r = divmod(op1.int, op2.int) - # remainder is r*10**ideal_exponent; other is +/-op2.int * - # 10**ideal_exponent. Apply correction to ensure that - # abs(remainder) <= abs(other)/2 - if 2*r + (q&1) > op2.int: - r -= op2.int - q += 1 - - if q >= 10**context.prec: - return context._raise_error(DivisionImpossible) - - # result has same sign as self unless r is negative - sign = self._sign - if r < 0: - sign = 1-sign - r = -r - - ans = _dec_from_triple(sign, str(r), ideal_exponent) - return ans._fix(context) - - def __floordiv__(self, other, context=None): - """self // other""" - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - if self._isinfinity(): - if other._isinfinity(): - return context._raise_error(InvalidOperation, 'INF // INF') - else: - return _SignedInfinity[self._sign ^ other._sign] - - if not other: - if self: - return context._raise_error(DivisionByZero, 'x // 0', - self._sign ^ other._sign) - else: - return context._raise_error(DivisionUndefined, '0 // 0') - - return self._divide(other, context)[0] - - def __rfloordiv__(self, other, context=None): - """Swaps self/other and returns __floordiv__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__floordiv__(self, context=context) - - def __float__(self): - """Float representation.""" - if self._isnan(): - if self.is_snan(): - raise ValueError("Cannot convert signaling NaN to float") - s = "-nan" if self._sign else "nan" - else: - s = str(self) - return float(s) - - def __int__(self): - """Converts self to an int, truncating if necessary.""" - if self._is_special: - if self._isnan(): - raise ValueError("Cannot convert NaN to integer") - elif self._isinfinity(): - raise OverflowError("Cannot convert infinity to integer") - s = (-1)**self._sign - if self._exp >= 0: - return s*int(self._int)*10**self._exp - else: - return s*int(self._int[:self._exp] or '0') - - __trunc__ = __int__ - - def real(self): - return self - real = property(real) - - def imag(self): - return Decimal(0) - imag = property(imag) - - def conjugate(self): - return self - - def __complex__(self): - return complex(float(self)) - - def _fix_nan(self, context): - """Decapitate the payload of a NaN to fit the context""" - payload = self._int - - # maximum length of payload is precision if clamp=0, - # precision-1 if clamp=1. - max_payload_len = context.prec - context.clamp - if len(payload) > max_payload_len: - payload = payload[len(payload)-max_payload_len:].lstrip('0') - return _dec_from_triple(self._sign, payload, self._exp, True) - return Decimal(self) - - def _fix(self, context): - """Round if it is necessary to keep self within prec precision. - - Rounds and fixes the exponent. Does not raise on a sNaN. - - Arguments: - self - Decimal instance - context - context used. - """ - - if self._is_special: - if self._isnan(): - # decapitate payload if necessary - return self._fix_nan(context) - else: - # self is +/-Infinity; return unaltered - return Decimal(self) - - # if self is zero then exponent should be between Etiny and - # Emax if clamp==0, and between Etiny and Etop if clamp==1. - Etiny = context.Etiny() - Etop = context.Etop() - if not self: - exp_max = [context.Emax, Etop][context.clamp] - new_exp = min(max(self._exp, Etiny), exp_max) - if new_exp != self._exp: - context._raise_error(Clamped) - return _dec_from_triple(self._sign, '0', new_exp) - else: - return Decimal(self) - - # exp_min is the smallest allowable exponent of the result, - # equal to max(self.adjusted()-context.prec+1, Etiny) - exp_min = len(self._int) + self._exp - context.prec - if exp_min > Etop: - # overflow: exp_min > Etop iff self.adjusted() > Emax - ans = context._raise_error(Overflow, 'above Emax', self._sign) - context._raise_error(Inexact) - context._raise_error(Rounded) - return ans - - self_is_subnormal = exp_min < Etiny - if self_is_subnormal: - exp_min = Etiny - - # round if self has too many digits - if self._exp < exp_min: - digits = len(self._int) + self._exp - exp_min - if digits < 0: - self = _dec_from_triple(self._sign, '1', exp_min-1) - digits = 0 - rounding_method = self._pick_rounding_function[context.rounding] - changed = rounding_method(self, digits) - coeff = self._int[:digits] or '0' - if changed > 0: - coeff = str(int(coeff)+1) - if len(coeff) > context.prec: - coeff = coeff[:-1] - exp_min += 1 - - # check whether the rounding pushed the exponent out of range - if exp_min > Etop: - ans = context._raise_error(Overflow, 'above Emax', self._sign) - else: - ans = _dec_from_triple(self._sign, coeff, exp_min) - - # raise the appropriate signals, taking care to respect - # the precedence described in the specification - if changed and self_is_subnormal: - context._raise_error(Underflow) - if self_is_subnormal: - context._raise_error(Subnormal) - if changed: - context._raise_error(Inexact) - context._raise_error(Rounded) - if not ans: - # raise Clamped on underflow to 0 - context._raise_error(Clamped) - return ans - - if self_is_subnormal: - context._raise_error(Subnormal) - - # fold down if clamp == 1 and self has too few digits - if context.clamp == 1 and self._exp > Etop: - context._raise_error(Clamped) - self_padded = self._int + '0'*(self._exp - Etop) - return _dec_from_triple(self._sign, self_padded, Etop) - - # here self was representable to begin with; return unchanged - return Decimal(self) - - # for each of the rounding functions below: - # self is a finite, nonzero Decimal - # prec is an integer satisfying 0 <= prec < len(self._int) - # - # each function returns either -1, 0, or 1, as follows: - # 1 indicates that self should be rounded up (away from zero) - # 0 indicates that self should be truncated, and that all the - # digits to be truncated are zeros (so the value is unchanged) - # -1 indicates that there are nonzero digits to be truncated - - def _round_down(self, prec): - """Also known as round-towards-0, truncate.""" - if _all_zeros(self._int, prec): - return 0 - else: - return -1 - - def _round_up(self, prec): - """Rounds away from 0.""" - return -self._round_down(prec) - - def _round_half_up(self, prec): - """Rounds 5 up (away from 0)""" - if self._int[prec] in '56789': - return 1 - elif _all_zeros(self._int, prec): - return 0 - else: - return -1 - - def _round_half_down(self, prec): - """Round 5 down""" - if _exact_half(self._int, prec): - return -1 - else: - return self._round_half_up(prec) - - def _round_half_even(self, prec): - """Round 5 to even, rest to nearest.""" - if _exact_half(self._int, prec) and \ - (prec == 0 or self._int[prec-1] in '02468'): - return -1 - else: - return self._round_half_up(prec) - - def _round_ceiling(self, prec): - """Rounds up (not away from 0 if negative.)""" - if self._sign: - return self._round_down(prec) - else: - return -self._round_down(prec) - - def _round_floor(self, prec): - """Rounds down (not towards 0 if negative)""" - if not self._sign: - return self._round_down(prec) - else: - return -self._round_down(prec) - - def _round_05up(self, prec): - """Round down unless digit prec-1 is 0 or 5.""" - if prec and self._int[prec-1] not in '05': - return self._round_down(prec) - else: - return -self._round_down(prec) - - _pick_rounding_function = dict( - ROUND_DOWN = _round_down, - ROUND_UP = _round_up, - ROUND_HALF_UP = _round_half_up, - ROUND_HALF_DOWN = _round_half_down, - ROUND_HALF_EVEN = _round_half_even, - ROUND_CEILING = _round_ceiling, - ROUND_FLOOR = _round_floor, - ROUND_05UP = _round_05up, - ) - - def __round__(self, n=None): - """Round self to the nearest integer, or to a given precision. - - If only one argument is supplied, round a finite Decimal - instance self to the nearest integer. If self is infinite or - a NaN then a Python exception is raised. If self is finite - and lies exactly halfway between two integers then it is - rounded to the integer with even last digit. - - >>> round(Decimal('123.456')) - 123 - >>> round(Decimal('-456.789')) - -457 - >>> round(Decimal('-3.0')) - -3 - >>> round(Decimal('2.5')) - 2 - >>> round(Decimal('3.5')) - 4 - >>> round(Decimal('Inf')) - Traceback (most recent call last): - ... - OverflowError: cannot round an infinity - >>> round(Decimal('NaN')) - Traceback (most recent call last): - ... - ValueError: cannot round a NaN - - If a second argument n is supplied, self is rounded to n - decimal places using the rounding mode for the current - context. - - For an integer n, round(self, -n) is exactly equivalent to - self.quantize(Decimal('1En')). - - >>> round(Decimal('123.456'), 0) - Decimal('123') - >>> round(Decimal('123.456'), 2) - Decimal('123.46') - >>> round(Decimal('123.456'), -2) - Decimal('1E+2') - >>> round(Decimal('-Infinity'), 37) - Decimal('NaN') - >>> round(Decimal('sNaN123'), 0) - Decimal('NaN123') - - """ - if n is not None: - # two-argument form: use the equivalent quantize call - if not isinstance(n, int): - raise TypeError('Second argument to round should be integral') - exp = _dec_from_triple(0, '1', -n) - return self.quantize(exp) - - # one-argument form - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_HALF_EVEN)) - - def __floor__(self): - """Return the floor of self, as an integer. - - For a finite Decimal instance self, return the greatest - integer n such that n <= self. If self is infinite or a NaN - then a Python exception is raised. - - """ - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_FLOOR)) - - def __ceil__(self): - """Return the ceiling of self, as an integer. - - For a finite Decimal instance self, return the least integer n - such that n >= self. If self is infinite or a NaN then a - Python exception is raised. - - """ - if self._is_special: - if self.is_nan(): - raise ValueError("cannot round a NaN") - else: - raise OverflowError("cannot round an infinity") - return int(self._rescale(0, ROUND_CEILING)) - - def fma(self, other, third, context=None): - """Fused multiply-add. - - Returns self*other+third with no rounding of the intermediate - product self*other. - - self and other are multiplied together, with no rounding of - the result. The third operand is then added to the result, - and a single final rounding is performed. - """ - - other = _convert_other(other, raiseit=True) - third = _convert_other(third, raiseit=True) - - # compute product; raise InvalidOperation if either operand is - # a signaling NaN or if the product is zero times infinity. - if self._is_special or other._is_special: - if context is None: - context = getcontext() - if self._exp == 'N': - return context._raise_error(InvalidOperation, 'sNaN', self) - if other._exp == 'N': - return context._raise_error(InvalidOperation, 'sNaN', other) - if self._exp == 'n': - product = self - elif other._exp == 'n': - product = other - elif self._exp == 'F': - if not other: - return context._raise_error(InvalidOperation, - 'INF * 0 in fma') - product = _SignedInfinity[self._sign ^ other._sign] - elif other._exp == 'F': - if not self: - return context._raise_error(InvalidOperation, - '0 * INF in fma') - product = _SignedInfinity[self._sign ^ other._sign] - else: - product = _dec_from_triple(self._sign ^ other._sign, - str(int(self._int) * int(other._int)), - self._exp + other._exp) - - return product.__add__(third, context) - - def _power_modulo(self, other, modulo, context=None): - """Three argument version of __pow__""" - - other = _convert_other(other) - if other is NotImplemented: - return other - modulo = _convert_other(modulo) - if modulo is NotImplemented: - return modulo - - if context is None: - context = getcontext() - - # deal with NaNs: if there are any sNaNs then first one wins, - # (i.e. behaviour for NaNs is identical to that of fma) - self_is_nan = self._isnan() - other_is_nan = other._isnan() - modulo_is_nan = modulo._isnan() - if self_is_nan or other_is_nan or modulo_is_nan: - if self_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - self) - if other_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - other) - if modulo_is_nan == 2: - return context._raise_error(InvalidOperation, 'sNaN', - modulo) - if self_is_nan: - return self._fix_nan(context) - if other_is_nan: - return other._fix_nan(context) - return modulo._fix_nan(context) - - # check inputs: we apply same restrictions as Python's pow() - if not (self._isinteger() and - other._isinteger() and - modulo._isinteger()): - return context._raise_error(InvalidOperation, - 'pow() 3rd argument not allowed ' - 'unless all arguments are integers') - if other < 0: - return context._raise_error(InvalidOperation, - 'pow() 2nd argument cannot be ' - 'negative when 3rd argument specified') - if not modulo: - return context._raise_error(InvalidOperation, - 'pow() 3rd argument cannot be 0') - - # additional restriction for decimal: the modulus must be less - # than 10**prec in absolute value - if modulo.adjusted() >= context.prec: - return context._raise_error(InvalidOperation, - 'insufficient precision: pow() 3rd ' - 'argument must not have more than ' - 'precision digits') - - # define 0**0 == NaN, for consistency with two-argument pow - # (even though it hurts!) - if not other and not self: - return context._raise_error(InvalidOperation, - 'at least one of pow() 1st argument ' - 'and 2nd argument must be nonzero ;' - '0**0 is not defined') - - # compute sign of result - if other._iseven(): - sign = 0 - else: - sign = self._sign - - # convert modulo to a Python integer, and self and other to - # Decimal integers (i.e. force their exponents to be >= 0) - modulo = abs(int(modulo)) - base = _WorkRep(self.to_integral_value()) - exponent = _WorkRep(other.to_integral_value()) - - # compute result using integer pow() - base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo - for i in range(exponent.exp): - base = pow(base, 10, modulo) - base = pow(base, exponent.int, modulo) - - return _dec_from_triple(sign, str(base), 0) - - def _power_exact(self, other, p): - """Attempt to compute self**other exactly. - - Given Decimals self and other and an integer p, attempt to - compute an exact result for the power self**other, with p - digits of precision. Return None if self**other is not - exactly representable in p digits. - - Assumes that elimination of special cases has already been - performed: self and other must both be nonspecial; self must - be positive and not numerically equal to 1; other must be - nonzero. For efficiency, other._exp should not be too large, - so that 10**abs(other._exp) is a feasible calculation.""" - - # In the comments below, we write x for the value of self and y for the - # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc - # and yc positive integers not divisible by 10. - - # The main purpose of this method is to identify the *failure* - # of x**y to be exactly representable with as little effort as - # possible. So we look for cheap and easy tests that - # eliminate the possibility of x**y being exact. Only if all - # these tests are passed do we go on to actually compute x**y. - - # Here's the main idea. Express y as a rational number m/n, with m and - # n relatively prime and n>0. Then for x**y to be exactly - # representable (at *any* precision), xc must be the nth power of a - # positive integer and xe must be divisible by n. If y is negative - # then additionally xc must be a power of either 2 or 5, hence a power - # of 2**n or 5**n. - # - # There's a limit to how small |y| can be: if y=m/n as above - # then: - # - # (1) if xc != 1 then for the result to be representable we - # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So - # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <= - # 2**(1/|y|), hence xc**|y| < 2 and the result is not - # representable. - # - # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if - # |y| < 1/|xe| then the result is not representable. - # - # Note that since x is not equal to 1, at least one of (1) and - # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) < - # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye. - # - # There's also a limit to how large y can be, at least if it's - # positive: the normalized result will have coefficient xc**y, - # so if it's representable then xc**y < 10**p, and y < - # p/log10(xc). Hence if y*log10(xc) >= p then the result is - # not exactly representable. - - # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye, - # so |y| < 1/xe and the result is not representable. - # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y| - # < 1/nbits(xc). - - x = _WorkRep(self) - xc, xe = x.int, x.exp - while xc % 10 == 0: - xc //= 10 - xe += 1 - - y = _WorkRep(other) - yc, ye = y.int, y.exp - while yc % 10 == 0: - yc //= 10 - ye += 1 - - # case where xc == 1: result is 10**(xe*y), with xe*y - # required to be an integer - if xc == 1: - xe *= yc - # result is now 10**(xe * 10**ye); xe * 10**ye must be integral - while xe % 10 == 0: - xe //= 10 - ye += 1 - if ye < 0: - return None - exponent = xe * 10**ye - if y.sign == 1: - exponent = -exponent - # if other is a nonnegative integer, use ideal exponent - if other._isinteger() and other._sign == 0: - ideal_exponent = self._exp*int(other) - zeros = min(exponent-ideal_exponent, p-1) - else: - zeros = 0 - return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros) - - # case where y is negative: xc must be either a power - # of 2 or a power of 5. - if y.sign == 1: - last_digit = xc % 10 - if last_digit in (2,4,6,8): - # quick test for power of 2 - if xc & -xc != xc: - return None - # now xc is a power of 2; e is its exponent - e = _nbits(xc)-1 - - # We now have: - # - # x = 2**e * 10**xe, e > 0, and y < 0. - # - # The exact result is: - # - # x**y = 5**(-e*y) * 10**(e*y + xe*y) - # - # provided that both e*y and xe*y are integers. Note that if - # 5**(-e*y) >= 10**p, then the result can't be expressed - # exactly with p digits of precision. - # - # Using the above, we can guard against large values of ye. - # 93/65 is an upper bound for log(10)/log(5), so if - # - # ye >= len(str(93*p//65)) - # - # then - # - # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), - # - # so 5**(-e*y) >= 10**p, and the coefficient of the result - # can't be expressed in p digits. - - # emax >= largest e such that 5**e < 10**p. - emax = p*93//65 - if ye >= len(str(emax)): - return None - - # Find -e*y and -xe*y; both must be integers - e = _decimal_lshift_exact(e * yc, ye) - xe = _decimal_lshift_exact(xe * yc, ye) - if e is None or xe is None: - return None - - if e > emax: - return None - xc = 5**e - - elif last_digit == 5: - # e >= log_5(xc) if xc is a power of 5; we have - # equality all the way up to xc=5**2658 - e = _nbits(xc)*28//65 - xc, remainder = divmod(5**e, xc) - if remainder: - return None - while xc % 5 == 0: - xc //= 5 - e -= 1 - - # Guard against large values of ye, using the same logic as in - # the 'xc is a power of 2' branch. 10/3 is an upper bound for - # log(10)/log(2). - emax = p*10//3 - if ye >= len(str(emax)): - return None - - e = _decimal_lshift_exact(e * yc, ye) - xe = _decimal_lshift_exact(xe * yc, ye) - if e is None or xe is None: - return None - - if e > emax: - return None - xc = 2**e - else: - return None - - if xc >= 10**p: - return None - xe = -e-xe - return _dec_from_triple(0, str(xc), xe) - - # now y is positive; find m and n such that y = m/n - if ye >= 0: - m, n = yc*10**ye, 1 - else: - if xe != 0 and len(str(abs(yc*xe))) <= -ye: - return None - xc_bits = _nbits(xc) - if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: - return None - m, n = yc, 10**(-ye) - while m % 2 == n % 2 == 0: - m //= 2 - n //= 2 - while m % 5 == n % 5 == 0: - m //= 5 - n //= 5 - - # compute nth root of xc*10**xe - if n > 1: - # if 1 < xc < 2**n then xc isn't an nth power - if xc != 1 and xc_bits <= n: - return None - - xe, rem = divmod(xe, n) - if rem != 0: - return None - - # compute nth root of xc using Newton's method - a = 1 << -(-_nbits(xc)//n) # initial estimate - while True: - q, r = divmod(xc, a**(n-1)) - if a <= q: - break - else: - a = (a*(n-1) + q)//n - if not (a == q and r == 0): - return None - xc = a - - # now xc*10**xe is the nth root of the original xc*10**xe - # compute mth power of xc*10**xe - - # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > - # 10**p and the result is not representable. - if xc > 1 and m > p*100//_log10_lb(xc): - return None - xc = xc**m - xe *= m - if xc > 10**p: - return None - - # by this point the result *is* exactly representable - # adjust the exponent to get as close as possible to the ideal - # exponent, if necessary - str_xc = str(xc) - if other._isinteger() and other._sign == 0: - ideal_exponent = self._exp*int(other) - zeros = min(xe-ideal_exponent, p-len(str_xc)) - else: - zeros = 0 - return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros) - - def __pow__(self, other, modulo=None, context=None): - """Return self ** other [ % modulo]. - - With two arguments, compute self**other. - - With three arguments, compute (self**other) % modulo. For the - three argument form, the following restrictions on the - arguments hold: - - - all three arguments must be integral - - other must be nonnegative - - either self or other (or both) must be nonzero - - modulo must be nonzero and must have at most p digits, - where p is the context precision. - - If any of these restrictions is violated the InvalidOperation - flag is raised. - - The result of pow(self, other, modulo) is identical to the - result that would be obtained by computing (self**other) % - modulo with unbounded precision, but is computed more - efficiently. It is always exact. - """ - - if modulo is not None: - return self._power_modulo(other, modulo, context) - - other = _convert_other(other) - if other is NotImplemented: - return other - - if context is None: - context = getcontext() - - # either argument is a NaN => result is NaN - ans = self._check_nans(other, context) - if ans: - return ans - - # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity) - if not other: - if not self: - return context._raise_error(InvalidOperation, '0 ** 0') - else: - return _One - - # result has sign 1 iff self._sign is 1 and other is an odd integer - result_sign = 0 - if self._sign == 1: - if other._isinteger(): - if not other._iseven(): - result_sign = 1 - else: - # -ve**noninteger = NaN - # (-0)**noninteger = 0**noninteger - if self: - return context._raise_error(InvalidOperation, - 'x ** y with x negative and y not an integer') - # negate self, without doing any unwanted rounding - self = self.copy_negate() - - # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity - if not self: - if other._sign == 0: - return _dec_from_triple(result_sign, '0', 0) - else: - return _SignedInfinity[result_sign] - - # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0 - if self._isinfinity(): - if other._sign == 0: - return _SignedInfinity[result_sign] - else: - return _dec_from_triple(result_sign, '0', 0) - - # 1**other = 1, but the choice of exponent and the flags - # depend on the exponent of self, and on whether other is a - # positive integer, a negative integer, or neither - if self == _One: - if other._isinteger(): - # exp = max(self._exp*max(int(other), 0), - # 1-context.prec) but evaluating int(other) directly - # is dangerous until we know other is small (other - # could be 1e999999999) - if other._sign == 1: - multiplier = 0 - elif other > context.prec: - multiplier = context.prec - else: - multiplier = int(other) - - exp = self._exp * multiplier - if exp < 1-context.prec: - exp = 1-context.prec - context._raise_error(Rounded) - else: - context._raise_error(Inexact) - context._raise_error(Rounded) - exp = 1-context.prec - - return _dec_from_triple(result_sign, '1'+'0'*-exp, exp) - - # compute adjusted exponent of self - self_adj = self.adjusted() - - # self ** infinity is infinity if self > 1, 0 if self < 1 - # self ** -infinity is infinity if self < 1, 0 if self > 1 - if other._isinfinity(): - if (other._sign == 0) == (self_adj < 0): - return _dec_from_triple(result_sign, '0', 0) - else: - return _SignedInfinity[result_sign] - - # from here on, the result always goes through the call - # to _fix at the end of this function. - ans = None - exact = False - - # crude test to catch cases of extreme overflow/underflow. If - # log10(self)*other >= 10**bound and bound >= len(str(Emax)) - # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence - # self**other >= 10**(Emax+1), so overflow occurs. The test - # for underflow is similar. - bound = self._log10_exp_bound() + other.adjusted() - if (self_adj >= 0) == (other._sign == 0): - # self > 1 and other +ve, or self < 1 and other -ve - # possibility of overflow - if bound >= len(str(context.Emax)): - ans = _dec_from_triple(result_sign, '1', context.Emax+1) - else: - # self > 1 and other -ve, or self < 1 and other +ve - # possibility of underflow to 0 - Etiny = context.Etiny() - if bound >= len(str(-Etiny)): - ans = _dec_from_triple(result_sign, '1', Etiny-1) - - # try for an exact result with precision +1 - if ans is None: - ans = self._power_exact(other, context.prec + 1) - if ans is not None: - if result_sign == 1: - ans = _dec_from_triple(1, ans._int, ans._exp) - exact = True - - # usual case: inexact result, x**y computed directly as exp(y*log(x)) - if ans is None: - p = context.prec - x = _WorkRep(self) - xc, xe = x.int, x.exp - y = _WorkRep(other) - yc, ye = y.int, y.exp - if y.sign == 1: - yc = -yc - - # compute correctly rounded result: start with precision +3, - # then increase precision until result is unambiguously roundable - extra = 3 - while True: - coeff, exp = _dpower(xc, xe, yc, ye, p+extra) - if coeff % (5*10**(len(str(coeff))-p-1)): - break - extra += 3 - - ans = _dec_from_triple(result_sign, str(coeff), exp) - - # unlike exp, ln and log10, the power function respects the - # rounding mode; no need to switch to ROUND_HALF_EVEN here - - # There's a difficulty here when 'other' is not an integer and - # the result is exact. In this case, the specification - # requires that the Inexact flag be raised (in spite of - # exactness), but since the result is exact _fix won't do this - # for us. (Correspondingly, the Underflow signal should also - # be raised for subnormal results.) We can't directly raise - # these signals either before or after calling _fix, since - # that would violate the precedence for signals. So we wrap - # the ._fix call in a temporary context, and reraise - # afterwards. - if exact and not other._isinteger(): - # pad with zeros up to length context.prec+1 if necessary; this - # ensures that the Rounded signal will be raised. - if len(ans._int) <= context.prec: - expdiff = context.prec + 1 - len(ans._int) - ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff, - ans._exp-expdiff) - - # create a copy of the current context, with cleared flags/traps - newcontext = context.copy() - newcontext.clear_flags() - for exception in _signals: - newcontext.traps[exception] = 0 - - # round in the new context - ans = ans._fix(newcontext) - - # raise Inexact, and if necessary, Underflow - newcontext._raise_error(Inexact) - if newcontext.flags[Subnormal]: - newcontext._raise_error(Underflow) - - # propagate signals to the original context; _fix could - # have raised any of Overflow, Underflow, Subnormal, - # Inexact, Rounded, Clamped. Overflow needs the correct - # arguments. Note that the order of the exceptions is - # important here. - if newcontext.flags[Overflow]: - context._raise_error(Overflow, 'above Emax', ans._sign) - for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: - if newcontext.flags[exception]: - context._raise_error(exception) - - else: - ans = ans._fix(context) - - return ans - - def __rpow__(self, other, context=None): - """Swaps self/other and returns __pow__.""" - other = _convert_other(other) - if other is NotImplemented: - return other - return other.__pow__(self, context=context) - - def normalize(self, context=None): - """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" - - if context is None: - context = getcontext() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - dup = self._fix(context) - if dup._isinfinity(): - return dup - - if not dup: - return _dec_from_triple(dup._sign, '0', 0) - exp_max = [context.Emax, context.Etop()][context.clamp] - end = len(dup._int) - exp = dup._exp - while dup._int[end-1] == '0' and exp < exp_max: - exp += 1 - end -= 1 - return _dec_from_triple(dup._sign, dup._int[:end], exp) - - def quantize(self, exp, rounding=None, context=None): - """Quantize self so its exponent is the same as that of exp. - - Similar to self._rescale(exp._exp) but with error checking. - """ - exp = _convert_other(exp, raiseit=True) - - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - - if self._is_special or exp._is_special: - ans = self._check_nans(exp, context) - if ans: - return ans - - if exp._isinfinity() or self._isinfinity(): - if exp._isinfinity() and self._isinfinity(): - return Decimal(self) # if both are inf, it is OK - return context._raise_error(InvalidOperation, - 'quantize with one INF') - - # exp._exp should be between Etiny and Emax - if not (context.Etiny() <= exp._exp <= context.Emax): - return context._raise_error(InvalidOperation, - 'target exponent out of bounds in quantize') - - if not self: - ans = _dec_from_triple(self._sign, '0', exp._exp) - return ans._fix(context) - - self_adjusted = self.adjusted() - if self_adjusted > context.Emax: - return context._raise_error(InvalidOperation, - 'exponent of quantize result too large for current context') - if self_adjusted - exp._exp + 1 > context.prec: - return context._raise_error(InvalidOperation, - 'quantize result has too many digits for current context') - - ans = self._rescale(exp._exp, rounding) - if ans.adjusted() > context.Emax: - return context._raise_error(InvalidOperation, - 'exponent of quantize result too large for current context') - if len(ans._int) > context.prec: - return context._raise_error(InvalidOperation, - 'quantize result has too many digits for current context') - - # raise appropriate flags - if ans and ans.adjusted() < context.Emin: - context._raise_error(Subnormal) - if ans._exp > self._exp: - if ans != self: - context._raise_error(Inexact) - context._raise_error(Rounded) - - # call to fix takes care of any necessary folddown, and - # signals Clamped if necessary - ans = ans._fix(context) - return ans - - def same_quantum(self, other, context=None): - """Return True if self and other have the same exponent; otherwise - return False. - - If either operand is a special value, the following rules are used: - * return True if both operands are infinities - * return True if both operands are NaNs - * otherwise, return False. - """ - other = _convert_other(other, raiseit=True) - if self._is_special or other._is_special: - return (self.is_nan() and other.is_nan() or - self.is_infinite() and other.is_infinite()) - return self._exp == other._exp - - def _rescale(self, exp, rounding): - """Rescale self so that the exponent is exp, either by padding with zeros - or by truncating digits, using the given rounding mode. - - Specials are returned without change. This operation is - quiet: it raises no flags, and uses no information from the - context. - - exp = exp to scale to (an integer) - rounding = rounding mode - """ - if self._is_special: - return Decimal(self) - if not self: - return _dec_from_triple(self._sign, '0', exp) - - if self._exp >= exp: - # pad answer with zeros if necessary - return _dec_from_triple(self._sign, - self._int + '0'*(self._exp - exp), exp) - - # too many digits; round and lose data. If self.adjusted() < - # exp-1, replace self by 10**(exp-1) before rounding - digits = len(self._int) + self._exp - exp - if digits < 0: - self = _dec_from_triple(self._sign, '1', exp-1) - digits = 0 - this_function = self._pick_rounding_function[rounding] - changed = this_function(self, digits) - coeff = self._int[:digits] or '0' - if changed == 1: - coeff = str(int(coeff)+1) - return _dec_from_triple(self._sign, coeff, exp) - - def _round(self, places, rounding): - """Round a nonzero, nonspecial Decimal to a fixed number of - significant figures, using the given rounding mode. - - Infinities, NaNs and zeros are returned unaltered. - - This operation is quiet: it raises no flags, and uses no - information from the context. - - """ - if places <= 0: - raise ValueError("argument should be at least 1 in _round") - if self._is_special or not self: - return Decimal(self) - ans = self._rescale(self.adjusted()+1-places, rounding) - # it can happen that the rescale alters the adjusted exponent; - # for example when rounding 99.97 to 3 significant figures. - # When this happens we end up with an extra 0 at the end of - # the number; a second rescale fixes this. - if ans.adjusted() != self.adjusted(): - ans = ans._rescale(ans.adjusted()+1-places, rounding) - return ans - - def to_integral_exact(self, rounding=None, context=None): - """Rounds to a nearby integer. - - If no rounding mode is specified, take the rounding mode from - the context. This method raises the Rounded and Inexact flags - when appropriate. - - See also: to_integral_value, which does exactly the same as - this method except that it doesn't raise Inexact or Rounded. - """ - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - return Decimal(self) - if self._exp >= 0: - return Decimal(self) - if not self: - return _dec_from_triple(self._sign, '0', 0) - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - ans = self._rescale(0, rounding) - if ans != self: - context._raise_error(Inexact) - context._raise_error(Rounded) - return ans - - def to_integral_value(self, rounding=None, context=None): - """Rounds to the nearest integer, without raising inexact, rounded.""" - if context is None: - context = getcontext() - if rounding is None: - rounding = context.rounding - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - return Decimal(self) - if self._exp >= 0: - return Decimal(self) - else: - return self._rescale(0, rounding) - - # the method name changed, but we provide also the old one, for compatibility - to_integral = to_integral_value - - def sqrt(self, context=None): - """Return the square root of self.""" - if context is None: - context = getcontext() - - if self._is_special: - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() and self._sign == 0: - return Decimal(self) - - if not self: - # exponent = self._exp // 2. sqrt(-0) = -0 - ans = _dec_from_triple(self._sign, '0', self._exp // 2) - return ans._fix(context) - - if self._sign == 1: - return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') - - # At this point self represents a positive number. Let p be - # the desired precision and express self in the form c*100**e - # with c a positive real number and e an integer, c and e - # being chosen so that 100**(p-1) <= c < 100**p. Then the - # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1) - # <= sqrt(c) < 10**p, so the closest representable Decimal at - # precision p is n*10**e where n = round_half_even(sqrt(c)), - # the closest integer to sqrt(c) with the even integer chosen - # in the case of a tie. - # - # To ensure correct rounding in all cases, we use the - # following trick: we compute the square root to an extra - # place (precision p+1 instead of precision p), rounding down. - # Then, if the result is inexact and its last digit is 0 or 5, - # we increase the last digit to 1 or 6 respectively; if it's - # exact we leave the last digit alone. Now the final round to - # p places (or fewer in the case of underflow) will round - # correctly and raise the appropriate flags. - - # use an extra digit of precision - prec = context.prec+1 - - # write argument in the form c*100**e where e = self._exp//2 - # is the 'ideal' exponent, to be used if the square root is - # exactly representable. l is the number of 'digits' of c in - # base 100, so that 100**(l-1) <= c < 100**l. - op = _WorkRep(self) - e = op.exp >> 1 - if op.exp & 1: - c = op.int * 10 - l = (len(self._int) >> 1) + 1 - else: - c = op.int - l = len(self._int)+1 >> 1 - - # rescale so that c has exactly prec base 100 'digits' - shift = prec-l - if shift >= 0: - c *= 100**shift - exact = True - else: - c, remainder = divmod(c, 100**-shift) - exact = not remainder - e -= shift - - # find n = floor(sqrt(c)) using Newton's method - n = 10**prec - while True: - q = c//n - if n <= q: - break - else: - n = n + q >> 1 - exact = exact and n*n == c - - if exact: - # result is exact; rescale to use ideal exponent e - if shift >= 0: - # assert n % 10**shift == 0 - n //= 10**shift - else: - n *= 10**-shift - e += shift - else: - # result is not exact; fix last digit as described above - if n % 5 == 0: - n += 1 - - ans = _dec_from_triple(0, str(n), e) - - # round, and fit to current context - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - - return ans - - def max(self, other, context=None): - """Returns the larger value. - - Like max(self, other) except if one is not a number, returns - NaN (and signals if one is sNaN). Also rounds. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self._cmp(other) - if c == 0: - # If both operands are finite and equal in numerical value - # then an ordering is applied: - # - # If the signs differ then max returns the operand with the - # positive sign and min returns the operand with the negative sign - # - # If the signs are the same then the exponent is used to select - # the result. This is exactly the ordering used in compare_total. - c = self.compare_total(other) - - if c == -1: - ans = other - else: - ans = self - - return ans._fix(context) - - def min(self, other, context=None): - """Returns the smaller value. - - Like min(self, other) except if one is not a number, returns - NaN (and signals if one is sNaN). Also rounds. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self._cmp(other) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = self - else: - ans = other - - return ans._fix(context) - - def _isinteger(self): - """Returns whether self is an integer""" - if self._is_special: - return False - if self._exp >= 0: - return True - rest = self._int[self._exp:] - return rest == '0'*len(rest) - - def _iseven(self): - """Returns True if self is even. Assumes self is an integer.""" - if not self or self._exp > 0: - return True - return self._int[-1+self._exp] in '02468' - - def adjusted(self): - """Return the adjusted exponent of self""" - try: - return self._exp + len(self._int) - 1 - # If NaN or Infinity, self._exp is string - except TypeError: - return 0 - - def canonical(self): - """Returns the same Decimal object. - - As we do not have different encodings for the same number, the - received object already is in its canonical form. - """ - return self - - def compare_signal(self, other, context=None): - """Compares self to the other operand numerically. - - It's pretty much like compare(), but all NaNs signal, with signaling - NaNs taking precedence over quiet NaNs. - """ - other = _convert_other(other, raiseit = True) - ans = self._compare_check_nans(other, context) - if ans: - return ans - return self.compare(other, context=context) - - def compare_total(self, other, context=None): - """Compares self to other using the abstract representations. - - This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. - """ - other = _convert_other(other, raiseit=True) - - # if one is negative and the other is positive, it's easy - if self._sign and not other._sign: - return _NegativeOne - if not self._sign and other._sign: - return _One - sign = self._sign - - # let's handle both NaN types - self_nan = self._isnan() - other_nan = other._isnan() - if self_nan or other_nan: - if self_nan == other_nan: - # compare payloads as though they're integers - self_key = len(self._int), self._int - other_key = len(other._int), other._int - if self_key < other_key: - if sign: - return _One - else: - return _NegativeOne - if self_key > other_key: - if sign: - return _NegativeOne - else: - return _One - return _Zero - - if sign: - if self_nan == 1: - return _NegativeOne - if other_nan == 1: - return _One - if self_nan == 2: - return _NegativeOne - if other_nan == 2: - return _One - else: - if self_nan == 1: - return _One - if other_nan == 1: - return _NegativeOne - if self_nan == 2: - return _One - if other_nan == 2: - return _NegativeOne - - if self < other: - return _NegativeOne - if self > other: - return _One - - if self._exp < other._exp: - if sign: - return _One - else: - return _NegativeOne - if self._exp > other._exp: - if sign: - return _NegativeOne - else: - return _One - return _Zero - - - def compare_total_mag(self, other, context=None): - """Compares self to other using abstract repr., ignoring sign. - - Like compare_total, but with operand's sign ignored and assumed to be 0. - """ - other = _convert_other(other, raiseit=True) - - s = self.copy_abs() - o = other.copy_abs() - return s.compare_total(o) - - def copy_abs(self): - """Returns a copy with the sign set to 0. """ - return _dec_from_triple(0, self._int, self._exp, self._is_special) - - def copy_negate(self): - """Returns a copy with the sign inverted.""" - if self._sign: - return _dec_from_triple(0, self._int, self._exp, self._is_special) - else: - return _dec_from_triple(1, self._int, self._exp, self._is_special) - - def copy_sign(self, other, context=None): - """Returns self with the sign of other.""" - other = _convert_other(other, raiseit=True) - return _dec_from_triple(other._sign, self._int, - self._exp, self._is_special) - - def exp(self, context=None): - """Returns e ** self.""" - - if context is None: - context = getcontext() - - # exp(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # exp(-Infinity) = 0 - if self._isinfinity() == -1: - return _Zero - - # exp(0) = 1 - if not self: - return _One - - # exp(Infinity) = Infinity - if self._isinfinity() == 1: - return Decimal(self) - - # the result is now guaranteed to be inexact (the true - # mathematical result is transcendental). There's no need to - # raise Rounded and Inexact here---they'll always be raised as - # a result of the call to _fix. - p = context.prec - adj = self.adjusted() - - # we only need to do any computation for quite a small range - # of adjusted exponents---for example, -29 <= adj <= 10 for - # the default context. For smaller exponent the result is - # indistinguishable from 1 at the given precision, while for - # larger exponent the result either overflows or underflows. - if self._sign == 0 and adj > len(str((context.Emax+1)*3)): - # overflow - ans = _dec_from_triple(0, '1', context.Emax+1) - elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)): - # underflow to 0 - ans = _dec_from_triple(0, '1', context.Etiny()-1) - elif self._sign == 0 and adj < -p: - # p+1 digits; final round will raise correct flags - ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p) - elif self._sign == 1 and adj < -p-1: - # p+1 digits; final round will raise correct flags - ans = _dec_from_triple(0, '9'*(p+1), -p-1) - # general case - else: - op = _WorkRep(self) - c, e = op.int, op.exp - if op.sign == 1: - c = -c - - # compute correctly rounded result: increase precision by - # 3 digits at a time until we get an unambiguously - # roundable result - extra = 3 - while True: - coeff, exp = _dexp(c, e, p+extra) - if coeff % (5*10**(len(str(coeff))-p-1)): - break - extra += 3 - - ans = _dec_from_triple(0, str(coeff), exp) - - # at this stage, ans should round correctly with *any* - # rounding mode, not just with ROUND_HALF_EVEN - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - - return ans - - def is_canonical(self): - """Return True if self is canonical; otherwise return False. - - Currently, the encoding of a Decimal instance is always - canonical, so this method returns True for any Decimal. - """ - return True - - def is_finite(self): - """Return True if self is finite; otherwise return False. - - A Decimal instance is considered finite if it is neither - infinite nor a NaN. - """ - return not self._is_special - - def is_infinite(self): - """Return True if self is infinite; otherwise return False.""" - return self._exp == 'F' - - def is_nan(self): - """Return True if self is a qNaN or sNaN; otherwise return False.""" - return self._exp in ('n', 'N') - - def is_normal(self, context=None): - """Return True if self is a normal number; otherwise return False.""" - if self._is_special or not self: - return False - if context is None: - context = getcontext() - return context.Emin <= self.adjusted() - - def is_qnan(self): - """Return True if self is a quiet NaN; otherwise return False.""" - return self._exp == 'n' - - def is_signed(self): - """Return True if self is negative; otherwise return False.""" - return self._sign == 1 - - def is_snan(self): - """Return True if self is a signaling NaN; otherwise return False.""" - return self._exp == 'N' - - def is_subnormal(self, context=None): - """Return True if self is subnormal; otherwise return False.""" - if self._is_special or not self: - return False - if context is None: - context = getcontext() - return self.adjusted() < context.Emin - - def is_zero(self): - """Return True if self is a zero; otherwise return False.""" - return not self._is_special and self._int == '0' - - def _ln_exp_bound(self): - """Compute a lower bound for the adjusted exponent of self.ln(). - In other words, compute r such that self.ln() >= 10**r. Assumes - that self is finite and positive and that self != 1. - """ - - # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 - adj = self._exp + len(self._int) - 1 - if adj >= 1: - # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) - return len(str(adj*23//10)) - 1 - if adj <= -2: - # argument <= 0.1 - return len(str((-1-adj)*23//10)) - 1 - op = _WorkRep(self) - c, e = op.int, op.exp - if adj == 0: - # 1 < self < 10 - num = str(c-10**-e) - den = str(c) - return len(num) - len(den) - (num < den) - # adj == -1, 0.1 <= self < 1 - return e + len(str(10**-e - c)) - 1 - - - def ln(self, context=None): - """Returns the natural (base e) logarithm of self.""" - - if context is None: - context = getcontext() - - # ln(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # ln(0.0) == -Infinity - if not self: - return _NegativeInfinity - - # ln(Infinity) = Infinity - if self._isinfinity() == 1: - return _Infinity - - # ln(1.0) == 0.0 - if self == _One: - return _Zero - - # ln(negative) raises InvalidOperation - if self._sign == 1: - return context._raise_error(InvalidOperation, - 'ln of a negative value') - - # result is irrational, so necessarily inexact - op = _WorkRep(self) - c, e = op.int, op.exp - p = context.prec - - # correctly rounded result: repeatedly increase precision by 3 - # until we get an unambiguously roundable result - places = p - self._ln_exp_bound() + 2 # at least p+3 places - while True: - coeff = _dlog(c, e, places) - # assert len(str(abs(coeff)))-p >= 1 - if coeff % (5*10**(len(str(abs(coeff)))-p-1)): - break - places += 3 - ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) - - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - return ans - - def _log10_exp_bound(self): - """Compute a lower bound for the adjusted exponent of self.log10(). - In other words, find r such that self.log10() >= 10**r. - Assumes that self is finite and positive and that self != 1. - """ - - # For x >= 10 or x < 0.1 we only need a bound on the integer - # part of log10(self), and this comes directly from the - # exponent of x. For 0.1 <= x <= 10 we use the inequalities - # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| > - # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0 - - adj = self._exp + len(self._int) - 1 - if adj >= 1: - # self >= 10 - return len(str(adj))-1 - if adj <= -2: - # self < 0.1 - return len(str(-1-adj))-1 - op = _WorkRep(self) - c, e = op.int, op.exp - if adj == 0: - # 1 < self < 10 - num = str(c-10**-e) - den = str(231*c) - return len(num) - len(den) - (num < den) + 2 - # adj == -1, 0.1 <= self < 1 - num = str(10**-e-c) - return len(num) + e - (num < "231") - 1 - - def log10(self, context=None): - """Returns the base 10 logarithm of self.""" - - if context is None: - context = getcontext() - - # log10(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - # log10(0.0) == -Infinity - if not self: - return _NegativeInfinity - - # log10(Infinity) = Infinity - if self._isinfinity() == 1: - return _Infinity - - # log10(negative or -Infinity) raises InvalidOperation - if self._sign == 1: - return context._raise_error(InvalidOperation, - 'log10 of a negative value') - - # log10(10**n) = n - if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1): - # answer may need rounding - ans = Decimal(self._exp + len(self._int) - 1) - else: - # result is irrational, so necessarily inexact - op = _WorkRep(self) - c, e = op.int, op.exp - p = context.prec - - # correctly rounded result: repeatedly increase precision - # until result is unambiguously roundable - places = p-self._log10_exp_bound()+2 - while True: - coeff = _dlog10(c, e, places) - # assert len(str(abs(coeff)))-p >= 1 - if coeff % (5*10**(len(str(abs(coeff)))-p-1)): - break - places += 3 - ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) - - context = context._shallow_copy() - rounding = context._set_rounding(ROUND_HALF_EVEN) - ans = ans._fix(context) - context.rounding = rounding - return ans - - def logb(self, context=None): - """ Returns the exponent of the magnitude of self's MSD. - - The result is the integer which is the exponent of the magnitude - of the most significant digit of self (as though it were truncated - to a single digit while maintaining the value of that digit and - without limiting the resulting exponent). - """ - # logb(NaN) = NaN - ans = self._check_nans(context=context) - if ans: - return ans - - if context is None: - context = getcontext() - - # logb(+/-Inf) = +Inf - if self._isinfinity(): - return _Infinity - - # logb(0) = -Inf, DivisionByZero - if not self: - return context._raise_error(DivisionByZero, 'logb(0)', 1) - - # otherwise, simply return the adjusted exponent of self, as a - # Decimal. Note that no attempt is made to fit the result - # into the current context. - ans = Decimal(self.adjusted()) - return ans._fix(context) - - def _islogical(self): - """Return True if self is a logical operand. - - For being logical, it must be a finite number with a sign of 0, - an exponent of 0, and a coefficient whose digits must all be - either 0 or 1. - """ - if self._sign != 0 or self._exp != 0: - return False - for dig in self._int: - if dig not in '01': - return False - return True - - def _fill_logical(self, context, opa, opb): - dif = context.prec - len(opa) - if dif > 0: - opa = '0'*dif + opa - elif dif < 0: - opa = opa[-context.prec:] - dif = context.prec - len(opb) - if dif > 0: - opb = '0'*dif + opb - elif dif < 0: - opb = opb[-context.prec:] - return opa, opb - - def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def logical_invert(self, context=None): - """Invert all its digits.""" - if context is None: - context = getcontext() - return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), - context) - - def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - if not self._islogical() or not other._islogical(): - return context._raise_error(InvalidOperation) - - # fill to context.prec - (opa, opb) = self._fill_logical(context, self._int, other._int) - - # make the operation, and clean starting zeroes - result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) - return _dec_from_triple(0, result.lstrip('0') or '0', 0) - - def max_mag(self, other, context=None): - """Compares the values numerically with their sign ignored.""" - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self.copy_abs()._cmp(other.copy_abs()) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = other - else: - ans = self - - return ans._fix(context) - - def min_mag(self, other, context=None): - """Compares the values numerically with their sign ignored.""" - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - if self._is_special or other._is_special: - # If one operand is a quiet NaN and the other is number, then the - # number is always returned - sn = self._isnan() - on = other._isnan() - if sn or on: - if on == 1 and sn == 0: - return self._fix(context) - if sn == 1 and on == 0: - return other._fix(context) - return self._check_nans(other, context) - - c = self.copy_abs()._cmp(other.copy_abs()) - if c == 0: - c = self.compare_total(other) - - if c == -1: - ans = self - else: - ans = other - - return ans._fix(context) - - def next_minus(self, context=None): - """Returns the largest representable number smaller than itself.""" - if context is None: - context = getcontext() - - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() == -1: - return _NegativeInfinity - if self._isinfinity() == 1: - return _dec_from_triple(0, '9'*context.prec, context.Etop()) - - context = context.copy() - context._set_rounding(ROUND_FLOOR) - context._ignore_all_flags() - new_self = self._fix(context) - if new_self != self: - return new_self - return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), - context) - - def next_plus(self, context=None): - """Returns the smallest representable number larger than itself.""" - if context is None: - context = getcontext() - - ans = self._check_nans(context=context) - if ans: - return ans - - if self._isinfinity() == 1: - return _Infinity - if self._isinfinity() == -1: - return _dec_from_triple(1, '9'*context.prec, context.Etop()) - - context = context.copy() - context._set_rounding(ROUND_CEILING) - context._ignore_all_flags() - new_self = self._fix(context) - if new_self != self: - return new_self - return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), - context) - - def next_toward(self, other, context=None): - """Returns the number closest to self, in the direction towards other. - - The result is the closest representable number to self - (excluding self) that is in the direction towards other, - unless both have the same value. If the two operands are - numerically equal, then the result is a copy of self with the - sign set to be the same as the sign of other. - """ - other = _convert_other(other, raiseit=True) - - if context is None: - context = getcontext() - - ans = self._check_nans(other, context) - if ans: - return ans - - comparison = self._cmp(other) - if comparison == 0: - return self.copy_sign(other) - - if comparison == -1: - ans = self.next_plus(context) - else: # comparison == 1 - ans = self.next_minus(context) - - # decide which flags to raise using value of ans - if ans._isinfinity(): - context._raise_error(Overflow, - 'Infinite result from next_toward', - ans._sign) - context._raise_error(Inexact) - context._raise_error(Rounded) - elif ans.adjusted() < context.Emin: - context._raise_error(Underflow) - context._raise_error(Subnormal) - context._raise_error(Inexact) - context._raise_error(Rounded) - # if precision == 1 then we don't raise Clamped for a - # result 0E-Etiny. - if not ans: - context._raise_error(Clamped) - - return ans - - def number_class(self, context=None): - """Returns an indication of the class of self. - - The class is one of the following strings: - sNaN - NaN - -Infinity - -Normal - -Subnormal - -Zero - +Zero - +Subnormal - +Normal - +Infinity - """ - if self.is_snan(): - return "sNaN" - if self.is_qnan(): - return "NaN" - inf = self._isinfinity() - if inf == 1: - return "+Infinity" - if inf == -1: - return "-Infinity" - if self.is_zero(): - if self._sign: - return "-Zero" - else: - return "+Zero" - if context is None: - context = getcontext() - if self.is_subnormal(context=context): - if self._sign: - return "-Subnormal" - else: - return "+Subnormal" - # just a normal, regular, boring number, :) - if self._sign: - return "-Normal" - else: - return "+Normal" - - def radix(self): - """Just returns 10, as this is Decimal, :)""" - return Decimal(10) - - def rotate(self, other, context=None): - """Returns a rotated copy of self, value-of-other times.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - if not (-context.prec <= int(other) <= context.prec): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - # get values, pad if necessary - torot = int(other) - rotdig = self._int - topad = context.prec - len(rotdig) - if topad > 0: - rotdig = '0'*topad + rotdig - elif topad < 0: - rotdig = rotdig[-topad:] - - # let's rotate! - rotated = rotdig[torot:] + rotdig[:torot] - return _dec_from_triple(self._sign, - rotated.lstrip('0') or '0', self._exp) - - def scaleb(self, other, context=None): - """Returns self operand after adding the second value to its exp.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - liminf = -2 * (context.Emax + context.prec) - limsup = 2 * (context.Emax + context.prec) - if not (liminf <= int(other) <= limsup): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) - d = d._fix(context) - return d - - def shift(self, other, context=None): - """Returns a shifted copy of self, value-of-other times.""" - if context is None: - context = getcontext() - - other = _convert_other(other, raiseit=True) - - ans = self._check_nans(other, context) - if ans: - return ans - - if other._exp != 0: - return context._raise_error(InvalidOperation) - if not (-context.prec <= int(other) <= context.prec): - return context._raise_error(InvalidOperation) - - if self._isinfinity(): - return Decimal(self) - - # get values, pad if necessary - torot = int(other) - rotdig = self._int - topad = context.prec - len(rotdig) - if topad > 0: - rotdig = '0'*topad + rotdig - elif topad < 0: - rotdig = rotdig[-topad:] - - # let's shift! - if torot < 0: - shifted = rotdig[:torot] - else: - shifted = rotdig + '0'*torot - shifted = shifted[-context.prec:] - - return _dec_from_triple(self._sign, - shifted.lstrip('0') or '0', self._exp) - - # Support for pickling, copy, and deepcopy - def __reduce__(self): - return (self.__class__, (str(self),)) - - def __copy__(self): - if type(self) is Decimal: - return self # I'm immutable; therefore I am my own clone - return self.__class__(str(self)) - - def __deepcopy__(self, memo): - if type(self) is Decimal: - return self # My components are also immutable - return self.__class__(str(self)) - - # PEP 3101 support. the _localeconv keyword argument should be - # considered private: it's provided for ease of testing only. - def __format__(self, specifier, context=None, _localeconv=None): - """Format a Decimal instance according to the given specifier. - - The specifier should be a standard format specifier, with the - form described in PEP 3101. Formatting types 'e', 'E', 'f', - 'F', 'g', 'G', 'n' and '%' are supported. If the formatting - type is omitted it defaults to 'g' or 'G', depending on the - value of context.capitals. - """ - - # Note: PEP 3101 says that if the type is not present then - # there should be at least one digit after the decimal point. - # We take the liberty of ignoring this requirement for - # Decimal---it's presumably there to make sure that - # format(float, '') behaves similarly to str(float). - if context is None: - context = getcontext() - - spec = _parse_format_specifier(specifier, _localeconv=_localeconv) - - # special values don't care about the type or precision - if self._is_special: - sign = _format_sign(self._sign, spec) - body = str(self.copy_abs()) - if spec['type'] == '%': - body += '%' - return _format_align(sign, body, spec) - - # a type of None defaults to 'g' or 'G', depending on context - if spec['type'] is None: - spec['type'] = ['g', 'G'][context.capitals] - - # if type is '%', adjust exponent of self accordingly - if spec['type'] == '%': - self = _dec_from_triple(self._sign, self._int, self._exp+2) - - # round if necessary, taking rounding mode from the context - rounding = context.rounding - precision = spec['precision'] - if precision is not None: - if spec['type'] in 'eE': - self = self._round(precision+1, rounding) - elif spec['type'] in 'fF%': - self = self._rescale(-precision, rounding) - elif spec['type'] in 'gG' and len(self._int) > precision: - self = self._round(precision, rounding) - # special case: zeros with a positive exponent can't be - # represented in fixed point; rescale them to 0e0. - if not self and self._exp > 0 and spec['type'] in 'fF%': - self = self._rescale(0, rounding) - - # figure out placement of the decimal point - leftdigits = self._exp + len(self._int) - if spec['type'] in 'eE': - if not self and precision is not None: - dotplace = 1 - precision - else: - dotplace = 1 - elif spec['type'] in 'fF%': - dotplace = leftdigits - elif spec['type'] in 'gG': - if self._exp <= 0 and leftdigits > -6: - dotplace = leftdigits - else: - dotplace = 1 - - # find digits before and after decimal point, and get exponent - if dotplace < 0: - intpart = '0' - fracpart = '0'*(-dotplace) + self._int - elif dotplace > len(self._int): - intpart = self._int + '0'*(dotplace-len(self._int)) - fracpart = '' - else: - intpart = self._int[:dotplace] or '0' - fracpart = self._int[dotplace:] - exp = leftdigits-dotplace - - # done with the decimal-specific stuff; hand over the rest - # of the formatting to the _format_number function - return _format_number(self._sign, intpart, fracpart, exp, spec) - -def _dec_from_triple(sign, coefficient, exponent, special=False): - """Create a decimal instance directly, without any validation, - normalization (e.g. removal of leading zeros) or argument - conversion. - - This function is for *internal use only*. - """ - - self = object.__new__(Decimal) - self._sign = sign - self._int = coefficient - self._exp = exponent - self._is_special = special - - return self - -# Register Decimal as a kind of Number (an abstract base class). -# However, do not register it as Real (because Decimals are not -# interoperable with floats). -_numbers.Number.register(Decimal) - - -##### Context class ####################################################### - -class _ContextManager(object): - """Context manager class to support localcontext(). - - Sets a copy of the supplied context in __enter__() and restores - the previous decimal context in __exit__() - """ - def __init__(self, new_context): - self.new_context = new_context.copy() - def __enter__(self): - self.saved_context = getcontext() - setcontext(self.new_context) - return self.new_context - def __exit__(self, t, v, tb): - setcontext(self.saved_context) - -class Context(object): - """Contains the context for a Decimal instance. - - Contains: - prec - precision (for use in rounding, division, square roots..) - rounding - rounding type (how you round) - traps - If traps[exception] = 1, then the exception is - raised when it is caused. Otherwise, a value is - substituted in. - flags - When an exception is caused, flags[exception] is set. - (Whether or not the trap_enabler is set) - Should be reset by user of Decimal instance. - Emin - Minimum exponent - Emax - Maximum exponent - capitals - If 1, 1*10^1 is printed as 1E+1. - If 0, printed as 1e1 - clamp - If 1, change exponents if too high (Default 0) - """ - - def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, - capitals=None, clamp=None, flags=None, traps=None, - _ignored_flags=None): - # Set defaults; for everything except flags and _ignored_flags, - # inherit from DefaultContext. - try: - dc = DefaultContext - except NameError: - pass - - self.prec = prec if prec is not None else dc.prec - self.rounding = rounding if rounding is not None else dc.rounding - self.Emin = Emin if Emin is not None else dc.Emin - self.Emax = Emax if Emax is not None else dc.Emax - self.capitals = capitals if capitals is not None else dc.capitals - self.clamp = clamp if clamp is not None else dc.clamp - - if _ignored_flags is None: - self._ignored_flags = [] - else: - self._ignored_flags = _ignored_flags - - if traps is None: - self.traps = dc.traps.copy() - elif not isinstance(traps, dict): - self.traps = dict((s, int(s in traps)) for s in _signals + traps) - else: - self.traps = traps - - if flags is None: - self.flags = dict.fromkeys(_signals, 0) - elif not isinstance(flags, dict): - self.flags = dict((s, int(s in flags)) for s in _signals + flags) - else: - self.flags = flags - - def _set_integer_check(self, name, value, vmin, vmax): - if not isinstance(value, int): - raise TypeError("%s must be an integer" % name) - if vmin == '-inf': - if value > vmax: - raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) - elif vmax == 'inf': - if value < vmin: - raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) - else: - if value < vmin or value > vmax: - raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) - return object.__setattr__(self, name, value) - - def _set_signal_dict(self, name, d): - if not isinstance(d, dict): - raise TypeError("%s must be a signal dict" % d) - for key in d: - if not key in _signals: - raise KeyError("%s is not a valid signal dict" % d) - for key in _signals: - if not key in d: - raise KeyError("%s is not a valid signal dict" % d) - return object.__setattr__(self, name, d) - - def __setattr__(self, name, value): - if name == 'prec': - return self._set_integer_check(name, value, 1, 'inf') - elif name == 'Emin': - return self._set_integer_check(name, value, '-inf', 0) - elif name == 'Emax': - return self._set_integer_check(name, value, 0, 'inf') - elif name == 'capitals': - return self._set_integer_check(name, value, 0, 1) - elif name == 'clamp': - return self._set_integer_check(name, value, 0, 1) - elif name == 'rounding': - if not value in _rounding_modes: - # raise TypeError even for strings to have consistency - # among various implementations. - raise TypeError("%s: invalid rounding mode" % value) - return object.__setattr__(self, name, value) - elif name == 'flags' or name == 'traps': - return self._set_signal_dict(name, value) - elif name == '_ignored_flags': - return object.__setattr__(self, name, value) - else: - raise AttributeError( - "'decimal.Context' object has no attribute '%s'" % name) - - def __delattr__(self, name): - raise AttributeError("%s cannot be deleted" % name) - - # Support for pickling, copy, and deepcopy - def __reduce__(self): - flags = [sig for sig, v in self.flags.items() if v] - traps = [sig for sig, v in self.traps.items() if v] - return (self.__class__, - (self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, flags, traps)) - - def __repr__(self): - """Show the current context.""" - s = [] - s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' - 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' - 'clamp=%(clamp)d' - % vars(self)) - names = [f.__name__ for f, v in self.flags.items() if v] - s.append('flags=[' + ', '.join(names) + ']') - names = [t.__name__ for t, v in self.traps.items() if v] - s.append('traps=[' + ', '.join(names) + ']') - return ', '.join(s) + ')' - - def clear_flags(self): - """Reset all flags to zero""" - for flag in self.flags: - self.flags[flag] = 0 - - def clear_traps(self): - """Reset all traps to zero""" - for flag in self.traps: - self.traps[flag] = 0 - - def _shallow_copy(self): - """Returns a shallow copy from self.""" - nc = Context(self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, self.flags, self.traps, - self._ignored_flags) - return nc - - def copy(self): - """Returns a deep copy from self.""" - nc = Context(self.prec, self.rounding, self.Emin, self.Emax, - self.capitals, self.clamp, - self.flags.copy(), self.traps.copy(), - self._ignored_flags) - return nc - __copy__ = copy - - def _raise_error(self, condition, explanation = None, *args): - """Handles an error - - If the flag is in _ignored_flags, returns the default response. - Otherwise, it sets the flag, then, if the corresponding - trap_enabler is set, it reraises the exception. Otherwise, it returns - the default value after setting the flag. - """ - error = _condition_map.get(condition, condition) - if error in self._ignored_flags: - # Don't touch the flag - return error().handle(self, *args) - - self.flags[error] = 1 - if not self.traps[error]: - # The errors define how to handle themselves. - return condition().handle(self, *args) - - # Errors should only be risked on copies of the context - # self._ignored_flags = [] - raise error(explanation) - - def _ignore_all_flags(self): - """Ignore all flags, if they are raised""" - return self._ignore_flags(*_signals) - - def _ignore_flags(self, *flags): - """Ignore the flags, if they are raised""" - # Do not mutate-- This way, copies of a context leave the original - # alone. - self._ignored_flags = (self._ignored_flags + list(flags)) - return list(flags) - - def _regard_flags(self, *flags): - """Stop ignoring the flags, if they are raised""" - if flags and isinstance(flags[0], (tuple,list)): - flags = flags[0] - for flag in flags: - self._ignored_flags.remove(flag) - - # We inherit object.__hash__, so we must deny this explicitly - __hash__ = None - - def Etiny(self): - """Returns Etiny (= Emin - prec + 1)""" - return int(self.Emin - self.prec + 1) - - def Etop(self): - """Returns maximum exponent (= Emax - prec + 1)""" - return int(self.Emax - self.prec + 1) - - def _set_rounding(self, type): - """Sets the rounding type. - - Sets the rounding type, and returns the current (previous) - rounding type. Often used like: - - context = context.copy() - # so you don't change the calling context - # if an error occurs in the middle. - rounding = context._set_rounding(ROUND_UP) - val = self.__sub__(other, context=context) - context._set_rounding(rounding) - - This will make it round up for that operation. - """ - rounding = self.rounding - self.rounding= type - return rounding - - def create_decimal(self, num='0'): - """Creates a new Decimal instance but using self as context. - - This method implements the to-number operation of the - IBM Decimal specification.""" - - if isinstance(num, str) and num != num.strip(): - return self._raise_error(ConversionSyntax, - "no trailing or leading whitespace is " - "permitted.") - - d = Decimal(num, context=self) - if d._isnan() and len(d._int) > self.prec - self.clamp: - return self._raise_error(ConversionSyntax, - "diagnostic info too long in NaN") - return d._fix(self) - - def create_decimal_from_float(self, f): - """Creates a new Decimal instance from a float but rounding using self - as the context. - - >>> context = Context(prec=5, rounding=ROUND_DOWN) - >>> context.create_decimal_from_float(3.1415926535897932) - Decimal('3.1415') - >>> context = Context(prec=5, traps=[Inexact]) - >>> context.create_decimal_from_float(3.1415926535897932) - Traceback (most recent call last): - ... - decimal.Inexact - - """ - d = Decimal.from_float(f) # An exact conversion - return d._fix(self) # Apply the context rounding - - # Methods - def abs(self, a): - """Returns the absolute value of the operand. - - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. - - >>> ExtendedContext.abs(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.abs(Decimal('-100')) - Decimal('100') - >>> ExtendedContext.abs(Decimal('101.5')) - Decimal('101.5') - >>> ExtendedContext.abs(Decimal('-101.5')) - Decimal('101.5') - >>> ExtendedContext.abs(-1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.__abs__(context=self) - - def add(self, a, b): - """Return the sum of the two operands. - - >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) - Decimal('19.00') - >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) - Decimal('1.02E+4') - >>> ExtendedContext.add(1, Decimal(2)) - Decimal('3') - >>> ExtendedContext.add(Decimal(8), 5) - Decimal('13') - >>> ExtendedContext.add(5, 5) - Decimal('10') - """ - a = _convert_other(a, raiseit=True) - r = a.__add__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def _apply(self, a): - return str(a._fix(self)) - - def canonical(self, a): - """Returns the same Decimal object. - - As we do not have different encodings for the same number, the - received object already is in its canonical form. - - >>> ExtendedContext.canonical(Decimal('2.50')) - Decimal('2.50') - """ - if not isinstance(a, Decimal): - raise TypeError("canonical requires a Decimal as an argument.") - return a.canonical() - - def compare(self, a, b): - """Compares values numerically. - - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. - - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. - - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) - Decimal('0') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) - Decimal('0') - >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) - Decimal('1') - >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) - Decimal('1') - >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) - Decimal('-1') - >>> ExtendedContext.compare(1, 2) - Decimal('-1') - >>> ExtendedContext.compare(Decimal(1), 2) - Decimal('-1') - >>> ExtendedContext.compare(1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare(b, context=self) - - def compare_signal(self, a, b): - """Compares the values of the two operands numerically. - - It's pretty much like compare(), but all NaNs signal, with signaling - NaNs taking precedence over quiet NaNs. - - >>> c = ExtendedContext - >>> c.compare_signal(Decimal('2.1'), Decimal('3')) - Decimal('-1') - >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) - Decimal('0') - >>> c.flags[InvalidOperation] = 0 - >>> print(c.flags[InvalidOperation]) - 0 - >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) - Decimal('NaN') - >>> print(c.flags[InvalidOperation]) - 1 - >>> c.flags[InvalidOperation] = 0 - >>> print(c.flags[InvalidOperation]) - 0 - >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) - Decimal('NaN') - >>> print(c.flags[InvalidOperation]) - 1 - >>> c.compare_signal(-1, 2) - Decimal('-1') - >>> c.compare_signal(Decimal(-1), 2) - Decimal('-1') - >>> c.compare_signal(-1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare_signal(b, context=self) - - def compare_total(self, a, b): - """Compares two operands using their abstract representation. - - This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. - - >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) - Decimal('0') - >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) - Decimal('1') - >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) - Decimal('-1') - >>> ExtendedContext.compare_total(1, 2) - Decimal('-1') - >>> ExtendedContext.compare_total(Decimal(1), 2) - Decimal('-1') - >>> ExtendedContext.compare_total(1, Decimal(2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.compare_total(b) - - def compare_total_mag(self, a, b): - """Compares two operands using their abstract representation ignoring sign. - - Like compare_total, but with operand's sign ignored and assumed to be 0. - """ - a = _convert_other(a, raiseit=True) - return a.compare_total_mag(b) - - def copy_abs(self, a): - """Returns a copy of the operand with the sign set to 0. - - >>> ExtendedContext.copy_abs(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.copy_abs(Decimal('-100')) - Decimal('100') - >>> ExtendedContext.copy_abs(-1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_abs() - - def copy_decimal(self, a): - """Returns a copy of the decimal object. - - >>> ExtendedContext.copy_decimal(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.copy_decimal(Decimal('-1.00')) - Decimal('-1.00') - >>> ExtendedContext.copy_decimal(1) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return Decimal(a) - - def copy_negate(self, a): - """Returns a copy of the operand with the sign inverted. - - >>> ExtendedContext.copy_negate(Decimal('101.5')) - Decimal('-101.5') - >>> ExtendedContext.copy_negate(Decimal('-101.5')) - Decimal('101.5') - >>> ExtendedContext.copy_negate(1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_negate() - - def copy_sign(self, a, b): - """Copies the second operand's sign to the first one. - - In detail, it returns a copy of the first operand with the sign - equal to the sign of the second operand. - - >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) - Decimal('1.50') - >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) - Decimal('1.50') - >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) - Decimal('-1.50') - >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) - Decimal('-1.50') - >>> ExtendedContext.copy_sign(1, -2) - Decimal('-1') - >>> ExtendedContext.copy_sign(Decimal(1), -2) - Decimal('-1') - >>> ExtendedContext.copy_sign(1, Decimal(-2)) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.copy_sign(b) - - def divide(self, a, b): - """Decimal division in a specified context. - - >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) - Decimal('0.333333333') - >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) - Decimal('0.666666667') - >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) - Decimal('2.5') - >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) - Decimal('0.1') - >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) - Decimal('1') - >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) - Decimal('4.00') - >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) - Decimal('1.20') - >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) - Decimal('10') - >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) - Decimal('1000') - >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) - Decimal('1.20E+6') - >>> ExtendedContext.divide(5, 5) - Decimal('1') - >>> ExtendedContext.divide(Decimal(5), 5) - Decimal('1') - >>> ExtendedContext.divide(5, Decimal(5)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - r = a.__truediv__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def divide_int(self, a, b): - """Divides two numbers and returns the integer part of the result. - - >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) - Decimal('0') - >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) - Decimal('3') - >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) - Decimal('3') - >>> ExtendedContext.divide_int(10, 3) - Decimal('3') - >>> ExtendedContext.divide_int(Decimal(10), 3) - Decimal('3') - >>> ExtendedContext.divide_int(10, Decimal(3)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - r = a.__floordiv__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def divmod(self, a, b): - """Return (a // b, a % b). - - >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) - (Decimal('2'), Decimal('2')) - >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(8, 4) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(Decimal(8), 4) - (Decimal('2'), Decimal('0')) - >>> ExtendedContext.divmod(8, Decimal(4)) - (Decimal('2'), Decimal('0')) - """ - a = _convert_other(a, raiseit=True) - r = a.__divmod__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def exp(self, a): - """Returns e ** a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.exp(Decimal('-Infinity')) - Decimal('0') - >>> c.exp(Decimal('-1')) - Decimal('0.367879441') - >>> c.exp(Decimal('0')) - Decimal('1') - >>> c.exp(Decimal('1')) - Decimal('2.71828183') - >>> c.exp(Decimal('0.693147181')) - Decimal('2.00000000') - >>> c.exp(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.exp(10) - Decimal('22026.4658') - """ - a =_convert_other(a, raiseit=True) - return a.exp(context=self) - - def fma(self, a, b, c): - """Returns a multiplied by b, plus c. - - The first two operands are multiplied together, using multiply, - the third operand is then added to the result of that - multiplication, using add, all with only one final rounding. - - >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) - Decimal('22') - >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) - Decimal('-8') - >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) - Decimal('1.38435736E+12') - >>> ExtendedContext.fma(1, 3, 4) - Decimal('7') - >>> ExtendedContext.fma(1, Decimal(3), 4) - Decimal('7') - >>> ExtendedContext.fma(1, 3, Decimal(4)) - Decimal('7') - """ - a = _convert_other(a, raiseit=True) - return a.fma(b, c, context=self) - - def is_canonical(self, a): - """Return True if the operand is canonical; otherwise return False. - - Currently, the encoding of a Decimal instance is always - canonical, so this method returns True for any Decimal. - - >>> ExtendedContext.is_canonical(Decimal('2.50')) - True - """ - if not isinstance(a, Decimal): - raise TypeError("is_canonical requires a Decimal as an argument.") - return a.is_canonical() - - def is_finite(self, a): - """Return True if the operand is finite; otherwise return False. - - A Decimal instance is considered finite if it is neither - infinite nor a NaN. - - >>> ExtendedContext.is_finite(Decimal('2.50')) - True - >>> ExtendedContext.is_finite(Decimal('-0.3')) - True - >>> ExtendedContext.is_finite(Decimal('0')) - True - >>> ExtendedContext.is_finite(Decimal('Inf')) - False - >>> ExtendedContext.is_finite(Decimal('NaN')) - False - >>> ExtendedContext.is_finite(1) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_finite() - - def is_infinite(self, a): - """Return True if the operand is infinite; otherwise return False. - - >>> ExtendedContext.is_infinite(Decimal('2.50')) - False - >>> ExtendedContext.is_infinite(Decimal('-Inf')) - True - >>> ExtendedContext.is_infinite(Decimal('NaN')) - False - >>> ExtendedContext.is_infinite(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_infinite() - - def is_nan(self, a): - """Return True if the operand is a qNaN or sNaN; - otherwise return False. - - >>> ExtendedContext.is_nan(Decimal('2.50')) - False - >>> ExtendedContext.is_nan(Decimal('NaN')) - True - >>> ExtendedContext.is_nan(Decimal('-sNaN')) - True - >>> ExtendedContext.is_nan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_nan() - - def is_normal(self, a): - """Return True if the operand is a normal number; - otherwise return False. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.is_normal(Decimal('2.50')) - True - >>> c.is_normal(Decimal('0.1E-999')) - False - >>> c.is_normal(Decimal('0.00')) - False - >>> c.is_normal(Decimal('-Inf')) - False - >>> c.is_normal(Decimal('NaN')) - False - >>> c.is_normal(1) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_normal(context=self) - - def is_qnan(self, a): - """Return True if the operand is a quiet NaN; otherwise return False. - - >>> ExtendedContext.is_qnan(Decimal('2.50')) - False - >>> ExtendedContext.is_qnan(Decimal('NaN')) - True - >>> ExtendedContext.is_qnan(Decimal('sNaN')) - False - >>> ExtendedContext.is_qnan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_qnan() - - def is_signed(self, a): - """Return True if the operand is negative; otherwise return False. - - >>> ExtendedContext.is_signed(Decimal('2.50')) - False - >>> ExtendedContext.is_signed(Decimal('-12')) - True - >>> ExtendedContext.is_signed(Decimal('-0')) - True - >>> ExtendedContext.is_signed(8) - False - >>> ExtendedContext.is_signed(-8) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_signed() - - def is_snan(self, a): - """Return True if the operand is a signaling NaN; - otherwise return False. - - >>> ExtendedContext.is_snan(Decimal('2.50')) - False - >>> ExtendedContext.is_snan(Decimal('NaN')) - False - >>> ExtendedContext.is_snan(Decimal('sNaN')) - True - >>> ExtendedContext.is_snan(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_snan() - - def is_subnormal(self, a): - """Return True if the operand is subnormal; otherwise return False. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.is_subnormal(Decimal('2.50')) - False - >>> c.is_subnormal(Decimal('0.1E-999')) - True - >>> c.is_subnormal(Decimal('0.00')) - False - >>> c.is_subnormal(Decimal('-Inf')) - False - >>> c.is_subnormal(Decimal('NaN')) - False - >>> c.is_subnormal(1) - False - """ - a = _convert_other(a, raiseit=True) - return a.is_subnormal(context=self) - - def is_zero(self, a): - """Return True if the operand is a zero; otherwise return False. - - >>> ExtendedContext.is_zero(Decimal('0')) - True - >>> ExtendedContext.is_zero(Decimal('2.50')) - False - >>> ExtendedContext.is_zero(Decimal('-0E+2')) - True - >>> ExtendedContext.is_zero(1) - False - >>> ExtendedContext.is_zero(0) - True - """ - a = _convert_other(a, raiseit=True) - return a.is_zero() - - def ln(self, a): - """Returns the natural (base e) logarithm of the operand. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.ln(Decimal('0')) - Decimal('-Infinity') - >>> c.ln(Decimal('1.000')) - Decimal('0') - >>> c.ln(Decimal('2.71828183')) - Decimal('1.00000000') - >>> c.ln(Decimal('10')) - Decimal('2.30258509') - >>> c.ln(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.ln(1) - Decimal('0') - """ - a = _convert_other(a, raiseit=True) - return a.ln(context=self) - - def log10(self, a): - """Returns the base 10 logarithm of the operand. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.log10(Decimal('0')) - Decimal('-Infinity') - >>> c.log10(Decimal('0.001')) - Decimal('-3') - >>> c.log10(Decimal('1.000')) - Decimal('0') - >>> c.log10(Decimal('2')) - Decimal('0.301029996') - >>> c.log10(Decimal('10')) - Decimal('1') - >>> c.log10(Decimal('70')) - Decimal('1.84509804') - >>> c.log10(Decimal('+Infinity')) - Decimal('Infinity') - >>> c.log10(0) - Decimal('-Infinity') - >>> c.log10(1) - Decimal('0') - """ - a = _convert_other(a, raiseit=True) - return a.log10(context=self) - - def logb(self, a): - """ Returns the exponent of the magnitude of the operand's MSD. - - The result is the integer which is the exponent of the magnitude - of the most significant digit of the operand (as though the - operand were truncated to a single digit while maintaining the - value of that digit and without limiting the resulting exponent). - - >>> ExtendedContext.logb(Decimal('250')) - Decimal('2') - >>> ExtendedContext.logb(Decimal('2.50')) - Decimal('0') - >>> ExtendedContext.logb(Decimal('0.03')) - Decimal('-2') - >>> ExtendedContext.logb(Decimal('0')) - Decimal('-Infinity') - >>> ExtendedContext.logb(1) - Decimal('0') - >>> ExtendedContext.logb(10) - Decimal('1') - >>> ExtendedContext.logb(100) - Decimal('2') - """ - a = _convert_other(a, raiseit=True) - return a.logb(context=self) - - def logical_and(self, a, b): - """Applies the logical operation 'and' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) - Decimal('1000') - >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) - Decimal('10') - >>> ExtendedContext.logical_and(110, 1101) - Decimal('100') - >>> ExtendedContext.logical_and(Decimal(110), 1101) - Decimal('100') - >>> ExtendedContext.logical_and(110, Decimal(1101)) - Decimal('100') - """ - a = _convert_other(a, raiseit=True) - return a.logical_and(b, context=self) - - def logical_invert(self, a): - """Invert all the digits in the operand. - - The operand must be a logical number. - - >>> ExtendedContext.logical_invert(Decimal('0')) - Decimal('111111111') - >>> ExtendedContext.logical_invert(Decimal('1')) - Decimal('111111110') - >>> ExtendedContext.logical_invert(Decimal('111111111')) - Decimal('0') - >>> ExtendedContext.logical_invert(Decimal('101010101')) - Decimal('10101010') - >>> ExtendedContext.logical_invert(1101) - Decimal('111110010') - """ - a = _convert_other(a, raiseit=True) - return a.logical_invert(context=self) - - def logical_or(self, a, b): - """Applies the logical operation 'or' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) - Decimal('1110') - >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) - Decimal('1110') - >>> ExtendedContext.logical_or(110, 1101) - Decimal('1111') - >>> ExtendedContext.logical_or(Decimal(110), 1101) - Decimal('1111') - >>> ExtendedContext.logical_or(110, Decimal(1101)) - Decimal('1111') - """ - a = _convert_other(a, raiseit=True) - return a.logical_or(b, context=self) - - def logical_xor(self, a, b): - """Applies the logical operation 'xor' between each operand's digits. - - The operands must be both logical numbers. - - >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) - Decimal('0') - >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) - Decimal('1') - >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) - Decimal('0') - >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) - Decimal('110') - >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) - Decimal('1101') - >>> ExtendedContext.logical_xor(110, 1101) - Decimal('1011') - >>> ExtendedContext.logical_xor(Decimal(110), 1101) - Decimal('1011') - >>> ExtendedContext.logical_xor(110, Decimal(1101)) - Decimal('1011') - """ - a = _convert_other(a, raiseit=True) - return a.logical_xor(b, context=self) - - def max(self, a, b): - """max compares two values numerically and returns the maximum. - - If either operand is a NaN then the general rules apply. - Otherwise, the operands are compared as though by the compare - operation. If they are numerically equal then the left-hand operand - is chosen as the result. Otherwise the maximum (closer to positive - infinity) of the two operands is chosen as the result. - - >>> ExtendedContext.max(Decimal('3'), Decimal('2')) - Decimal('3') - >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) - Decimal('3') - >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) - Decimal('1') - >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.max(1, 2) - Decimal('2') - >>> ExtendedContext.max(Decimal(1), 2) - Decimal('2') - >>> ExtendedContext.max(1, Decimal(2)) - Decimal('2') - """ - a = _convert_other(a, raiseit=True) - return a.max(b, context=self) - - def max_mag(self, a, b): - """Compares the values numerically with their sign ignored. - - >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) - Decimal('-10') - >>> ExtendedContext.max_mag(1, -2) - Decimal('-2') - >>> ExtendedContext.max_mag(Decimal(1), -2) - Decimal('-2') - >>> ExtendedContext.max_mag(1, Decimal(-2)) - Decimal('-2') - """ - a = _convert_other(a, raiseit=True) - return a.max_mag(b, context=self) - - def min(self, a, b): - """min compares two values numerically and returns the minimum. - - If either operand is a NaN then the general rules apply. - Otherwise, the operands are compared as though by the compare - operation. If they are numerically equal then the left-hand operand - is chosen as the result. Otherwise the minimum (closer to negative - infinity) of the two operands is chosen as the result. - - >>> ExtendedContext.min(Decimal('3'), Decimal('2')) - Decimal('2') - >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) - Decimal('-10') - >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) - Decimal('1.0') - >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) - Decimal('7') - >>> ExtendedContext.min(1, 2) - Decimal('1') - >>> ExtendedContext.min(Decimal(1), 2) - Decimal('1') - >>> ExtendedContext.min(1, Decimal(29)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.min(b, context=self) - - def min_mag(self, a, b): - """Compares the values numerically with their sign ignored. - - >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) - Decimal('-2') - >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) - Decimal('-3') - >>> ExtendedContext.min_mag(1, -2) - Decimal('1') - >>> ExtendedContext.min_mag(Decimal(1), -2) - Decimal('1') - >>> ExtendedContext.min_mag(1, Decimal(-2)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.min_mag(b, context=self) - - def minus(self, a): - """Minus corresponds to unary prefix minus in Python. - - The operation is evaluated using the same rules as subtract; the - operation minus(a) is calculated as subtract('0', a) where the '0' - has the same exponent as the operand. - - >>> ExtendedContext.minus(Decimal('1.3')) - Decimal('-1.3') - >>> ExtendedContext.minus(Decimal('-1.3')) - Decimal('1.3') - >>> ExtendedContext.minus(1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.__neg__(context=self) - - def multiply(self, a, b): - """multiply multiplies two operands. - - If either operand is a special value then the general rules apply. - Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. - - >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) - Decimal('3.60') - >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) - Decimal('21') - >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) - Decimal('0.72') - >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) - Decimal('-0.0') - >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) - Decimal('4.28135971E+11') - >>> ExtendedContext.multiply(7, 7) - Decimal('49') - >>> ExtendedContext.multiply(Decimal(7), 7) - Decimal('49') - >>> ExtendedContext.multiply(7, Decimal(7)) - Decimal('49') - """ - a = _convert_other(a, raiseit=True) - r = a.__mul__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def next_minus(self, a): - """Returns the largest representable number smaller than a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> ExtendedContext.next_minus(Decimal('1')) - Decimal('0.999999999') - >>> c.next_minus(Decimal('1E-1007')) - Decimal('0E-1007') - >>> ExtendedContext.next_minus(Decimal('-1.00000003')) - Decimal('-1.00000004') - >>> c.next_minus(Decimal('Infinity')) - Decimal('9.99999999E+999') - >>> c.next_minus(1) - Decimal('0.999999999') - """ - a = _convert_other(a, raiseit=True) - return a.next_minus(context=self) - - def next_plus(self, a): - """Returns the smallest representable number larger than a. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> ExtendedContext.next_plus(Decimal('1')) - Decimal('1.00000001') - >>> c.next_plus(Decimal('-1E-1007')) - Decimal('-0E-1007') - >>> ExtendedContext.next_plus(Decimal('-1.00000003')) - Decimal('-1.00000002') - >>> c.next_plus(Decimal('-Infinity')) - Decimal('-9.99999999E+999') - >>> c.next_plus(1) - Decimal('1.00000001') - """ - a = _convert_other(a, raiseit=True) - return a.next_plus(context=self) - - def next_toward(self, a, b): - """Returns the number closest to a, in direction towards b. - - The result is the closest representable number from the first - operand (but not the first operand) that is in the direction - towards the second operand, unless the operands have the same - value. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.next_toward(Decimal('1'), Decimal('2')) - Decimal('1.00000001') - >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) - Decimal('-0E-1007') - >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) - Decimal('-1.00000002') - >>> c.next_toward(Decimal('1'), Decimal('0')) - Decimal('0.999999999') - >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) - Decimal('0E-1007') - >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) - Decimal('-1.00000004') - >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) - Decimal('-0.00') - >>> c.next_toward(0, 1) - Decimal('1E-1007') - >>> c.next_toward(Decimal(0), 1) - Decimal('1E-1007') - >>> c.next_toward(0, Decimal(1)) - Decimal('1E-1007') - """ - a = _convert_other(a, raiseit=True) - return a.next_toward(b, context=self) - - def normalize(self, a): - """normalize reduces an operand to its simplest form. - - Essentially a plus operation with all trailing zeros removed from the - result. - - >>> ExtendedContext.normalize(Decimal('2.1')) - Decimal('2.1') - >>> ExtendedContext.normalize(Decimal('-2.0')) - Decimal('-2') - >>> ExtendedContext.normalize(Decimal('1.200')) - Decimal('1.2') - >>> ExtendedContext.normalize(Decimal('-120')) - Decimal('-1.2E+2') - >>> ExtendedContext.normalize(Decimal('120.00')) - Decimal('1.2E+2') - >>> ExtendedContext.normalize(Decimal('0.00')) - Decimal('0') - >>> ExtendedContext.normalize(6) - Decimal('6') - """ - a = _convert_other(a, raiseit=True) - return a.normalize(context=self) - - def number_class(self, a): - """Returns an indication of the class of the operand. - - The class is one of the following strings: - -sNaN - -NaN - -Infinity - -Normal - -Subnormal - -Zero - +Zero - +Subnormal - +Normal - +Infinity - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.number_class(Decimal('Infinity')) - '+Infinity' - >>> c.number_class(Decimal('1E-10')) - '+Normal' - >>> c.number_class(Decimal('2.50')) - '+Normal' - >>> c.number_class(Decimal('0.1E-999')) - '+Subnormal' - >>> c.number_class(Decimal('0')) - '+Zero' - >>> c.number_class(Decimal('-0')) - '-Zero' - >>> c.number_class(Decimal('-0.1E-999')) - '-Subnormal' - >>> c.number_class(Decimal('-1E-10')) - '-Normal' - >>> c.number_class(Decimal('-2.50')) - '-Normal' - >>> c.number_class(Decimal('-Infinity')) - '-Infinity' - >>> c.number_class(Decimal('NaN')) - 'NaN' - >>> c.number_class(Decimal('-NaN')) - 'NaN' - >>> c.number_class(Decimal('sNaN')) - 'sNaN' - >>> c.number_class(123) - '+Normal' - """ - a = _convert_other(a, raiseit=True) - return a.number_class(context=self) - - def plus(self, a): - """Plus corresponds to unary prefix plus in Python. - - The operation is evaluated using the same rules as add; the - operation plus(a) is calculated as add('0', a) where the '0' - has the same exponent as the operand. - - >>> ExtendedContext.plus(Decimal('1.3')) - Decimal('1.3') - >>> ExtendedContext.plus(Decimal('-1.3')) - Decimal('-1.3') - >>> ExtendedContext.plus(-1) - Decimal('-1') - """ - a = _convert_other(a, raiseit=True) - return a.__pos__(context=self) - - def power(self, a, b, modulo=None): - """Raises a to the power of b, to modulo if given. - - With two arguments, compute a**b. If a is negative then b - must be integral. The result will be inexact unless b is - integral and the result is finite and can be expressed exactly - in 'precision' digits. - - With three arguments, compute (a**b) % modulo. For the - three argument form, the following restrictions on the - arguments hold: - - - all three arguments must be integral - - b must be nonnegative - - at least one of a or b must be nonzero - - modulo must be nonzero and have at most 'precision' digits - - The result of pow(a, b, modulo) is identical to the result - that would be obtained by computing (a**b) % modulo with - unbounded precision, but is computed more efficiently. It is - always exact. - - >>> c = ExtendedContext.copy() - >>> c.Emin = -999 - >>> c.Emax = 999 - >>> c.power(Decimal('2'), Decimal('3')) - Decimal('8') - >>> c.power(Decimal('-2'), Decimal('3')) - Decimal('-8') - >>> c.power(Decimal('2'), Decimal('-3')) - Decimal('0.125') - >>> c.power(Decimal('1.7'), Decimal('8')) - Decimal('69.7575744') - >>> c.power(Decimal('10'), Decimal('0.301029996')) - Decimal('2.00000000') - >>> c.power(Decimal('Infinity'), Decimal('-1')) - Decimal('0') - >>> c.power(Decimal('Infinity'), Decimal('0')) - Decimal('1') - >>> c.power(Decimal('Infinity'), Decimal('1')) - Decimal('Infinity') - >>> c.power(Decimal('-Infinity'), Decimal('-1')) - Decimal('-0') - >>> c.power(Decimal('-Infinity'), Decimal('0')) - Decimal('1') - >>> c.power(Decimal('-Infinity'), Decimal('1')) - Decimal('-Infinity') - >>> c.power(Decimal('-Infinity'), Decimal('2')) - Decimal('Infinity') - >>> c.power(Decimal('0'), Decimal('0')) - Decimal('NaN') - - >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) - Decimal('11') - >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) - Decimal('-11') - >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) - Decimal('1') - >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) - Decimal('11') - >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) - Decimal('11729830') - >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) - Decimal('-0') - >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) - Decimal('1') - >>> ExtendedContext.power(7, 7) - Decimal('823543') - >>> ExtendedContext.power(Decimal(7), 7) - Decimal('823543') - >>> ExtendedContext.power(7, Decimal(7), 2) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - r = a.__pow__(b, modulo, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def quantize(self, a, b): - """Returns a value equal to 'a' (rounded), having the exponent of 'b'. - - The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). - - Unlike other operations, if the length of the coefficient after the - quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. - - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. - - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) - Decimal('2.170') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) - Decimal('2.17') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) - Decimal('2.2') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) - Decimal('2') - >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) - Decimal('0E+1') - >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) - Decimal('-Infinity') - >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) - Decimal('-0') - >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) - Decimal('-0E+5') - >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) - Decimal('NaN') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) - Decimal('217.0') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) - Decimal('217') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) - Decimal('2.2E+2') - >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) - Decimal('2E+2') - >>> ExtendedContext.quantize(1, 2) - Decimal('1') - >>> ExtendedContext.quantize(Decimal(1), 2) - Decimal('1') - >>> ExtendedContext.quantize(1, Decimal(2)) - Decimal('1') - """ - a = _convert_other(a, raiseit=True) - return a.quantize(b, context=self) - - def radix(self): - """Just returns 10, as this is Decimal, :) - - >>> ExtendedContext.radix() - Decimal('10') - """ - return Decimal(10) - - def remainder(self, a, b): - """Returns the remainder from integer division. - - The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. - - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). - - >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) - Decimal('2.1') - >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) - Decimal('1') - >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) - Decimal('0.2') - >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) - Decimal('0.1') - >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) - Decimal('1.0') - >>> ExtendedContext.remainder(22, 6) - Decimal('4') - >>> ExtendedContext.remainder(Decimal(22), 6) - Decimal('4') - >>> ExtendedContext.remainder(22, Decimal(6)) - Decimal('4') - """ - a = _convert_other(a, raiseit=True) - r = a.__mod__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def remainder_near(self, a, b): - """Returns to be "a - b * n", where n is the integer nearest the exact - value of "x / b" (if two integers are equally near then the even one - is chosen). If the result is equal to 0 then its sign will be the - sign of a. - - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). - - >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) - Decimal('-0.9') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) - Decimal('-2') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) - Decimal('1') - >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) - Decimal('-1') - >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) - Decimal('0.2') - >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) - Decimal('0.1') - >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) - Decimal('-0.3') - >>> ExtendedContext.remainder_near(3, 11) - Decimal('3') - >>> ExtendedContext.remainder_near(Decimal(3), 11) - Decimal('3') - >>> ExtendedContext.remainder_near(3, Decimal(11)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - return a.remainder_near(b, context=self) - - def rotate(self, a, b): - """Returns a rotated copy of a, b times. - - The coefficient of the result is a rotated copy of the digits in - the coefficient of the first operand. The number of places of - rotation is taken from the absolute value of the second operand, - with the rotation being to the left if the second operand is - positive or to the right otherwise. - - >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) - Decimal('400000003') - >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) - Decimal('12') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) - Decimal('891234567') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) - Decimal('123456789') - >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) - Decimal('345678912') - >>> ExtendedContext.rotate(1333333, 1) - Decimal('13333330') - >>> ExtendedContext.rotate(Decimal(1333333), 1) - Decimal('13333330') - >>> ExtendedContext.rotate(1333333, Decimal(1)) - Decimal('13333330') - """ - a = _convert_other(a, raiseit=True) - return a.rotate(b, context=self) - - def same_quantum(self, a, b): - """Returns True if the two operands have the same exponent. - - The result is never affected by either the sign or the coefficient of - either operand. - - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) - False - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) - True - >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) - False - >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) - True - >>> ExtendedContext.same_quantum(10000, -1) - True - >>> ExtendedContext.same_quantum(Decimal(10000), -1) - True - >>> ExtendedContext.same_quantum(10000, Decimal(-1)) - True - """ - a = _convert_other(a, raiseit=True) - return a.same_quantum(b) - - def scaleb (self, a, b): - """Returns the first operand after adding the second value its exp. - - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) - Decimal('0.0750') - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) - Decimal('7.50') - >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) - Decimal('7.50E+3') - >>> ExtendedContext.scaleb(1, 4) - Decimal('1E+4') - >>> ExtendedContext.scaleb(Decimal(1), 4) - Decimal('1E+4') - >>> ExtendedContext.scaleb(1, Decimal(4)) - Decimal('1E+4') - """ - a = _convert_other(a, raiseit=True) - return a.scaleb(b, context=self) - - def shift(self, a, b): - """Returns a shifted copy of a, b times. - - The coefficient of the result is a shifted copy of the digits - in the coefficient of the first operand. The number of places - to shift is taken from the absolute value of the second operand, - with the shift being to the left if the second operand is - positive or to the right otherwise. Digits shifted into the - coefficient are zeros. - - >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) - Decimal('400000000') - >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) - Decimal('0') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) - Decimal('1234567') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) - Decimal('123456789') - >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) - Decimal('345678900') - >>> ExtendedContext.shift(88888888, 2) - Decimal('888888800') - >>> ExtendedContext.shift(Decimal(88888888), 2) - Decimal('888888800') - >>> ExtendedContext.shift(88888888, Decimal(2)) - Decimal('888888800') - """ - a = _convert_other(a, raiseit=True) - return a.shift(b, context=self) - - def sqrt(self, a): - """Square root of a non-negative number to context precision. - - If the result must be inexact, it is rounded using the round-half-even - algorithm. - - >>> ExtendedContext.sqrt(Decimal('0')) - Decimal('0') - >>> ExtendedContext.sqrt(Decimal('-0')) - Decimal('-0') - >>> ExtendedContext.sqrt(Decimal('0.39')) - Decimal('0.624499800') - >>> ExtendedContext.sqrt(Decimal('100')) - Decimal('10') - >>> ExtendedContext.sqrt(Decimal('1')) - Decimal('1') - >>> ExtendedContext.sqrt(Decimal('1.0')) - Decimal('1.0') - >>> ExtendedContext.sqrt(Decimal('1.00')) - Decimal('1.0') - >>> ExtendedContext.sqrt(Decimal('7')) - Decimal('2.64575131') - >>> ExtendedContext.sqrt(Decimal('10')) - Decimal('3.16227766') - >>> ExtendedContext.sqrt(2) - Decimal('1.41421356') - >>> ExtendedContext.prec - 9 - """ - a = _convert_other(a, raiseit=True) - return a.sqrt(context=self) - - def subtract(self, a, b): - """Return the difference between the two operands. - - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) - Decimal('0.23') - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) - Decimal('0.00') - >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) - Decimal('-0.77') - >>> ExtendedContext.subtract(8, 5) - Decimal('3') - >>> ExtendedContext.subtract(Decimal(8), 5) - Decimal('3') - >>> ExtendedContext.subtract(8, Decimal(5)) - Decimal('3') - """ - a = _convert_other(a, raiseit=True) - r = a.__sub__(b, context=self) - if r is NotImplemented: - raise TypeError("Unable to convert %s to Decimal" % b) - else: - return r - - def to_eng_string(self, a): - """Converts a number to a string, using scientific notation. - - The operation is not affected by the context. - """ - a = _convert_other(a, raiseit=True) - return a.to_eng_string(context=self) - - def to_sci_string(self, a): - """Converts a number to a string, using scientific notation. - - The operation is not affected by the context. - """ - a = _convert_other(a, raiseit=True) - return a.__str__(context=self) - - def to_integral_exact(self, a): - """Rounds to an integer. - - When the operand has a negative exponent, the result is the same - as using the quantize() operation using the given operand as the - left-hand-operand, 1E+0 as the right-hand-operand, and the precision - of the operand as the precision setting; Inexact and Rounded flags - are allowed in this operation. The rounding mode is taken from the - context. - - >>> ExtendedContext.to_integral_exact(Decimal('2.1')) - Decimal('2') - >>> ExtendedContext.to_integral_exact(Decimal('100')) - Decimal('100') - >>> ExtendedContext.to_integral_exact(Decimal('100.0')) - Decimal('100') - >>> ExtendedContext.to_integral_exact(Decimal('101.5')) - Decimal('102') - >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) - Decimal('-102') - >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) - Decimal('1.0E+6') - >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) - Decimal('7.89E+77') - >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) - Decimal('-Infinity') - """ - a = _convert_other(a, raiseit=True) - return a.to_integral_exact(context=self) - - def to_integral_value(self, a): - """Rounds to an integer. - - When the operand has a negative exponent, the result is the same - as using the quantize() operation using the given operand as the - left-hand-operand, 1E+0 as the right-hand-operand, and the precision - of the operand as the precision setting, except that no flags will - be set. The rounding mode is taken from the context. - - >>> ExtendedContext.to_integral_value(Decimal('2.1')) - Decimal('2') - >>> ExtendedContext.to_integral_value(Decimal('100')) - Decimal('100') - >>> ExtendedContext.to_integral_value(Decimal('100.0')) - Decimal('100') - >>> ExtendedContext.to_integral_value(Decimal('101.5')) - Decimal('102') - >>> ExtendedContext.to_integral_value(Decimal('-101.5')) - Decimal('-102') - >>> ExtendedContext.to_integral_value(Decimal('10E+5')) - Decimal('1.0E+6') - >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) - Decimal('7.89E+77') - >>> ExtendedContext.to_integral_value(Decimal('-Inf')) - Decimal('-Infinity') - """ - a = _convert_other(a, raiseit=True) - return a.to_integral_value(context=self) - - # the method name changed, but we provide also the old one, for compatibility - to_integral = to_integral_value - -class _WorkRep(object): - __slots__ = ('sign','int','exp') - # sign: 0 or 1 - # int: int - # exp: None, int, or string - - def __init__(self, value=None): - if value is None: - self.sign = None - self.int = 0 - self.exp = None - elif isinstance(value, Decimal): - self.sign = value._sign - self.int = int(value._int) - self.exp = value._exp - else: - # assert isinstance(value, tuple) - self.sign = value[0] - self.int = value[1] - self.exp = value[2] - - def __repr__(self): - return "(%r, %r, %r)" % (self.sign, self.int, self.exp) - - __str__ = __repr__ - - - -def _normalize(op1, op2, prec = 0): - """Normalizes op1, op2 to have the same exp and length of coefficient. - - Done during addition. - """ - if op1.exp < op2.exp: - tmp = op2 - other = op1 - else: - tmp = op1 - other = op2 - - # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). - # Then adding 10**exp to tmp has the same effect (after rounding) - # as adding any positive quantity smaller than 10**exp; similarly - # for subtraction. So if other is smaller than 10**exp we replace - # it with 10**exp. This avoids tmp.exp - other.exp getting too large. - tmp_len = len(str(tmp.int)) - other_len = len(str(other.int)) - exp = tmp.exp + min(-1, tmp_len - prec - 2) - if other_len + other.exp - 1 < exp: - other.int = 1 - other.exp = exp - - tmp.int *= 10 ** (tmp.exp - other.exp) - tmp.exp = other.exp - return op1, op2 - -##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### - -_nbits = int.bit_length - -def _decimal_lshift_exact(n, e): - """ Given integers n and e, return n * 10**e if it's an integer, else None. - - The computation is designed to avoid computing large powers of 10 - unnecessarily. - - >>> _decimal_lshift_exact(3, 4) - 30000 - >>> _decimal_lshift_exact(300, -999999999) # returns None - - """ - if n == 0: - return 0 - elif e >= 0: - return n * 10**e - else: - # val_n = largest power of 10 dividing n. - str_n = str(abs(n)) - val_n = len(str_n) - len(str_n.rstrip('0')) - return None if val_n < -e else n // 10**-e - -def _sqrt_nearest(n, a): - """Closest integer to the square root of the positive integer n. a is - an initial approximation to the square root. Any positive integer - will do for a, but the closer a is to the square root of n the - faster convergence will be. - - """ - if n <= 0 or a <= 0: - raise ValueError("Both arguments to _sqrt_nearest should be positive.") - - b=0 - while a != b: - b, a = a, a--n//a>>1 - return a - -def _rshift_nearest(x, shift): - """Given an integer x and a nonnegative integer shift, return closest - integer to x / 2**shift; use round-to-even in case of a tie. - - """ - b, q = 1 << shift, x >> shift - return q + (2*(x & (b-1)) + (q&1) > b) - -def _div_nearest(a, b): - """Closest integer to a/b, a and b positive integers; rounds to even - in the case of a tie. - - """ - q, r = divmod(a, b) - return q + (2*r + (q&1) > b) - -def _ilog(x, M, L = 8): - """Integer approximation to M*log(x/M), with absolute error boundable - in terms only of x/M. - - Given positive integers x and M, return an integer approximation to - M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference - between the approximation and the exact result is at most 22. For - L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In - both cases these are upper bounds on the error; it will usually be - much smaller.""" - - # The basic algorithm is the following: let log1p be the function - # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use - # the reduction - # - # log1p(y) = 2*log1p(y/(1+sqrt(1+y))) - # - # repeatedly until the argument to log1p is small (< 2**-L in - # absolute value). For small y we can use the Taylor series - # expansion - # - # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T - # - # truncating at T such that y**T is small enough. The whole - # computation is carried out in a form of fixed-point arithmetic, - # with a real number z being represented by an integer - # approximation to z*M. To avoid loss of precision, the y below - # is actually an integer approximation to 2**R*y*M, where R is the - # number of reductions performed so far. - - y = x-M - # argument reduction; R = number of reductions performed - R = 0 - while (R <= L and abs(y) << L-R >= M or - R > L and abs(y) >> R-L >= M): - y = _div_nearest((M*y) << 1, - M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M)) - R += 1 - - # Taylor series with T terms - T = -int(-10*len(str(M))//(3*L)) - yshift = _rshift_nearest(y, R) - w = _div_nearest(M, T) - for k in range(T-1, 0, -1): - w = _div_nearest(M, k) - _div_nearest(yshift*w, M) - - return _div_nearest(w*y, M) - -def _dlog10(c, e, p): - """Given integers c, e and p with c > 0, p >= 0, compute an integer - approximation to 10**p * log10(c*10**e), with an absolute error of - at most 1. Assumes that c*10**e is not exactly 1.""" - - # increase precision by 2; compensate for this by dividing - # final result by 100 - p += 2 - - # write c*10**e as d*10**f with either: - # f >= 0 and 1 <= d <= 10, or - # f <= 0 and 0.1 <= d <= 1. - # Thus for c*10**e close to 1, f = 0 - l = len(str(c)) - f = e+l - (e+l >= 1) - - if p > 0: - M = 10**p - k = e+p-f - if k >= 0: - c *= 10**k - else: - c = _div_nearest(c, 10**-k) - - log_d = _ilog(c, M) # error < 5 + 22 = 27 - log_10 = _log10_digits(p) # error < 1 - log_d = _div_nearest(log_d*M, log_10) - log_tenpower = f*M # exact - else: - log_d = 0 # error < 2.31 - log_tenpower = _div_nearest(f, 10**-p) # error < 0.5 - - return _div_nearest(log_tenpower+log_d, 100) - -def _dlog(c, e, p): - """Given integers c, e and p with c > 0, compute an integer - approximation to 10**p * log(c*10**e), with an absolute error of - at most 1. Assumes that c*10**e is not exactly 1.""" - - # Increase precision by 2. The precision increase is compensated - # for at the end with a division by 100. - p += 2 - - # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10, - # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e) - # as 10**p * log(d) + 10**p*f * log(10). - l = len(str(c)) - f = e+l - (e+l >= 1) - - # compute approximation to 10**p*log(d), with error < 27 - if p > 0: - k = e+p-f - if k >= 0: - c *= 10**k - else: - c = _div_nearest(c, 10**-k) # error of <= 0.5 in c - - # _ilog magnifies existing error in c by a factor of at most 10 - log_d = _ilog(c, 10**p) # error < 5 + 22 = 27 - else: - # p <= 0: just approximate the whole thing by 0; error < 2.31 - log_d = 0 - - # compute approximation to f*10**p*log(10), with error < 11. - if f: - extra = len(str(abs(f)))-1 - if p + extra >= 0: - # error in f * _log10_digits(p+extra) < |f| * 1 = |f| - # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11 - f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra) - else: - f_log_ten = 0 - else: - f_log_ten = 0 - - # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 - return _div_nearest(f_log_ten + log_d, 100) - -class _Log10Memoize(object): - """Class to compute, store, and allow retrieval of, digits of the - constant log(10) = 2.302585.... This constant is needed by - Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" - def __init__(self): - self.digits = "23025850929940456840179914546843642076011014886" - - def getdigits(self, p): - """Given an integer p >= 0, return floor(10**p)*log(10). - - For example, self.getdigits(3) returns 2302. - """ - # digits are stored as a string, for quick conversion to - # integer in the case that we've already computed enough - # digits; the stored digits should always be correct - # (truncated, not rounded to nearest). - if p < 0: - raise ValueError("p should be nonnegative") - - if p >= len(self.digits): - # compute p+3, p+6, p+9, ... digits; continue until at - # least one of the extra digits is nonzero - extra = 3 - while True: - # compute p+extra digits, correct to within 1ulp - M = 10**(p+extra+2) - digits = str(_div_nearest(_ilog(10*M, M), 100)) - if digits[-extra:] != '0'*extra: - break - extra += 3 - # keep all reliable digits so far; remove trailing zeros - # and next nonzero digit - self.digits = digits.rstrip('0')[:-1] - return int(self.digits[:p+1]) - -_log10_digits = _Log10Memoize().getdigits - -def _iexp(x, M, L=8): - """Given integers x and M, M > 0, such that x/M is small in absolute - value, compute an integer approximation to M*exp(x/M). For 0 <= - x/M <= 2.4, the absolute error in the result is bounded by 60 (and - is usually much smaller).""" - - # Algorithm: to compute exp(z) for a real number z, first divide z - # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then - # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor - # series - # - # expm1(x) = x + x**2/2! + x**3/3! + ... - # - # Now use the identity - # - # expm1(2x) = expm1(x)*(expm1(x)+2) - # - # R times to compute the sequence expm1(z/2**R), - # expm1(z/2**(R-1)), ... , exp(z/2), exp(z). - - # Find R such that x/2**R/M <= 2**-L - R = _nbits((x< M - T = -int(-10*len(str(M))//(3*L)) - y = _div_nearest(x, T) - Mshift = M<= 0: - cshift = c*10**shift - else: - cshift = c//10**-shift - quot, rem = divmod(cshift, _log10_digits(q)) - - # reduce remainder back to original precision - rem = _div_nearest(rem, 10**extra) - - # error in result of _iexp < 120; error after division < 0.62 - return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3 - -def _dpower(xc, xe, yc, ye, p): - """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and - y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that: - - 10**(p-1) <= c <= 10**p, and - (c-1)*10**e < x**y < (c+1)*10**e - - in other words, c*10**e is an approximation to x**y with p digits - of precision, and with an error in c of at most 1. (This is - almost, but not quite, the same as the error being < 1ulp: when c - == 10**(p-1) we can only guarantee error < 10ulp.) - - We assume that: x is positive and not equal to 1, and y is nonzero. - """ - - # Find b such that 10**(b-1) <= |y| <= 10**b - b = len(str(abs(yc))) + ye - - # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point - lxc = _dlog(xc, xe, p+b+1) - - # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1) - shift = ye-b - if shift >= 0: - pc = lxc*yc*10**shift - else: - pc = _div_nearest(lxc*yc, 10**-shift) - - if pc == 0: - # we prefer a result that isn't exactly 1; this makes it - # easier to compute a correctly rounded result in __pow__ - if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1: - coeff, exp = 10**(p-1)+1, 1-p - else: - coeff, exp = 10**p-1, -p - else: - coeff, exp = _dexp(pc, -(p+1), p+1) - coeff = _div_nearest(coeff, 10) - exp += 1 - - return coeff, exp - -def _log10_lb(c, correction = { - '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, - '6': 23, '7': 16, '8': 10, '9': 5}): - """Compute a lower bound for 100*log10(c) for a positive integer c.""" - if c <= 0: - raise ValueError("The argument to _log10_lb should be nonnegative.") - str_c = str(c) - return 100*len(str_c) - correction[str_c[0]] - -##### Helper Functions #################################################### - -def _convert_other(other, raiseit=False, allow_float=False): - """Convert other to Decimal. - - Verifies that it's ok to use in an implicit construction. - If allow_float is true, allow conversion from float; this - is used in the comparison methods (__eq__ and friends). - - """ - if isinstance(other, Decimal): - return other - if isinstance(other, int): - return Decimal(other) - if allow_float and isinstance(other, float): - return Decimal.from_float(other) - - if raiseit: - raise TypeError("Unable to convert %s to Decimal" % other) - return NotImplemented - -def _convert_for_comparison(self, other, equality_op=False): - """Given a Decimal instance self and a Python object other, return - a pair (s, o) of Decimal instances such that "s op o" is - equivalent to "self op other" for any of the 6 comparison - operators "op". - - """ - if isinstance(other, Decimal): - return self, other - - # Comparison with a Rational instance (also includes integers): - # self op n/d <=> self*d op n (for n and d integers, d positive). - # A NaN or infinity can be left unchanged without affecting the - # comparison result. - if isinstance(other, _numbers.Rational): - if not self._is_special: - self = _dec_from_triple(self._sign, - str(int(self._int) * other.denominator), - self._exp) - return self, Decimal(other.numerator) - - # Comparisons with float and complex types. == and != comparisons - # with complex numbers should succeed, returning either True or False - # as appropriate. Other comparisons return NotImplemented. - if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: - other = other.real - if isinstance(other, float): - context = getcontext() - if equality_op: - context.flags[FloatOperation] = 1 - else: - context._raise_error(FloatOperation, - "strict semantics for mixing floats and Decimals are enabled") - return self, Decimal.from_float(other) - return NotImplemented, NotImplemented - - -##### Setup Specific Contexts ############################################ - -# The default context prototype used by Context() -# Is mutable, so that new contexts can have different default values - -DefaultContext = Context( - prec=28, rounding=ROUND_HALF_EVEN, - traps=[DivisionByZero, Overflow, InvalidOperation], - flags=[], - Emax=999999, - Emin=-999999, - capitals=1, - clamp=0 -) - -# Pre-made alternate contexts offered by the specification -# Don't change these; the user should be able to select these -# contexts and be able to reproduce results from other implementations -# of the spec. - -BasicContext = Context( - prec=9, rounding=ROUND_HALF_UP, - traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], - flags=[], -) - -ExtendedContext = Context( - prec=9, rounding=ROUND_HALF_EVEN, - traps=[], - flags=[], -) - - -##### crud for parsing strings ############################################# -# -# Regular expression used for parsing numeric strings. Additional -# comments: -# -# 1. Uncomment the two '\s*' lines to allow leading and/or trailing -# whitespace. But note that the specification disallows whitespace in -# a numeric string. -# -# 2. For finite numbers (not infinities and NaNs) the body of the -# number between the optional sign and the optional exponent must have -# at least one decimal digit, possibly after the decimal point. The -# lookahead expression '(?=\d|\.\d)' checks this. - -import re -_parser = re.compile(r""" # A numeric string consists of: -# \s* - (?P[-+])? # an optional sign, followed by either... - ( - (?=\d|\.\d) # ...a number (with at least one digit) - (?P\d*) # having a (possibly empty) integer part - (\.(?P\d*))? # followed by an optional fractional part - (E(?P[-+]?\d+))? # followed by an optional exponent, or... - | - Inf(inity)? # ...an infinity, or... - | - (?Ps)? # ...an (optionally signaling) - NaN # NaN - (?P\d*) # with (possibly empty) diagnostic info. - ) -# \s* - \Z -""", re.VERBOSE | re.IGNORECASE).match - -_all_zeros = re.compile('0*$').match -_exact_half = re.compile('50*$').match - -##### PEP3101 support functions ############################################## -# The functions in this section have little to do with the Decimal -# class, and could potentially be reused or adapted for other pure -# Python numeric classes that want to implement __format__ -# -# A format specifier for Decimal looks like: -# -# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] - -_parse_format_specifier_regex = re.compile(r"""\A -(?: - (?P.)? - (?P[<>=^]) -)? -(?P[-+ ])? -(?P\#)? -(?P0)? -(?P(?!0)\d+)? -(?P,)? -(?:\.(?P0|(?!0)\d+))? -(?P[eEfFgGn%])? -\Z -""", re.VERBOSE|re.DOTALL) - -del re - -# The locale module is only needed for the 'n' format specifier. The -# rest of the PEP 3101 code functions quite happily without it, so we -# don't care too much if locale isn't present. -try: - import locale as _locale -except ImportError: - pass - -def _parse_format_specifier(format_spec, _localeconv=None): - """Parse and validate a format specifier. - - Turns a standard numeric format specifier into a dict, with the - following entries: - - fill: fill character to pad field to minimum width - align: alignment type, either '<', '>', '=' or '^' - sign: either '+', '-' or ' ' - minimumwidth: nonnegative integer giving minimum width - zeropad: boolean, indicating whether to pad with zeros - thousands_sep: string to use as thousands separator, or '' - grouping: grouping for thousands separators, in format - used by localeconv - decimal_point: string to use for decimal point - precision: nonnegative integer giving precision, or None - type: one of the characters 'eEfFgG%', or None - - """ - m = _parse_format_specifier_regex.match(format_spec) - if m is None: - raise ValueError("Invalid format specifier: " + format_spec) - - # get the dictionary - format_dict = m.groupdict() - - # zeropad; defaults for fill and alignment. If zero padding - # is requested, the fill and align fields should be absent. - fill = format_dict['fill'] - align = format_dict['align'] - format_dict['zeropad'] = (format_dict['zeropad'] is not None) - if format_dict['zeropad']: - if fill is not None: - raise ValueError("Fill character conflicts with '0'" - " in format specifier: " + format_spec) - if align is not None: - raise ValueError("Alignment conflicts with '0' in " - "format specifier: " + format_spec) - format_dict['fill'] = fill or ' ' - # PEP 3101 originally specified that the default alignment should - # be left; it was later agreed that right-aligned makes more sense - # for numeric types. See http://bugs.python.org/issue6857. - format_dict['align'] = align or '>' - - # default sign handling: '-' for negative, '' for positive - if format_dict['sign'] is None: - format_dict['sign'] = '-' - - # minimumwidth defaults to 0; precision remains None if not given - format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') - if format_dict['precision'] is not None: - format_dict['precision'] = int(format_dict['precision']) - - # if format type is 'g' or 'G' then a precision of 0 makes little - # sense; convert it to 1. Same if format type is unspecified. - if format_dict['precision'] == 0: - if format_dict['type'] is None or format_dict['type'] in 'gGn': - format_dict['precision'] = 1 - - # determine thousands separator, grouping, and decimal separator, and - # add appropriate entries to format_dict - if format_dict['type'] == 'n': - # apart from separators, 'n' behaves just like 'g' - format_dict['type'] = 'g' - if _localeconv is None: - _localeconv = _locale.localeconv() - if format_dict['thousands_sep'] is not None: - raise ValueError("Explicit thousands separator conflicts with " - "'n' type in format specifier: " + format_spec) - format_dict['thousands_sep'] = _localeconv['thousands_sep'] - format_dict['grouping'] = _localeconv['grouping'] - format_dict['decimal_point'] = _localeconv['decimal_point'] - else: - if format_dict['thousands_sep'] is None: - format_dict['thousands_sep'] = '' - format_dict['grouping'] = [3, 0] - format_dict['decimal_point'] = '.' - - return format_dict - -def _format_align(sign, body, spec): - """Given an unpadded, non-aligned numeric string 'body' and sign - string 'sign', add padding and alignment conforming to the given - format specifier dictionary 'spec' (as produced by - parse_format_specifier). - - """ - # how much extra space do we have to play with? - minimumwidth = spec['minimumwidth'] - fill = spec['fill'] - padding = fill*(minimumwidth - len(sign) - len(body)) - - align = spec['align'] - if align == '<': - result = sign + body + padding - elif align == '>': - result = padding + sign + body - elif align == '=': - result = sign + padding + body - elif align == '^': - half = len(padding)//2 - result = padding[:half] + sign + body + padding[half:] - else: - raise ValueError('Unrecognised alignment field') - - return result - -def _group_lengths(grouping): - """Convert a localeconv-style grouping into a (possibly infinite) - iterable of integers representing group lengths. - - """ - # The result from localeconv()['grouping'], and the input to this - # function, should be a list of integers in one of the - # following three forms: - # - # (1) an empty list, or - # (2) nonempty list of positive integers + [0] - # (3) list of positive integers + [locale.CHAR_MAX], or - - from itertools import chain, repeat - if not grouping: - return [] - elif grouping[-1] == 0 and len(grouping) >= 2: - return chain(grouping[:-1], repeat(grouping[-2])) - elif grouping[-1] == _locale.CHAR_MAX: - return grouping[:-1] - else: - raise ValueError('unrecognised format for grouping') - -def _insert_thousands_sep(digits, spec, min_width=1): - """Insert thousands separators into a digit string. - - spec is a dictionary whose keys should include 'thousands_sep' and - 'grouping'; typically it's the result of parsing the format - specifier using _parse_format_specifier. - - The min_width keyword argument gives the minimum length of the - result, which will be padded on the left with zeros if necessary. - - If necessary, the zero padding adds an extra '0' on the left to - avoid a leading thousands separator. For example, inserting - commas every three digits in '123456', with min_width=8, gives - '0,123,456', even though that has length 9. - - """ - - sep = spec['thousands_sep'] - grouping = spec['grouping'] - - groups = [] - for l in _group_lengths(grouping): - if l <= 0: - raise ValueError("group length should be positive") - # max(..., 1) forces at least 1 digit to the left of a separator - l = min(max(len(digits), min_width, 1), l) - groups.append('0'*(l - len(digits)) + digits[-l:]) - digits = digits[:-l] - min_width -= l - if not digits and min_width <= 0: - break - min_width -= len(sep) - else: - l = max(len(digits), min_width, 1) - groups.append('0'*(l - len(digits)) + digits[-l:]) - return sep.join(reversed(groups)) - -def _format_sign(is_negative, spec): - """Determine sign character.""" - - if is_negative: - return '-' - elif spec['sign'] in ' +': - return spec['sign'] - else: - return '' - -def _format_number(is_negative, intpart, fracpart, exp, spec): - """Format a number, given the following data: - - is_negative: true if the number is negative, else false - intpart: string of digits that must appear before the decimal point - fracpart: string of digits that must come after the point - exp: exponent, as an integer - spec: dictionary resulting from parsing the format specifier - - This function uses the information in spec to: - insert separators (decimal separator and thousands separators) - format the sign - format the exponent - add trailing '%' for the '%' type - zero-pad if necessary - fill and align if necessary - """ - - sign = _format_sign(is_negative, spec) - - if fracpart or spec['alt']: - fracpart = spec['decimal_point'] + fracpart - - if exp != 0 or spec['type'] in 'eE': - echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] - fracpart += "{0}{1:+}".format(echar, exp) - if spec['type'] == '%': - fracpart += '%' - - if spec['zeropad']: - min_width = spec['minimumwidth'] - len(fracpart) - len(sign) - else: - min_width = 0 - intpart = _insert_thousands_sep(intpart, spec, min_width) - - return _format_align(sign, intpart+fracpart, spec) - - -##### Useful Constants (internal use only) ################################ - -# Reusable defaults -_Infinity = Decimal('Inf') -_NegativeInfinity = Decimal('-Inf') -_NaN = Decimal('NaN') -_Zero = Decimal(0) -_One = Decimal(1) -_NegativeOne = Decimal(-1) - -# _SignedInfinity[sign] is infinity w/ that sign -_SignedInfinity = (_Infinity, _NegativeInfinity) - -# Constants related to the hash implementation; hash(x) is based -# on the reduction of x modulo _PyHASH_MODULUS -_PyHASH_MODULUS = sys.hash_info.modulus -# hash values to use for positive and negative infinities, and nans -_PyHASH_INF = sys.hash_info.inf -_PyHASH_NAN = sys.hash_info.nan - -# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS -_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) -del sys diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_pyio.py --- a/Lib/_pyio.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/_pyio.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,34 +5,23 @@ import os import abc import codecs +import warnings import errno -import array -import stat -import sys # Import _thread instead of threading to reduce startup cost try: from _thread import allocate_lock as Lock except ImportError: from _dummy_thread import allocate_lock as Lock -if sys.platform in {'win32', 'cygwin'}: - from msvcrt import setmode as _setmode -else: - _setmode = None import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) -valid_seek_flags = {0, 1, 2} # Hardwired values -if hasattr(os, 'SEEK_HOLE') : - valid_seek_flags.add(os.SEEK_HOLE) - valid_seek_flags.add(os.SEEK_DATA) - # open() uses st_blksize whenever we can DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs -# defined in io.py. We don't use real inheritance though, because we don't want -# to inherit the C implementations. +# defined in io.py. We don't use real inheritance though, because we don't +# want to inherit the C implementations. # Rebind for compatibility BlockingIOError = BlockingIOError @@ -41,7 +30,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): - r"""Open file and return a stream. Raise OSError upon failure. + r"""Open file and return a stream. Raise IOError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to @@ -69,7 +58,8 @@ 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - 'U' universal newline mode (deprecated) + 'U' universal newline mode (for backwards compatibility; unneeded + for new code) ========= =============================================================== The default mode is 'rt' (open for reading text). For binary random @@ -85,10 +75,6 @@ returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. - 'U' mode is deprecated and will raise an exception in future versions - of Python. It has no effect in Python 3. Use newline to control - universal newlines mode. - buffering is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate @@ -139,8 +125,6 @@ be kept open when the file is closed. This does not work when a file name is given and must be True in that case. - The newly created file is non-inheritable. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*file*, *flags*). *opener* must return an open file @@ -182,11 +166,8 @@ text = "t" in modes binary = "b" in modes if "U" in modes: - if creating or writing or appending or updating: - raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") - import warnings - warnings.warn("'U' mode is deprecated", - DeprecationWarning, 2) + if creating or writing or appending: + raise ValueError("can't use U and writing mode at once") reading = True if text and binary: raise ValueError("can't have text and binary mode at once") @@ -207,45 +188,38 @@ (appending and "a" or "") + (updating and "+" or ""), closefd, opener=opener) - result = raw - try: - line_buffering = False - if buffering == 1 or buffering < 0 and raw.isatty(): - buffering = -1 - line_buffering = True - if buffering < 0: - buffering = DEFAULT_BUFFER_SIZE - try: - bs = os.fstat(raw.fileno()).st_blksize - except (OSError, AttributeError): - pass - else: - if bs > 1: - buffering = bs - if buffering < 0: - raise ValueError("invalid buffering size") - if buffering == 0: - if binary: - return result - raise ValueError("can't have unbuffered text I/O") - if updating: - buffer = BufferedRandom(raw, buffering) - elif creating or writing or appending: - buffer = BufferedWriter(raw, buffering) - elif reading: - buffer = BufferedReader(raw, buffering) + line_buffering = False + if buffering == 1 or buffering < 0 and raw.isatty(): + buffering = -1 + line_buffering = True + if buffering < 0: + buffering = DEFAULT_BUFFER_SIZE + try: + bs = os.fstat(raw.fileno()).st_blksize + except (os.error, AttributeError): + pass else: - raise ValueError("unknown mode: %r" % mode) - result = buffer + if bs > 1: + buffering = bs + if buffering < 0: + raise ValueError("invalid buffering size") + if buffering == 0: if binary: - return result - text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) - result = text - text.mode = mode - return result - except: - result.close() - raise + return raw + raise ValueError("can't have unbuffered text I/O") + if updating: + buffer = BufferedRandom(raw, buffering) + elif creating or writing or appending: + buffer = BufferedWriter(raw, buffering) + elif reading: + buffer = BufferedReader(raw, buffering) + else: + raise ValueError("unknown mode: %r" % mode) + if binary: + return buffer + text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) + text.mode = mode + return text class DocDescriptor: @@ -263,7 +237,7 @@ Trick so that open won't become a bound method when stored as a class variable (as dbm.dumb does). - See initstdio() in Python/pylifecycle.c. + See initstdio() in Python/pythonrun.c. """ __doc__ = DocDescriptor() @@ -276,7 +250,7 @@ try: UnsupportedOperation = io.UnsupportedOperation except AttributeError: - class UnsupportedOperation(ValueError, OSError): + class UnsupportedOperation(ValueError, IOError): pass @@ -300,7 +274,7 @@ readinto) needed. Text I/O classes work with str data. Note that calling any method (even inquiries) on a closed stream is - undefined. Implementations may raise OSError in this case. + undefined. Implementations may raise IOError in this case. IOBase (and its subclasses) support the iterator protocol, meaning that an IOBase object can be iterated over yielding the lines in a @@ -316,7 +290,7 @@ ### Internal ### def _unsupported(self, name): - """Internal: raise an OSError exception for unsupported operations.""" + """Internal: raise an IOError exception for unsupported operations.""" raise UnsupportedOperation("%s.%s() not supported" % (self.__class__.__name__, name)) @@ -325,14 +299,13 @@ def seek(self, pos, whence=0): """Change stream position. - Change the stream position to byte offset pos. Argument pos is + Change the stream position to byte offset offset. offset is interpreted relative to the position indicated by whence. Values for whence are ints: * 0 -- start of stream (the default); offset should be zero or positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. Return an int indicating the new absolute position. """ @@ -368,10 +341,8 @@ This method has no effect if the file is already closed. """ if not self.__closed: - try: - self.flush() - finally: - self.__closed = True + self.flush() + self.__closed = True def __del__(self): """Destructor. Calls close().""" @@ -463,7 +434,7 @@ def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An IOError is raised if the IO object does not use a file descriptor. """ self._unsupported("fileno") @@ -477,11 +448,11 @@ ### Readline[s] and writelines ### - def readline(self, size=-1): + def readline(self, limit=-1): r"""Read and return a line of bytes from the stream. - If size is specified, at most size bytes will be read. - Size should be an int. + If limit is specified, at most limit bytes will be read. + Limit should be an int. The line terminator is always b'\n' for binary files; for text files, the newlines argument to open can be used to select the line @@ -494,18 +465,18 @@ if not readahead: return 1 n = (readahead.find(b"\n") + 1) or len(readahead) - if size >= 0: - n = min(n, size) + if limit >= 0: + n = min(n, limit) return n else: def nreadahead(): return 1 - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError("size must be an integer") + if limit is None: + limit = -1 + elif not isinstance(limit, int): + raise TypeError("limit must be an integer") res = bytearray() - while size < 0 or len(res) < size: + while limit < 0 or len(res) < limit: b = self.read(nreadahead()) if not b: break @@ -564,17 +535,17 @@ # primitive operation, but that would lead to nasty recursion in case # a subclass doesn't implement either.) - def read(self, size=-1): - """Read and return up to size bytes, where size is an int. + def read(self, n=-1): + """Read and return up to n bytes, where n is an int. Returns an empty bytes object on EOF, or None if the object is set not to block and has no data to read. """ - if size is None: - size = -1 - if size < 0: + if n is None: + n = -1 + if n < 0: return self.readall() - b = bytearray(size.__index__()) + b = bytearray(n.__index__()) n = self.readinto(b) if n is None: return None @@ -632,8 +603,8 @@ implementation, but wrap one. """ - def read(self, size=None): - """Read and return up to size bytes, where size is an int. + def read(self, n=None): + """Read and return up to n bytes, where n is an int. If the argument is omitted, None, or negative, reads and returns all data until EOF. @@ -652,9 +623,9 @@ """ self._unsupported("read") - def read1(self, size=None): - """Read up to size bytes with at most one read() system call, - where size is an int. + def read1(self, n=None): + """Read up to n bytes with at most one read() system call, + where n is an int. """ self._unsupported("read1") @@ -669,33 +640,16 @@ Raises BlockingIOError if the underlying raw stream has no data at the moment. """ - - return self._readinto(b, read1=False) - - def readinto1(self, b): - """Read up to len(b) bytes into *b*, using at most one system call - - Returns an int representing the number of bytes read (0 for EOF). - - Raises BlockingIOError if the underlying raw stream has no - data at the moment. - """ - - return self._readinto(b, read1=True) - - def _readinto(self, b, read1): - if not isinstance(b, memoryview): - b = memoryview(b) - b = b.cast('B') - - if read1: - data = self.read1(len(b)) - else: - data = self.read(len(b)) + # XXX This ought to work with anything that supports the buffer API + data = self.read(len(b)) n = len(data) - - b[:n] = data - + try: + b[:n] = data + except TypeError as err: + import array + if not isinstance(b, array.array): + raise err + b[:n] = array.array('b', data) return n def write(self, b): @@ -738,13 +692,13 @@ def seek(self, pos, whence=0): new_position = self.raw.seek(pos, whence) if new_position < 0: - raise OSError("seek() returned an invalid position") + raise IOError("seek() returned an invalid position") return new_position def tell(self): pos = self.raw.tell() if pos < 0: - raise OSError("tell() returned an invalid position") + raise IOError("tell() returned an invalid position") return pos def truncate(self, pos=None): @@ -814,14 +768,13 @@ .format(self.__class__.__name__)) def __repr__(self): - modname = self.__class__.__module__ - clsname = self.__class__.__qualname__ + clsname = self.__class__.__name__ try: name = self.name - except Exception: - return "<{}.{}>".format(modname, clsname) + except AttributeError: + return "<_pyio.{0}>".format(clsname) else: - return "<{}.{} name={!r}>".format(modname, clsname, name) + return "<_pyio.{0} name={1!r}>".format(clsname, name) ### Lower-level APIs ### @@ -858,32 +811,26 @@ def getbuffer(self): """Return a readable and writable view of the buffer. """ - if self.closed: - raise ValueError("getbuffer on closed file") return memoryview(self._buffer) - def close(self): - self._buffer.clear() - super().close() - - def read(self, size=None): + def read(self, n=None): if self.closed: raise ValueError("read from closed file") - if size is None: - size = -1 - if size < 0: - size = len(self._buffer) + if n is None: + n = -1 + if n < 0: + n = len(self._buffer) if len(self._buffer) <= self._pos: return b"" - newpos = min(len(self._buffer), self._pos + size) + newpos = min(len(self._buffer), self._pos + n) b = self._buffer[self._pos : newpos] self._pos = newpos return bytes(b) - def read1(self, size): + def read1(self, n): """This is the same as read. """ - return self.read(size) + return self.read(n) def write(self, b): if self.closed: @@ -919,7 +866,7 @@ elif whence == 2: self._pos = max(0, len(self._buffer) + pos) else: - raise ValueError("unsupported whence value") + raise ValueError("invalid whence value") return self._pos def tell(self): @@ -943,18 +890,12 @@ return pos def readable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") return True def writable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") return True def seekable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") return True @@ -973,7 +914,7 @@ """Create a new buffered reader using the given readable raw IO object. """ if not raw.readable(): - raise OSError('"raw" argument must be readable.') + raise IOError('"raw" argument must be readable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: @@ -986,18 +927,18 @@ self._read_buf = b"" self._read_pos = 0 - def read(self, size=None): - """Read size bytes. + def read(self, n=None): + """Read n bytes. - Returns exactly size bytes of data unless the underlying raw IO + Returns exactly n bytes of data unless the underlying raw IO stream reaches EOF or if the call would block in non-blocking - mode. If size is negative, read until EOF or until read() would + mode. If n is negative, read until EOF or until read() would block. """ - if size is not None and size < -1: + if n is not None and n < -1: raise ValueError("invalid number of bytes to read") with self._read_lock: - return self._read_unlocked(size) + return self._read_unlocked(n) def _read_unlocked(self, n=None): nodata_val = b"" @@ -1018,7 +959,10 @@ current_size = 0 while True: # Read until EOF or until read() would block. - chunk = self.raw.read() + try: + chunk = self.raw.read() + except InterruptedError: + continue if chunk in empty_values: nodata_val = chunk break @@ -1037,7 +981,10 @@ chunks = [buf[pos:]] wanted = max(self.buffer_size, n) while avail < n: - chunk = self.raw.read(wanted) + try: + chunk = self.raw.read(wanted) + except InterruptedError: + continue if chunk in empty_values: nodata_val = chunk break @@ -1051,7 +998,7 @@ self._read_pos = 0 return out[:n] if out else nodata_val - def peek(self, size=0): + def peek(self, n=0): """Returns buffered bytes without advancing the position. The argument indicates a desired minimal number of bytes; we @@ -1059,89 +1006,42 @@ than self.buffer_size. """ with self._read_lock: - return self._peek_unlocked(size) + return self._peek_unlocked(n) def _peek_unlocked(self, n=0): want = min(n, self.buffer_size) have = len(self._read_buf) - self._read_pos if have < want or have <= 0: to_read = self.buffer_size - have - current = self.raw.read(to_read) + while True: + try: + current = self.raw.read(to_read) + except InterruptedError: + continue + break if current: self._read_buf = self._read_buf[self._read_pos:] + current self._read_pos = 0 return self._read_buf[self._read_pos:] - def read1(self, size): - """Reads up to size bytes, with at most one read() system call.""" - # Returns up to size bytes. If at least one byte is buffered, we + def read1(self, n): + """Reads up to n bytes, with at most one read() system call.""" + # Returns up to n bytes. If at least one byte is buffered, we # only return buffered bytes. Otherwise, we do one raw read. - if size < 0: + if n < 0: raise ValueError("number of bytes to read must be positive") - if size == 0: + if n == 0: return b"" with self._read_lock: self._peek_unlocked(1) return self._read_unlocked( - min(size, len(self._read_buf) - self._read_pos)) - - # Implementing readinto() and readinto1() is not strictly necessary (we - # could rely on the base class that provides an implementation in terms of - # read() and read1()). We do it anyway to keep the _pyio implementation - # similar to the io implementation (which implements the methods for - # performance reasons). - def _readinto(self, buf, read1): - """Read data into *buf* with at most one system call.""" - - if len(buf) == 0: - return 0 - - # Need to create a memoryview object of type 'b', otherwise - # we may not be able to assign bytes to it, and slicing it - # would create a new object. - if not isinstance(buf, memoryview): - buf = memoryview(buf) - buf = buf.cast('B') - - written = 0 - with self._read_lock: - while written < len(buf): - - # First try to read from internal buffer - avail = min(len(self._read_buf) - self._read_pos, len(buf)) - if avail: - buf[written:written+avail] = \ - self._read_buf[self._read_pos:self._read_pos+avail] - self._read_pos += avail - written += avail - if written == len(buf): - break - - # If remaining space in callers buffer is larger than - # internal buffer, read directly into callers buffer - if len(buf) - written > self.buffer_size: - n = self.raw.readinto(buf[written:]) - if not n: - break # eof - written += n - - # Otherwise refill internal buffer - unless we're - # in read1 mode and already got some data - elif not (read1 and written): - if not self._peek_unlocked(1): - break # eof - - # In readinto1 mode, return as soon as we have some data - if read1 and written: - break - - return written + min(n, len(self._read_buf) - self._read_pos)) def tell(self): return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos def seek(self, pos, whence=0): - if whence not in valid_seek_flags: + if not (0 <= whence <= 2): raise ValueError("invalid whence value") with self._read_lock: if whence == 1: @@ -1159,13 +1059,19 @@ DEFAULT_BUFFER_SIZE. """ - def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): + _warning_stack_offset = 2 + + def __init__(self, raw, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): if not raw.writable(): - raise OSError('"raw" argument must be writable.') + raise IOError('"raw" argument must be writable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: raise ValueError("invalid buffer size") + if max_buffer_size is not None: + warnings.warn("max_buffer_size is deprecated", DeprecationWarning, + self._warning_stack_offset) self.buffer_size = buffer_size self._write_buf = bytearray() self._write_lock = Lock() @@ -1215,6 +1121,8 @@ while self._write_buf: try: n = self.raw.write(self._write_buf) + except InterruptedError: + continue except BlockingIOError: raise RuntimeError("self.raw should implement RawIOBase: it " "should not raise BlockingIOError") @@ -1223,15 +1131,15 @@ errno.EAGAIN, "write could not complete without blocking", 0) if n > len(self._write_buf) or n < 0: - raise OSError("write() returned incorrect number of bytes") + raise IOError("write() returned incorrect number of bytes") del self._write_buf[:n] def tell(self): return _BufferedIOMixin.tell(self) + len(self._write_buf) def seek(self, pos, whence=0): - if whence not in valid_seek_flags: - raise ValueError("invalid whence value") + if not (0 <= whence <= 2): + raise ValueError("invalid whence") with self._write_lock: self._flush_unlocked() return _BufferedIOMixin.seek(self, pos, whence) @@ -1253,24 +1161,28 @@ # XXX The usefulness of this (compared to having two separate IO # objects) is questionable. - def __init__(self, reader, writer, buffer_size=DEFAULT_BUFFER_SIZE): + def __init__(self, reader, writer, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): """Constructor. The arguments are two RawIO instances. """ + if max_buffer_size is not None: + warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2) + if not reader.readable(): - raise OSError('"reader" argument must be readable.') + raise IOError('"reader" argument must be readable.') if not writer.writable(): - raise OSError('"writer" argument must be writable.') + raise IOError('"writer" argument must be writable.') self.reader = BufferedReader(reader, buffer_size) self.writer = BufferedWriter(writer, buffer_size) - def read(self, size=None): - if size is None: - size = -1 - return self.reader.read(size) + def read(self, n=None): + if n is None: + n = -1 + return self.reader.read(n) def readinto(self, b): return self.reader.readinto(b) @@ -1278,14 +1190,11 @@ def write(self, b): return self.writer.write(b) - def peek(self, size=0): - return self.reader.peek(size) + def peek(self, n=0): + return self.reader.peek(n) - def read1(self, size): - return self.reader.read1(size) - - def readinto1(self, b): - return self.reader.readinto1(b) + def read1(self, n): + return self.reader.read1(n) def readable(self): return self.reader.readable() @@ -1297,10 +1206,8 @@ return self.writer.flush() def close(self): - try: - self.writer.close() - finally: - self.reader.close() + self.writer.close() + self.reader.close() def isatty(self): return self.reader.isatty() or self.writer.isatty() @@ -1319,14 +1226,17 @@ defaults to DEFAULT_BUFFER_SIZE. """ - def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): + _warning_stack_offset = 3 + + def __init__(self, raw, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): raw._checkSeekable() BufferedReader.__init__(self, raw, buffer_size) - BufferedWriter.__init__(self, raw, buffer_size) + BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size) def seek(self, pos, whence=0): - if whence not in valid_seek_flags: - raise ValueError("invalid whence value") + if not (0 <= whence <= 2): + raise ValueError("invalid whence") self.flush() if self._read_buf: # Undo read ahead. @@ -1338,7 +1248,7 @@ with self._read_lock: self._reset_read_buf() if pos < 0: - raise OSError("seek() returned invalid position") + raise IOError("seek() returned invalid position") return pos def tell(self): @@ -1353,27 +1263,23 @@ # Use seek to flush the read buffer. return BufferedWriter.truncate(self, pos) - def read(self, size=None): - if size is None: - size = -1 + def read(self, n=None): + if n is None: + n = -1 self.flush() - return BufferedReader.read(self, size) + return BufferedReader.read(self, n) def readinto(self, b): self.flush() return BufferedReader.readinto(self, b) - def peek(self, size=0): + def peek(self, n=0): self.flush() - return BufferedReader.peek(self, size) + return BufferedReader.peek(self, n) - def read1(self, size): + def read1(self, n): self.flush() - return BufferedReader.read1(self, size) - - def readinto1(self, b): - self.flush() - return BufferedReader.readinto1(self, b) + return BufferedReader.read1(self, n) def write(self, b): if self._read_buf: @@ -1384,345 +1290,6 @@ return BufferedWriter.write(self, b) -class FileIO(RawIOBase): - _fd = -1 - _created = False - _readable = False - _writable = False - _appending = False - _seekable = None - _closefd = True - - def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). - """ - if self._fd >= 0: - # Have to close the existing file first. - try: - if self._closefd: - os.close(self._fd) - finally: - self._fd = -1 - - if isinstance(file, float): - raise TypeError('integer argument expected, got float') - if isinstance(file, int): - fd = file - if fd < 0: - raise ValueError('negative file descriptor') - else: - fd = -1 - - if not isinstance(mode, str): - raise TypeError('invalid mode: %s' % (mode,)) - if not set(mode) <= set('xrwab+'): - raise ValueError('invalid mode: %s' % (mode,)) - if sum(c in 'rwax' for c in mode) != 1 or mode.count('+') > 1: - raise ValueError('Must have exactly one of create/read/write/append ' - 'mode and at most one plus') - - if 'x' in mode: - self._created = True - self._writable = True - flags = os.O_EXCL | os.O_CREAT - elif 'r' in mode: - self._readable = True - flags = 0 - elif 'w' in mode: - self._writable = True - flags = os.O_CREAT | os.O_TRUNC - elif 'a' in mode: - self._writable = True - self._appending = True - flags = os.O_APPEND | os.O_CREAT - - if '+' in mode: - self._readable = True - self._writable = True - - if self._readable and self._writable: - flags |= os.O_RDWR - elif self._readable: - flags |= os.O_RDONLY - else: - flags |= os.O_WRONLY - - flags |= getattr(os, 'O_BINARY', 0) - - noinherit_flag = (getattr(os, 'O_NOINHERIT', 0) or - getattr(os, 'O_CLOEXEC', 0)) - flags |= noinherit_flag - - owned_fd = None - try: - if fd < 0: - if not closefd: - raise ValueError('Cannot use closefd=False with file name') - if opener is None: - fd = os.open(file, flags, 0o666) - else: - fd = opener(file, flags) - if not isinstance(fd, int): - raise TypeError('expected integer from opener') - if fd < 0: - raise OSError('Negative file descriptor') - owned_fd = fd - if not noinherit_flag: - os.set_inheritable(fd, False) - - self._closefd = closefd - fdfstat = os.fstat(fd) - try: - if stat.S_ISDIR(fdfstat.st_mode): - raise IsADirectoryError(errno.EISDIR, - os.strerror(errno.EISDIR), file) - except AttributeError: - # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR - # don't exist. - pass - self._blksize = getattr(fdfstat, 'st_blksize', 0) - if self._blksize <= 1: - self._blksize = DEFAULT_BUFFER_SIZE - - if _setmode: - # don't translate newlines (\r\n <=> \n) - _setmode(fd, os.O_BINARY) - - self.name = file - if self._appending: - # For consistent behaviour, we explicitly seek to the - # end of file (otherwise, it might be done only on the - # first write()). - os.lseek(fd, 0, SEEK_END) - except: - if owned_fd is not None: - os.close(owned_fd) - raise - self._fd = fd - - def __del__(self): - if self._fd >= 0 and self._closefd and not self.closed: - import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, - stacklevel=2) - self.close() - - def __getstate__(self): - raise TypeError("cannot serialize '%s' object", self.__class__.__name__) - - def __repr__(self): - class_name = '%s.%s' % (self.__class__.__module__, - self.__class__.__qualname__) - if self.closed: - return '<%s [closed]>' % class_name - try: - name = self.name - except AttributeError: - return ('<%s fd=%d mode=%r closefd=%r>' % - (class_name, self._fd, self.mode, self._closefd)) - else: - return ('<%s name=%r mode=%r closefd=%r>' % - (class_name, name, self.mode, self._closefd)) - - def _checkReadable(self): - if not self._readable: - raise UnsupportedOperation('File not open for reading') - - def _checkWritable(self, msg=None): - if not self._writable: - raise UnsupportedOperation('File not open for writing') - - def read(self, size=None): - """Read at most size bytes, returned as bytes. - - Only makes one system call, so less data may be returned than requested - In non-blocking mode, returns None if no data is available. - Return an empty bytes object at EOF. - """ - self._checkClosed() - self._checkReadable() - if size is None or size < 0: - return self.readall() - try: - return os.read(self._fd, size) - except BlockingIOError: - return None - - def readall(self): - """Read all data from the file, returned as bytes. - - In non-blocking mode, returns as much as is immediately available, - or None if no data is available. Return an empty bytes object at EOF. - """ - self._checkClosed() - self._checkReadable() - bufsize = DEFAULT_BUFFER_SIZE - try: - pos = os.lseek(self._fd, 0, SEEK_CUR) - end = os.fstat(self._fd).st_size - if end >= pos: - bufsize = end - pos + 1 - except OSError: - pass - - result = bytearray() - while True: - if len(result) >= bufsize: - bufsize = len(result) - bufsize += max(bufsize, DEFAULT_BUFFER_SIZE) - n = bufsize - len(result) - try: - chunk = os.read(self._fd, n) - except BlockingIOError: - if result: - break - return None - if not chunk: # reached the end of the file - break - result += chunk - - return bytes(result) - - def readinto(self, b): - """Same as RawIOBase.readinto().""" - m = memoryview(b).cast('B') - data = self.read(len(m)) - n = len(data) - m[:n] = data - return n - - def write(self, b): - """Write bytes b to file, return number written. - - Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. - """ - self._checkClosed() - self._checkWritable() - try: - return os.write(self._fd, b) - except BlockingIOError: - return None - - def seek(self, pos, whence=SEEK_SET): - """Move to new file position. - - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). - - Note that not all file objects are seekable. - """ - if isinstance(pos, float): - raise TypeError('an integer is required') - self._checkClosed() - return os.lseek(self._fd, pos, whence) - - def tell(self): - """tell() -> int. Current file position. - - Can raise OSError for non seekable files.""" - self._checkClosed() - return os.lseek(self._fd, 0, SEEK_CUR) - - def truncate(self, size=None): - """Truncate the file to at most size bytes. - - Size defaults to the current file position, as returned by tell(). - The current file position is changed to the value of size. - """ - self._checkClosed() - self._checkWritable() - if size is None: - size = self.tell() - os.ftruncate(self._fd, size) - return size - - def close(self): - """Close the file. - - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. - """ - if not self.closed: - try: - if self._closefd: - os.close(self._fd) - finally: - super().close() - - def seekable(self): - """True if file supports random-access.""" - self._checkClosed() - if self._seekable is None: - try: - self.tell() - except OSError: - self._seekable = False - else: - self._seekable = True - return self._seekable - - def readable(self): - """True if file was opened in a read mode.""" - self._checkClosed() - return self._readable - - def writable(self): - """True if file was opened in a write mode.""" - self._checkClosed() - return self._writable - - def fileno(self): - """Return the underlying file descriptor (an integer).""" - self._checkClosed() - return self._fd - - def isatty(self): - """True if the file is connected to a TTY device.""" - self._checkClosed() - return os.isatty(self._fd) - - @property - def closefd(self): - """True if the file descriptor will be closed by close().""" - return self._closefd - - @property - def mode(self): - """String giving the file mode""" - if self._created: - if self._readable: - return 'xb+' - else: - return 'xb' - elif self._appending: - if self._readable: - return 'ab+' - else: - return 'ab' - elif self._readable: - if self._writable: - return 'rb+' - else: - return 'rb' - else: - return 'wb' - - class TextIOBase(IOBase): """Base class for text I/O. @@ -1732,11 +1299,11 @@ are immutable. There is no public constructor. """ - def read(self, size=-1): - """Read at most size characters from stream, where size is an int. + def read(self, n=-1): + """Read at most n characters from stream, where n is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have n characters or we hit EOF. + If n is negative or omitted, read until EOF. Returns a string. """ @@ -1881,7 +1448,7 @@ r"""Character and line based layer over a BufferedIOBase object, buffer. encoding gives the name of the encoding that the stream will be - decoded or encoded with. It defaults to locale.getpreferredencoding(False). + decoded or encoded with. It defaults to locale.getpreferredencoding. errors determines the strictness of encoding and decoding (see the codecs.register) and defaults to "strict". @@ -1902,9 +1469,6 @@ _CHUNK_SIZE = 2048 - # The write_through argument has no effect here since this - # implementation always writes through. The argument is present only - # so that the signature can match the signature of the C version. def __init__(self, buffer, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False): if newline is not None and not isinstance(newline, str): @@ -1923,16 +1487,11 @@ # Importing locale may fail if Python is being built encoding = "ascii" else: - encoding = locale.getpreferredencoding(False) + encoding = locale.getpreferredencoding() if not isinstance(encoding, str): raise ValueError("invalid encoding: %r" % encoding) - if not codecs.lookup(encoding)._is_text_encoding: - msg = ("%r is not a text encoding; " - "use codecs.open() to handle arbitrary codecs") - raise LookupError(msg % encoding) - if errors is None: errors = "strict" else: @@ -1976,17 +1535,16 @@ # - "chars_..." for integer variables that count decoded characters def __repr__(self): - result = "<{}.{}".format(self.__class__.__module__, - self.__class__.__qualname__) + result = "<_pyio.TextIOWrapper" try: name = self.name - except Exception: + except AttributeError: pass else: result += " name={0!r}".format(name) try: mode = self.mode - except Exception: + except AttributeError: pass else: result += " mode={0!r}".format(mode) @@ -2009,8 +1567,6 @@ return self._buffer def seekable(self): - if self.closed: - raise ValueError("I/O operation on closed file.") return self._seekable def readable(self): @@ -2025,10 +1581,8 @@ def close(self): if self.buffer is not None and not self.closed: - try: - self.flush() - finally: - self.buffer.close() + self.flush() + self.buffer.close() @property def closed(self): @@ -2166,7 +1720,7 @@ if not self._seekable: raise UnsupportedOperation("underlying stream is not seekable") if not self._telling: - raise OSError("telling position disabled by next() call") + raise IOError("telling position disabled by next() call") self.flush() position = self.buffer.tell() decoder = self._decoder @@ -2253,7 +1807,7 @@ chars_decoded += len(decoder.decode(b'', final=True)) need_eof = 1 if chars_decoded < chars_to_skip: - raise OSError("can't reconstruct logical file position") + raise IOError("can't reconstruct logical file position") # The returned cookie corresponds to the last safe start point. return self._pack_cookie( @@ -2276,19 +1830,6 @@ return buffer def seek(self, cookie, whence=0): - def _reset_encoder(position): - """Reset the encoder (merely useful for proper BOM handling)""" - try: - encoder = self._encoder or self._get_encoder() - except LookupError: - # Sometimes the encoder doesn't exist - pass - else: - if position != 0: - encoder.setstate(0) - else: - encoder.reset() - if self.closed: raise ValueError("tell on closed file") if not self._seekable: @@ -2309,10 +1850,10 @@ self._snapshot = None if self._decoder: self._decoder.reset() - _reset_encoder(position) return position if whence != 0: - raise ValueError("unsupported whence (%r)" % (whence,)) + raise ValueError("invalid whence (%r, should be 0, 1 or 2)" % + (whence,)) if cookie < 0: raise ValueError("negative seek position %r" % (cookie,)) self.flush() @@ -2344,22 +1885,32 @@ # Skip chars_to_skip of the decoded characters. if len(self._decoded_chars) < chars_to_skip: - raise OSError("can't restore logical file position") + raise IOError("can't restore logical file position") self._decoded_chars_used = chars_to_skip - _reset_encoder(cookie) + # Finally, reset the encoder (merely useful for proper BOM handling) + try: + encoder = self._encoder or self._get_encoder() + except LookupError: + # Sometimes the encoder doesn't exist + pass + else: + if cookie != 0: + encoder.setstate(0) + else: + encoder.reset() return cookie - def read(self, size=None): + def read(self, n=None): self._checkReadable() - if size is None: - size = -1 + if n is None: + n = -1 decoder = self._decoder or self._get_decoder() try: - size.__index__ + n.__index__ except AttributeError as err: raise TypeError("an integer is required") from err - if size < 0: + if n < 0: # Read everything. result = (self._get_decoded_chars() + decoder.decode(self.buffer.read(), final=True)) @@ -2367,12 +1918,12 @@ self._snapshot = None return result else: - # Keep reading chunks until we have size characters to return. + # Keep reading chunks until we have n characters to return. eof = False - result = self._get_decoded_chars(size) - while len(result) < size and not eof: + result = self._get_decoded_chars(n) + while len(result) < n and not eof: eof = not self._read_chunk() - result += self._get_decoded_chars(size - len(result)) + result += self._get_decoded_chars(n - len(result)) return result def __next__(self): @@ -2384,13 +1935,13 @@ raise StopIteration return line - def readline(self, size=None): + def readline(self, limit=None): if self.closed: raise ValueError("read from closed file") - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError("size must be an integer") + if limit is None: + limit = -1 + elif not isinstance(limit, int): + raise TypeError("limit must be an integer") # Grab all the decoded text (we will rewind any extra bits later). line = self._get_decoded_chars() @@ -2449,8 +2000,8 @@ endpos = pos + len(self._readnl) break - if size >= 0 and len(line) >= size: - endpos = size # reached length size + if limit >= 0 and len(line) >= limit: + endpos = limit # reached length limit break # No line ending seen yet - get more data' @@ -2465,8 +2016,8 @@ self._snapshot = None return line - if size >= 0 and endpos > size: - endpos = size # don't exceed size + if limit >= 0 and endpos > limit: + endpos = limit # don't exceed limit # Rewind _decoded_chars to just after the line ending we found. self._rewind_decoded_chars(len(line) - endpos) @@ -2487,7 +2038,7 @@ def __init__(self, initial_value="", newline="\n"): super(StringIO, self).__init__(BytesIO(), encoding="utf-8", - errors="surrogatepass", + errors="strict", newline=newline) # Issue #5645: make universal newlines semantics the same as in the # C version, even under Windows. @@ -2497,22 +2048,17 @@ if not isinstance(initial_value, str): raise TypeError("initial_value must be str or None, not {0}" .format(type(initial_value).__name__)) + initial_value = str(initial_value) self.write(initial_value) self.seek(0) def getvalue(self): self.flush() - decoder = self._decoder or self._get_decoder() - old_state = decoder.getstate() - decoder.reset() - try: - return decoder.decode(self.buffer.getvalue(), final=True) - finally: - decoder.setstate(old_state) + return self.buffer.getvalue().decode(self._encoding, self._errors) def __repr__(self): # TextIOWrapper tells the encoding in its repr. In StringIO, - # that's an implementation detail. + # that's a implementation detail. return object.__repr__(self) @property diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_sitebuiltins.py --- a/Lib/_sitebuiltins.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,103 +0,0 @@ -""" -The objects used by the site module to add custom builtins. -""" - -# Those objects are almost immortal and they keep a reference to their module -# globals. Defining them in the site module would keep too many references -# alive. -# Note this means this module should also avoid keep things alive in its -# globals. - -import sys - -class Quitter(object): - def __init__(self, name, eof): - self.name = name - self.eof = eof - def __repr__(self): - return 'Use %s() or %s to exit' % (self.name, self.eof) - def __call__(self, code=None): - # Shells like IDLE catch the SystemExit, but listen when their - # stdin wrapper is closed. - try: - sys.stdin.close() - except: - pass - raise SystemExit(code) - - -class _Printer(object): - """interactive prompt objects for printing the license text, a list of - contributors and the copyright notice.""" - - MAXLINES = 23 - - def __init__(self, name, data, files=(), dirs=()): - import os - self.__name = name - self.__data = data - self.__lines = None - self.__filenames = [os.path.join(dir, filename) - for dir in dirs - for filename in files] - - def __setup(self): - if self.__lines: - return - data = None - for filename in self.__filenames: - try: - with open(filename, "r") as fp: - data = fp.read() - break - except OSError: - pass - if not data: - data = self.__data - self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) - - def __repr__(self): - self.__setup() - if len(self.__lines) <= self.MAXLINES: - return "\n".join(self.__lines) - else: - return "Type %s() to see the full %s text" % ((self.__name,)*2) - - def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: - try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break - - -class _Helper(object): - """Define the builtin 'help'. - - This is a wrapper around pydoc.help that provides a helpful message - when 'help' is typed at the Python interactive prompt. - - Calling help() at the Python prompt starts an interactive help session. - Calling help(thing) prints help for the python object 'thing'. - """ - - def __repr__(self): - return "Type help() for interactive help, " \ - "or help(object) for help about object." - def __call__(self, *args, **kwds): - import pydoc - return pydoc.help(*args, **kwds) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_strptime.py --- a/Lib/_strptime.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/_strptime.py Mon Jan 25 17:05:13 2016 +0100 @@ -14,14 +14,14 @@ import locale import calendar from re import compile as re_compile -from re import IGNORECASE +from re import IGNORECASE, ASCII from re import escape as re_escape from datetime import (date as datetime_date, timedelta as datetime_timedelta, timezone as datetime_timezone) try: from _thread import allocate_lock as _thread_allocate_lock -except ImportError: +except: from _dummy_thread import allocate_lock as _thread_allocate_lock __all__ = [] @@ -77,8 +77,6 @@ self.__calc_date_time() if _getlang() != self.lang: raise ValueError("locale changed during initialization") - if time.tzname != self.tzname or time.daylight != self.daylight: - raise ValueError("timezone changed during initialization") def __pad(self, seq, front): # Add '' to seq to either the front (is True), else the back. @@ -163,17 +161,15 @@ def __calc_timezone(self): # Set self.timezone by using time.tzname. - # Do not worry about possibility of time.tzname[0] == time.tzname[1] - # and time.daylight; handle that in strptime. + # Do not worry about possibility of time.tzname[0] == timetzname[1] + # and time.daylight; handle that in strptime . try: time.tzset() except AttributeError: pass - self.tzname = time.tzname - self.daylight = time.daylight - no_saving = frozenset({"utc", "gmt", self.tzname[0].lower()}) - if self.daylight: - has_saving = frozenset({self.tzname[1].lower()}) + no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()]) + if time.daylight: + has_saving = frozenset([time.tzname[1].lower()]) else: has_saving = frozenset() self.timezone = (no_saving, has_saving) @@ -199,15 +195,12 @@ 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9])", - 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", 'M': r"(?P[0-5]\d|\d)", 'S': r"(?P6[0-1]|[0-5]\d|\d)", 'U': r"(?P5[0-3]|[0-4]\d|\d)", 'w': r"(?P[0-6])", - 'u': r"(?P[1-7])", - 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", #XXX: Does 'Y' need to worry about having less or more than @@ -232,7 +225,7 @@ """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This - prevents the possibility of a match occurring for a value that also + prevents the possibility of a match occuring for a value that also a substring of a larger value that should have matched (e.g., 'abc' matching when 'abcdef' should have been the match). @@ -260,8 +253,8 @@ # format directives (%m, etc.). regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) + whitespace_replacement = re_compile('\s+') + format = whitespace_replacement.sub('\s+', format) while '%' in format: directive_index = format.index('%')+1 processed_format = "%s%s%s" % (processed_format, @@ -302,22 +295,6 @@ return 1 + days_to_week + day_of_week -def _calc_julian_from_V(iso_year, iso_week, iso_weekday): - """Calculate the Julian day based on the ISO 8601 year, week, and weekday. - ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. - ISO week days range from 1 (Monday) to 7 (Sunday). - """ - correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 - ordinal = (iso_week * 7) + iso_weekday - correction - # ordinal may be negative or 0 now, which means the date is in the previous - # calendar year - if ordinal < 1: - ordinal += datetime_date(iso_year, 1, 1).toordinal() - iso_year -= 1 - ordinal -= datetime_date(iso_year, 1, 1).toordinal() - return iso_year, ordinal - - def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the @@ -330,15 +307,13 @@ global _TimeRE_cache, _regex_cache with _cache_lock: - locale_time = _TimeRE_cache.locale_time - if (_getlang() != locale_time.lang or - time.tzname != locale_time.tzname or - time.daylight != locale_time.daylight): + + if _getlang() != _TimeRE_cache.locale_time.lang: _TimeRE_cache = TimeRE() _regex_cache.clear() - locale_time = _TimeRE_cache.locale_time if len(_regex_cache) > _CACHE_MAX_SIZE: _regex_cache.clear() + locale_time = _TimeRE_cache.locale_time format_regex = _regex_cache.get(format) if not format_regex: try: @@ -351,10 +326,10 @@ bad_directive = "%" del err raise ValueError("'%s' is a bad directive in format '%s'" % - (bad_directive, format)) from None + (bad_directive, format)) # IndexError only occurs when the format string is "%" except IndexError: - raise ValueError("stray %% in format '%s'" % format) from None + raise ValueError("stray %% in format '%s'" % format) _regex_cache[format] = format_regex found = format_regex.match(data_string) if not found: @@ -364,18 +339,18 @@ raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - iso_year = year = None + year = 1900 month = day = 1 hour = minute = second = fraction = 0 tz = -1 tzoffset = None # Default to -1 to signify that values not known; not critical to have, # though - iso_week = week_of_year = None - week_of_year_start = None - # weekday and julian defaulted to None so as to signal need to calculate + week_of_year = -1 + week_of_year_start = -1 + # weekday and julian defaulted to -1 so as to signal need to calculate # values - weekday = julian = None + weekday = julian = -1 found_dict = found.groupdict() for group_key in found_dict.keys(): # Directives not explicitly handled below: @@ -394,8 +369,6 @@ year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) - elif group_key == 'G': - iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': @@ -441,9 +414,6 @@ weekday = 6 else: weekday -= 1 - elif group_key == 'u': - weekday = int(found_dict['u']) - weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): @@ -454,8 +424,6 @@ else: # W starts week on Monday. week_of_year_start = 0 - elif group_key == 'V': - iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] tzoffset = int(z[1:3]) * 60 + int(z[3:5]) @@ -476,61 +444,26 @@ else: tz = value break - # Deal with the cases where ambiguities arize - # don't assume default values for ISO week/year - if year is None and iso_year is not None: - if iso_week is None or weekday is None: - raise ValueError("ISO year directive '%G' must be used with " - "the ISO week directive '%V' and a weekday " - "directive ('%A', '%a', '%w', or '%u').") - if julian is not None: - raise ValueError("Day of the year directive '%j' is not " - "compatible with ISO year directive '%G'. " - "Use '%Y' instead.") - elif week_of_year is None and iso_week is not None: - if weekday is None: - raise ValueError("ISO week directive '%V' must be used with " - "the ISO year directive '%G' and a weekday " - "directive ('%A', '%a', '%w', or '%u').") - else: - raise ValueError("ISO week directive '%V' is incompatible with " - "the year directive '%Y'. Use the ISO year '%G' " - "instead.") - - leap_year_fix = False - if year is None and month == 2 and day == 29: - year = 1904 # 1904 is first leap year of 20th century - leap_year_fix = True - elif year is None: - year = 1900 - - # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian is None and weekday is not None: - if week_of_year is not None: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - elif iso_year is not None and iso_week is not None: - year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) - - if julian is None: - # Cannot pre-calculate datetime_date() since can change in Julian - # calculation and thus could have different value for the day of - # the week calculation. + if julian == -1 and week_of_year != -1 and weekday != -1: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of the week + # calculation. + if julian == -1: # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 - else: # Assume that if they bothered to include Julian day (or if it was - # calculated above with year/week/weekday) it will be accurate. - datetime_result = datetime_date.fromordinal( - (julian - 1) + - datetime_date(year, 1, 1).toordinal()) + else: # Assume that if they bothered to include Julian day it will + # be accurate. + datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day - if weekday is None: + if weekday == -1: weekday = datetime_date(year, month, day).weekday() # Add timezone info tzname = found_dict.get("Z") @@ -539,27 +472,21 @@ else: gmtoff = None - if leap_year_fix: - # the caller didn't supply a year but asked for Feb 29th. We couldn't - # use the default of 1900 for computations. We set it back to ensure - # that February 29th is smaller than March 1st. - year = 1900 - return (year, month, day, hour, minute, second, - weekday, julian, tz, tzname, gmtoff), fraction + weekday, julian, tz, gmtoff, tzname), fraction def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a time struct based on the input string and the format string.""" tt = _strptime(data_string, format)[0] - return time.struct_time(tt[:time._STRUCT_TM_ITEMS]) + return time.struct_time(tt[:9]) def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"): """Return a class cls instance based on the input string and the format string.""" tt, fraction = _strptime(data_string, format) - tzname, gmtoff = tt[-2:] + gmtoff, tzname = tt[-2:] args = tt[:6] + (fraction,) if gmtoff is not None: tzdelta = datetime_timedelta(seconds=gmtoff) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/_weakrefset.py --- a/Lib/_weakrefset.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/_weakrefset.py Mon Jan 25 17:05:13 2016 +0100 @@ -60,8 +60,6 @@ for itemref in self.data: item = itemref() if item is not None: - # Caveat: the iterator will keep a strong reference to - # `item` until it is resumed or closed. yield item def __len__(self): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/abc.py --- a/Lib/abc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/abc.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,6 @@ from _weakrefset import WeakSet - def abstractmethod(funcobj): """A decorator indicating abstract methods. @@ -125,8 +124,6 @@ # A global counter that is incremented each time a class is # registered as a virtual subclass of anything. It forces the # negative cache to be cleared before its next use. - # Note: this counter is private. Use `abc.get_cache_token()` for - # external code. _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace): @@ -168,7 +165,7 @@ def _dump_registry(cls, file=None): """Debug helper to print the ABC registry.""" - print("Class: %s.%s" % (cls.__module__, cls.__qualname__), file=file) + print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file) print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file) for name in sorted(cls.__dict__.keys()): if name.startswith("_abc_"): @@ -229,20 +226,3 @@ # No dice; update negative cache cls._abc_negative_cache.add(subclass) return False - - -class ABC(metaclass=ABCMeta): - """Helper class that provides a standard way to create an ABC using - inheritance. - """ - pass - - -def get_cache_token(): - """Returns the current ABC cache token. - - The token is an opaque object (supporting equality testing) identifying the - current version of the ABC cache for virtual subclasses. The token changes - with every call to ``register()`` on any ABC. - """ - return ABCMeta._abc_invalidation_counter diff -r 6db40a9955dc -r 0d413f60cc23 Lib/aifc.py --- a/Lib/aifc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/aifc.py Mon Jan 25 17:05:13 2016 +0100 @@ -69,7 +69,7 @@ getcomptype() -- returns compression type ('NONE' for AIFF files) getcompname() -- returns human-readable version of compression type ('not compressed' for AIFF files) - getparams() -- returns a namedtuple consisting of all of the + getparams() -- returns a tuple consisting of all of the above in the above order getmarkers() -- get the list of marks in the audio file or None if there are no marks @@ -121,9 +121,9 @@ be patched up. It is best to first set all parameters, perhaps possibly the compression type, and then write audio frames using writeframesraw. -When all frames have been written, either call writeframes(b'') or +When all frames have been written, either call writeframes('') or close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, you must call +Marks can be added anytime. If there are any marks, ypu must call close() after all frames have been written. The close() method is called automatically when the class instance is destroyed. @@ -252,20 +252,6 @@ _write_ulong(f, lomant) from chunk import Chunk -from collections import namedtuple - -_aifc_params = namedtuple('_aifc_params', - 'nchannels sampwidth framerate nframes comptype compname') - -_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' -_aifc_params.framerate.__doc__ = 'Sampling frequency' -_aifc_params.nframes.__doc__ = 'Number of audio frames' -_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""\ -A human-readable version of the compression type -('not compressed' for AIFF files)""") - class Aifc_read: # Variables used in this class: @@ -348,12 +334,6 @@ # else, assume it is an open file object already self.initfp(f) - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - # # User visible methods. # @@ -365,10 +345,7 @@ self._soundpos = 0 def close(self): - file = self._file - if file is not None: - self._file = None - file.close() + self._file.close() def tell(self): return self._soundpos @@ -395,9 +372,9 @@ ## return self._version def getparams(self): - return _aifc_params(self.getnchannels(), self.getsampwidth(), - self.getframerate(), self.getnframes(), - self.getcomptype(), self.getcompname()) + return self.getnchannels(), self.getsampwidth(), \ + self.getframerate(), self.getnframes(), \ + self.getcomptype(), self.getcompname() def getmarkers(self): if len(self._markers) == 0: @@ -480,13 +457,15 @@ if self._comptype != b'NONE': if self._comptype == b'G722': self._convert = self._adpcm2lin + self._framesize = self._framesize // 4 elif self._comptype in (b'ulaw', b'ULAW'): self._convert = self._ulaw2lin + self._framesize = self._framesize // 2 elif self._comptype in (b'alaw', b'ALAW'): self._convert = self._alaw2lin + self._framesize = self._framesize // 2 else: raise Error('unsupported compression type') - self._sampwidth = 2 else: self._comptype = b'NONE' self._compname = b'not compressed' @@ -574,12 +553,6 @@ def __del__(self): self.close() - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - # # User visible methods. # @@ -673,8 +646,8 @@ def getparams(self): if not self._nchannels or not self._sampwidth or not self._framerate: raise Error('not all parameters set') - return _aifc_params(self._nchannels, self._sampwidth, self._framerate, - self._nframes, self._comptype, self._compname) + return self._nchannels, self._sampwidth, self._framerate, \ + self._nframes, self._comptype, self._compname def setmark(self, id, pos, name): if id <= 0: @@ -704,8 +677,6 @@ return self._nframeswritten def writeframesraw(self, data): - if not isinstance(data, (bytes, bytearray)): - data = memoryview(data).cast('B') self._ensure_header_written(len(data)) nframes = len(data) // (self._sampwidth * self._nchannels) if self._convert: @@ -721,9 +692,7 @@ self._patchheader() def close(self): - if self._file is None: - return - try: + if self._file: self._ensure_header_written(0) if self._datawritten & 1: # quick pad to even size @@ -734,12 +703,10 @@ self._datalength != self._datawritten or \ self._marklength: self._patchheader() - finally: # Prevent ref cycles self._convert = None - f = self._file + self._file.close() self._file = None - f.close() # # Internal methods. @@ -802,10 +769,7 @@ self._datalength = (self._datalength + 3) // 4 if self._datalength & 1: self._datalength = self._datalength + 1 - try: - self._form_length_pos = self._file.tell() - except (AttributeError, OSError): - self._form_length_pos = None + self._form_length_pos = self._file.tell() commlength = self._write_form_length(self._datalength) if self._aifc: self._file.write(b'AIFC') @@ -817,20 +781,15 @@ self._file.write(b'COMM') _write_ulong(self._file, commlength) _write_short(self._file, self._nchannels) - if self._form_length_pos is not None: - self._nframes_pos = self._file.tell() + self._nframes_pos = self._file.tell() _write_ulong(self._file, self._nframes) - if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'): - _write_short(self._file, 8) - else: - _write_short(self._file, self._sampwidth * 8) + _write_short(self._file, self._sampwidth * 8) _write_float(self._file, self._framerate) if self._aifc: self._file.write(self._comptype) _write_string(self._file, self._compname) self._file.write(b'SSND') - if self._form_length_pos is not None: - self._ssnd_length_pos = self._file.tell() + self._ssnd_length_pos = self._file.tell() _write_ulong(self._file, self._datalength + 8) _write_ulong(self._file, 0) _write_ulong(self._file, 0) @@ -909,22 +868,24 @@ if not sys.argv[1:]: sys.argv.append('/usr/demos/data/audio/bach.aiff') fn = sys.argv[1] - with open(fn, 'r') as f: - print("Reading", fn) - print("nchannels =", f.getnchannels()) - print("nframes =", f.getnframes()) - print("sampwidth =", f.getsampwidth()) - print("framerate =", f.getframerate()) - print("comptype =", f.getcomptype()) - print("compname =", f.getcompname()) - if sys.argv[2:]: - gn = sys.argv[2] - print("Writing", gn) - with open(gn, 'w') as g: - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - print("Done.") + f = open(fn, 'r') + print("Reading", fn) + print("nchannels =", f.getnchannels()) + print("nframes =", f.getnframes()) + print("sampwidth =", f.getsampwidth()) + print("framerate =", f.getframerate()) + print("comptype =", f.getcomptype()) + print("compname =", f.getcompname()) + if sys.argv[2:]: + gn = sys.argv[2] + print("Writing", gn) + g = open(gn, 'w') + g.setparams(f.getparams()) + while 1: + data = f.readframes(1024) + if not data: + break + g.writeframes(data) + g.close() + f.close() + print("Done.") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/argparse.py --- a/Lib/argparse.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/argparse.py Mon Jan 25 17:05:13 2016 +0100 @@ -118,16 +118,10 @@ def __repr__(self): type_name = type(self).__name__ arg_strings = [] - star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): - if name.isidentifier(): - arg_strings.append('%s=%r' % (name, value)) - else: - star_args[name] = value - if star_args: - arg_strings.append('**%s' % repr(star_args)) + arg_strings.append('%s=%r' % (name, value)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): @@ -171,8 +165,6 @@ self._prog = prog self._indent_increment = indent_increment self._max_help_position = max_help_position - self._max_help_position = min(max_help_position, - max(width - 20, indent_increment * 2)) self._width = width self._current_indent = 0 @@ -344,7 +336,7 @@ else: line_len = len(indent) - 1 for part in parts: - if line_len + 1 + len(part) > text_width and line: + if line_len + 1 + len(part) > text_width: lines.append(indent + ' '.join(line)) line = [] line_len = len(indent) - 1 @@ -484,7 +476,7 @@ def _format_text(self, text): if '%(prog)' in text: text = text % dict(prog=self._prog) - text_width = max(self._width - self._current_indent, 11) + text_width = self._width - self._current_indent indent = ' ' * self._current_indent return self._fill_text(text, text_width, indent) + '\n\n' @@ -492,11 +484,11 @@ # determine the required width and the entry label help_position = min(self._action_max_length + 2, self._max_help_position) - help_width = max(self._width - help_position, 11) + help_width = self._width - help_position action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) - # no help; start on same line and add a final newline + # ho nelp; start on same line and add a final newline if not action.help: tup = self._current_indent, '', action_header action_header = '%*s%s\n' % tup @@ -614,7 +606,8 @@ pass else: self._indent() - yield from get_subactions() + for subaction in get_subactions(): + yield subaction self._dedent() def _split_lines(self, text, width): @@ -768,10 +761,10 @@ - default -- The value to be produced if the option is not specified. - - type -- A callable that accepts a single string argument, and - returns the converted value. The standard Python types str, int, - float, and complex are useful examples of such callables. If None, - str is used. + - type -- The type which the command-line arguments should be converted + to, should be one of 'string', 'int', 'float', 'complex' or a + callable object that accepts a single string argument. If None, + 'string' is assumed. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate @@ -1045,8 +1038,7 @@ version = parser.version formatter = parser._get_formatter() formatter.add_text(version) - parser._print_message(formatter.format_help(), _sys.stdout) - parser.exit() + parser.exit(message=formatter.format_help()) class _SubParsersAction(Action): @@ -1128,14 +1120,7 @@ # parse all the remaining options into the namespace # store any unrecognized options on the object, so that the top # level parser can decide what to do with them - - # In case this subparser defines new defaults, we parse them - # in a new namespace object and then update the original - # namespace for the relevant parts. - subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) - for key, value in vars(subnamespace).items(): - setattr(namespace, key, value) - + namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) if arg_strings: vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) @@ -1156,17 +1141,11 @@ same values as the builtin open() function. - bufsize -- The file's desired buffer size. Accepts the same values as the builtin open() function. - - encoding -- The file's encoding. Accepts the same values as the - builtin open() function. - - errors -- A string indicating how encoding and decoding errors are to - be handled. Accepts the same value as the builtin open() function. """ - def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + def __init__(self, mode='r', bufsize=-1): self._mode = mode self._bufsize = bufsize - self._encoding = encoding - self._errors = errors def __call__(self, string): # the special argument "-" means sys.std{in,out} @@ -1181,18 +1160,14 @@ # all other arguments are used as file names try: - return open(string, self._mode, self._bufsize, self._encoding, - self._errors) - except OSError as e: + return open(string, self._mode, self._bufsize) + except IOError as e: message = _("can't open '%s': %s") raise ArgumentTypeError(message % (string, e)) def __repr__(self): args = self._mode, self._bufsize - kwargs = [('encoding', self._encoding), ('errors', self._errors)] - args_str = ', '.join([repr(arg) for arg in args if arg != -1] + - ['%s=%r' % (kw, arg) for kw, arg in kwargs - if arg is not None]) + args_str = ', '.join(repr(arg) for arg in args if arg != -1) return '%s(%s)' % (type(self).__name__, args_str) # =========================== @@ -1211,10 +1186,11 @@ setattr(self, name, kwargs[name]) def __eq__(self, other): - if not isinstance(other, Namespace): - return NotImplemented return vars(self) == vars(other) + def __ne__(self, other): + return not (self == other) + def __contains__(self, key): return key in self.__dict__ @@ -1596,7 +1572,6 @@ - argument_default -- The default value for all arguments - conflict_handler -- String indicating how to handle conflicts - add_help -- Add a -h/-help option - - allow_abbrev -- Allow long options to be abbreviated unambiguously """ def __init__(self, @@ -1604,14 +1579,22 @@ usage=None, description=None, epilog=None, + version=None, parents=[], formatter_class=HelpFormatter, prefix_chars='-', fromfile_prefix_chars=None, argument_default=None, conflict_handler='error', - add_help=True, - allow_abbrev=True): + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) superinit = super(ArgumentParser, self).__init__ superinit(description=description, @@ -1626,10 +1609,10 @@ self.prog = prog self.usage = usage self.epilog = epilog + self.version = version self.formatter_class = formatter_class self.fromfile_prefix_chars = fromfile_prefix_chars self.add_help = add_help - self.allow_abbrev = allow_abbrev add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) @@ -1641,7 +1624,7 @@ return string self.register('type', None, identity) - # add help argument if necessary + # add help and version arguments if necessary # (using explicit default to override global argument_default) default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] if self.add_help: @@ -1649,6 +1632,12 @@ default_prefix+'h', default_prefix*2+'help', action='help', default=SUPPRESS, help=_('show this help message and exit')) + if self.version: + self.add_argument( + default_prefix+'v', default_prefix*2+'version', + action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) # add parent arguments and defaults for parent in parents: @@ -1668,6 +1657,7 @@ 'prog', 'usage', 'description', + 'version', 'formatter_class', 'conflict_handler', 'add_help', @@ -1736,12 +1726,9 @@ return args def parse_known_args(self, args=None, namespace=None): + # args default to the system args if args is None: - # args default to the system args args = _sys.argv[1:] - else: - # make sure that args are mutable - args = list(args) # default Namespace built from parser defaults if namespace is None: @@ -1752,7 +1739,10 @@ if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: - setattr(namespace, action.dest, action.default) + default = action.default + if isinstance(action.default, str): + default = self._get_value(action, default) + setattr(namespace, action.dest, default) # add any parser defaults that aren't present for dest in self._defaults: @@ -1975,25 +1965,9 @@ # if we didn't consume all the argument strings, there were extras extras.extend(arg_strings[stop_index:]) - # make sure all required actions were present and also convert - # action defaults which were not given as arguments - required_actions = [] - for action in self._actions: - if action not in seen_actions: - if action.required: - required_actions.append(_get_action_name(action)) - else: - # Convert action default now instead of doing it before - # parsing arguments to avoid calling convert functions - # twice (which may fail) if the argument was given, but - # only if it was defined already in the namespace - if (action.default is not None and - isinstance(action.default, str) and - hasattr(namespace, action.dest) and - action.default is getattr(namespace, action.dest)): - setattr(namespace, action.dest, - self._get_value(action, action.default)) - + # make sure all required actions were present + required_actions = [_get_action_name(action) for action in self._actions + if action.required and action not in seen_actions] if required_actions: self.error(_('the following arguments are required: %s') % ', '.join(required_actions)) @@ -2022,20 +1996,23 @@ for arg_string in arg_strings: # for regular arguments, just add them back into the list - if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: + if arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content else: try: - with open(arg_string[1:]) as args_file: + args_file = open(arg_string[1:]) + try: arg_strings = [] for arg_line in args_file.read().splitlines(): for arg in self.convert_arg_line_to_args(arg_line): arg_strings.append(arg) arg_strings = self._read_args_from_files(arg_strings) new_arg_strings.extend(arg_strings) - except OSError: + finally: + args_file.close() + except IOError: err = _sys.exc_info()[1] self.error(str(err)) @@ -2107,24 +2084,23 @@ action = self._option_string_actions[option_string] return action, option_string, explicit_arg - if self.allow_abbrev: - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = self._get_option_tuples(arg_string) - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + args = {'option': arg_string, 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + self.error(msg % args) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple # if it was not found as an option, but it looks like a negative # number, it was meant to be positional @@ -2230,12 +2206,9 @@ # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' + # for everything but PARSER args, strip out '--' if action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass + arg_strings = [s for s in arg_strings if s != '--'] # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: @@ -2347,6 +2320,16 @@ # determine help from format above return formatter.format_help() + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + def _get_formatter(self): return self.formatter_class(prog=self.prog) @@ -2363,6 +2346,14 @@ file = _sys.stdout self._print_message(self.format_help(), file) + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + def _print_message(self, message, file=None): if message: if file is None: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ast.py --- a/Lib/ast.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ast.py Mon Jan 25 17:05:13 2016 +0100 @@ -42,6 +42,7 @@ Python literal structures: strings, bytes, numbers, tuples, lists, dicts, sets, booleans, and None. """ + _safe_names = {'None': None, 'True': True, 'False': False} if isinstance(node_or_string, str): node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): @@ -60,8 +61,9 @@ elif isinstance(node, Dict): return dict((_convert(k), _convert(v)) for k, v in zip(node.keys, node.values)) - elif isinstance(node, NameConstant): - return node.value + elif isinstance(node, Name): + if node.id in _safe_names: + return _safe_names[node.id] elif isinstance(node, UnaryOp) and \ isinstance(node.op, (UAdd, USub)) and \ isinstance(node.operand, (Num, UnaryOp, BinOp)): @@ -194,7 +196,7 @@ be found. If the node provided does not have docstrings a TypeError will be raised. """ - if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): + if not isinstance(node, (FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) if node.body and isinstance(node.body[0], Expr) and \ isinstance(node.body[0].value, Str): @@ -293,6 +295,7 @@ def generic_visit(self, node): for field, old_value in iter_fields(node): + old_value = getattr(node, field, None) if isinstance(old_value, list): new_values = [] for value in old_value: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asynchat.py --- a/Lib/asynchat.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/asynchat.py Mon Jan 25 17:05:13 2016 +0100 @@ -45,38 +45,54 @@ method) up to the terminator, and then control will be returned to you - by calling your self.found_terminator() method. """ +import socket import asyncore from collections import deque +def buffer(obj, start=None, stop=None): + # if memoryview objects gain slicing semantics, + # this function will change for the better + # memoryview used for the TypeError + memoryview(obj) + if start == None: + start = 0 + if stop == None: + stop = len(obj) + x = obj[start:stop] + ## print("buffer type is: %s"%(type(x),)) + return x -class async_chat(asyncore.dispatcher): +class async_chat (asyncore.dispatcher): """This is an abstract class. You must derive from this class, and add the two methods collect_incoming_data() and found_terminator()""" # these are overridable defaults - ac_in_buffer_size = 65536 - ac_out_buffer_size = 65536 + ac_in_buffer_size = 4096 + ac_out_buffer_size = 4096 # we don't want to enable the use of encoding by default, because that is a # sign of an application bug that we don't want to pass silently - use_encoding = 0 - encoding = 'latin-1' + use_encoding = 0 + encoding = 'latin-1' - def __init__(self, sock=None, map=None): + def __init__ (self, sock=None, map=None): # for string terminator matching self.ac_in_buffer = b'' - # we use a list here rather than io.BytesIO for a few reasons... - # del lst[:] is faster than bio.truncate(0) - # lst = [] is faster than bio.truncate(0) + # we use a list here rather than cStringIO for a few reasons... + # del lst[:] is faster than sio.truncate(0) + # lst = [] is faster than sio.truncate(0) + # cStringIO will be gaining unicode support in py3k, which + # will negatively affect the performance of bytes compared to + # a ''.join() equivalent self.incoming = [] # we toss the use of the "simple producer" and replace it with # a pure deque, which the original fifo was a wrapping of self.producer_fifo = deque() - asyncore.dispatcher.__init__(self, sock, map) + asyncore.dispatcher.__init__ (self, sock, map) def collect_incoming_data(self, data): raise NotImplementedError("must be implemented in subclass") @@ -92,18 +108,13 @@ def found_terminator(self): raise NotImplementedError("must be implemented in subclass") - def set_terminator(self, term): - """Set the input delimiter. - - Can be a fixed string of any length, an integer, or None. - """ + def set_terminator (self, term): + "Set the input delimiter. Can be a fixed string of any length, an integer, or None" if isinstance(term, str) and self.use_encoding: term = bytes(term, self.encoding) - elif isinstance(term, int) and term < 0: - raise ValueError('the number of received bytes must be positive') self.terminator = term - def get_terminator(self): + def get_terminator (self): return self.terminator # grab some more data from the socket, @@ -111,13 +122,11 @@ # check for the terminator, # if found, transition to the next state. - def handle_read(self): + def handle_read (self): try: - data = self.recv(self.ac_in_buffer_size) - except BlockingIOError: - return - except OSError as why: + data = self.recv (self.ac_in_buffer_size) + except socket.error as why: self.handle_error() return @@ -135,17 +144,17 @@ terminator = self.get_terminator() if not terminator: # no terminator, collect it all - self.collect_incoming_data(self.ac_in_buffer) + self.collect_incoming_data (self.ac_in_buffer) self.ac_in_buffer = b'' elif isinstance(terminator, int): # numeric terminator n = terminator if lb < n: - self.collect_incoming_data(self.ac_in_buffer) + self.collect_incoming_data (self.ac_in_buffer) self.ac_in_buffer = b'' self.terminator = self.terminator - lb else: - self.collect_incoming_data(self.ac_in_buffer[:n]) + self.collect_incoming_data (self.ac_in_buffer[:n]) self.ac_in_buffer = self.ac_in_buffer[n:] self.terminator = 0 self.found_terminator() @@ -162,37 +171,32 @@ if index != -1: # we found the terminator if index > 0: - # don't bother reporting the empty string - # (source of subtle bugs) - self.collect_incoming_data(self.ac_in_buffer[:index]) + # don't bother reporting the empty string (source of subtle bugs) + self.collect_incoming_data (self.ac_in_buffer[:index]) self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] - # This does the Right Thing if the terminator - # is changed here. + # This does the Right Thing if the terminator is changed here. self.found_terminator() else: # check for a prefix of the terminator - index = find_prefix_at_end(self.ac_in_buffer, terminator) + index = find_prefix_at_end (self.ac_in_buffer, terminator) if index: if index != lb: # we found a prefix, collect up to the prefix - self.collect_incoming_data(self.ac_in_buffer[:-index]) + self.collect_incoming_data (self.ac_in_buffer[:-index]) self.ac_in_buffer = self.ac_in_buffer[-index:] break else: # no prefix, collect it all - self.collect_incoming_data(self.ac_in_buffer) + self.collect_incoming_data (self.ac_in_buffer) self.ac_in_buffer = b'' - def handle_write(self): + def handle_write (self): self.initiate_send() - def handle_close(self): + def handle_close (self): self.close() - def push(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) + def push (self, data): sabs = self.ac_out_buffer_size if len(data) > sabs: for i in range(0, len(data), sabs): @@ -201,11 +205,11 @@ self.producer_fifo.append(data) self.initiate_send() - def push_with_producer(self, producer): + def push_with_producer (self, producer): self.producer_fifo.append(producer) self.initiate_send() - def readable(self): + def readable (self): "predicate for inclusion in the readable for select()" # cannot use the old predicate, it violates the claim of the # set_terminator method. @@ -213,11 +217,11 @@ # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) return 1 - def writable(self): + def writable (self): "predicate for inclusion in the writable for select()" return self.producer_fifo or (not self.connected) - def close_when_done(self): + def close_when_done (self): "automatically close this channel once the outgoing queue is empty" self.producer_fifo.append(None) @@ -228,13 +232,15 @@ if not first: del self.producer_fifo[0] if first is None: + ## print("first is None") self.handle_close() return + ## print("first is not None") # handle classic producer behavior obs = self.ac_out_buffer_size try: - data = first[:obs] + data = buffer(first, 0, obs) except TypeError: data = first.more() if data: @@ -249,7 +255,7 @@ # send the data try: num_sent = self.send(data) - except OSError: + except socket.error: self.handle_error() return @@ -261,21 +267,20 @@ # we tried to send some actual data return - def discard_buffers(self): + def discard_buffers (self): # Emergencies only! self.ac_in_buffer = b'' del self.incoming[:] self.producer_fifo.clear() - class simple_producer: - def __init__(self, data, buffer_size=512): + def __init__ (self, data, buffer_size=512): self.data = data self.buffer_size = buffer_size - def more(self): - if len(self.data) > self.buffer_size: + def more (self): + if len (self.data) > self.buffer_size: result = self.data[:self.buffer_size] self.data = self.data[self.buffer_size:] return result @@ -284,43 +289,38 @@ self.data = b'' return result - class fifo: - def __init__(self, list=None): - import warnings - warnings.warn('fifo class will be removed in Python 3.6', - DeprecationWarning, stacklevel=2) + def __init__ (self, list=None): if not list: self.list = deque() else: self.list = deque(list) - def __len__(self): + def __len__ (self): return len(self.list) - def is_empty(self): + def is_empty (self): return not self.list - def first(self): + def first (self): return self.list[0] - def push(self, data): + def push (self, data): self.list.append(data) - def pop(self): + def pop (self): if self.list: return (1, self.list.popleft()) else: return (0, None) - # Given 'haystack', see if any prefix of 'needle' is at its end. This # assumes an exact match has already been checked. Return the number of # characters matched. # for example: -# f_p_a_e("qwerty\r", "\r\n") => 1 -# f_p_a_e("qwertydkjf", "\r\n") => 0 -# f_p_a_e("qwerty\r\n", "\r\n") => +# f_p_a_e ("qwerty\r", "\r\n") => 1 +# f_p_a_e ("qwertydkjf", "\r\n") => 0 +# f_p_a_e ("qwerty\r\n", "\r\n") => # this could maybe be made faster with a computed regex? # [answer: no; circa Python-2.0, Jan 2001] @@ -329,7 +329,7 @@ # re: 12820/s # regex: 14035/s -def find_prefix_at_end(haystack, needle): +def find_prefix_at_end (haystack, needle): l = len(needle) - 1 while l and not haystack.endswith(needle[:l]): l -= 1 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/__init__.py --- a/Lib/asyncio/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,50 +0,0 @@ -"""The asyncio package, tracking PEP 3156.""" - -import sys - -# The selectors module is in the stdlib in Python 3.4 but not in 3.3. -# Do this first, so the other submodules can use "from . import selectors". -# Prefer asyncio/selectors.py over the stdlib one, as ours may be newer. -try: - from . import selectors -except ImportError: - import selectors # Will also be exported. - -if sys.platform == 'win32': - # Similar thing for _overlapped. - try: - from . import _overlapped - except ImportError: - import _overlapped # Will also be exported. - -# This relies on each of the submodules having an __all__ variable. -from .base_events import * -from .coroutines import * -from .events import * -from .futures import * -from .locks import * -from .protocols import * -from .queues import * -from .streams import * -from .subprocess import * -from .tasks import * -from .transports import * - -__all__ = (base_events.__all__ + - coroutines.__all__ + - events.__all__ + - futures.__all__ + - locks.__all__ + - protocols.__all__ + - queues.__all__ + - streams.__all__ + - subprocess.__all__ + - tasks.__all__ + - transports.__all__) - -if sys.platform == 'win32': # pragma: no cover - from .windows_events import * - __all__ += windows_events.__all__ -else: - from .unix_events import * # pragma: no cover - __all__ += unix_events.__all__ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/base_events.py --- a/Lib/asyncio/base_events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1338 +0,0 @@ -"""Base implementation of event loop. - -The event loop can be broken up into a multiplexer (the part -responsible for notifying us of I/O events) and the event loop proper, -which wraps a multiplexer with functionality for scheduling callbacks, -immediately or at a given time in the future. - -Whenever a public API takes a callback, subsequent positional -arguments will be passed to the callback if/when it is called. This -avoids the proliferation of trivial lambdas implementing closures. -Keyword arguments for the callback are not supported; this is a -conscious design decision, leaving the door open for keyword arguments -to modify the meaning of the API call itself. -""" - - -import collections -import concurrent.futures -import functools -import heapq -import inspect -import ipaddress -import itertools -import logging -import os -import socket -import subprocess -import threading -import time -import traceback -import sys -import warnings - -from . import compat -from . import coroutines -from . import events -from . import futures -from . import tasks -from .coroutines import coroutine -from .log import logger - - -__all__ = ['BaseEventLoop'] - - -# Argument for default thread pool executor creation. -_MAX_WORKERS = 5 - -# Minimum number of _scheduled timer handles before cleanup of -# cancelled handles is performed. -_MIN_SCHEDULED_TIMER_HANDLES = 100 - -# Minimum fraction of _scheduled timer handles that are cancelled -# before cleanup of cancelled handles is performed. -_MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5 - -def _format_handle(handle): - cb = handle._callback - if inspect.ismethod(cb) and isinstance(cb.__self__, tasks.Task): - # format the task - return repr(cb.__self__) - else: - return str(handle) - - -def _format_pipe(fd): - if fd == subprocess.PIPE: - return '' - elif fd == subprocess.STDOUT: - return '' - else: - return repr(fd) - - -# Linux's sock.type is a bitmask that can include extra info about socket. -_SOCKET_TYPE_MASK = 0 -if hasattr(socket, 'SOCK_NONBLOCK'): - _SOCKET_TYPE_MASK |= socket.SOCK_NONBLOCK -if hasattr(socket, 'SOCK_CLOEXEC'): - _SOCKET_TYPE_MASK |= socket.SOCK_CLOEXEC - - -@functools.lru_cache(maxsize=1024) -def _ipaddr_info(host, port, family, type, proto): - # Try to skip getaddrinfo if "host" is already an IP. Since getaddrinfo - # blocks on an exclusive lock on some platforms, users might handle name - # resolution in their own code and pass in resolved IPs. - if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or host is None: - return None - - type &= ~_SOCKET_TYPE_MASK - if type == socket.SOCK_STREAM: - proto = socket.IPPROTO_TCP - elif type == socket.SOCK_DGRAM: - proto = socket.IPPROTO_UDP - else: - return None - - if hasattr(socket, 'inet_pton'): - if family == socket.AF_UNSPEC: - afs = [socket.AF_INET, socket.AF_INET6] - else: - afs = [family] - - for af in afs: - # Linux's inet_pton doesn't accept an IPv6 zone index after host, - # like '::1%lo0', so strip it. If we happen to make an invalid - # address look valid, we fail later in sock.connect or sock.bind. - try: - if af == socket.AF_INET6: - socket.inet_pton(af, host.partition('%')[0]) - else: - socket.inet_pton(af, host) - return af, type, proto, '', (host, port) - except OSError: - pass - - # "host" is not an IP address. - return None - - # No inet_pton. (On Windows it's only available since Python 3.4.) - # Even though getaddrinfo with AI_NUMERICHOST would be non-blocking, it - # still requires a lock on some platforms, and waiting for that lock could - # block the event loop. Use ipaddress instead, it's just text parsing. - try: - addr = ipaddress.IPv4Address(host) - except ValueError: - try: - addr = ipaddress.IPv6Address(host.partition('%')[0]) - except ValueError: - return None - - af = socket.AF_INET if addr.version == 4 else socket.AF_INET6 - if family not in (socket.AF_UNSPEC, af): - # "host" is wrong IP version for "family". - return None - - return af, type, proto, '', (host, port) - - -def _check_resolved_address(sock, address): - # Ensure that the address is already resolved to avoid the trap of hanging - # the entire event loop when the address requires doing a DNS lookup. - - if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX: - return - - host, port = address[:2] - if _ipaddr_info(host, port, sock.family, sock.type, sock.proto) is None: - raise ValueError("address must be resolved (IP address)," - " got host %r" % host) - - -def _run_until_complete_cb(fut): - exc = fut._exception - if (isinstance(exc, BaseException) - and not isinstance(exc, Exception)): - # Issue #22429: run_forever() already finished, no need to - # stop it. - return - fut._loop.stop() - - -class Server(events.AbstractServer): - - def __init__(self, loop, sockets): - self._loop = loop - self.sockets = sockets - self._active_count = 0 - self._waiters = [] - - def __repr__(self): - return '<%s sockets=%r>' % (self.__class__.__name__, self.sockets) - - def _attach(self): - assert self.sockets is not None - self._active_count += 1 - - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self.sockets is None: - self._wakeup() - - def close(self): - sockets = self.sockets - if sockets is None: - return - self.sockets = None - for sock in sockets: - self._loop._stop_serving(sock) - if self._active_count == 0: - self._wakeup() - - def _wakeup(self): - waiters = self._waiters - self._waiters = None - for waiter in waiters: - if not waiter.done(): - waiter.set_result(waiter) - - @coroutine - def wait_closed(self): - if self.sockets is None or self._waiters is None: - return - waiter = futures.Future(loop=self._loop) - self._waiters.append(waiter) - yield from waiter - - -class BaseEventLoop(events.AbstractEventLoop): - - def __init__(self): - self._timer_cancelled_count = 0 - self._closed = False - self._stopping = False - self._ready = collections.deque() - self._scheduled = [] - self._default_executor = None - self._internal_fds = 0 - # Identifier of the thread running the event loop, or None if the - # event loop is not running - self._thread_id = None - self._clock_resolution = time.get_clock_info('monotonic').resolution - self._exception_handler = None - self.set_debug((not sys.flags.ignore_environment - and bool(os.environ.get('PYTHONASYNCIODEBUG')))) - # In debug mode, if the execution of a callback or a step of a task - # exceed this duration in seconds, the slow callback/task is logged. - self.slow_callback_duration = 0.1 - self._current_handle = None - self._task_factory = None - self._coroutine_wrapper_set = False - - def __repr__(self): - return ('<%s running=%s closed=%s debug=%s>' - % (self.__class__.__name__, self.is_running(), - self.is_closed(), self.get_debug())) - - def create_task(self, coro): - """Schedule a coroutine object. - - Return a task object. - """ - self._check_closed() - if self._task_factory is None: - task = tasks.Task(coro, loop=self) - if task._source_traceback: - del task._source_traceback[-1] - else: - task = self._task_factory(self, coro) - return task - - def set_task_factory(self, factory): - """Set a task factory that will be used by loop.create_task(). - - If factory is None the default task factory will be set. - - If factory is a callable, it should have a signature matching - '(loop, coro)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object. The callable - must return a Future. - """ - if factory is not None and not callable(factory): - raise TypeError('task factory must be a callable or None') - self._task_factory = factory - - def get_task_factory(self): - """Return a task factory, or None if the default one is in use.""" - return self._task_factory - - def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): - """Create socket transport.""" - raise NotImplementedError - - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - """Create SSL transport.""" - raise NotImplementedError - - def _make_datagram_transport(self, sock, protocol, - address=None, waiter=None, extra=None): - """Create datagram transport.""" - raise NotImplementedError - - def _make_read_pipe_transport(self, pipe, protocol, waiter=None, - extra=None): - """Create read pipe transport.""" - raise NotImplementedError - - def _make_write_pipe_transport(self, pipe, protocol, waiter=None, - extra=None): - """Create write pipe transport.""" - raise NotImplementedError - - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): - """Create subprocess transport.""" - raise NotImplementedError - - def _write_to_self(self): - """Write a byte to self-pipe, to wake up the event loop. - - This may be called from a different thread. - - The subclass is responsible for implementing the self-pipe. - """ - raise NotImplementedError - - def _process_events(self, event_list): - """Process selector events.""" - raise NotImplementedError - - def _check_closed(self): - if self._closed: - raise RuntimeError('Event loop is closed') - - def run_forever(self): - """Run until stop() is called.""" - self._check_closed() - if self.is_running(): - raise RuntimeError('Event loop is running.') - self._set_coroutine_wrapper(self._debug) - self._thread_id = threading.get_ident() - try: - while True: - self._run_once() - if self._stopping: - break - finally: - self._stopping = False - self._thread_id = None - self._set_coroutine_wrapper(False) - - def run_until_complete(self, future): - """Run until the Future is done. - - If the argument is a coroutine, it is wrapped in a Task. - - WARNING: It would be disastrous to call run_until_complete() - with the same coroutine twice -- it would wrap it in two - different Tasks and that can't be good. - - Return the Future's result, or raise its exception. - """ - self._check_closed() - - new_task = not isinstance(future, futures.Future) - future = tasks.ensure_future(future, loop=self) - if new_task: - # An exception is raised if the future didn't complete, so there - # is no need to log the "destroy pending task" message - future._log_destroy_pending = False - - future.add_done_callback(_run_until_complete_cb) - try: - self.run_forever() - except: - if new_task and future.done() and not future.cancelled(): - # The coroutine raised a BaseException. Consume the exception - # to not log a warning, the caller doesn't have access to the - # local task. - future.exception() - raise - future.remove_done_callback(_run_until_complete_cb) - if not future.done(): - raise RuntimeError('Event loop stopped before Future completed.') - - return future.result() - - def stop(self): - """Stop running the event loop. - - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. - """ - self._stopping = True - - def close(self): - """Close the event loop. - - This clears the queues and shuts down the executor, - but does not wait for the executor to finish. - - The event loop must not be running. - """ - if self.is_running(): - raise RuntimeError("Cannot close a running event loop") - if self._closed: - return - if self._debug: - logger.debug("Close %r", self) - self._closed = True - self._ready.clear() - self._scheduled.clear() - executor = self._default_executor - if executor is not None: - self._default_executor = None - executor.shutdown(wait=False) - - def is_closed(self): - """Returns True if the event loop was closed.""" - return self._closed - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning) - if not self.is_running(): - self.close() - - def is_running(self): - """Returns True if the event loop is running.""" - return (self._thread_id is not None) - - def time(self): - """Return the time according to the event loop's clock. - - This is a float expressed in seconds since an epoch, but the - epoch, precision, accuracy and drift are unspecified and may - differ per event loop. - """ - return time.monotonic() - - def call_later(self, delay, callback, *args): - """Arrange for a callback to be called at a given time. - - Return a Handle: an opaque object with a cancel() method that - can be used to cancel the call. - - The delay can be an int or float, expressed in seconds. It is - always relative to the current time. - - Each callback will be called exactly once. If two callbacks - are scheduled for exactly the same time, it undefined which - will be called first. - - Any positional arguments after the callback will be passed to - the callback when it is called. - """ - timer = self.call_at(self.time() + delay, callback, *args) - if timer._source_traceback: - del timer._source_traceback[-1] - return timer - - def call_at(self, when, callback, *args): - """Like call_later(), but uses an absolute time. - - Absolute time corresponds to the event loop's time() method. - """ - if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): - raise TypeError("coroutines cannot be used with call_at()") - self._check_closed() - if self._debug: - self._check_thread() - timer = events.TimerHandle(when, callback, args, self) - if timer._source_traceback: - del timer._source_traceback[-1] - heapq.heappush(self._scheduled, timer) - timer._scheduled = True - return timer - - def call_soon(self, callback, *args): - """Arrange for a callback to be called as soon as possible. - - This operates as a FIFO queue: callbacks are called in the - order in which they are registered. Each callback will be - called exactly once. - - Any positional arguments after the callback will be passed to - the callback when it is called. - """ - if self._debug: - self._check_thread() - handle = self._call_soon(callback, args) - if handle._source_traceback: - del handle._source_traceback[-1] - return handle - - def _call_soon(self, callback, args): - if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): - raise TypeError("coroutines cannot be used with call_soon()") - self._check_closed() - handle = events.Handle(callback, args, self) - if handle._source_traceback: - del handle._source_traceback[-1] - self._ready.append(handle) - return handle - - def _check_thread(self): - """Check that the current thread is the thread running the event loop. - - Non-thread-safe methods of this class make this assumption and will - likely behave incorrectly when the assumption is violated. - - Should only be called when (self._debug == True). The caller is - responsible for checking this condition for performance reasons. - """ - if self._thread_id is None: - return - thread_id = threading.get_ident() - if thread_id != self._thread_id: - raise RuntimeError( - "Non-thread-safe operation invoked on an event loop other " - "than the current one") - - def call_soon_threadsafe(self, callback, *args): - """Like call_soon(), but thread-safe.""" - handle = self._call_soon(callback, args) - if handle._source_traceback: - del handle._source_traceback[-1] - self._write_to_self() - return handle - - def run_in_executor(self, executor, func, *args): - if (coroutines.iscoroutine(func) - or coroutines.iscoroutinefunction(func)): - raise TypeError("coroutines cannot be used with run_in_executor()") - self._check_closed() - if isinstance(func, events.Handle): - assert not args - assert not isinstance(func, events.TimerHandle) - if func._cancelled: - f = futures.Future(loop=self) - f.set_result(None) - return f - func, args = func._callback, func._args - if executor is None: - executor = self._default_executor - if executor is None: - executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS) - self._default_executor = executor - return futures.wrap_future(executor.submit(func, *args), loop=self) - - def set_default_executor(self, executor): - self._default_executor = executor - - def _getaddrinfo_debug(self, host, port, family, type, proto, flags): - msg = ["%s:%r" % (host, port)] - if family: - msg.append('family=%r' % family) - if type: - msg.append('type=%r' % type) - if proto: - msg.append('proto=%r' % proto) - if flags: - msg.append('flags=%r' % flags) - msg = ', '.join(msg) - logger.debug('Get address info %s', msg) - - t0 = self.time() - addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags) - dt = self.time() - t0 - - msg = ('Getting address info %s took %.3f ms: %r' - % (msg, dt * 1e3, addrinfo)) - if dt >= self.slow_callback_duration: - logger.info(msg) - else: - logger.debug(msg) - return addrinfo - - def getaddrinfo(self, host, port, *, - family=0, type=0, proto=0, flags=0): - info = _ipaddr_info(host, port, family, type, proto) - if info is not None: - fut = futures.Future(loop=self) - fut.set_result([info]) - return fut - elif self._debug: - return self.run_in_executor(None, self._getaddrinfo_debug, - host, port, family, type, proto, flags) - else: - return self.run_in_executor(None, socket.getaddrinfo, - host, port, family, type, proto, flags) - - def getnameinfo(self, sockaddr, flags=0): - return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags) - - @coroutine - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): - """Connect to a TCP server. - - Create a streaming transport connection to a given Internet host and - port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. - - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. - """ - if server_hostname is not None and not ssl: - raise ValueError('server_hostname is only meaningful with ssl') - - if server_hostname is None and ssl: - # Use host as default for server_hostname. It is an error - # if host is empty or not set, e.g. when an - # already-connected socket was passed or when only a port - # is given. To avoid this error, you can pass - # server_hostname='' -- this will bypass the hostname - # check. (This also means that if host is a numeric - # IP/IPv6 address, we will attempt to verify that exact - # address; this will probably fail, but it is possible to - # create a certificate for a specific IP address, so we - # don't judge it here.) - if not host: - raise ValueError('You must set server_hostname ' - 'when using ssl without a host') - server_hostname = host - - if host is not None or port is not None: - if sock is not None: - raise ValueError( - 'host/port and sock can not be specified at the same time') - - f1 = self.getaddrinfo( - host, port, family=family, - type=socket.SOCK_STREAM, proto=proto, flags=flags) - fs = [f1] - if local_addr is not None: - f2 = self.getaddrinfo( - *local_addr, family=family, - type=socket.SOCK_STREAM, proto=proto, flags=flags) - fs.append(f2) - else: - f2 = None - - yield from tasks.wait(fs, loop=self) - - infos = f1.result() - if not infos: - raise OSError('getaddrinfo() returned empty list') - if f2 is not None: - laddr_infos = f2.result() - if not laddr_infos: - raise OSError('getaddrinfo() returned empty list') - - exceptions = [] - for family, type, proto, cname, address in infos: - try: - sock = socket.socket(family=family, type=type, proto=proto) - sock.setblocking(False) - if f2 is not None: - for _, _, _, _, laddr in laddr_infos: - try: - sock.bind(laddr) - break - except OSError as exc: - exc = OSError( - exc.errno, 'error while ' - 'attempting to bind on address ' - '{!r}: {}'.format( - laddr, exc.strerror.lower())) - exceptions.append(exc) - else: - sock.close() - sock = None - continue - if self._debug: - logger.debug("connect %r to %r", sock, address) - yield from self.sock_connect(sock, address) - except OSError as exc: - if sock is not None: - sock.close() - exceptions.append(exc) - except: - if sock is not None: - sock.close() - raise - else: - break - else: - if len(exceptions) == 1: - raise exceptions[0] - else: - # If they all have the same str(), raise one. - model = str(exceptions[0]) - if all(str(exc) == model for exc in exceptions): - raise exceptions[0] - # Raise a combined exception so the user can see all - # the various error messages. - raise OSError('Multiple exceptions: {}'.format( - ', '.join(str(exc) for exc in exceptions))) - - elif sock is None: - raise ValueError( - 'host and port was not specified and no sock specified') - - sock.setblocking(False) - - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) - if self._debug: - # Get the socket from the transport because SSL transport closes - # the old socket and creates a new SSL socket - sock = transport.get_extra_info('socket') - logger.debug("%r connected to %s:%r: (%r, %r)", - sock, host, port, transport, protocol) - return transport, protocol - - @coroutine - def _create_connection_transport(self, sock, protocol_factory, ssl, - server_hostname): - protocol = protocol_factory() - waiter = futures.Future(loop=self) - if ssl: - sslcontext = None if isinstance(ssl, bool) else ssl - transport = self._make_ssl_transport( - sock, protocol, sslcontext, waiter, - server_side=False, server_hostname=server_hostname) - else: - transport = self._make_socket_transport(sock, protocol, waiter) - - try: - yield from waiter - except: - transport.close() - raise - - return transport, protocol - - @coroutine - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): - """Create datagram connection.""" - if sock is not None: - if (local_addr or remote_addr or - family or proto or flags or - reuse_address or reuse_port or allow_broadcast): - # show the problematic kwargs in exception msg - opts = dict(local_addr=local_addr, remote_addr=remote_addr, - family=family, proto=proto, flags=flags, - reuse_address=reuse_address, reuse_port=reuse_port, - allow_broadcast=allow_broadcast) - problems = ', '.join( - '{}={}'.format(k, v) for k, v in opts.items() if v) - raise ValueError( - 'socket modifier keyword arguments can not be used ' - 'when sock is specified. ({})'.format(problems)) - sock.setblocking(False) - r_addr = None - else: - if not (local_addr or remote_addr): - if family == 0: - raise ValueError('unexpected address family') - addr_pairs_info = (((family, proto), (None, None)),) - else: - # join address by (family, protocol) - addr_infos = collections.OrderedDict() - for idx, addr in ((0, local_addr), (1, remote_addr)): - if addr is not None: - assert isinstance(addr, tuple) and len(addr) == 2, ( - '2-tuple is expected') - - infos = yield from self.getaddrinfo( - *addr, family=family, type=socket.SOCK_DGRAM, - proto=proto, flags=flags) - if not infos: - raise OSError('getaddrinfo() returned empty list') - - for fam, _, pro, _, address in infos: - key = (fam, pro) - if key not in addr_infos: - addr_infos[key] = [None, None] - addr_infos[key][idx] = address - - # each addr has to have info for each (family, proto) pair - addr_pairs_info = [ - (key, addr_pair) for key, addr_pair in addr_infos.items() - if not ((local_addr and addr_pair[0] is None) or - (remote_addr and addr_pair[1] is None))] - - if not addr_pairs_info: - raise ValueError('can not get address information') - - exceptions = [] - - if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' - - for ((family, proto), - (local_address, remote_address)) in addr_pairs_info: - sock = None - r_addr = None - try: - sock = socket.socket( - family=family, type=socket.SOCK_DGRAM, proto=proto) - if reuse_address: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if reuse_port: - if not hasattr(socket, 'SO_REUSEPORT'): - raise ValueError( - 'reuse_port not supported by socket module') - else: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if allow_broadcast: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - sock.setblocking(False) - - if local_addr: - sock.bind(local_address) - if remote_addr: - yield from self.sock_connect(sock, remote_address) - r_addr = remote_address - except OSError as exc: - if sock is not None: - sock.close() - exceptions.append(exc) - except: - if sock is not None: - sock.close() - raise - else: - break - else: - raise exceptions[0] - - protocol = protocol_factory() - waiter = futures.Future(loop=self) - transport = self._make_datagram_transport( - sock, protocol, r_addr, waiter) - if self._debug: - if local_addr: - logger.info("Datagram endpoint local_addr=%r remote_addr=%r " - "created: (%r, %r)", - local_addr, remote_addr, transport, protocol) - else: - logger.debug("Datagram endpoint remote_addr=%r created: " - "(%r, %r)", - remote_addr, transport, protocol) - - try: - yield from waiter - except: - transport.close() - raise - - return transport, protocol - - @coroutine - def _create_server_getaddrinfo(self, host, port, family, flags): - infos = yield from self.getaddrinfo(host, port, family=family, - type=socket.SOCK_STREAM, - flags=flags) - if not infos: - raise OSError('getaddrinfo({!r}) returned empty list'.format(host)) - return infos - - @coroutine - def create_server(self, protocol_factory, host=None, port=None, - *, - family=socket.AF_UNSPEC, - flags=socket.AI_PASSIVE, - sock=None, - backlog=100, - ssl=None, - reuse_address=None, - reuse_port=None): - """Create a TCP server. - - The host parameter can be a string, in that case the TCP server is bound - to host and port. - - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. - - Return a Server object which can be used to stop the service. - - This method is a coroutine. - """ - if isinstance(ssl, bool): - raise TypeError('ssl argument must be an SSLContext or None') - if host is not None or port is not None: - if sock is not None: - raise ValueError( - 'host/port and sock can not be specified at the same time') - - AF_INET6 = getattr(socket, 'AF_INET6', 0) - if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' - sockets = [] - if host == '': - hosts = [None] - elif (isinstance(host, str) or - not isinstance(host, collections.Iterable)): - hosts = [host] - else: - hosts = host - - fs = [self._create_server_getaddrinfo(host, port, family=family, - flags=flags) - for host in hosts] - infos = yield from tasks.gather(*fs, loop=self) - infos = itertools.chain.from_iterable(infos) - - completed = False - try: - for res in infos: - af, socktype, proto, canonname, sa = res - try: - sock = socket.socket(af, socktype, proto) - except socket.error: - # Assume it's a bad family/type/protocol combination. - if self._debug: - logger.warning('create_server() failed to create ' - 'socket.socket(%r, %r, %r)', - af, socktype, proto, exc_info=True) - continue - sockets.append(sock) - if reuse_address: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, True) - if reuse_port: - if not hasattr(socket, 'SO_REUSEPORT'): - raise ValueError( - 'reuse_port not supported by socket module') - else: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEPORT, True) - # Disable IPv4/IPv6 dual stack support (enabled by - # default on Linux) which makes a single socket - # listen on both address families. - if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'): - sock.setsockopt(socket.IPPROTO_IPV6, - socket.IPV6_V6ONLY, - True) - try: - sock.bind(sa) - except OSError as err: - raise OSError(err.errno, 'error while attempting ' - 'to bind on address %r: %s' - % (sa, err.strerror.lower())) - completed = True - finally: - if not completed: - for sock in sockets: - sock.close() - else: - if sock is None: - raise ValueError('Neither host/port nor sock were specified') - sockets = [sock] - - server = Server(self, sockets) - for sock in sockets: - sock.listen(backlog) - sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server) - if self._debug: - logger.info("%r is serving", server) - return server - - @coroutine - def connect_read_pipe(self, protocol_factory, pipe): - protocol = protocol_factory() - waiter = futures.Future(loop=self) - transport = self._make_read_pipe_transport(pipe, protocol, waiter) - - try: - yield from waiter - except: - transport.close() - raise - - if self._debug: - logger.debug('Read pipe %r connected: (%r, %r)', - pipe.fileno(), transport, protocol) - return transport, protocol - - @coroutine - def connect_write_pipe(self, protocol_factory, pipe): - protocol = protocol_factory() - waiter = futures.Future(loop=self) - transport = self._make_write_pipe_transport(pipe, protocol, waiter) - - try: - yield from waiter - except: - transport.close() - raise - - if self._debug: - logger.debug('Write pipe %r connected: (%r, %r)', - pipe.fileno(), transport, protocol) - return transport, protocol - - def _log_subprocess(self, msg, stdin, stdout, stderr): - info = [msg] - if stdin is not None: - info.append('stdin=%s' % _format_pipe(stdin)) - if stdout is not None and stderr == subprocess.STDOUT: - info.append('stdout=stderr=%s' % _format_pipe(stdout)) - else: - if stdout is not None: - info.append('stdout=%s' % _format_pipe(stdout)) - if stderr is not None: - info.append('stderr=%s' % _format_pipe(stderr)) - logger.debug(' '.join(info)) - - @coroutine - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=False, shell=True, bufsize=0, - **kwargs): - if not isinstance(cmd, (bytes, str)): - raise ValueError("cmd must be a string") - if universal_newlines: - raise ValueError("universal_newlines must be False") - if not shell: - raise ValueError("shell must be True") - if bufsize != 0: - raise ValueError("bufsize must be 0") - protocol = protocol_factory() - if self._debug: - # don't log parameters: they may contain sensitive information - # (password) and may be too long - debug_log = 'run shell command %r' % cmd - self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( - protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs) - if self._debug: - logger.info('%s: %r' % (debug_log, transport)) - return transport, protocol - - @coroutine - def subprocess_exec(self, protocol_factory, program, *args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=False, - shell=False, bufsize=0, **kwargs): - if universal_newlines: - raise ValueError("universal_newlines must be False") - if shell: - raise ValueError("shell must be False") - if bufsize != 0: - raise ValueError("bufsize must be 0") - popen_args = (program,) + args - for arg in popen_args: - if not isinstance(arg, (str, bytes)): - raise TypeError("program arguments must be " - "a bytes or text string, not %s" - % type(arg).__name__) - protocol = protocol_factory() - if self._debug: - # don't log parameters: they may contain sensitive information - # (password) and may be too long - debug_log = 'execute program %r' % program - self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( - protocol, popen_args, False, stdin, stdout, stderr, - bufsize, **kwargs) - if self._debug: - logger.info('%s: %r' % (debug_log, transport)) - return transport, protocol - - def set_exception_handler(self, handler): - """Set handler as the new event loop exception handler. - - If handler is None, the default exception handler will - be set. - - If handler is a callable object, it should have a - signature matching '(loop, context)', where 'loop' - will be a reference to the active event loop, 'context' - will be a dict object (see `call_exception_handler()` - documentation for details about context). - """ - if handler is not None and not callable(handler): - raise TypeError('A callable object or None is expected, ' - 'got {!r}'.format(handler)) - self._exception_handler = handler - - def default_exception_handler(self, context): - """Default exception handler. - - This is called when an exception occurs and no exception - handler is set, and can be called by a custom exception - handler that wants to defer to the default behavior. - - The context parameter has the same meaning as in - `call_exception_handler()`. - """ - message = context.get('message') - if not message: - message = 'Unhandled exception in event loop' - - exception = context.get('exception') - if exception is not None: - exc_info = (type(exception), exception, exception.__traceback__) - else: - exc_info = False - - if ('source_traceback' not in context - and self._current_handle is not None - and self._current_handle._source_traceback): - context['handle_traceback'] = self._current_handle._source_traceback - - log_lines = [message] - for key in sorted(context): - if key in {'message', 'exception'}: - continue - value = context[key] - if key == 'source_traceback': - tb = ''.join(traceback.format_list(value)) - value = 'Object created at (most recent call last):\n' - value += tb.rstrip() - elif key == 'handle_traceback': - tb = ''.join(traceback.format_list(value)) - value = 'Handle created at (most recent call last):\n' - value += tb.rstrip() - else: - value = repr(value) - log_lines.append('{}: {}'.format(key, value)) - - logger.error('\n'.join(log_lines), exc_info=exc_info) - - def call_exception_handler(self, context): - """Call the current event loop's exception handler. - - The context argument is a dict containing the following keys: - - - 'message': Error message; - - 'exception' (optional): Exception object; - - 'future' (optional): Future instance; - - 'handle' (optional): Handle instance; - - 'protocol' (optional): Protocol instance; - - 'transport' (optional): Transport instance; - - 'socket' (optional): Socket instance. - - New keys maybe introduced in the future. - - Note: do not overload this method in an event loop subclass. - For custom exception handling, use the - `set_exception_handler()` method. - """ - if self._exception_handler is None: - try: - self.default_exception_handler(context) - except Exception: - # Second protection layer for unexpected errors - # in the default implementation, as well as for subclassed - # event loops with overloaded "default_exception_handler". - logger.error('Exception in default exception handler', - exc_info=True) - else: - try: - self._exception_handler(self, context) - except Exception as exc: - # Exception in the user set custom exception handler. - try: - # Let's try default handler. - self.default_exception_handler({ - 'message': 'Unhandled error in exception handler', - 'exception': exc, - 'context': context, - }) - except Exception: - # Guard 'default_exception_handler' in case it is - # overloaded. - logger.error('Exception in default exception handler ' - 'while handling an unexpected error ' - 'in custom exception handler', - exc_info=True) - - def _add_callback(self, handle): - """Add a Handle to _scheduled (TimerHandle) or _ready.""" - assert isinstance(handle, events.Handle), 'A Handle is required here' - if handle._cancelled: - return - assert not isinstance(handle, events.TimerHandle) - self._ready.append(handle) - - def _add_callback_signalsafe(self, handle): - """Like _add_callback() but called from a signal handler.""" - self._add_callback(handle) - self._write_to_self() - - def _timer_handle_cancelled(self, handle): - """Notification that a TimerHandle has been cancelled.""" - if handle._scheduled: - self._timer_cancelled_count += 1 - - def _run_once(self): - """Run one full iteration of the event loop. - - This calls all currently ready callbacks, polls for I/O, - schedules the resulting callbacks, and finally schedules - 'call_later' callbacks. - """ - - sched_count = len(self._scheduled) - if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and - self._timer_cancelled_count / sched_count > - _MIN_CANCELLED_TIMER_HANDLES_FRACTION): - # Remove delayed calls that were cancelled if their number - # is too high - new_scheduled = [] - for handle in self._scheduled: - if handle._cancelled: - handle._scheduled = False - else: - new_scheduled.append(handle) - - heapq.heapify(new_scheduled) - self._scheduled = new_scheduled - self._timer_cancelled_count = 0 - else: - # Remove delayed calls that were cancelled from head of queue. - while self._scheduled and self._scheduled[0]._cancelled: - self._timer_cancelled_count -= 1 - handle = heapq.heappop(self._scheduled) - handle._scheduled = False - - timeout = None - if self._ready or self._stopping: - timeout = 0 - elif self._scheduled: - # Compute the desired timeout. - when = self._scheduled[0]._when - timeout = max(0, when - self.time()) - - if self._debug and timeout != 0: - t0 = self.time() - event_list = self._selector.select(timeout) - dt = self.time() - t0 - if dt >= 1.0: - level = logging.INFO - else: - level = logging.DEBUG - nevent = len(event_list) - if timeout is None: - logger.log(level, 'poll took %.3f ms: %s events', - dt * 1e3, nevent) - elif nevent: - logger.log(level, - 'poll %.3f ms took %.3f ms: %s events', - timeout * 1e3, dt * 1e3, nevent) - elif dt >= 1.0: - logger.log(level, - 'poll %.3f ms took %.3f ms: timeout', - timeout * 1e3, dt * 1e3) - else: - event_list = self._selector.select(timeout) - self._process_events(event_list) - - # Handle 'later' callbacks that are ready. - end_time = self.time() + self._clock_resolution - while self._scheduled: - handle = self._scheduled[0] - if handle._when >= end_time: - break - handle = heapq.heappop(self._scheduled) - handle._scheduled = False - self._ready.append(handle) - - # This is the only place where callbacks are actually *called*. - # All other places just add them to ready. - # Note: We run all currently scheduled callbacks, but not any - # callbacks scheduled by callbacks run this time around -- - # they will be run the next time (after another I/O poll). - # Use an idiom that is thread-safe without using locks. - ntodo = len(self._ready) - for i in range(ntodo): - handle = self._ready.popleft() - if handle._cancelled: - continue - if self._debug: - try: - self._current_handle = handle - t0 = self.time() - handle._run() - dt = self.time() - t0 - if dt >= self.slow_callback_duration: - logger.warning('Executing %s took %.3f seconds', - _format_handle(handle), dt) - finally: - self._current_handle = None - else: - handle._run() - handle = None # Needed to break cycles when an exception occurs. - - def _set_coroutine_wrapper(self, enabled): - try: - set_wrapper = sys.set_coroutine_wrapper - get_wrapper = sys.get_coroutine_wrapper - except AttributeError: - return - - enabled = bool(enabled) - if self._coroutine_wrapper_set == enabled: - return - - wrapper = coroutines.debug_wrapper - current_wrapper = get_wrapper() - - if enabled: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(True): cannot set debug coroutine " - "wrapper; another wrapper is already set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(wrapper) - self._coroutine_wrapper_set = True - else: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(False): cannot unset debug coroutine " - "wrapper; another wrapper was set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(None) - self._coroutine_wrapper_set = False - - def get_debug(self): - return self._debug - - def set_debug(self, enabled): - self._debug = enabled - - if self.is_running(): - self._set_coroutine_wrapper(enabled) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/base_subprocess.py --- a/Lib/asyncio/base_subprocess.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,283 +0,0 @@ -import collections -import subprocess -import warnings - -from . import compat -from . import futures -from . import protocols -from . import transports -from .coroutines import coroutine -from .log import logger - - -class BaseSubprocessTransport(transports.SubprocessTransport): - - def __init__(self, loop, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=None, extra=None, **kwargs): - super().__init__(extra) - self._closed = False - self._protocol = protocol - self._loop = loop - self._proc = None - self._pid = None - self._returncode = None - self._exit_waiters = [] - self._pending_calls = collections.deque() - self._pipes = {} - self._finished = False - - if stdin == subprocess.PIPE: - self._pipes[0] = None - if stdout == subprocess.PIPE: - self._pipes[1] = None - if stderr == subprocess.PIPE: - self._pipes[2] = None - - # Create the child process: set the _proc attribute - try: - self._start(args=args, shell=shell, stdin=stdin, stdout=stdout, - stderr=stderr, bufsize=bufsize, **kwargs) - except: - self.close() - raise - - self._pid = self._proc.pid - self._extra['subprocess'] = self._proc - - if self._loop.get_debug(): - if isinstance(args, (bytes, str)): - program = args - else: - program = args[0] - logger.debug('process %r created: pid %s', - program, self._pid) - - self._loop.create_task(self._connect_pipes(waiter)) - - def __repr__(self): - info = [self.__class__.__name__] - if self._closed: - info.append('closed') - if self._pid is not None: - info.append('pid=%s' % self._pid) - if self._returncode is not None: - info.append('returncode=%s' % self._returncode) - elif self._pid is not None: - info.append('running') - else: - info.append('not started') - - stdin = self._pipes.get(0) - if stdin is not None: - info.append('stdin=%s' % stdin.pipe) - - stdout = self._pipes.get(1) - stderr = self._pipes.get(2) - if stdout is not None and stderr is stdout: - info.append('stdout=stderr=%s' % stdout.pipe) - else: - if stdout is not None: - info.append('stdout=%s' % stdout.pipe) - if stderr is not None: - info.append('stderr=%s' % stderr.pipe) - - return '<%s>' % ' '.join(info) - - def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): - raise NotImplementedError - - def is_closing(self): - return self._closed - - def close(self): - if self._closed: - return - self._closed = True - - for proto in self._pipes.values(): - if proto is None: - continue - proto.pipe.close() - - if (self._proc is not None - # the child process finished? - and self._returncode is None - # the child process finished but the transport was not notified yet? - and self._proc.poll() is None - ): - if self._loop.get_debug(): - logger.warning('Close running child process: kill %r', self) - - try: - self._proc.kill() - except ProcessLookupError: - pass - - # Don't clear the _proc reference yet: _post_init() may still run - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self.close() - - def get_pid(self): - return self._pid - - def get_returncode(self): - return self._returncode - - def get_pipe_transport(self, fd): - if fd in self._pipes: - return self._pipes[fd].pipe - else: - return None - - def _check_proc(self): - if self._proc is None: - raise ProcessLookupError() - - def send_signal(self, signal): - self._check_proc() - self._proc.send_signal(signal) - - def terminate(self): - self._check_proc() - self._proc.terminate() - - def kill(self): - self._check_proc() - self._proc.kill() - - @coroutine - def _connect_pipes(self, waiter): - try: - proc = self._proc - loop = self._loop - - if proc.stdin is not None: - _, pipe = yield from loop.connect_write_pipe( - lambda: WriteSubprocessPipeProto(self, 0), - proc.stdin) - self._pipes[0] = pipe - - if proc.stdout is not None: - _, pipe = yield from loop.connect_read_pipe( - lambda: ReadSubprocessPipeProto(self, 1), - proc.stdout) - self._pipes[1] = pipe - - if proc.stderr is not None: - _, pipe = yield from loop.connect_read_pipe( - lambda: ReadSubprocessPipeProto(self, 2), - proc.stderr) - self._pipes[2] = pipe - - assert self._pending_calls is not None - - loop.call_soon(self._protocol.connection_made, self) - for callback, data in self._pending_calls: - loop.call_soon(callback, *data) - self._pending_calls = None - except Exception as exc: - if waiter is not None and not waiter.cancelled(): - waiter.set_exception(exc) - else: - if waiter is not None and not waiter.cancelled(): - waiter.set_result(None) - - def _call(self, cb, *data): - if self._pending_calls is not None: - self._pending_calls.append((cb, data)) - else: - self._loop.call_soon(cb, *data) - - def _pipe_connection_lost(self, fd, exc): - self._call(self._protocol.pipe_connection_lost, fd, exc) - self._try_finish() - - def _pipe_data_received(self, fd, data): - self._call(self._protocol.pipe_data_received, fd, data) - - def _process_exited(self, returncode): - assert returncode is not None, returncode - assert self._returncode is None, self._returncode - if self._loop.get_debug(): - logger.info('%r exited with return code %r', - self, returncode) - self._returncode = returncode - self._call(self._protocol.process_exited) - self._try_finish() - - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.cancelled(): - waiter.set_result(returncode) - self._exit_waiters = None - - @coroutine - def _wait(self): - """Wait until the process exit and return the process return code. - - This method is a coroutine.""" - if self._returncode is not None: - return self._returncode - - waiter = futures.Future(loop=self._loop) - self._exit_waiters.append(waiter) - return (yield from waiter) - - def _try_finish(self): - assert not self._finished - if self._returncode is None: - return - if all(p is not None and p.disconnected - for p in self._pipes.values()): - self._finished = True - self._call(self._call_connection_lost, None) - - def _call_connection_lost(self, exc): - try: - self._protocol.connection_lost(exc) - finally: - self._loop = None - self._proc = None - self._protocol = None - - -class WriteSubprocessPipeProto(protocols.BaseProtocol): - - def __init__(self, proc, fd): - self.proc = proc - self.fd = fd - self.pipe = None - self.disconnected = False - - def connection_made(self, transport): - self.pipe = transport - - def __repr__(self): - return ('<%s fd=%s pipe=%r>' - % (self.__class__.__name__, self.fd, self.pipe)) - - def connection_lost(self, exc): - self.disconnected = True - self.proc._pipe_connection_lost(self.fd, exc) - self.proc = None - - def pause_writing(self): - self.proc._protocol.pause_writing() - - def resume_writing(self): - self.proc._protocol.resume_writing() - - -class ReadSubprocessPipeProto(WriteSubprocessPipeProto, - protocols.Protocol): - - def data_received(self, data): - self.proc._pipe_data_received(self.fd, data) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/compat.py --- a/Lib/asyncio/compat.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -"""Compatibility helpers for the different Python versions.""" - -import sys - -PY34 = sys.version_info >= (3, 4) -PY35 = sys.version_info >= (3, 5) - - -def flatten_list_bytes(list_of_data): - """Concatenate a sequence of bytes-like objects.""" - if not PY34: - # On Python 3.3 and older, bytes.join() doesn't handle - # memoryview. - list_of_data = ( - bytes(data) if isinstance(data, memoryview) else data - for data in list_of_data) - return b''.join(list_of_data) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/constants.py --- a/Lib/asyncio/constants.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""Constants.""" - -# After the connection is lost, log warnings after this many write()s. -LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5 - -# Seconds to wait before retrying accept(). -ACCEPT_RETRY_DELAY = 1 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/coroutines.py --- a/Lib/asyncio/coroutines.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,308 +0,0 @@ -__all__ = ['coroutine', - 'iscoroutinefunction', 'iscoroutine'] - -import functools -import inspect -import opcode -import os -import sys -import traceback -import types - -from . import compat -from . import events -from . import futures -from .log import logger - - -# Opcode of "yield from" instruction -_YIELD_FROM = opcode.opmap['YIELD_FROM'] - -# If you set _DEBUG to true, @coroutine will wrap the resulting -# generator objects in a CoroWrapper instance (defined below). That -# instance will log a message when the generator is never iterated -# over, which may happen when you forget to use "yield from" with a -# coroutine call. Note that the value of the _DEBUG flag is taken -# when the decorator is used, so to be of any use it must be set -# before you define your coroutines. A downside of using this feature -# is that tracebacks show entries for the CoroWrapper.__next__ method -# when _DEBUG is true. -_DEBUG = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - - -try: - _types_coroutine = types.coroutine -except AttributeError: - _types_coroutine = None - -try: - _inspect_iscoroutinefunction = inspect.iscoroutinefunction -except AttributeError: - _inspect_iscoroutinefunction = lambda func: False - -try: - from collections.abc import Coroutine as _CoroutineABC, \ - Awaitable as _AwaitableABC -except ImportError: - _CoroutineABC = _AwaitableABC = None - - -# Check for CPython issue #21209 -def has_yield_from_bug(): - class MyGen: - def __init__(self): - self.send_args = None - def __iter__(self): - return self - def __next__(self): - return 42 - def send(self, *what): - self.send_args = what - return None - def yield_from_gen(gen): - yield from gen - value = (1, 2, 3) - gen = MyGen() - coro = yield_from_gen(gen) - next(coro) - coro.send(value) - return gen.send_args != (value,) -_YIELD_FROM_BUG = has_yield_from_bug() -del has_yield_from_bug - - -def debug_wrapper(gen): - # This function is called from 'sys.set_coroutine_wrapper'. - # We only wrap here coroutines defined via 'async def' syntax. - # Generator-based coroutines are wrapped in @coroutine - # decorator. - return CoroWrapper(gen, None) - - -class CoroWrapper: - # Wrapper for coroutine object in _DEBUG mode. - - def __init__(self, gen, func=None): - assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen - self.gen = gen - self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = traceback.extract_stack(sys._getframe(1)) - self.__name__ = getattr(gen, '__name__', None) - self.__qualname__ = getattr(gen, '__qualname__', None) - - def __repr__(self): - coro_repr = _format_coroutine(self) - if self._source_traceback: - frame = self._source_traceback[-1] - coro_repr += ', created at %s:%s' % (frame[0], frame[1]) - return '<%s %s>' % (self.__class__.__name__, coro_repr) - - def __iter__(self): - return self - - def __next__(self): - return self.gen.send(None) - - if _YIELD_FROM_BUG: - # For for CPython issue #21209: using "yield from" and a custom - # generator, generator.send(tuple) unpacks the tuple instead of passing - # the tuple unchanged. Check if the caller is a generator using "yield - # from" to decide if the parameter should be unpacked or not. - def send(self, *value): - frame = sys._getframe() - caller = frame.f_back - assert caller.f_lasti >= 0 - if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM: - value = value[0] - return self.gen.send(value) - else: - def send(self, value): - return self.gen.send(value) - - def throw(self, exc): - return self.gen.throw(exc) - - def close(self): - return self.gen.close() - - @property - def gi_frame(self): - return self.gen.gi_frame - - @property - def gi_running(self): - return self.gen.gi_running - - @property - def gi_code(self): - return self.gen.gi_code - - if compat.PY35: - - def __await__(self): - cr_await = getattr(self.gen, 'cr_await', None) - if cr_await is not None: - raise RuntimeError( - "Cannot await on coroutine {!r} while it's " - "awaiting for {!r}".format(self.gen, cr_await)) - return self - - @property - def gi_yieldfrom(self): - return self.gen.gi_yieldfrom - - @property - def cr_await(self): - return self.gen.cr_await - - @property - def cr_running(self): - return self.gen.cr_running - - @property - def cr_code(self): - return self.gen.cr_code - - @property - def cr_frame(self): - return self.gen.cr_frame - - def __del__(self): - # Be careful accessing self.gen.frame -- self.gen might not exist. - gen = getattr(self, 'gen', None) - frame = getattr(gen, 'gi_frame', None) - if frame is None: - frame = getattr(gen, 'cr_frame', None) - if frame is not None and frame.f_lasti == -1: - msg = '%r was never yielded from' % self - tb = getattr(self, '_source_traceback', ()) - if tb: - tb = ''.join(traceback.format_list(tb)) - msg += ('\nCoroutine object created at ' - '(most recent call last):\n') - msg += tb.rstrip() - logger.error(msg) - - -def coroutine(func): - """Decorator to mark coroutines. - - If the coroutine is not yielded from before it is destroyed, - an error message is logged. - """ - if _inspect_iscoroutinefunction(func): - # In Python 3.5 that's all we need to do for coroutines - # defiend with "async def". - # Wrapping in CoroWrapper will happen via - # 'sys.set_coroutine_wrapper' function. - return func - - if inspect.isgeneratorfunction(func): - coro = func - else: - @functools.wraps(func) - def coro(*args, **kw): - res = func(*args, **kw) - if isinstance(res, futures.Future) or inspect.isgenerator(res): - res = yield from res - elif _AwaitableABC is not None: - # If 'func' returns an Awaitable (new in 3.5) we - # want to run it. - try: - await_meth = res.__await__ - except AttributeError: - pass - else: - if isinstance(res, _AwaitableABC): - res = yield from await_meth() - return res - - if not _DEBUG: - if _types_coroutine is None: - wrapper = coro - else: - wrapper = _types_coroutine(coro) - else: - @functools.wraps(func) - def wrapper(*args, **kwds): - w = CoroWrapper(coro(*args, **kwds), func=func) - if w._source_traceback: - del w._source_traceback[-1] - # Python < 3.5 does not implement __qualname__ - # on generator objects, so we set it manually. - # We use getattr as some callables (such as - # functools.partial may lack __qualname__). - w.__name__ = getattr(func, '__name__', None) - w.__qualname__ = getattr(func, '__qualname__', None) - return w - - wrapper._is_coroutine = True # For iscoroutinefunction(). - return wrapper - - -def iscoroutinefunction(func): - """Return True if func is a decorated coroutine function.""" - return (getattr(func, '_is_coroutine', False) or - _inspect_iscoroutinefunction(func)) - - -_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper) -if _CoroutineABC is not None: - _COROUTINE_TYPES += (_CoroutineABC,) - - -def iscoroutine(obj): - """Return True if obj is a coroutine object.""" - return isinstance(obj, _COROUTINE_TYPES) - - -def _format_coroutine(coro): - assert iscoroutine(coro) - - coro_name = None - if isinstance(coro, CoroWrapper): - func = coro.func - coro_name = coro.__qualname__ - if coro_name is not None: - coro_name = '{}()'.format(coro_name) - else: - func = coro - - if coro_name is None: - coro_name = events._format_callback(func, ()) - - try: - coro_code = coro.gi_code - except AttributeError: - coro_code = coro.cr_code - - try: - coro_frame = coro.gi_frame - except AttributeError: - coro_frame = coro.cr_frame - - filename = coro_code.co_filename - lineno = 0 - if (isinstance(coro, CoroWrapper) and - not inspect.isgeneratorfunction(coro.func) and - coro.func is not None): - source = events._get_function_source(coro.func) - if source is not None: - filename, lineno = source - if coro_frame is None: - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) - else: - coro_repr = ('%s running, defined at %s:%s' - % (coro_name, filename, lineno)) - elif coro_frame is not None: - lineno = coro_frame.f_lineno - coro_repr = ('%s running at %s:%s' - % (coro_name, filename, lineno)) - else: - lineno = coro_code.co_firstlineno - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) - - return coro_repr diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/events.py --- a/Lib/asyncio/events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,647 +0,0 @@ -"""Event loop and event loop policy.""" - -__all__ = ['AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - ] - -import functools -import inspect -import reprlib -import socket -import subprocess -import sys -import threading -import traceback - -from asyncio import compat - - -def _get_function_source(func): - if compat.PY34: - func = inspect.unwrap(func) - elif hasattr(func, '__wrapped__'): - func = func.__wrapped__ - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if compat.PY34 and isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None - - -def _format_args(args): - """Format function arguments. - - Special case for a single parameter: ('hello',) is formatted as ('hello'). - """ - # use reprlib to limit the length of the output - args_repr = reprlib.repr(args) - if len(args) == 1 and args_repr.endswith(',)'): - args_repr = args_repr[:-2] + ')' - return args_repr - - -def _format_callback(func, args, suffix=''): - if isinstance(func, functools.partial): - if args is not None: - suffix = _format_args(args) + suffix - return _format_callback(func.func, func.args, suffix) - - if hasattr(func, '__qualname__'): - func_repr = getattr(func, '__qualname__') - elif hasattr(func, '__name__'): - func_repr = getattr(func, '__name__') - else: - func_repr = repr(func) - - if args is not None: - func_repr += _format_args(args) - if suffix: - func_repr += suffix - return func_repr - -def _format_callback_source(func, args): - func_repr = _format_callback(func, args) - source = _get_function_source(func) - if source: - func_repr += ' at %s:%s' % source - return func_repr - - -class Handle: - """Object returned by callback registration methods.""" - - __slots__ = ('_callback', '_args', '_cancelled', '_loop', - '_source_traceback', '_repr', '__weakref__') - - def __init__(self, callback, args, loop): - assert not isinstance(callback, Handle), 'A Handle is not a callback' - self._loop = loop - self._callback = callback - self._args = args - self._cancelled = False - self._repr = None - if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) - else: - self._source_traceback = None - - def _repr_info(self): - info = [self.__class__.__name__] - if self._cancelled: - info.append('cancelled') - if self._callback is not None: - info.append(_format_callback_source(self._callback, self._args)) - if self._source_traceback: - frame = self._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) - return info - - def __repr__(self): - if self._repr is not None: - return self._repr - info = self._repr_info() - return '<%s>' % ' '.join(info) - - def cancel(self): - if not self._cancelled: - self._cancelled = True - if self._loop.get_debug(): - # Keep a representation in debug mode to keep callback and - # parameters. For example, to log the warning - # "Executing took 2.5 second" - self._repr = repr(self) - self._callback = None - self._args = None - - def _run(self): - try: - self._callback(*self._args) - except Exception as exc: - cb = _format_callback_source(self._callback, self._args) - msg = 'Exception in callback {}'.format(cb) - context = { - 'message': msg, - 'exception': exc, - 'handle': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - self = None # Needed to break cycles when an exception occurs. - - -class TimerHandle(Handle): - """Object returned by timed callback registration methods.""" - - __slots__ = ['_scheduled', '_when'] - - def __init__(self, when, callback, args, loop): - assert when is not None - super().__init__(callback, args, loop) - if self._source_traceback: - del self._source_traceback[-1] - self._when = when - self._scheduled = False - - def _repr_info(self): - info = super()._repr_info() - pos = 2 if self._cancelled else 1 - info.insert(pos, 'when=%s' % self._when) - return info - - def __hash__(self): - return hash(self._when) - - def __lt__(self, other): - return self._when < other._when - - def __le__(self, other): - if self._when < other._when: - return True - return self.__eq__(other) - - def __gt__(self, other): - return self._when > other._when - - def __ge__(self, other): - if self._when > other._when: - return True - return self.__eq__(other) - - def __eq__(self, other): - if isinstance(other, TimerHandle): - return (self._when == other._when and - self._callback == other._callback and - self._args == other._args and - self._cancelled == other._cancelled) - return NotImplemented - - def __ne__(self, other): - equal = self.__eq__(other) - return NotImplemented if equal is NotImplemented else not equal - - def cancel(self): - if not self._cancelled: - self._loop._timer_handle_cancelled(self) - super().cancel() - - -class AbstractServer: - """Abstract server returned by create_server().""" - - def close(self): - """Stop serving. This leaves existing connections open.""" - return NotImplemented - - def wait_closed(self): - """Coroutine to wait until service is closed.""" - return NotImplemented - - -class AbstractEventLoop: - """Abstract event loop.""" - - # Running and stopping the event loop. - - def run_forever(self): - """Run the event loop until stop() is called.""" - raise NotImplementedError - - def run_until_complete(self, future): - """Run the event loop until a Future is done. - - Return the Future's result, or raise its exception. - """ - raise NotImplementedError - - def stop(self): - """Stop the event loop as soon as reasonable. - - Exactly how soon that is may depend on the implementation, but - no more I/O callbacks should be scheduled. - """ - raise NotImplementedError - - def is_running(self): - """Return whether the event loop is currently running.""" - raise NotImplementedError - - def is_closed(self): - """Returns True if the event loop was closed.""" - raise NotImplementedError - - def close(self): - """Close the loop. - - The loop should not be running. - - This is idempotent and irreversible. - - No other methods should be called after this one. - """ - raise NotImplementedError - - # Methods scheduling callbacks. All these return Handles. - - def _timer_handle_cancelled(self, handle): - """Notification that a TimerHandle has been cancelled.""" - raise NotImplementedError - - def call_soon(self, callback, *args): - return self.call_later(0, callback, *args) - - def call_later(self, delay, callback, *args): - raise NotImplementedError - - def call_at(self, when, callback, *args): - raise NotImplementedError - - def time(self): - raise NotImplementedError - - # Method scheduling a coroutine object: create a task. - - def create_task(self, coro): - raise NotImplementedError - - # Methods for interacting with threads. - - def call_soon_threadsafe(self, callback, *args): - raise NotImplementedError - - def run_in_executor(self, executor, func, *args): - raise NotImplementedError - - def set_default_executor(self, executor): - raise NotImplementedError - - # Network I/O methods returning Futures. - - def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0): - raise NotImplementedError - - def getnameinfo(self, sockaddr, flags=0): - raise NotImplementedError - - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): - raise NotImplementedError - - def create_server(self, protocol_factory, host=None, port=None, *, - family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, - sock=None, backlog=100, ssl=None, reuse_address=None, - reuse_port=None): - """A coroutine which creates a TCP server bound to host and port. - - The return value is a Server object which can be used to stop - the service. - - If host is an empty string or None all interfaces are assumed - and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be a - sequence (e.g. list) of hosts to bind to. - - family can be set to either AF_INET or AF_INET6 to force the - socket to use IPv4 or IPv6. If not set it will be determined - from host (defaults to AF_UNSPEC). - - flags is a bitmask for getaddrinfo(). - - sock can optionally be specified in order to use a preexisting - socket object. - - backlog is the maximum number of queued connections passed to - listen() (defaults to 100). - - ssl can be set to an SSLContext to enable SSL over the - accepted connections. - - reuse_address tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified will automatically be set to True on - UNIX. - - reuse_port tells the kernel to allow this endpoint to be bound to - the same port as other existing endpoints are bound to, so long as - they all set this flag when being created. This option is not - supported on Windows. - """ - raise NotImplementedError - - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): - raise NotImplementedError - - def create_unix_server(self, protocol_factory, path, *, - sock=None, backlog=100, ssl=None): - """A coroutine which creates a UNIX Domain Socket server. - - The return value is a Server object, which can be used to stop - the service. - - path is a str, representing a file systsem path to bind the - server socket to. - - sock can optionally be specified in order to use a preexisting - socket object. - - backlog is the maximum number of queued connections passed to - listen() (defaults to 100). - - ssl can be set to an SSLContext to enable SSL over the - accepted connections. - """ - raise NotImplementedError - - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): - """A coroutine which creates a datagram endpoint. - - This method will try to establish the endpoint in the background. - When successful, the coroutine returns a (transport, protocol) pair. - - protocol_factory must be a callable returning a protocol instance. - - socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_DGRAM. - - reuse_address tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified it will automatically be set to True on - UNIX. - - reuse_port tells the kernel to allow this endpoint to be bound to - the same port as other existing endpoints are bound to, so long as - they all set this flag when being created. This option is not - supported on Windows and some UNIX's. If the - :py:data:`~socket.SO_REUSEPORT` constant is not defined then this - capability is unsupported. - - allow_broadcast tells the kernel to allow this endpoint to send - messages to the broadcast address. - - sock can optionally be specified in order to use a preexisting - socket object. - """ - raise NotImplementedError - - # Pipes and subprocesses. - - def connect_read_pipe(self, protocol_factory, pipe): - """Register read pipe in event loop. Set the pipe to non-blocking mode. - - protocol_factory should instantiate object with Protocol interface. - pipe is a file-like object. - Return pair (transport, protocol), where transport supports the - ReadTransport interface.""" - # The reason to accept file-like object instead of just file descriptor - # is: we need to own pipe and close it at transport finishing - # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. - raise NotImplementedError - - def connect_write_pipe(self, protocol_factory, pipe): - """Register write pipe in event loop. - - protocol_factory should instantiate object with BaseProtocol interface. - Pipe is file-like object already switched to nonblocking. - Return pair (transport, protocol), where transport support - WriteTransport interface.""" - # The reason to accept file-like object instead of just file descriptor - # is: we need to own pipe and close it at transport finishing - # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. - raise NotImplementedError - - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): - raise NotImplementedError - - def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): - raise NotImplementedError - - # Ready-based callback registration methods. - # The add_*() methods return None. - # The remove_*() methods return True if something was removed, - # False if there was nothing to delete. - - def add_reader(self, fd, callback, *args): - raise NotImplementedError - - def remove_reader(self, fd): - raise NotImplementedError - - def add_writer(self, fd, callback, *args): - raise NotImplementedError - - def remove_writer(self, fd): - raise NotImplementedError - - # Completion based I/O methods returning Futures. - - def sock_recv(self, sock, nbytes): - raise NotImplementedError - - def sock_sendall(self, sock, data): - raise NotImplementedError - - def sock_connect(self, sock, address): - raise NotImplementedError - - def sock_accept(self, sock): - raise NotImplementedError - - # Signal handling. - - def add_signal_handler(self, sig, callback, *args): - raise NotImplementedError - - def remove_signal_handler(self, sig): - raise NotImplementedError - - # Task factory. - - def set_task_factory(self, factory): - raise NotImplementedError - - def get_task_factory(self): - raise NotImplementedError - - # Error handlers. - - def set_exception_handler(self, handler): - raise NotImplementedError - - def default_exception_handler(self, context): - raise NotImplementedError - - def call_exception_handler(self, context): - raise NotImplementedError - - # Debug flag management. - - def get_debug(self): - raise NotImplementedError - - def set_debug(self, enabled): - raise NotImplementedError - - -class AbstractEventLoopPolicy: - """Abstract policy for accessing the event loop.""" - - def get_event_loop(self): - """Get the event loop for the current context. - - Returns an event loop object implementing the BaseEventLoop interface, - or raises an exception in case no event loop has been set for the - current context and the current policy does not specify to create one. - - It should never return None.""" - raise NotImplementedError - - def set_event_loop(self, loop): - """Set the event loop for the current context to loop.""" - raise NotImplementedError - - def new_event_loop(self): - """Create and return a new event loop object according to this - policy's rules. If there's need to set this loop as the event loop for - the current context, set_event_loop must be called explicitly.""" - raise NotImplementedError - - # Child processes handling (Unix only). - - def get_child_watcher(self): - "Get the watcher for child processes." - raise NotImplementedError - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - raise NotImplementedError - - -class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): - """Default policy implementation for accessing the event loop. - - In this policy, each thread has its own event loop. However, we - only automatically create an event loop by default for the main - thread; other threads by default have no event loop. - - Other policies may have different rules (e.g. a single global - event loop, or automatically creating an event loop per thread, or - using some other notion of context to which an event loop is - associated). - """ - - _loop_factory = None - - class _Local(threading.local): - _loop = None - _set_called = False - - def __init__(self): - self._local = self._Local() - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - if (self._local._loop is None and - not self._local._set_called and - isinstance(threading.current_thread(), threading._MainThread)): - self.set_event_loop(self.new_event_loop()) - if self._local._loop is None: - raise RuntimeError('There is no current event loop in thread %r.' - % threading.current_thread().name) - return self._local._loop - - def set_event_loop(self, loop): - """Set the event loop.""" - self._local._set_called = True - assert loop is None or isinstance(loop, AbstractEventLoop) - self._local._loop = loop - - def new_event_loop(self): - """Create a new event loop. - - You must call set_event_loop() to make this the current event - loop. - """ - return self._loop_factory() - - -# Event loop policy. The policy itself is always global, even if the -# policy's rules say that there is an event loop per thread (or other -# notion of context). The default policy is installed by the first -# call to get_event_loop_policy(). -_event_loop_policy = None - -# Lock for protecting the on-the-fly creation of the event loop policy. -_lock = threading.Lock() - - -def _init_event_loop_policy(): - global _event_loop_policy - with _lock: - if _event_loop_policy is None: # pragma: no branch - from . import DefaultEventLoopPolicy - _event_loop_policy = DefaultEventLoopPolicy() - - -def get_event_loop_policy(): - """Get the current event loop policy.""" - if _event_loop_policy is None: - _init_event_loop_policy() - return _event_loop_policy - - -def set_event_loop_policy(policy): - """Set the current event loop policy. - - If policy is None, the default policy is restored.""" - global _event_loop_policy - assert policy is None or isinstance(policy, AbstractEventLoopPolicy) - _event_loop_policy = policy - - -def get_event_loop(): - """Equivalent to calling get_event_loop_policy().get_event_loop().""" - return get_event_loop_policy().get_event_loop() - - -def set_event_loop(loop): - """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - get_event_loop_policy().set_event_loop(loop) - - -def new_event_loop(): - """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return get_event_loop_policy().new_event_loop() - - -def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() - - -def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/futures.py --- a/Lib/asyncio/futures.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,453 +0,0 @@ -"""A Future class similar to the one in PEP 3148.""" - -__all__ = ['CancelledError', 'TimeoutError', - 'InvalidStateError', - 'Future', 'wrap_future', - ] - -import concurrent.futures._base -import logging -import reprlib -import sys -import traceback - -from . import compat -from . import events - -# States for Future. -_PENDING = 'PENDING' -_CANCELLED = 'CANCELLED' -_FINISHED = 'FINISHED' - -Error = concurrent.futures._base.Error -CancelledError = concurrent.futures.CancelledError -TimeoutError = concurrent.futures.TimeoutError - -STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging - - -class InvalidStateError(Error): - """The operation is not allowed in this state.""" - - -class _TracebackLogger: - """Helper to log a traceback upon destruction if not cleared. - - This solves a nasty problem with Futures and Tasks that have an - exception set: if nobody asks for the exception, the exception is - never logged. This violates the Zen of Python: 'Errors should - never pass silently. Unless explicitly silenced.' - - However, we don't want to log the exception as soon as - set_exception() is called: if the calling code is written - properly, it will get the exception and handle it properly. But - we *do* want to log it if result() or exception() was never called - -- otherwise developers waste a lot of time wondering why their - buggy code fails silently. - - An earlier attempt added a __del__() method to the Future class - itself, but this backfired because the presence of __del__() - prevents garbage collection from breaking cycles. A way out of - this catch-22 is to avoid having a __del__() method on the Future - class itself, but instead to have a reference to a helper object - with a __del__() method that logs the traceback, where we ensure - that the helper object doesn't participate in cycles, and only the - Future has a reference to it. - - The helper object is added when set_exception() is called. When - the Future is collected, and the helper is present, the helper - object is also collected, and its __del__() method will log the - traceback. When the Future's result() or exception() method is - called (and a helper object is present), it removes the helper - object, after calling its clear() method to prevent it from - logging. - - One downside is that we do a fair amount of work to extract the - traceback from the exception, even when it is never logged. It - would seem cheaper to just store the exception object, but that - references the traceback, which references stack frames, which may - reference the Future, which references the _TracebackLogger, and - then the _TracebackLogger would be included in a cycle, which is - what we're trying to avoid! As an optimization, we don't - immediately format the exception; we only do the work when - activate() is called, which call is delayed until after all the - Future's callbacks have run. Since usually a Future has at least - one callback (typically set by 'yield from') and usually that - callback extracts the callback, thereby removing the need to - format the exception. - - PS. I don't claim credit for this solution. I first heard of it - in a discussion about closing files when they are collected. - """ - - __slots__ = ('loop', 'source_traceback', 'exc', 'tb') - - def __init__(self, future, exc): - self.loop = future._loop - self.source_traceback = future._source_traceback - self.exc = exc - self.tb = None - - def activate(self): - exc = self.exc - if exc is not None: - self.exc = None - self.tb = traceback.format_exception(exc.__class__, exc, - exc.__traceback__) - - def clear(self): - self.exc = None - self.tb = None - - def __del__(self): - if self.tb: - msg = 'Future/Task exception was never retrieved\n' - if self.source_traceback: - src = ''.join(traceback.format_list(self.source_traceback)) - msg += 'Future/Task created at (most recent call last):\n' - msg += '%s\n' % src.rstrip() - msg += ''.join(self.tb).rstrip() - self.loop.call_exception_handler({'message': msg}) - - -class Future: - """This class is *almost* compatible with concurrent.futures.Future. - - Differences: - - - result() and exception() do not take a timeout argument and - raise an exception when the future isn't done yet. - - - Callbacks registered with add_done_callback() are always called - via the event loop's call_soon_threadsafe(). - - - This class is not compatible with the wait() and as_completed() - methods in the concurrent.futures package. - - (In Python 3.4 or later we may be able to unify the implementations.) - """ - - # Class variables serving as defaults for instance variables. - _state = _PENDING - _result = None - _exception = None - _loop = None - _source_traceback = None - - _blocking = False # proper use of future (yield vs yield from) - - _log_traceback = False # Used for Python 3.4 and later - _tb_logger = None # Used for Python 3.3 only - - def __init__(self, *, loop=None): - """Initialize the future. - - The optional event_loop argument allows to explicitly set the event - loop object used by the future. If it's not provided, the future uses - the default event loop. - """ - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - self._callbacks = [] - if self._loop.get_debug(): - self._source_traceback = traceback.extract_stack(sys._getframe(1)) - - def __format_callbacks(self): - cb = self._callbacks - size = len(cb) - if not size: - cb = '' - - def format_cb(callback): - return events._format_callback_source(callback, ()) - - if size == 1: - cb = format_cb(cb[0]) - elif size == 2: - cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) - elif size > 2: - cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), - size-2, - format_cb(cb[-1])) - return 'cb=[%s]' % cb - - def _repr_info(self): - info = [self._state.lower()] - if self._state == _FINISHED: - if self._exception is not None: - info.append('exception={!r}'.format(self._exception)) - else: - # use reprlib to limit the length of the output, especially - # for very long strings - result = reprlib.repr(self._result) - info.append('result={}'.format(result)) - if self._callbacks: - info.append(self.__format_callbacks()) - if self._source_traceback: - frame = self._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) - return info - - def __repr__(self): - info = self._repr_info() - return '<%s %s>' % (self.__class__.__name__, ' '.join(info)) - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._log_traceback: - # set_exception() was not called, or result() or exception() - # has consumed the exception - return - exc = self._exception - context = { - 'message': ('%s exception was never retrieved' - % self.__class__.__name__), - 'exception': exc, - 'future': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - - def cancel(self): - """Cancel the future and schedule callbacks. - - If the future is already done or cancelled, return False. Otherwise, - change the future's state to cancelled, schedule the callbacks and - return True. - """ - if self._state != _PENDING: - return False - self._state = _CANCELLED - self._schedule_callbacks() - return True - - def _schedule_callbacks(self): - """Internal: Ask the event loop to call all callbacks. - - The callbacks are scheduled to be called as soon as possible. Also - clears the callback list. - """ - callbacks = self._callbacks[:] - if not callbacks: - return - - self._callbacks[:] = [] - for callback in callbacks: - self._loop.call_soon(callback, self) - - def cancelled(self): - """Return True if the future was cancelled.""" - return self._state == _CANCELLED - - # Don't implement running(); see http://bugs.python.org/issue18699 - - def done(self): - """Return True if the future is done. - - Done means either that a result / exception are available, or that the - future was cancelled. - """ - return self._state != _PENDING - - def result(self): - """Return the result this future represents. - - If the future has been cancelled, raises CancelledError. If the - future's result isn't yet available, raises InvalidStateError. If - the future is done and has an exception set, this exception is raised. - """ - if self._state == _CANCELLED: - raise CancelledError - if self._state != _FINISHED: - raise InvalidStateError('Result is not ready.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None - if self._exception is not None: - raise self._exception - return self._result - - def exception(self): - """Return the exception that was set on this future. - - The exception (or None if no exception was set) is returned only if - the future is done. If the future has been cancelled, raises - CancelledError. If the future isn't done yet, raises - InvalidStateError. - """ - if self._state == _CANCELLED: - raise CancelledError - if self._state != _FINISHED: - raise InvalidStateError('Exception is not set.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None - return self._exception - - def add_done_callback(self, fn): - """Add a callback to be run when the future becomes done. - - The callback is called with a single argument - the future object. If - the future is already done when this is called, the callback is - scheduled with call_soon. - """ - if self._state != _PENDING: - self._loop.call_soon(fn, self) - else: - self._callbacks.append(fn) - - # New method not in PEP 3148. - - def remove_done_callback(self, fn): - """Remove all instances of a callback from the "call when done" list. - - Returns the number of callbacks removed. - """ - filtered_callbacks = [f for f in self._callbacks if f != fn] - removed_count = len(self._callbacks) - len(filtered_callbacks) - if removed_count: - self._callbacks[:] = filtered_callbacks - return removed_count - - # So-called internal methods (note: no set_running_or_notify_cancel()). - - def set_result(self, result): - """Mark the future done and set its result. - - If the future is already done when this method is called, raises - InvalidStateError. - """ - if self._state != _PENDING: - raise InvalidStateError('{}: {!r}'.format(self._state, self)) - self._result = result - self._state = _FINISHED - self._schedule_callbacks() - - def set_exception(self, exception): - """Mark the future done and set an exception. - - If the future is already done when this method is called, raises - InvalidStateError. - """ - if self._state != _PENDING: - raise InvalidStateError('{}: {!r}'.format(self._state, self)) - if isinstance(exception, type): - exception = exception() - self._exception = exception - self._state = _FINISHED - self._schedule_callbacks() - if compat.PY34: - self._log_traceback = True - else: - self._tb_logger = _TracebackLogger(self, exception) - # Arrange for the logger to be activated after all callbacks - # have had a chance to call result() or exception(). - self._loop.call_soon(self._tb_logger.activate) - - def __iter__(self): - if not self.done(): - self._blocking = True - yield self # This tells Task to wait for completion. - assert self.done(), "yield from wasn't used with future" - return self.result() # May raise too. - - if compat.PY35: - __await__ = __iter__ # make compatible with 'await' expression - - -def _set_result_unless_cancelled(fut, result): - """Helper setting the result only if the future was not cancelled.""" - if fut.cancelled(): - return - fut.set_result(result) - - -def _set_concurrent_future_state(concurrent, source): - """Copy state from a future to a concurrent.futures.Future.""" - assert source.done() - if source.cancelled(): - concurrent.cancel() - if not concurrent.set_running_or_notify_cancel(): - return - exception = source.exception() - if exception is not None: - concurrent.set_exception(exception) - else: - result = source.result() - concurrent.set_result(result) - - -def _copy_future_state(source, dest): - """Internal helper to copy state from another Future. - - The other Future may be a concurrent.futures.Future. - """ - assert source.done() - if dest.cancelled(): - return - assert not dest.done() - if source.cancelled(): - dest.cancel() - else: - exception = source.exception() - if exception is not None: - dest.set_exception(exception) - else: - result = source.result() - dest.set_result(result) - - -def _chain_future(source, destination): - """Chain two futures so that when one completes, so does the other. - - The result (or exception) of source will be copied to destination. - If destination is cancelled, source gets cancelled too. - Compatible with both asyncio.Future and concurrent.futures.Future. - """ - if not isinstance(source, (Future, concurrent.futures.Future)): - raise TypeError('A future is required for source argument') - if not isinstance(destination, (Future, concurrent.futures.Future)): - raise TypeError('A future is required for destination argument') - source_loop = source._loop if isinstance(source, Future) else None - dest_loop = destination._loop if isinstance(destination, Future) else None - - def _set_state(future, other): - if isinstance(future, Future): - _copy_future_state(other, future) - else: - _set_concurrent_future_state(future, other) - - def _call_check_cancel(destination): - if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: - source.cancel() - else: - source_loop.call_soon_threadsafe(source.cancel) - - def _call_set_state(source): - if dest_loop is None or dest_loop is source_loop: - _set_state(destination, source) - else: - dest_loop.call_soon_threadsafe(_set_state, destination, source) - - destination.add_done_callback(_call_check_cancel) - source.add_done_callback(_call_set_state) - - -def wrap_future(future, *, loop=None): - """Wrap concurrent.futures.Future object.""" - if isinstance(future, Future): - return future - assert isinstance(future, concurrent.futures.Future), \ - 'concurrent.futures.Future is expected, got {!r}'.format(future) - new_future = Future(loop=loop) - _chain_future(future, new_future) - return new_future diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/locks.py --- a/Lib/asyncio/locks.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,472 +0,0 @@ -"""Synchronization primitives.""" - -__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore'] - -import collections - -from . import compat -from . import events -from . import futures -from .coroutines import coroutine - - -class _ContextManager: - """Context manager. - - This enables the following idiom for acquiring and releasing a - lock around a block: - - with (yield from lock): - - - while failing loudly when accidentally using: - - with lock: - - """ - - def __init__(self, lock): - self._lock = lock - - def __enter__(self): - # We have no use for the "as ..." clause in the with - # statement for locks. - return None - - def __exit__(self, *args): - try: - self._lock.release() - finally: - self._lock = None # Crudely prevent reuse. - - -class _ContextManagerMixin: - def __enter__(self): - raise RuntimeError( - '"yield from" should be used as context manager expression') - - def __exit__(self, *args): - # This must exist because __enter__ exists, even though that - # always raises; that's how the with-statement works. - pass - - @coroutine - def __iter__(self): - # This is not a coroutine. It is meant to enable the idiom: - # - # with (yield from lock): - # - # - # as an alternative to: - # - # yield from lock.acquire() - # try: - # - # finally: - # lock.release() - yield from self.acquire() - return _ContextManager(self) - - if compat.PY35: - - def __await__(self): - # To make "with await lock" work. - yield from self.acquire() - return _ContextManager(self) - - @coroutine - def __aenter__(self): - yield from self.acquire() - # We have no use for the "as ..." clause in the with - # statement for locks. - return None - - @coroutine - def __aexit__(self, exc_type, exc, tb): - self.release() - - -class Lock(_ContextManagerMixin): - """Primitive lock objects. - - A primitive lock is a synchronization primitive that is not owned - by a particular coroutine when locked. A primitive lock is in one - of two states, 'locked' or 'unlocked'. - - It is created in the unlocked state. It has two basic methods, - acquire() and release(). When the state is unlocked, acquire() - changes the state to locked and returns immediately. When the - state is locked, acquire() blocks until a call to release() in - another coroutine changes it to unlocked, then the acquire() call - resets it to locked and returns. The release() method should only - be called in the locked state; it changes the state to unlocked - and returns immediately. If an attempt is made to release an - unlocked lock, a RuntimeError will be raised. - - When more than one coroutine is blocked in acquire() waiting for - the state to turn to unlocked, only one coroutine proceeds when a - release() call resets the state to unlocked; first coroutine which - is blocked in acquire() is being processed. - - acquire() is a coroutine and should be called with 'yield from'. - - Locks also support the context management protocol. '(yield from lock)' - should be used as context manager expression. - - Usage: - - lock = Lock() - ... - yield from lock - try: - ... - finally: - lock.release() - - Context manager usage: - - lock = Lock() - ... - with (yield from lock): - ... - - Lock objects can be tested for locking state: - - if not lock.locked(): - yield from lock - else: - # lock is acquired - ... - - """ - - def __init__(self, *, loop=None): - self._waiters = collections.deque() - self._locked = False - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() - - def __repr__(self): - res = super().__repr__() - extra = 'locked' if self._locked else 'unlocked' - if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) - - def locked(self): - """Return True if lock is acquired.""" - return self._locked - - @coroutine - def acquire(self): - """Acquire a lock. - - This method blocks until the lock is unlocked, then sets it to - locked and returns True. - """ - if not self._waiters and not self._locked: - self._locked = True - return True - - fut = futures.Future(loop=self._loop) - self._waiters.append(fut) - try: - yield from fut - self._locked = True - return True - finally: - self._waiters.remove(fut) - - def release(self): - """Release a lock. - - When the lock is locked, reset it to unlocked, and return. - If any other coroutines are blocked waiting for the lock to become - unlocked, allow exactly one of them to proceed. - - When invoked on an unlocked lock, a RuntimeError is raised. - - There is no return value. - """ - if self._locked: - self._locked = False - # Wake up the first waiter who isn't cancelled. - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - break - else: - raise RuntimeError('Lock is not acquired.') - - -class Event: - """Asynchronous equivalent to threading.Event. - - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. - """ - - def __init__(self, *, loop=None): - self._waiters = collections.deque() - self._value = False - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() - - def __repr__(self): - res = super().__repr__() - extra = 'set' if self._value else 'unset' - if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) - - def is_set(self): - """Return True if and only if the internal flag is true.""" - return self._value - - def set(self): - """Set the internal flag to true. All coroutines waiting for it to - become true are awakened. Coroutine that call wait() once the flag is - true will not block at all. - """ - if not self._value: - self._value = True - - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - - def clear(self): - """Reset the internal flag to false. Subsequently, coroutines calling - wait() will block until set() is called to set the internal flag - to true again.""" - self._value = False - - @coroutine - def wait(self): - """Block until the internal flag is true. - - If the internal flag is true on entry, return True - immediately. Otherwise, block until another coroutine calls - set() to set the flag to true, then return True. - """ - if self._value: - return True - - fut = futures.Future(loop=self._loop) - self._waiters.append(fut) - try: - yield from fut - return True - finally: - self._waiters.remove(fut) - - -class Condition(_ContextManagerMixin): - """Asynchronous equivalent to threading.Condition. - - This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. - - A new Lock object is created and used as the underlying lock. - """ - - def __init__(self, lock=None, *, loop=None): - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() - - if lock is None: - lock = Lock(loop=self._loop) - elif lock._loop is not self._loop: - raise ValueError("loop argument must agree with lock") - - self._lock = lock - # Export the lock's locked(), acquire() and release() methods. - self.locked = lock.locked - self.acquire = lock.acquire - self.release = lock.release - - self._waiters = collections.deque() - - def __repr__(self): - res = super().__repr__() - extra = 'locked' if self.locked() else 'unlocked' - if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) - - @coroutine - def wait(self): - """Wait until notified. - - If the calling coroutine has not acquired the lock when this - method is called, a RuntimeError is raised. - - This method releases the underlying lock, and then blocks - until it is awakened by a notify() or notify_all() call for - the same condition variable in another coroutine. Once - awakened, it re-acquires the lock and returns True. - """ - if not self.locked(): - raise RuntimeError('cannot wait on un-acquired lock') - - self.release() - try: - fut = futures.Future(loop=self._loop) - self._waiters.append(fut) - try: - yield from fut - return True - finally: - self._waiters.remove(fut) - - finally: - yield from self.acquire() - - @coroutine - def wait_for(self, predicate): - """Wait until a predicate becomes true. - - The predicate should be a callable which result will be - interpreted as a boolean value. The final predicate value is - the return value. - """ - result = predicate() - while not result: - yield from self.wait() - result = predicate() - return result - - def notify(self, n=1): - """By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method - is called, a RuntimeError is raised. - - This method wakes up at most n of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. - - Note: an awakened coroutine does not actually return from its - wait() call until it can reacquire the lock. Since notify() does - not release the lock, its caller should. - """ - if not self.locked(): - raise RuntimeError('cannot notify on un-acquired lock') - - idx = 0 - for fut in self._waiters: - if idx >= n: - break - - if not fut.done(): - idx += 1 - fut.set_result(False) - - def notify_all(self): - """Wake up all threads waiting on this condition. This method acts - like notify(), but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, - a RuntimeError is raised. - """ - self.notify(len(self._waiters)) - - -class Semaphore(_ContextManagerMixin): - """A Semaphore implementation. - - A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). - - Semaphores also support the context management protocol. - - The optional argument gives the initial value for the internal - counter; it defaults to 1. If the value given is less than 0, - ValueError is raised. - """ - - def __init__(self, value=1, *, loop=None): - if value < 0: - raise ValueError("Semaphore initial value must be >= 0") - self._value = value - self._waiters = collections.deque() - if loop is not None: - self._loop = loop - else: - self._loop = events.get_event_loop() - - def __repr__(self): - res = super().__repr__() - extra = 'locked' if self.locked() else 'unlocked,value:{}'.format( - self._value) - if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) - - def _wake_up_next(self): - while self._waiters: - waiter = self._waiters.popleft() - if not waiter.done(): - waiter.set_result(None) - return - - def locked(self): - """Returns True if semaphore can not be acquired immediately.""" - return self._value == 0 - - @coroutine - def acquire(self): - """Acquire a semaphore. - - If the internal counter is larger than zero on entry, - decrement it by one and return True immediately. If it is - zero on entry, block, waiting until some other coroutine has - called release() to make it larger than 0, and then return - True. - """ - while self._value <= 0: - fut = futures.Future(loop=self._loop) - self._waiters.append(fut) - try: - yield from fut - except: - # See the similar code in Queue.get. - fut.cancel() - if self._value > 0 and not fut.cancelled(): - self._wake_up_next() - raise - self._value -= 1 - return True - - def release(self): - """Release a semaphore, incrementing the internal counter by one. - When it was zero on entry and another coroutine is waiting for it to - become larger than zero again, wake up that coroutine. - """ - self._value += 1 - self._wake_up_next() - - -class BoundedSemaphore(Semaphore): - """A bounded semaphore implementation. - - This raises ValueError in release() if it would increase the value - above the initial value. - """ - - def __init__(self, value=1, *, loop=None): - self._bound_value = value - super().__init__(value, loop=loop) - - def release(self): - if self._value >= self._bound_value: - raise ValueError('BoundedSemaphore released too many times') - super().release() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/log.py --- a/Lib/asyncio/log.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,7 +0,0 @@ -"""Logging configuration.""" - -import logging - - -# Name the logger after the package. -logger = logging.getLogger(__package__) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/proactor_events.py --- a/Lib/asyncio/proactor_events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,550 +0,0 @@ -"""Event loop using a proactor and related classes. - -A proactor is a "notify-on-completion" multiplexer. Currently a -proactor is only implemented on Windows with IOCP. -""" - -__all__ = ['BaseProactorEventLoop'] - -import socket -import warnings - -from . import base_events -from . import compat -from . import constants -from . import futures -from . import sslproto -from . import transports -from .log import logger - - -class _ProactorBasePipeTransport(transports._FlowControlMixin, - transports.BaseTransport): - """Base class for pipe and socket transports.""" - - def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - super().__init__(extra, loop) - self._set_extra(sock) - self._sock = sock - self._protocol = protocol - self._server = server - self._buffer = None # None or bytearray. - self._read_fut = None - self._write_fut = None - self._pending_write = 0 - self._conn_lost = 0 - self._closing = False # Set when close() called. - self._eof_written = False - if self._server is not None: - self._server._attach() - self._loop.call_soon(self._protocol.connection_made, self) - if waiter is not None: - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) - - def __repr__(self): - info = [self.__class__.__name__] - if self._sock is None: - info.append('closed') - elif self._closing: - info.append('closing') - if self._sock is not None: - info.append('fd=%s' % self._sock.fileno()) - if self._read_fut is not None: - info.append('read=%s' % self._read_fut) - if self._write_fut is not None: - info.append("write=%r" % self._write_fut) - if self._buffer: - bufsize = len(self._buffer) - info.append('write_bufsize=%s' % bufsize) - if self._eof_written: - info.append('EOF written') - return '<%s>' % ' '.join(info) - - def _set_extra(self, sock): - self._extra['pipe'] = sock - - def is_closing(self): - return self._closing - - def close(self): - if self._closing: - return - self._closing = True - self._conn_lost += 1 - if not self._buffer and self._write_fut is None: - self._loop.call_soon(self._call_connection_lost, None) - if self._read_fut is not None: - self._read_fut.cancel() - self._read_fut = None - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self.close() - - def _fatal_error(self, exc, message='Fatal error on pipe transport'): - if isinstance(exc, (BrokenPipeError, ConnectionResetError)): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._force_close(exc) - - def _force_close(self, exc): - if self._closing: - return - self._closing = True - self._conn_lost += 1 - if self._write_fut: - self._write_fut.cancel() - self._write_fut = None - if self._read_fut: - self._read_fut.cancel() - self._read_fut = None - self._pending_write = 0 - self._buffer = None - self._loop.call_soon(self._call_connection_lost, exc) - - def _call_connection_lost(self, exc): - try: - self._protocol.connection_lost(exc) - finally: - # XXX If there is a pending overlapped read on the other - # end then it may fail with ERROR_NETNAME_DELETED if we - # just close our end. First calling shutdown() seems to - # cure it, but maybe using DisconnectEx() would be better. - if hasattr(self._sock, 'shutdown'): - self._sock.shutdown(socket.SHUT_RDWR) - self._sock.close() - self._sock = None - server = self._server - if server is not None: - server._detach() - self._server = None - - def get_write_buffer_size(self): - size = self._pending_write - if self._buffer is not None: - size += len(self._buffer) - return size - - -class _ProactorReadPipeTransport(_ProactorBasePipeTransport, - transports.ReadTransport): - """Transport for read pipes.""" - - def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - super().__init__(loop, sock, protocol, waiter, extra, server) - self._paused = False - self._loop.call_soon(self._loop_reading) - - def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) - - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop.call_soon(self._loop_reading, self._read_fut) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) - - def _loop_reading(self, fut=None): - if self._paused: - return - data = None - - try: - if fut is not None: - assert self._read_fut is fut or (self._read_fut is None and - self._closing) - self._read_fut = None - data = fut.result() # deliver data later in "finally" clause - - if self._closing: - # since close() has been called we ignore any read data - data = None - return - - if data == b'': - # we got end-of-file so no need to reschedule a new read - return - - # reschedule a new read - self._read_fut = self._loop._proactor.recv(self._sock, 4096) - except ConnectionAbortedError as exc: - if not self._closing: - self._fatal_error(exc, 'Fatal read error on pipe transport') - elif self._loop.get_debug(): - logger.debug("Read error on pipe transport while closing", - exc_info=True) - except ConnectionResetError as exc: - self._force_close(exc) - except OSError as exc: - self._fatal_error(exc, 'Fatal read error on pipe transport') - except futures.CancelledError: - if not self._closing: - raise - else: - self._read_fut.add_done_callback(self._loop_reading) - finally: - if data: - self._protocol.data_received(data) - elif data is not None: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if not keep_open: - self.close() - - -class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, - transports.WriteTransport): - """Transport for write pipes.""" - - def write(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - if self._eof_written: - raise RuntimeError('write_eof() already called') - - if not data: - return - - if self._conn_lost: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('socket.send() raised exception.') - self._conn_lost += 1 - return - - # Observable states: - # 1. IDLE: _write_fut and _buffer both None - # 2. WRITING: _write_fut set; _buffer None - # 3. BACKED UP: _write_fut set; _buffer a bytearray - # We always copy the data, so the caller can't modify it - # while we're still waiting for the I/O to happen. - if self._write_fut is None: # IDLE -> WRITING - assert self._buffer is None - # Pass a copy, except if it's already immutable. - self._loop_writing(data=bytes(data)) - elif not self._buffer: # WRITING -> BACKED UP - # Make a mutable copy which we can extend. - self._buffer = bytearray(data) - self._maybe_pause_protocol() - else: # BACKED UP - # Append to buffer (also copies). - self._buffer.extend(data) - self._maybe_pause_protocol() - - def _loop_writing(self, f=None, data=None): - try: - assert f is self._write_fut - self._write_fut = None - self._pending_write = 0 - if f: - f.result() - if data is None: - data = self._buffer - self._buffer = None - if not data: - if self._closing: - self._loop.call_soon(self._call_connection_lost, None) - if self._eof_written: - self._sock.shutdown(socket.SHUT_WR) - # Now that we've reduced the buffer size, tell the - # protocol to resume writing if it was paused. Note that - # we do this last since the callback is called immediately - # and it may add more data to the buffer (even causing the - # protocol to be paused again). - self._maybe_resume_protocol() - else: - self._write_fut = self._loop._proactor.send(self._sock, data) - if not self._write_fut.done(): - assert self._pending_write == 0 - self._pending_write = len(data) - self._write_fut.add_done_callback(self._loop_writing) - self._maybe_pause_protocol() - else: - self._write_fut.add_done_callback(self._loop_writing) - except ConnectionResetError as exc: - self._force_close(exc) - except OSError as exc: - self._fatal_error(exc, 'Fatal write error on pipe transport') - - def can_write_eof(self): - return True - - def write_eof(self): - self.close() - - def abort(self): - self._force_close(None) - - -class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport): - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self._read_fut = self._loop._proactor.recv(self._sock, 16) - self._read_fut.add_done_callback(self._pipe_closed) - - def _pipe_closed(self, fut): - if fut.cancelled(): - # the transport has been closed - return - assert fut.result() == b'' - if self._closing: - assert self._read_fut is None - return - assert fut is self._read_fut, (fut, self._read_fut) - self._read_fut = None - if self._write_fut is not None: - self._force_close(BrokenPipeError()) - else: - self.close() - - -class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport, - _ProactorBaseWritePipeTransport, - transports.Transport): - """Transport for duplex pipes.""" - - def can_write_eof(self): - return False - - def write_eof(self): - raise NotImplementedError - - -class _ProactorSocketTransport(_ProactorReadPipeTransport, - _ProactorBaseWritePipeTransport, - transports.Transport): - """Transport for connected sockets.""" - - def _set_extra(self, sock): - self._extra['socket'] = sock - try: - self._extra['sockname'] = sock.getsockname() - except (socket.error, AttributeError): - if self._loop.get_debug(): - logger.warning("getsockname() failed on %r", - sock, exc_info=True) - if 'peername' not in self._extra: - try: - self._extra['peername'] = sock.getpeername() - except (socket.error, AttributeError): - if self._loop.get_debug(): - logger.warning("getpeername() failed on %r", - sock, exc_info=True) - - def can_write_eof(self): - return True - - def write_eof(self): - if self._closing or self._eof_written: - return - self._eof_written = True - if self._write_fut is None: - self._sock.shutdown(socket.SHUT_WR) - - -class BaseProactorEventLoop(base_events.BaseEventLoop): - - def __init__(self, proactor): - super().__init__() - logger.debug('Using proactor: %s', proactor.__class__.__name__) - self._proactor = proactor - self._selector = proactor # convenient alias - self._self_reading_future = None - self._accept_futures = {} # socket file descriptor => Future - proactor.set_loop(self) - self._make_self_pipe() - - def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): - return _ProactorSocketTransport(self, sock, protocol, waiter, - extra, server) - - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - if not sslproto._is_sslproto_available(): - raise NotImplementedError("Proactor event loop requires Python 3.5" - " or newer (ssl.MemoryBIO) to support " - "SSL") - - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) - _ProactorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) - return ssl_protocol._app_transport - - def _make_duplex_pipe_transport(self, sock, protocol, waiter=None, - extra=None): - return _ProactorDuplexPipeTransport(self, - sock, protocol, waiter, extra) - - def _make_read_pipe_transport(self, sock, protocol, waiter=None, - extra=None): - return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra) - - def _make_write_pipe_transport(self, sock, protocol, waiter=None, - extra=None): - # We want connection_lost() to be called when other end closes - return _ProactorWritePipeTransport(self, - sock, protocol, waiter, extra) - - def close(self): - if self.is_running(): - raise RuntimeError("Cannot close a running event loop") - if self.is_closed(): - return - - # Call these methods before closing the event loop (before calling - # BaseEventLoop.close), because they can schedule callbacks with - # call_soon(), which is forbidden when the event loop is closed. - self._stop_accept_futures() - self._close_self_pipe() - self._proactor.close() - self._proactor = None - self._selector = None - - # Close the event loop - super().close() - - def sock_recv(self, sock, n): - return self._proactor.recv(sock, n) - - def sock_sendall(self, sock, data): - return self._proactor.send(sock, data) - - def sock_connect(self, sock, address): - try: - base_events._check_resolved_address(sock, address) - except ValueError as err: - fut = futures.Future(loop=self) - fut.set_exception(err) - return fut - else: - return self._proactor.connect(sock, address) - - def sock_accept(self, sock): - return self._proactor.accept(sock) - - def _socketpair(self): - raise NotImplementedError - - def _close_self_pipe(self): - if self._self_reading_future is not None: - self._self_reading_future.cancel() - self._self_reading_future = None - self._ssock.close() - self._ssock = None - self._csock.close() - self._csock = None - self._internal_fds -= 1 - - def _make_self_pipe(self): - # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() - self._ssock.setblocking(False) - self._csock.setblocking(False) - self._internal_fds += 1 - self.call_soon(self._loop_self_reading) - - def _loop_self_reading(self, f=None): - try: - if f is not None: - f.result() # may raise - f = self._proactor.recv(self._ssock, 4096) - except futures.CancelledError: - # _close_self_pipe() has been called, stop waiting for data - return - except Exception as exc: - self.call_exception_handler({ - 'message': 'Error on reading from the event loop self pipe', - 'exception': exc, - 'loop': self, - }) - else: - self._self_reading_future = f - f.add_done_callback(self._loop_self_reading) - - def _write_to_self(self): - self._csock.send(b'\0') - - def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None): - - def loop(f=None): - try: - if f is not None: - conn, addr = f.result() - if self._debug: - logger.debug("%r got a new connection from %r: %r", - server, addr, conn) - protocol = protocol_factory() - if sslcontext is not None: - self._make_ssl_transport( - conn, protocol, sslcontext, server_side=True, - extra={'peername': addr}, server=server) - else: - self._make_socket_transport( - conn, protocol, - extra={'peername': addr}, server=server) - if self.is_closed(): - return - f = self._proactor.accept(sock) - except OSError as exc: - if sock.fileno() != -1: - self.call_exception_handler({ - 'message': 'Accept failed on a socket', - 'exception': exc, - 'socket': sock, - }) - sock.close() - elif self._debug: - logger.debug("Accept failed on socket %r", - sock, exc_info=True) - except futures.CancelledError: - sock.close() - else: - self._accept_futures[sock.fileno()] = f - f.add_done_callback(loop) - - self.call_soon(loop) - - def _process_events(self, event_list): - # Events are processed in the IocpProactor._poll() method - pass - - def _stop_accept_futures(self): - for future in self._accept_futures.values(): - future.cancel() - self._accept_futures.clear() - - def _stop_serving(self, sock): - self._stop_accept_futures() - self._proactor._stop_serving(sock) - sock.close() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/protocols.py --- a/Lib/asyncio/protocols.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -"""Abstract Protocol class.""" - -__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol', - 'SubprocessProtocol'] - - -class BaseProtocol: - """Common base class for protocol interfaces. - - Usually user implements protocols that derived from BaseProtocol - like Protocol or ProcessProtocol. - - The only case when BaseProtocol should be implemented directly is - write-only transport like write pipe - """ - - def connection_made(self, transport): - """Called when a connection is made. - - The argument is the transport representing the pipe connection. - To receive data, wait for data_received() calls. - When the connection is closed, connection_lost() is called. - """ - - def connection_lost(self, exc): - """Called when the connection is lost or closed. - - The argument is an exception object or None (the latter - meaning a regular EOF is received or the connection was - aborted or closed). - """ - - def pause_writing(self): - """Called when the transport's buffer goes over the high-water mark. - - Pause and resume calls are paired -- pause_writing() is called - once when the buffer goes strictly over the high-water mark - (even if subsequent writes increases the buffer size even - more), and eventually resume_writing() is called once when the - buffer size reaches the low-water mark. - - Note that if the buffer size equals the high-water mark, - pause_writing() is not called -- it must go strictly over. - Conversely, resume_writing() is called when the buffer size is - equal or lower than the low-water mark. These end conditions - are important to ensure that things go as expected when either - mark is zero. - - NOTE: This is the only Protocol callback that is not called - through EventLoop.call_soon() -- if it were, it would have no - effect when it's most needed (when the app keeps writing - without yielding until pause_writing() is called). - """ - - def resume_writing(self): - """Called when the transport's buffer drains below the low-water mark. - - See pause_writing() for details. - """ - - -class Protocol(BaseProtocol): - """Interface for stream protocol. - - The user should implement this interface. They can inherit from - this class but don't need to. The implementations here do - nothing (they don't raise exceptions). - - When the user wants to requests a transport, they pass a protocol - factory to a utility function (e.g., EventLoop.create_connection()). - - When the connection is made successfully, connection_made() is - called with a suitable transport object. Then data_received() - will be called 0 or more times with data (bytes) received from the - transport; finally, connection_lost() will be called exactly once - with either an exception object or None as an argument. - - State machine of calls: - - start -> CM [-> DR*] [-> ER?] -> CL -> end - - * CM: connection_made() - * DR: data_received() - * ER: eof_received() - * CL: connection_lost() - """ - - def data_received(self, data): - """Called when some data is received. - - The argument is a bytes object. - """ - - def eof_received(self): - """Called when the other end calls write_eof() or equivalent. - - If this returns a false value (including None), the transport - will close itself. If it returns a true value, closing the - transport is up to the protocol. - """ - - -class DatagramProtocol(BaseProtocol): - """Interface for datagram protocol.""" - - def datagram_received(self, data, addr): - """Called when some datagram is received.""" - - def error_received(self, exc): - """Called when a send or receive operation raises an OSError. - - (Other than BlockingIOError or InterruptedError.) - """ - - -class SubprocessProtocol(BaseProtocol): - """Interface for protocol for subprocess calls.""" - - def pipe_data_received(self, fd, data): - """Called when the subprocess writes data into stdout/stderr pipe. - - fd is int file descriptor. - data is bytes object. - """ - - def pipe_connection_lost(self, fd, exc): - """Called when a file descriptor associated with the child process is - closed. - - fd is the int file descriptor that was closed. - """ - - def process_exited(self): - """Called when subprocess has exited.""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/queues.py --- a/Lib/asyncio/queues.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,254 +0,0 @@ -"""Queues""" - -__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] - -import collections -import heapq - -from . import compat -from . import events -from . import futures -from . import locks -from .coroutines import coroutine - - -class QueueEmpty(Exception): - """Exception raised when Queue.get_nowait() is called on a Queue object - which is empty. - """ - pass - - -class QueueFull(Exception): - """Exception raised when the Queue.put_nowait() method is called on a Queue - object which is full. - """ - pass - - -class Queue: - """A queue, useful for coordinating producer and consumer coroutines. - - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "yield from put()" will block when the - queue reaches maxsize, until an item is removed by get(). - - Unlike the standard library Queue, you can reliably know this Queue's size - with qsize(), since your single-threaded asyncio application won't be - interrupted between calling qsize() and doing an operation on the Queue. - """ - - def __init__(self, maxsize=0, *, loop=None): - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - self._maxsize = maxsize - - # Futures. - self._getters = collections.deque() - # Futures. - self._putters = collections.deque() - self._unfinished_tasks = 0 - self._finished = locks.Event(loop=self._loop) - self._finished.set() - self._init(maxsize) - - # These three are overridable in subclasses. - - def _init(self, maxsize): - self._queue = collections.deque() - - def _get(self): - return self._queue.popleft() - - def _put(self, item): - self._queue.append(item) - - # End of the overridable methods. - - def _wakeup_next(self, waiters): - # Wake up the next waiter (if any) that isn't cancelled. - while waiters: - waiter = waiters.popleft() - if not waiter.done(): - waiter.set_result(None) - break - - def __repr__(self): - return '<{} at {:#x} {}>'.format( - type(self).__name__, id(self), self._format()) - - def __str__(self): - return '<{} {}>'.format(type(self).__name__, self._format()) - - def _format(self): - result = 'maxsize={!r}'.format(self._maxsize) - if getattr(self, '_queue', None): - result += ' _queue={!r}'.format(list(self._queue)) - if self._getters: - result += ' _getters[{}]'.format(len(self._getters)) - if self._putters: - result += ' _putters[{}]'.format(len(self._putters)) - if self._unfinished_tasks: - result += ' tasks={}'.format(self._unfinished_tasks) - return result - - def qsize(self): - """Number of items in the queue.""" - return len(self._queue) - - @property - def maxsize(self): - """Number of items allowed in the queue.""" - return self._maxsize - - def empty(self): - """Return True if the queue is empty, False otherwise.""" - return not self._queue - - def full(self): - """Return True if there are maxsize items in the queue. - - Note: if the Queue was initialized with maxsize=0 (the default), - then full() is never True. - """ - if self._maxsize <= 0: - return False - else: - return self.qsize() >= self._maxsize - - @coroutine - def put(self, item): - """Put an item into the queue. - - Put an item into the queue. If the queue is full, wait until a free - slot is available before adding item. - - This method is a coroutine. - """ - while self.full(): - putter = futures.Future(loop=self._loop) - self._putters.append(putter) - try: - yield from putter - except: - putter.cancel() # Just in case putter is not done yet. - if not self.full() and not putter.cancelled(): - # We were woken up by get_nowait(), but can't take - # the call. Wake up the next in line. - self._wakeup_next(self._putters) - raise - return self.put_nowait(item) - - def put_nowait(self, item): - """Put an item into the queue without blocking. - - If no free slot is immediately available, raise QueueFull. - """ - if self.full(): - raise QueueFull - self._put(item) - self._unfinished_tasks += 1 - self._finished.clear() - self._wakeup_next(self._getters) - - @coroutine - def get(self): - """Remove and return an item from the queue. - - If queue is empty, wait until an item is available. - - This method is a coroutine. - """ - while self.empty(): - getter = futures.Future(loop=self._loop) - self._getters.append(getter) - try: - yield from getter - except: - getter.cancel() # Just in case getter is not done yet. - if not self.empty() and not getter.cancelled(): - # We were woken up by put_nowait(), but can't take - # the call. Wake up the next in line. - self._wakeup_next(self._getters) - raise - return self.get_nowait() - - def get_nowait(self): - """Remove and return an item from the queue. - - Return an item if one is immediately available, else raise QueueEmpty. - """ - if self.empty(): - raise QueueEmpty - item = self._get() - self._wakeup_next(self._putters) - return item - - def task_done(self): - """Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each get() used to fetch a task, - a subsequent call to task_done() tells the queue that the processing - on the task is complete. - - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). - - Raises ValueError if called more times than there were items placed in - the queue. - """ - if self._unfinished_tasks <= 0: - raise ValueError('task_done() called too many times') - self._unfinished_tasks -= 1 - if self._unfinished_tasks == 0: - self._finished.set() - - @coroutine - def join(self): - """Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. - """ - if self._unfinished_tasks > 0: - yield from self._finished.wait() - - -class PriorityQueue(Queue): - """A subclass of Queue; retrieves entries in priority order (lowest first). - - Entries are typically tuples of the form: (priority number, data). - """ - - def _init(self, maxsize): - self._queue = [] - - def _put(self, item, heappush=heapq.heappush): - heappush(self._queue, item) - - def _get(self, heappop=heapq.heappop): - return heappop(self._queue) - - -class LifoQueue(Queue): - """A subclass of Queue that retrieves most recently added entries first.""" - - def _init(self, maxsize): - self._queue = [] - - def _put(self, item): - self._queue.append(item) - - def _get(self): - return self._queue.pop() - - -if not compat.PY35: - JoinableQueue = Queue - """Deprecated alias for Queue.""" - __all__.append('JoinableQueue') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/selector_events.py --- a/Lib/asyncio/selector_events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1073 +0,0 @@ -"""Event loop using a selector and related classes. - -A selector is a "notify-when-ready" multiplexer. For a subclass which -also includes support for signal handling, see the unix_events sub-module. -""" - -__all__ = ['BaseSelectorEventLoop'] - -import collections -import errno -import functools -import socket -import warnings -try: - import ssl -except ImportError: # pragma: no cover - ssl = None - -from . import base_events -from . import compat -from . import constants -from . import events -from . import futures -from . import selectors -from . import transports -from . import sslproto -from .coroutines import coroutine -from .log import logger - - -def _test_selector_event(selector, fd, event): - # Test if the selector is monitoring 'event' events - # for the file descriptor 'fd'. - try: - key = selector.get_key(fd) - except KeyError: - return False - else: - return bool(key.events & event) - - -class BaseSelectorEventLoop(base_events.BaseEventLoop): - """Selector event loop. - - See events.EventLoop for API specification. - """ - - def __init__(self, selector=None): - super().__init__() - - if selector is None: - selector = selectors.DefaultSelector() - logger.debug('Using selector: %s', selector.__class__.__name__) - self._selector = selector - self._make_self_pipe() - - def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): - return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) - - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - if not sslproto._is_sslproto_available(): - return self._make_legacy_ssl_transport( - rawsock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname, - extra=extra, server=server) - - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) - _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) - return ssl_protocol._app_transport - - def _make_legacy_ssl_transport(self, rawsock, protocol, sslcontext, - waiter, *, - server_side=False, server_hostname=None, - extra=None, server=None): - # Use the legacy API: SSL_write, SSL_read, etc. The legacy API is used - # on Python 3.4 and older, when ssl.MemoryBIO is not available. - return _SelectorSslTransport( - self, rawsock, protocol, sslcontext, waiter, - server_side, server_hostname, extra, server) - - def _make_datagram_transport(self, sock, protocol, - address=None, waiter=None, extra=None): - return _SelectorDatagramTransport(self, sock, protocol, - address, waiter, extra) - - def close(self): - if self.is_running(): - raise RuntimeError("Cannot close a running event loop") - if self.is_closed(): - return - self._close_self_pipe() - super().close() - if self._selector is not None: - self._selector.close() - self._selector = None - - def _socketpair(self): - raise NotImplementedError - - def _close_self_pipe(self): - self.remove_reader(self._ssock.fileno()) - self._ssock.close() - self._ssock = None - self._csock.close() - self._csock = None - self._internal_fds -= 1 - - def _make_self_pipe(self): - # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() - self._ssock.setblocking(False) - self._csock.setblocking(False) - self._internal_fds += 1 - self.add_reader(self._ssock.fileno(), self._read_from_self) - - def _process_self_data(self, data): - pass - - def _read_from_self(self): - while True: - try: - data = self._ssock.recv(4096) - if not data: - break - self._process_self_data(data) - except InterruptedError: - continue - except BlockingIOError: - break - - def _write_to_self(self): - # This may be called from a different thread, possibly after - # _close_self_pipe() has been called or even while it is - # running. Guard for self._csock being None or closed. When - # a socket is closed, send() raises OSError (with errno set to - # EBADF, but let's not rely on the exact error code). - csock = self._csock - if csock is not None: - try: - csock.send(b'\0') - except OSError: - if self._debug: - logger.debug("Fail to write a null byte into the " - "self-pipe socket", - exc_info=True) - - def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None): - self.add_reader(sock.fileno(), self._accept_connection, - protocol_factory, sock, sslcontext, server) - - def _accept_connection(self, protocol_factory, sock, - sslcontext=None, server=None): - try: - conn, addr = sock.accept() - if self._debug: - logger.debug("%r got a new connection from %r: %r", - server, addr, conn) - conn.setblocking(False) - except (BlockingIOError, InterruptedError, ConnectionAbortedError): - pass # False alarm. - except OSError as exc: - # There's nowhere to send the error, so just log it. - if exc.errno in (errno.EMFILE, errno.ENFILE, - errno.ENOBUFS, errno.ENOMEM): - # Some platforms (e.g. Linux keep reporting the FD as - # ready, so we remove the read handler temporarily. - # We'll try again in a while. - self.call_exception_handler({ - 'message': 'socket.accept() out of system resource', - 'exception': exc, - 'socket': sock, - }) - self.remove_reader(sock.fileno()) - self.call_later(constants.ACCEPT_RETRY_DELAY, - self._start_serving, - protocol_factory, sock, sslcontext, server) - else: - raise # The event loop will catch, log and ignore it. - else: - extra = {'peername': addr} - accept = self._accept_connection2(protocol_factory, conn, extra, - sslcontext, server) - self.create_task(accept) - - @coroutine - def _accept_connection2(self, protocol_factory, conn, extra, - sslcontext=None, server=None): - protocol = None - transport = None - try: - protocol = protocol_factory() - waiter = futures.Future(loop=self) - if sslcontext: - transport = self._make_ssl_transport( - conn, protocol, sslcontext, waiter=waiter, - server_side=True, extra=extra, server=server) - else: - transport = self._make_socket_transport( - conn, protocol, waiter=waiter, extra=extra, - server=server) - - try: - yield from waiter - except: - transport.close() - raise - - # It's now up to the protocol to handle the connection. - except Exception as exc: - if self._debug: - context = { - 'message': ('Error on transport creation ' - 'for incoming connection'), - 'exception': exc, - } - if protocol is not None: - context['protocol'] = protocol - if transport is not None: - context['transport'] = transport - self.call_exception_handler(context) - - def add_reader(self, fd, callback, *args): - """Add a reader callback.""" - self._check_closed() - handle = events.Handle(callback, args, self) - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, selectors.EVENT_READ, - (handle, None)) - else: - mask, (reader, writer) = key.events, key.data - self._selector.modify(fd, mask | selectors.EVENT_READ, - (handle, writer)) - if reader is not None: - reader.cancel() - - def remove_reader(self, fd): - """Remove a reader callback.""" - if self.is_closed(): - return False - try: - key = self._selector.get_key(fd) - except KeyError: - return False - else: - mask, (reader, writer) = key.events, key.data - mask &= ~selectors.EVENT_READ - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (None, writer)) - - if reader is not None: - reader.cancel() - return True - else: - return False - - def add_writer(self, fd, callback, *args): - """Add a writer callback..""" - self._check_closed() - handle = events.Handle(callback, args, self) - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, selectors.EVENT_WRITE, - (None, handle)) - else: - mask, (reader, writer) = key.events, key.data - self._selector.modify(fd, mask | selectors.EVENT_WRITE, - (reader, handle)) - if writer is not None: - writer.cancel() - - def remove_writer(self, fd): - """Remove a writer callback.""" - if self.is_closed(): - return False - try: - key = self._selector.get_key(fd) - except KeyError: - return False - else: - mask, (reader, writer) = key.events, key.data - # Remove both writer and connector. - mask &= ~selectors.EVENT_WRITE - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (reader, None)) - - if writer is not None: - writer.cancel() - return True - else: - return False - - def sock_recv(self, sock, n): - """Receive data from the socket. - - The return value is a bytes object representing the data received. - The maximum amount of data to be received at once is specified by - nbytes. - - This method is a coroutine. - """ - if self._debug and sock.gettimeout() != 0: - raise ValueError("the socket must be non-blocking") - fut = futures.Future(loop=self) - self._sock_recv(fut, False, sock, n) - return fut - - def _sock_recv(self, fut, registered, sock, n): - # _sock_recv() can add itself as an I/O callback if the operation can't - # be done immediately. Don't use it directly, call sock_recv(). - fd = sock.fileno() - if registered: - # Remove the callback early. It should be rare that the - # selector says the fd is ready but the call still returns - # EAGAIN, and I am willing to take a hit in that case in - # order to simplify the common case. - self.remove_reader(fd) - if fut.cancelled(): - return - try: - data = sock.recv(n) - except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_recv, fut, True, sock, n) - except Exception as exc: - fut.set_exception(exc) - else: - fut.set_result(data) - - def sock_sendall(self, sock, data): - """Send data to the socket. - - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. - - This method is a coroutine. - """ - if self._debug and sock.gettimeout() != 0: - raise ValueError("the socket must be non-blocking") - fut = futures.Future(loop=self) - if data: - self._sock_sendall(fut, False, sock, data) - else: - fut.set_result(None) - return fut - - def _sock_sendall(self, fut, registered, sock, data): - fd = sock.fileno() - - if registered: - self.remove_writer(fd) - if fut.cancelled(): - return - - try: - n = sock.send(data) - except (BlockingIOError, InterruptedError): - n = 0 - except Exception as exc: - fut.set_exception(exc) - return - - if n == len(data): - fut.set_result(None) - else: - if n: - data = data[n:] - self.add_writer(fd, self._sock_sendall, fut, True, sock, data) - - def sock_connect(self, sock, address): - """Connect to a remote socket at address. - - The address must be already resolved to avoid the trap of hanging the - entire event loop when the address requires doing a DNS lookup. For - example, it must be an IP address, not an hostname, for AF_INET and - AF_INET6 address families. Use getaddrinfo() to resolve the hostname - asynchronously. - - This method is a coroutine. - """ - if self._debug and sock.gettimeout() != 0: - raise ValueError("the socket must be non-blocking") - fut = futures.Future(loop=self) - try: - base_events._check_resolved_address(sock, address) - except ValueError as err: - fut.set_exception(err) - else: - self._sock_connect(fut, sock, address) - return fut - - def _sock_connect(self, fut, sock, address): - fd = sock.fileno() - try: - sock.connect(address) - except (BlockingIOError, InterruptedError): - # Issue #23618: When the C function connect() fails with EINTR, the - # connection runs in background. We have to wait until the socket - # becomes writable to be notified when the connection succeed or - # fails. - fut.add_done_callback(functools.partial(self._sock_connect_done, - fd)) - self.add_writer(fd, self._sock_connect_cb, fut, sock, address) - except Exception as exc: - fut.set_exception(exc) - else: - fut.set_result(None) - - def _sock_connect_done(self, fd, fut): - self.remove_writer(fd) - - def _sock_connect_cb(self, fut, sock, address): - if fut.cancelled(): - return - - try: - err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - # Jump to any except clause below. - raise OSError(err, 'Connect call failed %s' % (address,)) - except (BlockingIOError, InterruptedError): - # socket is still registered, the callback will be retried later - pass - except Exception as exc: - fut.set_exception(exc) - else: - fut.set_result(None) - - def sock_accept(self, sock): - """Accept a connection. - - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. - - This method is a coroutine. - """ - if self._debug and sock.gettimeout() != 0: - raise ValueError("the socket must be non-blocking") - fut = futures.Future(loop=self) - self._sock_accept(fut, False, sock) - return fut - - def _sock_accept(self, fut, registered, sock): - fd = sock.fileno() - if registered: - self.remove_reader(fd) - if fut.cancelled(): - return - try: - conn, address = sock.accept() - conn.setblocking(False) - except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_accept, fut, True, sock) - except Exception as exc: - fut.set_exception(exc) - else: - fut.set_result((conn, address)) - - def _process_events(self, event_list): - for key, mask in event_list: - fileobj, (reader, writer) = key.fileobj, key.data - if mask & selectors.EVENT_READ and reader is not None: - if reader._cancelled: - self.remove_reader(fileobj) - else: - self._add_callback(reader) - if mask & selectors.EVENT_WRITE and writer is not None: - if writer._cancelled: - self.remove_writer(fileobj) - else: - self._add_callback(writer) - - def _stop_serving(self, sock): - self.remove_reader(sock.fileno()) - sock.close() - - -class _SelectorTransport(transports._FlowControlMixin, - transports.Transport): - - max_size = 256 * 1024 # Buffer size passed to recv(). - - _buffer_factory = bytearray # Constructs initial value for self._buffer. - - # Attribute used in the destructor: it must be set even if the constructor - # is not called (see _SelectorSslTransport which may start by raising an - # exception) - _sock = None - - def __init__(self, loop, sock, protocol, extra=None, server=None): - super().__init__(extra, loop) - self._extra['socket'] = sock - self._extra['sockname'] = sock.getsockname() - if 'peername' not in self._extra: - try: - self._extra['peername'] = sock.getpeername() - except socket.error: - self._extra['peername'] = None - self._sock = sock - self._sock_fd = sock.fileno() - self._protocol = protocol - self._protocol_connected = True - self._server = server - self._buffer = self._buffer_factory() - self._conn_lost = 0 # Set when call to connection_lost scheduled. - self._closing = False # Set when close() called. - if self._server is not None: - self._server._attach() - - def __repr__(self): - info = [self.__class__.__name__] - if self._sock is None: - info.append('closed') - elif self._closing: - info.append('closing') - info.append('fd=%s' % self._sock_fd) - # test if the transport was closed - if self._loop is not None and not self._loop.is_closed(): - polling = _test_selector_event(self._loop._selector, - self._sock_fd, selectors.EVENT_READ) - if polling: - info.append('read=polling') - else: - info.append('read=idle') - - polling = _test_selector_event(self._loop._selector, - self._sock_fd, - selectors.EVENT_WRITE) - if polling: - state = 'polling' - else: - state = 'idle' - - bufsize = self.get_write_buffer_size() - info.append('write=<%s, bufsize=%s>' % (state, bufsize)) - return '<%s>' % ' '.join(info) - - def abort(self): - self._force_close(None) - - def is_closing(self): - return self._closing - - def close(self): - if self._closing: - return - self._closing = True - self._loop.remove_reader(self._sock_fd) - if not self._buffer: - self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, None) - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self._sock.close() - - def _fatal_error(self, exc, message='Fatal error on transport'): - # Should be called from exception handler only. - if isinstance(exc, (BrokenPipeError, - ConnectionResetError, ConnectionAbortedError)): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._force_close(exc) - - def _force_close(self, exc): - if self._conn_lost: - return - if self._buffer: - self._buffer.clear() - self._loop.remove_writer(self._sock_fd) - if not self._closing: - self._closing = True - self._loop.remove_reader(self._sock_fd) - self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) - - def _call_connection_lost(self, exc): - try: - if self._protocol_connected: - self._protocol.connection_lost(exc) - finally: - self._sock.close() - self._sock = None - self._protocol = None - self._loop = None - server = self._server - if server is not None: - server._detach() - self._server = None - - def get_write_buffer_size(self): - return len(self._buffer) - - -class _SelectorSocketTransport(_SelectorTransport): - - def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - super().__init__(loop, sock, protocol, extra, server) - self._eof = False - self._paused = False - - self._loop.call_soon(self._protocol.connection_made, self) - # only start reading when connection_made() has been called - self._loop.call_soon(self._loop.add_reader, - self._sock_fd, self._read_ready) - if waiter is not None: - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) - - def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - self._loop.remove_reader(self._sock_fd) - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) - - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop.add_reader(self._sock_fd, self._read_ready) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) - - def _read_ready(self): - try: - data = self._sock.recv(self.max_size) - except (BlockingIOError, InterruptedError): - pass - except Exception as exc: - self._fatal_error(exc, 'Fatal read error on socket transport') - else: - if data: - self._protocol.data_received(data) - else: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - # We're keeping the connection open so the - # protocol can write more, but we still can't - # receive more, so remove the reader callback. - self._loop.remove_reader(self._sock_fd) - else: - self.close() - - def write(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - if self._eof: - raise RuntimeError('Cannot call write() after write_eof()') - if not data: - return - - if self._conn_lost: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('socket.send() raised exception.') - self._conn_lost += 1 - return - - if not self._buffer: - # Optimization: try to send now. - try: - n = self._sock.send(data) - except (BlockingIOError, InterruptedError): - pass - except Exception as exc: - self._fatal_error(exc, 'Fatal write error on socket transport') - return - else: - data = data[n:] - if not data: - return - # Not all was written; register write handler. - self._loop.add_writer(self._sock_fd, self._write_ready) - - # Add it to the buffer. - self._buffer.extend(data) - self._maybe_pause_protocol() - - def _write_ready(self): - assert self._buffer, 'Data should not be empty' - - try: - n = self._sock.send(self._buffer) - except (BlockingIOError, InterruptedError): - pass - except Exception as exc: - self._loop.remove_writer(self._sock_fd) - self._buffer.clear() - self._fatal_error(exc, 'Fatal write error on socket transport') - else: - if n: - del self._buffer[:n] - self._maybe_resume_protocol() # May append to buffer. - if not self._buffer: - self._loop.remove_writer(self._sock_fd) - if self._closing: - self._call_connection_lost(None) - elif self._eof: - self._sock.shutdown(socket.SHUT_WR) - - def write_eof(self): - if self._eof: - return - self._eof = True - if not self._buffer: - self._sock.shutdown(socket.SHUT_WR) - - def can_write_eof(self): - return True - - -class _SelectorSslTransport(_SelectorTransport): - - _buffer_factory = bytearray - - def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None, - server_side=False, server_hostname=None, - extra=None, server=None): - if ssl is None: - raise RuntimeError('stdlib ssl module not available') - - if not sslcontext: - sslcontext = sslproto._create_transport_context(server_side, server_hostname) - - wrap_kwargs = { - 'server_side': server_side, - 'do_handshake_on_connect': False, - } - if server_hostname and not server_side: - wrap_kwargs['server_hostname'] = server_hostname - sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs) - - super().__init__(loop, sslsock, protocol, extra, server) - # the protocol connection is only made after the SSL handshake - self._protocol_connected = False - - self._server_hostname = server_hostname - self._waiter = waiter - self._sslcontext = sslcontext - self._paused = False - - # SSL-specific extra info. (peercert is set later) - self._extra.update(sslcontext=sslcontext) - - if self._loop.get_debug(): - logger.debug("%r starts SSL handshake", self) - start_time = self._loop.time() - else: - start_time = None - self._on_handshake(start_time) - - def _wakeup_waiter(self, exc=None): - if self._waiter is None: - return - if not self._waiter.cancelled(): - if exc is not None: - self._waiter.set_exception(exc) - else: - self._waiter.set_result(None) - self._waiter = None - - def _on_handshake(self, start_time): - try: - self._sock.do_handshake() - except ssl.SSLWantReadError: - self._loop.add_reader(self._sock_fd, - self._on_handshake, start_time) - return - except ssl.SSLWantWriteError: - self._loop.add_writer(self._sock_fd, - self._on_handshake, start_time) - return - except BaseException as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed", - self, exc_info=True) - self._loop.remove_reader(self._sock_fd) - self._loop.remove_writer(self._sock_fd) - self._sock.close() - self._wakeup_waiter(exc) - if isinstance(exc, Exception): - return - else: - raise - - self._loop.remove_reader(self._sock_fd) - self._loop.remove_writer(self._sock_fd) - - peercert = self._sock.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname and - self._sslcontext.verify_mode != ssl.CERT_NONE): - try: - ssl.match_hostname(peercert, self._server_hostname) - except Exception as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed " - "on matching the hostname", - self, exc_info=True) - self._sock.close() - self._wakeup_waiter(exc) - return - - # Add extra info that becomes available after handshake. - self._extra.update(peercert=peercert, - cipher=self._sock.cipher(), - compression=self._sock.compression(), - ssl_object=self._sock, - ) - - self._read_wants_write = False - self._write_wants_read = False - self._loop.add_reader(self._sock_fd, self._read_ready) - self._protocol_connected = True - self._loop.call_soon(self._protocol.connection_made, self) - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(self._wakeup_waiter) - - if self._loop.get_debug(): - dt = self._loop.time() - start_time - logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) - - def pause_reading(self): - # XXX This is a bit icky, given the comment at the top of - # _read_ready(). Is it possible to evoke a deadlock? I don't - # know, although it doesn't look like it; write() will still - # accept more data for the buffer and eventually the app will - # call resume_reading() again, and things will flow again. - - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - self._loop.remove_reader(self._sock_fd) - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) - - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop.add_reader(self._sock_fd, self._read_ready) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) - - def _read_ready(self): - if self._write_wants_read: - self._write_wants_read = False - self._write_ready() - - if self._buffer: - self._loop.add_writer(self._sock_fd, self._write_ready) - - try: - data = self._sock.recv(self.max_size) - except (BlockingIOError, InterruptedError, ssl.SSLWantReadError): - pass - except ssl.SSLWantWriteError: - self._read_wants_write = True - self._loop.remove_reader(self._sock_fd) - self._loop.add_writer(self._sock_fd, self._write_ready) - except Exception as exc: - self._fatal_error(exc, 'Fatal read error on SSL transport') - else: - if data: - self._protocol.data_received(data) - else: - try: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - logger.warning('returning true from eof_received() ' - 'has no effect when using ssl') - finally: - self.close() - - def _write_ready(self): - if self._read_wants_write: - self._read_wants_write = False - self._read_ready() - - if not (self._paused or self._closing): - self._loop.add_reader(self._sock_fd, self._read_ready) - - if self._buffer: - try: - n = self._sock.send(self._buffer) - except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError): - n = 0 - except ssl.SSLWantReadError: - n = 0 - self._loop.remove_writer(self._sock_fd) - self._write_wants_read = True - except Exception as exc: - self._loop.remove_writer(self._sock_fd) - self._buffer.clear() - self._fatal_error(exc, 'Fatal write error on SSL transport') - return - - if n: - del self._buffer[:n] - - self._maybe_resume_protocol() # May append to buffer. - - if not self._buffer: - self._loop.remove_writer(self._sock_fd) - if self._closing: - self._call_connection_lost(None) - - def write(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - if not data: - return - - if self._conn_lost: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('socket.send() raised exception.') - self._conn_lost += 1 - return - - if not self._buffer: - self._loop.add_writer(self._sock_fd, self._write_ready) - - # Add it to the buffer. - self._buffer.extend(data) - self._maybe_pause_protocol() - - def can_write_eof(self): - return False - - -class _SelectorDatagramTransport(_SelectorTransport): - - _buffer_factory = collections.deque - - def __init__(self, loop, sock, protocol, address=None, - waiter=None, extra=None): - super().__init__(loop, sock, protocol, extra) - self._address = address - self._loop.call_soon(self._protocol.connection_made, self) - # only start reading when connection_made() has been called - self._loop.call_soon(self._loop.add_reader, - self._sock_fd, self._read_ready) - if waiter is not None: - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) - - def get_write_buffer_size(self): - return sum(len(data) for data, _ in self._buffer) - - def _read_ready(self): - try: - data, addr = self._sock.recvfrom(self.max_size) - except (BlockingIOError, InterruptedError): - pass - except OSError as exc: - self._protocol.error_received(exc) - except Exception as exc: - self._fatal_error(exc, 'Fatal read error on datagram transport') - else: - self._protocol.datagram_received(data, addr) - - def sendto(self, data, addr=None): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - if not data: - return - - if self._address and addr not in (None, self._address): - raise ValueError('Invalid address: must be None or %s' % - (self._address,)) - - if self._conn_lost and self._address: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('socket.send() raised exception.') - self._conn_lost += 1 - return - - if not self._buffer: - # Attempt to send it right away first. - try: - if self._address: - self._sock.send(data) - else: - self._sock.sendto(data, addr) - return - except (BlockingIOError, InterruptedError): - self._loop.add_writer(self._sock_fd, self._sendto_ready) - except OSError as exc: - self._protocol.error_received(exc) - return - except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') - return - - # Ensure that what we buffer is immutable. - self._buffer.append((bytes(data), addr)) - self._maybe_pause_protocol() - - def _sendto_ready(self): - while self._buffer: - data, addr = self._buffer.popleft() - try: - if self._address: - self._sock.send(data) - else: - self._sock.sendto(data, addr) - except (BlockingIOError, InterruptedError): - self._buffer.appendleft((data, addr)) # Try again later. - break - except OSError as exc: - self._protocol.error_received(exc) - return - except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') - return - - self._maybe_resume_protocol() # May append to buffer. - if not self._buffer: - self._loop.remove_writer(self._sock_fd) - if self._closing: - self._call_connection_lost(None) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/sslproto.py --- a/Lib/asyncio/sslproto.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,680 +0,0 @@ -import collections -import warnings -try: - import ssl -except ImportError: # pragma: no cover - ssl = None - -from . import compat -from . import protocols -from . import transports -from .log import logger - - -def _create_transport_context(server_side, server_hostname): - if server_side: - raise ValueError('Server side SSL needs a valid SSLContext') - - # Client side may pass ssl=True to use a default - # context; in that case the sslcontext passed is None. - # The default is secure for client connections. - if hasattr(ssl, 'create_default_context'): - # Python 3.4+: use up-to-date strong settings. - sslcontext = ssl.create_default_context() - if not server_hostname: - sslcontext.check_hostname = False - else: - # Fallback for Python 3.3. - sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - sslcontext.options |= ssl.OP_NO_SSLv2 - sslcontext.options |= ssl.OP_NO_SSLv3 - sslcontext.set_default_verify_paths() - sslcontext.verify_mode = ssl.CERT_REQUIRED - return sslcontext - - -def _is_sslproto_available(): - return hasattr(ssl, "MemoryBIO") - - -# States of an _SSLPipe. -_UNWRAPPED = "UNWRAPPED" -_DO_HANDSHAKE = "DO_HANDSHAKE" -_WRAPPED = "WRAPPED" -_SHUTDOWN = "SHUTDOWN" - - -class _SSLPipe(object): - """An SSL "Pipe". - - An SSL pipe allows you to communicate with an SSL/TLS protocol instance - through memory buffers. It can be used to implement a security layer for an - existing connection where you don't have access to the connection's file - descriptor, or for some reason you don't want to use it. - - An SSL pipe can be in "wrapped" and "unwrapped" mode. In unwrapped mode, - data is passed through untransformed. In wrapped mode, application level - data is encrypted to SSL record level data and vice versa. The SSL record - level is the lowest level in the SSL protocol suite and is what travels - as-is over the wire. - - An SslPipe initially is in "unwrapped" mode. To start SSL, call - do_handshake(). To shutdown SSL again, call unwrap(). - """ - - max_size = 256 * 1024 # Buffer size passed to read() - - def __init__(self, context, server_side, server_hostname=None): - """ - The *context* argument specifies the ssl.SSLContext to use. - - The *server_side* argument indicates whether this is a server side or - client side transport. - - The optional *server_hostname* argument can be used to specify the - hostname you are connecting to. You may only specify this parameter if - the _ssl module supports Server Name Indication (SNI). - """ - self._context = context - self._server_side = server_side - self._server_hostname = server_hostname - self._state = _UNWRAPPED - self._incoming = ssl.MemoryBIO() - self._outgoing = ssl.MemoryBIO() - self._sslobj = None - self._need_ssldata = False - self._handshake_cb = None - self._shutdown_cb = None - - @property - def context(self): - """The SSL context passed to the constructor.""" - return self._context - - @property - def ssl_object(self): - """The internal ssl.SSLObject instance. - - Return None if the pipe is not wrapped. - """ - return self._sslobj - - @property - def need_ssldata(self): - """Whether more record level data is needed to complete a handshake - that is currently in progress.""" - return self._need_ssldata - - @property - def wrapped(self): - """ - Whether a security layer is currently in effect. - - Return False during handshake. - """ - return self._state == _WRAPPED - - def do_handshake(self, callback=None): - """Start the SSL handshake. - - Return a list of ssldata. A ssldata element is a list of buffers - - The optional *callback* argument can be used to install a callback that - will be called when the handshake is complete. The callback will be - called with None if successful, else an exception instance. - """ - if self._state != _UNWRAPPED: - raise RuntimeError('handshake in progress or completed') - self._sslobj = self._context.wrap_bio( - self._incoming, self._outgoing, - server_side=self._server_side, - server_hostname=self._server_hostname) - self._state = _DO_HANDSHAKE - self._handshake_cb = callback - ssldata, appdata = self.feed_ssldata(b'', only_handshake=True) - assert len(appdata) == 0 - return ssldata - - def shutdown(self, callback=None): - """Start the SSL shutdown sequence. - - Return a list of ssldata. A ssldata element is a list of buffers - - The optional *callback* argument can be used to install a callback that - will be called when the shutdown is complete. The callback will be - called without arguments. - """ - if self._state == _UNWRAPPED: - raise RuntimeError('no security layer present') - if self._state == _SHUTDOWN: - raise RuntimeError('shutdown in progress') - assert self._state in (_WRAPPED, _DO_HANDSHAKE) - self._state = _SHUTDOWN - self._shutdown_cb = callback - ssldata, appdata = self.feed_ssldata(b'') - assert appdata == [] or appdata == [b''] - return ssldata - - def feed_eof(self): - """Send a potentially "ragged" EOF. - - This method will raise an SSL_ERROR_EOF exception if the EOF is - unexpected. - """ - self._incoming.write_eof() - ssldata, appdata = self.feed_ssldata(b'') - assert appdata == [] or appdata == [b''] - - def feed_ssldata(self, data, only_handshake=False): - """Feed SSL record level data into the pipe. - - The data must be a bytes instance. It is OK to send an empty bytes - instance. This can be used to get ssldata for a handshake initiated by - this endpoint. - - Return a (ssldata, appdata) tuple. The ssldata element is a list of - buffers containing SSL data that needs to be sent to the remote SSL. - - The appdata element is a list of buffers containing plaintext data that - needs to be forwarded to the application. The appdata list may contain - an empty buffer indicating an SSL "close_notify" alert. This alert must - be acknowledged by calling shutdown(). - """ - if self._state == _UNWRAPPED: - # If unwrapped, pass plaintext data straight through. - if data: - appdata = [data] - else: - appdata = [] - return ([], appdata) - - self._need_ssldata = False - if data: - self._incoming.write(data) - - ssldata = [] - appdata = [] - try: - if self._state == _DO_HANDSHAKE: - # Call do_handshake() until it doesn't raise anymore. - self._sslobj.do_handshake() - self._state = _WRAPPED - if self._handshake_cb: - self._handshake_cb(None) - if only_handshake: - return (ssldata, appdata) - # Handshake done: execute the wrapped block - - if self._state == _WRAPPED: - # Main state: read data from SSL until close_notify - while True: - chunk = self._sslobj.read(self.max_size) - appdata.append(chunk) - if not chunk: # close_notify - break - - elif self._state == _SHUTDOWN: - # Call shutdown() until it doesn't raise anymore. - self._sslobj.unwrap() - self._sslobj = None - self._state = _UNWRAPPED - if self._shutdown_cb: - self._shutdown_cb() - - elif self._state == _UNWRAPPED: - # Drain possible plaintext data after close_notify. - appdata.append(self._incoming.read()) - except (ssl.SSLError, ssl.CertificateError) as exc: - if getattr(exc, 'errno', None) not in ( - ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE, - ssl.SSL_ERROR_SYSCALL): - if self._state == _DO_HANDSHAKE and self._handshake_cb: - self._handshake_cb(exc) - raise - self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) - - # Check for record level data that needs to be sent back. - # Happens for the initial handshake and renegotiations. - if self._outgoing.pending: - ssldata.append(self._outgoing.read()) - return (ssldata, appdata) - - def feed_appdata(self, data, offset=0): - """Feed plaintext data into the pipe. - - Return an (ssldata, offset) tuple. The ssldata element is a list of - buffers containing record level data that needs to be sent to the - remote SSL instance. The offset is the number of plaintext bytes that - were processed, which may be less than the length of data. - - NOTE: In case of short writes, this call MUST be retried with the SAME - buffer passed into the *data* argument (i.e. the id() must be the - same). This is an OpenSSL requirement. A further particularity is that - a short write will always have offset == 0, because the _ssl module - does not enable partial writes. And even though the offset is zero, - there will still be encrypted data in ssldata. - """ - assert 0 <= offset <= len(data) - if self._state == _UNWRAPPED: - # pass through data in unwrapped mode - if offset < len(data): - ssldata = [data[offset:]] - else: - ssldata = [] - return (ssldata, len(data)) - - ssldata = [] - view = memoryview(data) - while True: - self._need_ssldata = False - try: - if offset < len(view): - offset += self._sslobj.write(view[offset:]) - except ssl.SSLError as exc: - # It is not allowed to call write() after unwrap() until the - # close_notify is acknowledged. We return the condition to the - # caller as a short write. - if exc.reason == 'PROTOCOL_IS_SHUTDOWN': - exc.errno = ssl.SSL_ERROR_WANT_READ - if exc.errno not in (ssl.SSL_ERROR_WANT_READ, - ssl.SSL_ERROR_WANT_WRITE, - ssl.SSL_ERROR_SYSCALL): - raise - self._need_ssldata = (exc.errno == ssl.SSL_ERROR_WANT_READ) - - # See if there's any record level data back for us. - if self._outgoing.pending: - ssldata.append(self._outgoing.read()) - if offset == len(view) or self._need_ssldata: - break - return (ssldata, offset) - - -class _SSLProtocolTransport(transports._FlowControlMixin, - transports.Transport): - - def __init__(self, loop, ssl_protocol, app_protocol): - self._loop = loop - # SSLProtocol instance - self._ssl_protocol = ssl_protocol - self._app_protocol = app_protocol - self._closed = False - - def get_extra_info(self, name, default=None): - """Get optional transport information.""" - return self._ssl_protocol._get_extra_info(name, default) - - def is_closing(self): - return self._closed - - def close(self): - """Close the transport. - - Buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's connection_lost() method will (eventually) called - with None as its argument. - """ - self._closed = True - self._ssl_protocol._start_shutdown() - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self.close() - - def pause_reading(self): - """Pause the receiving end. - - No data will be passed to the protocol's data_received() - method until resume_reading() is called. - """ - self._ssl_protocol._transport.pause_reading() - - def resume_reading(self): - """Resume the receiving end. - - Data received will once again be passed to the protocol's - data_received() method. - """ - self._ssl_protocol._transport.resume_reading() - - def set_write_buffer_limits(self, high=None, low=None): - """Set the high- and low-water limits for write flow control. - - These two values control when to call the protocol's - pause_writing() and resume_writing() methods. If specified, - the low-water limit must be less than or equal to the - high-water limit. Neither value can be negative. - - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to an - implementation-specific value less than or equal to the - high-water limit. Setting high to zero forces low to zero as - well, and causes pause_writing() to be called whenever the - buffer becomes non-empty. Setting low to zero causes - resume_writing() to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. - """ - self._ssl_protocol._transport.set_write_buffer_limits(high, low) - - def get_write_buffer_size(self): - """Return the current size of the write buffer.""" - return self._ssl_protocol._transport.get_write_buffer_size() - - def write(self, data): - """Write some data bytes to the transport. - - This does not block; it buffers the data and arranges for it - to be sent out asynchronously. - """ - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError("data: expecting a bytes-like instance, got {!r}" - .format(type(data).__name__)) - if not data: - return - self._ssl_protocol._write_appdata(data) - - def can_write_eof(self): - """Return True if this transport supports write_eof(), False if not.""" - return False - - def abort(self): - """Close the transport immediately. - - Buffered data will be lost. No more data will be received. - The protocol's connection_lost() method will (eventually) be - called with None as its argument. - """ - self._ssl_protocol._abort() - - -class SSLProtocol(protocols.Protocol): - """SSL protocol. - - Implementation of SSL on top of a socket using incoming and outgoing - buffers which are ssl.MemoryBIO objects. - """ - - def __init__(self, loop, app_protocol, sslcontext, waiter, - server_side=False, server_hostname=None): - if ssl is None: - raise RuntimeError('stdlib ssl module not available') - - if not sslcontext: - sslcontext = _create_transport_context(server_side, server_hostname) - - self._server_side = server_side - if server_hostname and not server_side: - self._server_hostname = server_hostname - else: - self._server_hostname = None - self._sslcontext = sslcontext - # SSL-specific extra info. More info are set when the handshake - # completes. - self._extra = dict(sslcontext=sslcontext) - - # App data write buffering - self._write_backlog = collections.deque() - self._write_buffer_size = 0 - - self._waiter = waiter - self._loop = loop - self._app_protocol = app_protocol - self._app_transport = _SSLProtocolTransport(self._loop, - self, self._app_protocol) - # _SSLPipe instance (None until the connection is made) - self._sslpipe = None - self._session_established = False - self._in_handshake = False - self._in_shutdown = False - # transport, ex: SelectorSocketTransport - self._transport = None - - def _wakeup_waiter(self, exc=None): - if self._waiter is None: - return - if not self._waiter.cancelled(): - if exc is not None: - self._waiter.set_exception(exc) - else: - self._waiter.set_result(None) - self._waiter = None - - def connection_made(self, transport): - """Called when the low-level connection is made. - - Start the SSL handshake. - """ - self._transport = transport - self._sslpipe = _SSLPipe(self._sslcontext, - self._server_side, - self._server_hostname) - self._start_handshake() - - def connection_lost(self, exc): - """Called when the low-level connection is lost or closed. - - The argument is an exception object or None (the latter - meaning a regular EOF is received or the connection was - aborted or closed). - """ - if self._session_established: - self._session_established = False - self._loop.call_soon(self._app_protocol.connection_lost, exc) - self._transport = None - self._app_transport = None - - def pause_writing(self): - """Called when the low-level transport's buffer goes over - the high-water mark. - """ - self._app_protocol.pause_writing() - - def resume_writing(self): - """Called when the low-level transport's buffer drains below - the low-water mark. - """ - self._app_protocol.resume_writing() - - def data_received(self, data): - """Called when some SSL data is received. - - The argument is a bytes object. - """ - try: - ssldata, appdata = self._sslpipe.feed_ssldata(data) - except ssl.SSLError as e: - if self._loop.get_debug(): - logger.warning('%r: SSL error %s (reason %s)', - self, e.errno, e.reason) - self._abort() - return - - for chunk in ssldata: - self._transport.write(chunk) - - for chunk in appdata: - if chunk: - self._app_protocol.data_received(chunk) - else: - self._start_shutdown() - break - - def eof_received(self): - """Called when the other end of the low-level stream - is half-closed. - - If this returns a false value (including None), the transport - will close itself. If it returns a true value, closing the - transport is up to the protocol. - """ - try: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - - self._wakeup_waiter(ConnectionResetError) - - if not self._in_handshake: - keep_open = self._app_protocol.eof_received() - if keep_open: - logger.warning('returning true from eof_received() ' - 'has no effect when using ssl') - finally: - self._transport.close() - - def _get_extra_info(self, name, default=None): - if name in self._extra: - return self._extra[name] - else: - return self._transport.get_extra_info(name, default) - - def _start_shutdown(self): - if self._in_shutdown: - return - self._in_shutdown = True - self._write_appdata(b'') - - def _write_appdata(self, data): - self._write_backlog.append((data, 0)) - self._write_buffer_size += len(data) - self._process_write_backlog() - - def _start_handshake(self): - if self._loop.get_debug(): - logger.debug("%r starts SSL handshake", self) - self._handshake_start_time = self._loop.time() - else: - self._handshake_start_time = None - self._in_handshake = True - # (b'', 1) is a special value in _process_write_backlog() to do - # the SSL handshake - self._write_backlog.append((b'', 1)) - self._loop.call_soon(self._process_write_backlog) - - def _on_handshake_complete(self, handshake_exc): - self._in_handshake = False - - sslobj = self._sslpipe.ssl_object - try: - if handshake_exc is not None: - raise handshake_exc - - peercert = sslobj.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname - and self._sslcontext.verify_mode != ssl.CERT_NONE): - ssl.match_hostname(peercert, self._server_hostname) - except BaseException as exc: - if self._loop.get_debug(): - if isinstance(exc, ssl.CertificateError): - logger.warning("%r: SSL handshake failed " - "on verifying the certificate", - self, exc_info=True) - else: - logger.warning("%r: SSL handshake failed", - self, exc_info=True) - self._transport.close() - if isinstance(exc, Exception): - self._wakeup_waiter(exc) - return - else: - raise - - if self._loop.get_debug(): - dt = self._loop.time() - self._handshake_start_time - logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) - - # Add extra info that becomes available after handshake. - self._extra.update(peercert=peercert, - cipher=sslobj.cipher(), - compression=sslobj.compression(), - ssl_object=sslobj, - ) - self._app_protocol.connection_made(self._app_transport) - self._wakeup_waiter() - self._session_established = True - # In case transport.write() was already called. Don't call - # immediatly _process_write_backlog(), but schedule it: - # _on_handshake_complete() can be called indirectly from - # _process_write_backlog(), and _process_write_backlog() is not - # reentrant. - self._loop.call_soon(self._process_write_backlog) - - def _process_write_backlog(self): - # Try to make progress on the write backlog. - if self._transport is None: - return - - try: - for i in range(len(self._write_backlog)): - data, offset = self._write_backlog[0] - if data: - ssldata, offset = self._sslpipe.feed_appdata(data, offset) - elif offset: - ssldata = self._sslpipe.do_handshake( - self._on_handshake_complete) - offset = 1 - else: - ssldata = self._sslpipe.shutdown(self._finalize) - offset = 1 - - for chunk in ssldata: - self._transport.write(chunk) - - if offset < len(data): - self._write_backlog[0] = (data, offset) - # A short write means that a write is blocked on a read - # We need to enable reading if it is paused! - assert self._sslpipe.need_ssldata - if self._transport._paused: - self._transport.resume_reading() - break - - # An entire chunk from the backlog was processed. We can - # delete it and reduce the outstanding buffer size. - del self._write_backlog[0] - self._write_buffer_size -= len(data) - except BaseException as exc: - if self._in_handshake: - # BaseExceptions will be re-raised in _on_handshake_complete. - self._on_handshake_complete(exc) - else: - self._fatal_error(exc, 'Fatal error on SSL transport') - if not isinstance(exc, Exception): - # BaseException - raise - - def _fatal_error(self, exc, message='Fatal error on transport'): - # Should be called from exception handler only. - if isinstance(exc, (BrokenPipeError, ConnectionResetError)): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self._transport, - 'protocol': self, - }) - if self._transport: - self._transport._force_close(exc) - - def _finalize(self): - if self._transport is not None: - self._transport.close() - - def _abort(self): - if self._transport is not None: - try: - self._transport.abort() - finally: - self._finalize() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/streams.py --- a/Lib/asyncio/streams.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,680 +0,0 @@ -"""Stream-related things.""" - -__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol', - 'open_connection', 'start_server', - 'IncompleteReadError', - 'LimitOverrunError', - ] - -import socket - -if hasattr(socket, 'AF_UNIX'): - __all__.extend(['open_unix_connection', 'start_unix_server']) - -from . import coroutines -from . import compat -from . import events -from . import futures -from . import protocols -from .coroutines import coroutine -from .log import logger - - -_DEFAULT_LIMIT = 2**16 - - -class IncompleteReadError(EOFError): - """ - Incomplete read error. Attributes: - - - partial: read bytes string before the end of stream was reached - - expected: total number of expected bytes (or None if unknown) - """ - def __init__(self, partial, expected): - super().__init__("%d bytes read on a total of %r expected bytes" - % (len(partial), expected)) - self.partial = partial - self.expected = expected - - -class LimitOverrunError(Exception): - """Reached buffer limit while looking for the separator. - - Attributes: - - message: error message - - consumed: total number of bytes that should be consumed - """ - def __init__(self, message, consumed): - super().__init__(message) - self.message = message - self.consumed = consumed - - -@coroutine -def open_connection(host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): - """A wrapper for create_connection() returning a (reader, writer) pair. - - The reader returned is a StreamReader instance; the writer is a - StreamWriter instance. - - The arguments are all the usual arguments to create_connection() - except protocol_factory; most common are positional host and port, - with various optional keyword arguments following. - - Additional optional keyword arguments are loop (to set the event loop - instance to use) and limit (to set the buffer limit passed to the - StreamReader). - - (If you want to customize the StreamReader and/or - StreamReaderProtocol classes, just copy the code -- there's - really nothing special here except some convenience.) - """ - if loop is None: - loop = events.get_event_loop() - reader = StreamReader(limit=limit, loop=loop) - protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_connection( - lambda: protocol, host, port, **kwds) - writer = StreamWriter(transport, protocol, reader, loop) - return reader, writer - - -@coroutine -def start_server(client_connected_cb, host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): - """Start a socket server, call back for each client connected. - - The first parameter, `client_connected_cb`, takes two parameters: - client_reader, client_writer. client_reader is a StreamReader - object, while client_writer is a StreamWriter object. This - parameter can either be a plain callback function or a coroutine; - if it is a coroutine, it will be automatically converted into a - Task. - - The rest of the arguments are all the usual arguments to - loop.create_server() except protocol_factory; most common are - positional host and port, with various optional keyword arguments - following. The return value is the same as loop.create_server(). - - Additional optional keyword arguments are loop (to set the event loop - instance to use) and limit (to set the buffer limit passed to the - StreamReader). - - The return value is the same as loop.create_server(), i.e. a - Server object which can be used to stop the service. - """ - if loop is None: - loop = events.get_event_loop() - - def factory(): - reader = StreamReader(limit=limit, loop=loop) - protocol = StreamReaderProtocol(reader, client_connected_cb, - loop=loop) - return protocol - - return (yield from loop.create_server(factory, host, port, **kwds)) - - -if hasattr(socket, 'AF_UNIX'): - # UNIX Domain Sockets are supported on this platform - - @coroutine - def open_unix_connection(path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): - """Similar to `open_connection` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() - reader = StreamReader(limit=limit, loop=loop) - protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_unix_connection( - lambda: protocol, path, **kwds) - writer = StreamWriter(transport, protocol, reader, loop) - return reader, writer - - - @coroutine - def start_unix_server(client_connected_cb, path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): - """Similar to `start_server` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() - - def factory(): - reader = StreamReader(limit=limit, loop=loop) - protocol = StreamReaderProtocol(reader, client_connected_cb, - loop=loop) - return protocol - - return (yield from loop.create_unix_server(factory, path, **kwds)) - - -class FlowControlMixin(protocols.Protocol): - """Reusable flow control logic for StreamWriter.drain(). - - This implements the protocol methods pause_writing(), - resume_reading() and connection_lost(). If the subclass overrides - these it must call the super methods. - - StreamWriter.drain() must wait for _drain_helper() coroutine. - """ - - def __init__(self, loop=None): - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - self._paused = False - self._drain_waiter = None - self._connection_lost = False - - def pause_writing(self): - assert not self._paused - self._paused = True - if self._loop.get_debug(): - logger.debug("%r pauses writing", self) - - def resume_writing(self): - assert self._paused - self._paused = False - if self._loop.get_debug(): - logger.debug("%r resumes writing", self) - - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None - if not waiter.done(): - waiter.set_result(None) - - def connection_lost(self, exc): - self._connection_lost = True - # Wake up the writer if currently paused. - if not self._paused: - return - waiter = self._drain_waiter - if waiter is None: - return - self._drain_waiter = None - if waiter.done(): - return - if exc is None: - waiter.set_result(None) - else: - waiter.set_exception(exc) - - @coroutine - def _drain_helper(self): - if self._connection_lost: - raise ConnectionResetError('Connection lost') - if not self._paused: - return - waiter = self._drain_waiter - assert waiter is None or waiter.cancelled() - waiter = futures.Future(loop=self._loop) - self._drain_waiter = waiter - yield from waiter - - -class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): - """Helper class to adapt between Protocol and StreamReader. - - (This is a helper class instead of making StreamReader itself a - Protocol subclass, because the StreamReader has other potential - uses, and to prevent the user of the StreamReader to accidentally - call inappropriate methods of the protocol.) - """ - - def __init__(self, stream_reader, client_connected_cb=None, loop=None): - super().__init__(loop=loop) - self._stream_reader = stream_reader - self._stream_writer = None - self._client_connected_cb = client_connected_cb - - def connection_made(self, transport): - self._stream_reader.set_transport(transport) - if self._client_connected_cb is not None: - self._stream_writer = StreamWriter(transport, self, - self._stream_reader, - self._loop) - res = self._client_connected_cb(self._stream_reader, - self._stream_writer) - if coroutines.iscoroutine(res): - self._loop.create_task(res) - - def connection_lost(self, exc): - if exc is None: - self._stream_reader.feed_eof() - else: - self._stream_reader.set_exception(exc) - super().connection_lost(exc) - - def data_received(self, data): - self._stream_reader.feed_data(data) - - def eof_received(self): - self._stream_reader.feed_eof() - return True - - -class StreamWriter: - """Wraps a Transport. - - This exposes write(), writelines(), [can_]write_eof(), - get_extra_info() and close(). It adds drain() which returns an - optional Future on which you can wait for flow control. It also - adds a transport property which references the Transport - directly. - """ - - def __init__(self, transport, protocol, reader, loop): - self._transport = transport - self._protocol = protocol - # drain() expects that the reader has an exception() method - assert reader is None or isinstance(reader, StreamReader) - self._reader = reader - self._loop = loop - - def __repr__(self): - info = [self.__class__.__name__, 'transport=%r' % self._transport] - if self._reader is not None: - info.append('reader=%r' % self._reader) - return '<%s>' % ' '.join(info) - - @property - def transport(self): - return self._transport - - def write(self, data): - self._transport.write(data) - - def writelines(self, data): - self._transport.writelines(data) - - def write_eof(self): - return self._transport.write_eof() - - def can_write_eof(self): - return self._transport.can_write_eof() - - def close(self): - return self._transport.close() - - def get_extra_info(self, name, default=None): - return self._transport.get_extra_info(name, default) - - @coroutine - def drain(self): - """Flush the write buffer. - - The intended use is to write - - w.write(data) - yield from w.drain() - """ - if self._reader is not None: - exc = self._reader.exception() - if exc is not None: - raise exc - if self._transport is not None: - if self._transport.is_closing(): - # Yield to the event loop so connection_lost() may be - # called. Without this, _drain_helper() would return - # immediately, and code that calls - # write(...); yield from drain() - # in a loop would never call connection_lost(), so it - # would not see an error when the socket is closed. - yield - yield from self._protocol._drain_helper() - - -class StreamReader: - - def __init__(self, limit=_DEFAULT_LIMIT, loop=None): - # The line length limit is a security feature; - # it also doubles as half the buffer limit. - - if limit <= 0: - raise ValueError('Limit cannot be <= 0') - - self._limit = limit - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - self._buffer = bytearray() - self._eof = False # Whether we're done. - self._waiter = None # A future used by _wait_for_data() - self._exception = None - self._transport = None - self._paused = False - - def __repr__(self): - info = ['StreamReader'] - if self._buffer: - info.append('%d bytes' % len(self._buffer)) - if self._eof: - info.append('eof') - if self._limit != _DEFAULT_LIMIT: - info.append('l=%d' % self._limit) - if self._waiter: - info.append('w=%r' % self._waiter) - if self._exception: - info.append('e=%r' % self._exception) - if self._transport: - info.append('t=%r' % self._transport) - if self._paused: - info.append('paused') - return '<%s>' % ' '.join(info) - - def exception(self): - return self._exception - - def set_exception(self, exc): - self._exception = exc - - waiter = self._waiter - if waiter is not None: - self._waiter = None - if not waiter.cancelled(): - waiter.set_exception(exc) - - def _wakeup_waiter(self): - """Wakeup read*() functions waiting for data or EOF.""" - waiter = self._waiter - if waiter is not None: - self._waiter = None - if not waiter.cancelled(): - waiter.set_result(None) - - def set_transport(self, transport): - assert self._transport is None, 'Transport already set' - self._transport = transport - - def _maybe_resume_transport(self): - if self._paused and len(self._buffer) <= self._limit: - self._paused = False - self._transport.resume_reading() - - def feed_eof(self): - self._eof = True - self._wakeup_waiter() - - def at_eof(self): - """Return True if the buffer is empty and 'feed_eof' was called.""" - return self._eof and not self._buffer - - def feed_data(self, data): - assert not self._eof, 'feed_data after feed_eof' - - if not data: - return - - self._buffer.extend(data) - self._wakeup_waiter() - - if (self._transport is not None and - not self._paused and - len(self._buffer) > 2*self._limit): - try: - self._transport.pause_reading() - except NotImplementedError: - # The transport can't be paused. - # We'll just have to buffer all data. - # Forget the transport so we don't keep trying. - self._transport = None - else: - self._paused = True - - @coroutine - def _wait_for_data(self, func_name): - """Wait until feed_data() or feed_eof() is called. - - If stream was paused, automatically resume it. - """ - # StreamReader uses a future to link the protocol feed_data() method - # to a read coroutine. Running two read coroutines at the same time - # would have an unexpected behaviour. It would not possible to know - # which coroutine would get the next data. - if self._waiter is not None: - raise RuntimeError('%s() called while another coroutine is ' - 'already waiting for incoming data' % func_name) - - assert not self._eof, '_wait_for_data after EOF' - - # Waiting for data while paused will make deadlock, so prevent it. - if self._paused: - self._paused = False - self._transport.resume_reading() - - self._waiter = futures.Future(loop=self._loop) - try: - yield from self._waiter - finally: - self._waiter = None - - @coroutine - def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. - - On success, return chunk that ends with newline. If only partial - line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. - - If limit is reached, ValueError will be raised. In that case, if - newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. - - If stream was paused, this function will automatically resume it if - needed. - """ - sep = b'\n' - seplen = len(sep) - try: - line = yield from self.readuntil(sep) - except IncompleteReadError as e: - return e.partial - except LimitOverrunError as e: - if self._buffer.startswith(sep, e.consumed): - del self._buffer[:e.consumed + seplen] - else: - self._buffer.clear() - self._maybe_resume_transport() - raise ValueError(e.args[0]) - return line - - @coroutine - def readuntil(self, separator=b'\n'): - """Read chunk of data from the stream until `separator` is found. - - On success, chunk and its separator will be removed from internal buffer - (i.e. consumed). Returned chunk will include separator at the end. - - Configured stream limit is used to check result. Limit means maximal - length of chunk that can be returned, not counting the separator. - - If EOF occurs and complete separator still not found, - IncompleteReadError(, None) will be raised and internal - buffer becomes empty. This partial data may contain a partial separator. - - If chunk cannot be read due to overlimit, LimitOverrunError will be raised - and data will be left in internal buffer, so it can be read again, in - some different way. - - If stream was paused, this function will automatically resume it if - needed. - """ - seplen = len(separator) - if seplen == 0: - raise ValueError('Separator should be at least one-byte string') - - if self._exception is not None: - raise self._exception - - # Consume whole buffer except last bytes, which length is - # one less than seplen. Let's check corner cases with - # separator='SEPARATOR': - # * we have received almost complete separator (without last - # byte). i.e buffer='some textSEPARATO'. In this case we - # can safely consume len(separator) - 1 bytes. - # * last byte of buffer is first byte of separator, i.e. - # buffer='abcdefghijklmnopqrS'. We may safely consume - # everything except that last byte, but this require to - # analyze bytes of buffer that match partial separator. - # This is slow and/or require FSM. For this case our - # implementation is not optimal, since require rescanning - # of data that is known to not belong to separator. In - # real world, separator will not be so long to notice - # performance problems. Even when reading MIME-encoded - # messages :) - - # `offset` is the number of bytes from the beginning of the buffer where - # is no occurrence of `separator`. - offset = 0 - - # Loop until we find `separator` in the buffer, exceed the buffer size, - # or an EOF has happened. - while True: - buflen = len(self._buffer) - - # Check if we now have enough data in the buffer for `separator` to - # fit. - if buflen - offset >= seplen: - isep = self._buffer.find(separator, offset) - - if isep != -1: - # `separator` is in the buffer. `isep` will be used later to - # retrieve the data. - break - - # see upper comment for explanation. - offset = buflen + 1 - seplen - if offset > self._limit: - raise LimitOverrunError('Separator is not found, and chunk exceed the limit', offset) - - # Complete message (with full separator) may be present in buffer - # even when EOF flag is set. This may happen when the last chunk - # adds data which makes separator be found. That's why we check for - # EOF *ater* inspecting the buffer. - if self._eof: - chunk = bytes(self._buffer) - self._buffer.clear() - raise IncompleteReadError(chunk, None) - - # _wait_for_data() will resume reading if stream was paused. - yield from self._wait_for_data('readuntil') - - if isep > self._limit: - raise LimitOverrunError('Separator is found, but chunk is longer than limit', isep) - - chunk = self._buffer[:isep + seplen] - del self._buffer[:isep + seplen] - self._maybe_resume_transport() - return bytes(chunk) - - @coroutine - def read(self, n=-1): - """Read up to `n` bytes from the stream. - - If n is not provided, or set to -1, read until EOF and return all read - bytes. If the EOF was received and the internal buffer is empty, return - an empty bytes object. - - If n is zero, return empty bytes object immediatelly. - - If n is positive, this function try to read `n` bytes, and may return - less or equal bytes than requested, but at least one byte. If EOF was - received before any byte is read, this function returns empty byte - object. - - Returned value is not limited with limit, configured at stream creation. - - If stream was paused, this function will automatically resume it if - needed. - """ - - if self._exception is not None: - raise self._exception - - if n == 0: - return b'' - - if n < 0: - # This used to just loop creating a new waiter hoping to - # collect everything in self._buffer, but that would - # deadlock if the subprocess sends more than self.limit - # bytes. So just call self.read(self._limit) until EOF. - blocks = [] - while True: - block = yield from self.read(self._limit) - if not block: - break - blocks.append(block) - return b''.join(blocks) - - if not self._buffer and not self._eof: - yield from self._wait_for_data('read') - - # This will work right even if buffer is less than n bytes - data = bytes(self._buffer[:n]) - del self._buffer[:n] - - self._maybe_resume_transport() - return data - - @coroutine - def readexactly(self, n): - """Read exactly `n` bytes. - - Raise an `IncompleteReadError` if EOF is reached before `n` bytes can be - read. The `IncompleteReadError.partial` attribute of the exception will - contain the partial read bytes. - - if n is zero, return empty bytes object. - - Returned value is not limited with limit, configured at stream creation. - - If stream was paused, this function will automatically resume it if - needed. - """ - if n < 0: - raise ValueError('readexactly size can not be less than zero') - - if self._exception is not None: - raise self._exception - - if n == 0: - return b'' - - # There used to be "optimized" code here. It created its own - # Future and waited until self._buffer had at least the n - # bytes, then called read(n). Unfortunately, this could pause - # the transport if the argument was larger than the pause - # limit (which is twice self._limit). So now we just read() - # into a local buffer. - - blocks = [] - while n > 0: - block = yield from self.read(n) - if not block: - partial = b''.join(blocks) - raise IncompleteReadError(partial, len(partial) + n) - blocks.append(block) - n -= len(block) - - assert n == 0 - - return b''.join(blocks) - - if compat.PY35: - @coroutine - def __aiter__(self): - return self - - @coroutine - def __anext__(self): - val = yield from self.readline() - if val == b'': - raise StopAsyncIteration - return val diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/subprocess.py --- a/Lib/asyncio/subprocess.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,213 +0,0 @@ -__all__ = ['create_subprocess_exec', 'create_subprocess_shell'] - -import subprocess - -from . import events -from . import protocols -from . import streams -from . import tasks -from .coroutines import coroutine -from .log import logger - - -PIPE = subprocess.PIPE -STDOUT = subprocess.STDOUT -DEVNULL = subprocess.DEVNULL - - -class SubprocessStreamProtocol(streams.FlowControlMixin, - protocols.SubprocessProtocol): - """Like StreamReaderProtocol, but for a subprocess.""" - - def __init__(self, limit, loop): - super().__init__(loop=loop) - self._limit = limit - self.stdin = self.stdout = self.stderr = None - self._transport = None - - def __repr__(self): - info = [self.__class__.__name__] - if self.stdin is not None: - info.append('stdin=%r' % self.stdin) - if self.stdout is not None: - info.append('stdout=%r' % self.stdout) - if self.stderr is not None: - info.append('stderr=%r' % self.stderr) - return '<%s>' % ' '.join(info) - - def connection_made(self, transport): - self._transport = transport - - stdout_transport = transport.get_pipe_transport(1) - if stdout_transport is not None: - self.stdout = streams.StreamReader(limit=self._limit, - loop=self._loop) - self.stdout.set_transport(stdout_transport) - - stderr_transport = transport.get_pipe_transport(2) - if stderr_transport is not None: - self.stderr = streams.StreamReader(limit=self._limit, - loop=self._loop) - self.stderr.set_transport(stderr_transport) - - stdin_transport = transport.get_pipe_transport(0) - if stdin_transport is not None: - self.stdin = streams.StreamWriter(stdin_transport, - protocol=self, - reader=None, - loop=self._loop) - - def pipe_data_received(self, fd, data): - if fd == 1: - reader = self.stdout - elif fd == 2: - reader = self.stderr - else: - reader = None - if reader is not None: - reader.feed_data(data) - - def pipe_connection_lost(self, fd, exc): - if fd == 0: - pipe = self.stdin - if pipe is not None: - pipe.close() - self.connection_lost(exc) - return - if fd == 1: - reader = self.stdout - elif fd == 2: - reader = self.stderr - else: - reader = None - if reader != None: - if exc is None: - reader.feed_eof() - else: - reader.set_exception(exc) - - def process_exited(self): - self._transport.close() - self._transport = None - - -class Process: - def __init__(self, transport, protocol, loop): - self._transport = transport - self._protocol = protocol - self._loop = loop - self.stdin = protocol.stdin - self.stdout = protocol.stdout - self.stderr = protocol.stderr - self.pid = transport.get_pid() - - def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.pid) - - @property - def returncode(self): - return self._transport.get_returncode() - - @coroutine - def wait(self): - """Wait until the process exit and return the process return code. - - This method is a coroutine.""" - return (yield from self._transport._wait()) - - def send_signal(self, signal): - self._transport.send_signal(signal) - - def terminate(self): - self._transport.terminate() - - def kill(self): - self._transport.kill() - - @coroutine - def _feed_stdin(self, input): - debug = self._loop.get_debug() - self.stdin.write(input) - if debug: - logger.debug('%r communicate: feed stdin (%s bytes)', - self, len(input)) - try: - yield from self.stdin.drain() - except (BrokenPipeError, ConnectionResetError) as exc: - # communicate() ignores BrokenPipeError and ConnectionResetError - if debug: - logger.debug('%r communicate: stdin got %r', self, exc) - - if debug: - logger.debug('%r communicate: close stdin', self) - self.stdin.close() - - @coroutine - def _noop(self): - return None - - @coroutine - def _read_stream(self, fd): - transport = self._transport.get_pipe_transport(fd) - if fd == 2: - stream = self.stderr - else: - assert fd == 1 - stream = self.stdout - if self._loop.get_debug(): - name = 'stdout' if fd == 1 else 'stderr' - logger.debug('%r communicate: read %s', self, name) - output = yield from stream.read() - if self._loop.get_debug(): - name = 'stdout' if fd == 1 else 'stderr' - logger.debug('%r communicate: close %s', self, name) - transport.close() - return output - - @coroutine - def communicate(self, input=None): - if input: - stdin = self._feed_stdin(input) - else: - stdin = self._noop() - if self.stdout is not None: - stdout = self._read_stream(1) - else: - stdout = self._noop() - if self.stderr is not None: - stderr = self._read_stream(2) - else: - stderr = self._noop() - stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr, - loop=self._loop) - yield from self.wait() - return (stdout, stderr) - - -@coroutine -def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, - loop=None, limit=streams._DEFAULT_LIMIT, **kwds): - if loop is None: - loop = events.get_event_loop() - protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, - loop=loop) - transport, protocol = yield from loop.subprocess_shell( - protocol_factory, - cmd, stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) - return Process(transport, protocol, loop) - -@coroutine -def create_subprocess_exec(program, *args, stdin=None, stdout=None, - stderr=None, loop=None, - limit=streams._DEFAULT_LIMIT, **kwds): - if loop is None: - loop = events.get_event_loop() - protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, - loop=loop) - transport, protocol = yield from loop.subprocess_exec( - protocol_factory, - program, *args, - stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) - return Process(transport, protocol, loop) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/tasks.py --- a/Lib/asyncio/tasks.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,785 +0,0 @@ -"""Support for tasks, coroutines and the scheduler.""" - -__all__ = ['Task', - 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', - 'wait', 'wait_for', 'as_completed', 'sleep', 'async', - 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', - 'timeout', - ] - -import concurrent.futures -import functools -import inspect -import linecache -import traceback -import warnings -import weakref - -from . import compat -from . import coroutines -from . import events -from . import futures -from .coroutines import coroutine - - -class Task(futures.Future): - """A coroutine wrapped in a Future.""" - - # An important invariant maintained while a Task not done: - # - # - Either _fut_waiter is None, and _step() is scheduled; - # - or _fut_waiter is some Future, and _step() is *not* scheduled. - # - # The only transition from the latter to the former is through - # _wakeup(). When _fut_waiter is not None, one of its callbacks - # must be _wakeup(). - - # Weak set containing all tasks alive. - _all_tasks = weakref.WeakSet() - - # Dictionary containing tasks that are currently active in - # all running event loops. {EventLoop: Task} - _current_tasks = {} - - # If False, don't log a message if the task is destroyed whereas its - # status is still pending - _log_destroy_pending = True - - @classmethod - def current_task(cls, loop=None): - """Return the currently running task in an event loop or None. - - By default the current task for the current event loop is returned. - - None is returned when called not in the context of a Task. - """ - if loop is None: - loop = events.get_event_loop() - return cls._current_tasks.get(loop) - - @classmethod - def all_tasks(cls, loop=None): - """Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - """ - if loop is None: - loop = events.get_event_loop() - return {t for t in cls._all_tasks if t._loop is loop} - - def __init__(self, coro, *, loop=None): - assert coroutines.iscoroutine(coro), repr(coro) - super().__init__(loop=loop) - if self._source_traceback: - del self._source_traceback[-1] - self._coro = coro - self._fut_waiter = None - self._must_cancel = False - self._loop.call_soon(self._step) - self.__class__._all_tasks.add(self) - - # On Python 3.3 or older, objects with a destructor that are part of a - # reference cycle are never destroyed. That's not the case any more on - # Python 3.4 thanks to the PEP 442. - if compat.PY34: - def __del__(self): - if self._state == futures._PENDING and self._log_destroy_pending: - context = { - 'task': self, - 'message': 'Task was destroyed but it is pending!', - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - futures.Future.__del__(self) - - def _repr_info(self): - info = super()._repr_info() - - if self._must_cancel: - # replace status - info[0] = 'cancelling' - - coro = coroutines._format_coroutine(self._coro) - info.insert(1, 'coro=<%s>' % coro) - - if self._fut_waiter is not None: - info.insert(2, 'wait_for=%r' % self._fut_waiter) - return info - - def get_stack(self, *, limit=None): - """Return the list of stack frames for this task's coroutine. - - If the coroutine is not done, this returns the stack where it is - suspended. If the coroutine has completed successfully or was - cancelled, this returns an empty list. If the coroutine was - terminated by an exception, this returns the list of traceback - frames. - - The frames are always ordered from oldest to newest. - - The optional limit gives the maximum number of frames to - return; by default all available frames are returned. Its - meaning differs depending on whether a stack or a traceback is - returned: the newest frames of a stack are returned, but the - oldest frames of a traceback are returned. (This matches the - behavior of the traceback module.) - - For reasons beyond our control, only one stack frame is - returned for a suspended coroutine. - """ - frames = [] - try: - # 'async def' coroutines - f = self._coro.cr_frame - except AttributeError: - f = self._coro.gi_frame - if f is not None: - while f is not None: - if limit is not None: - if limit <= 0: - break - limit -= 1 - frames.append(f) - f = f.f_back - frames.reverse() - elif self._exception is not None: - tb = self._exception.__traceback__ - while tb is not None: - if limit is not None: - if limit <= 0: - break - limit -= 1 - frames.append(tb.tb_frame) - tb = tb.tb_next - return frames - - def print_stack(self, *, limit=None, file=None): - """Print the stack or traceback for this task's coroutine. - - This produces output similar to that of the traceback module, - for the frames retrieved by get_stack(). The limit argument - is passed to get_stack(). The file argument is an I/O stream - to which the output is written; by default output is written - to sys.stderr. - """ - extracted_list = [] - checked = set() - for f in self.get_stack(limit=limit): - lineno = f.f_lineno - co = f.f_code - filename = co.co_filename - name = co.co_name - if filename not in checked: - checked.add(filename) - linecache.checkcache(filename) - line = linecache.getline(filename, lineno, f.f_globals) - extracted_list.append((filename, lineno, name, line)) - exc = self._exception - if not extracted_list: - print('No stack for %r' % self, file=file) - elif exc is not None: - print('Traceback for %r (most recent call last):' % self, - file=file) - else: - print('Stack for %r (most recent call last):' % self, - file=file) - traceback.print_list(extracted_list, file=file) - if exc is not None: - for line in traceback.format_exception_only(exc.__class__, exc): - print(line, file=file, end='') - - def cancel(self): - """Request that this task cancel itself. - - This arranges for a CancelledError to be thrown into the - wrapped coroutine on the next cycle through the event loop. - The coroutine then has a chance to clean up or even deny - the request using try/except/finally. - - Unlike Future.cancel, this does not guarantee that the - task will be cancelled: the exception might be caught and - acted upon, delaying cancellation of the task or preventing - cancellation completely. The task may also return a value or - raise a different exception. - - Immediately after this method is called, Task.cancelled() will - not return True (unless the task was already cancelled). A - task will be marked as cancelled when the wrapped coroutine - terminates with a CancelledError exception (even if cancel() - was not called). - """ - if self.done(): - return False - if self._fut_waiter is not None: - if self._fut_waiter.cancel(): - # Leave self._fut_waiter; it may be a Task that - # catches and ignores the cancellation so we may have - # to cancel it again later. - return True - # It must be the case that self._step is already scheduled. - self._must_cancel = True - return True - - def _step(self, exc=None): - assert not self.done(), \ - '_step(): already done: {!r}, {!r}'.format(self, exc) - if self._must_cancel: - if not isinstance(exc, futures.CancelledError): - exc = futures.CancelledError() - self._must_cancel = False - coro = self._coro - self._fut_waiter = None - - self.__class__._current_tasks[self._loop] = self - # Call either coro.throw(exc) or coro.send(None). - try: - if exc is None: - # We use the `send` method directly, because coroutines - # don't have `__iter__` and `__next__` methods. - result = coro.send(None) - else: - result = coro.throw(exc) - except StopIteration as exc: - self.set_result(exc.value) - except futures.CancelledError as exc: - super().cancel() # I.e., Future.cancel(self). - except Exception as exc: - self.set_exception(exc) - except BaseException as exc: - self.set_exception(exc) - raise - else: - if isinstance(result, futures.Future): - # Yielded Future must come from Future.__iter__(). - if result._loop is not self._loop: - self._loop.call_soon( - self._step, - RuntimeError( - 'Task {!r} got Future {!r} attached to a ' - 'different loop'.format(self, result))) - elif result._blocking: - result._blocking = False - result.add_done_callback(self._wakeup) - self._fut_waiter = result - if self._must_cancel: - if self._fut_waiter.cancel(): - self._must_cancel = False - else: - self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from ' - 'in task {!r} with {!r}'.format(self, result))) - elif result is None: - # Bare yield relinquishes control for one event loop iteration. - self._loop.call_soon(self._step) - elif inspect.isgenerator(result): - # Yielding a generator is just wrong. - self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from for ' - 'generator in task {!r} with {}'.format( - self, result))) - else: - # Yielding something else is an error. - self._loop.call_soon( - self._step, - RuntimeError( - 'Task got bad yield: {!r}'.format(result))) - finally: - self.__class__._current_tasks.pop(self._loop) - self = None # Needed to break cycles when an exception occurs. - - def _wakeup(self, future): - try: - future.result() - except Exception as exc: - # This may also be a cancellation. - self._step(exc) - else: - # Don't pass the value of `future.result()` explicitly, - # as `Future.__iter__` and `Future.__await__` don't need it. - # If we call `_step(value, None)` instead of `_step()`, - # Python eval loop would use `.send(value)` method call, - # instead of `__next__()`, which is slower for futures - # that return non-generator iterators from their `__iter__`. - self._step() - self = None # Needed to break cycles when an exception occurs. - - -# wait() and as_completed() similar to those in PEP 3148. - -FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED -FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION -ALL_COMPLETED = concurrent.futures.ALL_COMPLETED - - -@coroutine -def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): - """Wait for the Futures and coroutines given by fs to complete. - - The sequence futures must not be empty. - - Coroutines will be wrapped in Tasks. - - Returns two sets of Future: (done, pending). - - Usage: - - done, pending = yield from asyncio.wait(fs) - - Note: This does not raise TimeoutError! Futures that aren't done - when the timeout occurs are returned in the second set. - """ - if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) - if not fs: - raise ValueError('Set of coroutines/Futures is empty.') - if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED): - raise ValueError('Invalid return_when value: {}'.format(return_when)) - - if loop is None: - loop = events.get_event_loop() - - fs = {ensure_future(f, loop=loop) for f in set(fs)} - - return (yield from _wait(fs, timeout, return_when, loop)) - - -def _release_waiter(waiter, *args): - if not waiter.done(): - waiter.set_result(None) - - -@coroutine -def wait_for(fut, timeout, *, loop=None): - """Wait for the single Future or coroutine to complete, with timeout. - - Coroutine will be wrapped in Task. - - Returns result of the Future or coroutine. When a timeout occurs, - it cancels the task and raises TimeoutError. To avoid the task - cancellation, wrap it in shield(). - - If the wait is cancelled, the task is also cancelled. - - This function is a coroutine. - """ - if loop is None: - loop = events.get_event_loop() - - if timeout is None: - return (yield from fut) - - waiter = futures.Future(loop=loop) - timeout_handle = loop.call_later(timeout, _release_waiter, waiter) - cb = functools.partial(_release_waiter, waiter) - - fut = ensure_future(fut, loop=loop) - fut.add_done_callback(cb) - - try: - # wait until the future completes or the timeout - try: - yield from waiter - except futures.CancelledError: - fut.remove_done_callback(cb) - fut.cancel() - raise - - if fut.done(): - return fut.result() - else: - fut.remove_done_callback(cb) - fut.cancel() - raise futures.TimeoutError() - finally: - timeout_handle.cancel() - - -@coroutine -def _wait(fs, timeout, return_when, loop): - """Internal helper for wait() and _wait_for(). - - The fs argument must be a collection of Futures. - """ - assert fs, 'Set of Futures is empty.' - waiter = futures.Future(loop=loop) - timeout_handle = None - if timeout is not None: - timeout_handle = loop.call_later(timeout, _release_waiter, waiter) - counter = len(fs) - - def _on_completion(f): - nonlocal counter - counter -= 1 - if (counter <= 0 or - return_when == FIRST_COMPLETED or - return_when == FIRST_EXCEPTION and (not f.cancelled() and - f.exception() is not None)): - if timeout_handle is not None: - timeout_handle.cancel() - if not waiter.done(): - waiter.set_result(None) - - for f in fs: - f.add_done_callback(_on_completion) - - try: - yield from waiter - finally: - if timeout_handle is not None: - timeout_handle.cancel() - - done, pending = set(), set() - for f in fs: - f.remove_done_callback(_on_completion) - if f.done(): - done.add(f) - else: - pending.add(f) - return done, pending - - -# This is *not* a @coroutine! It is just an iterator (yielding Futures). -def as_completed(fs, *, loop=None, timeout=None): - """Return an iterator whose values are coroutines. - - When waiting for the yielded coroutines you'll get the results (or - exceptions!) of the original Futures (or coroutines), in the order - in which and as soon as they complete. - - This differs from PEP 3148; the proper way to use this is: - - for f in as_completed(fs): - result = yield from f # The 'yield from' may raise. - # Use result. - - If a timeout is specified, the 'yield from' will raise - TimeoutError when the timeout occurs before all Futures are done. - - Note: The futures 'f' are not necessarily members of fs. - """ - if isinstance(fs, futures.Future) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) - loop = loop if loop is not None else events.get_event_loop() - todo = {ensure_future(f, loop=loop) for f in set(fs)} - from .queues import Queue # Import here to avoid circular import problem. - done = Queue(loop=loop) - timeout_handle = None - - def _on_timeout(): - for f in todo: - f.remove_done_callback(_on_completion) - done.put_nowait(None) # Queue a dummy value for _wait_for_one(). - todo.clear() # Can't do todo.remove(f) in the loop. - - def _on_completion(f): - if not todo: - return # _on_timeout() was here first. - todo.remove(f) - done.put_nowait(f) - if not todo and timeout_handle is not None: - timeout_handle.cancel() - - @coroutine - def _wait_for_one(): - f = yield from done.get() - if f is None: - # Dummy value from _on_timeout(). - raise futures.TimeoutError - return f.result() # May raise f.exception(). - - for f in todo: - f.add_done_callback(_on_completion) - if todo and timeout is not None: - timeout_handle = loop.call_later(timeout, _on_timeout) - for _ in range(len(todo)): - yield _wait_for_one() - - -@coroutine -def sleep(delay, result=None, *, loop=None): - """Coroutine that completes after a given time (in seconds).""" - if delay == 0: - yield - return result - - future = futures.Future(loop=loop) - h = future._loop.call_later(delay, - futures._set_result_unless_cancelled, - future, result) - try: - return (yield from future) - finally: - h.cancel() - - -def async(coro_or_future, *, loop=None): - """Wrap a coroutine in a future. - - If the argument is a Future, it is returned directly. - - This function is deprecated in 3.5. Use asyncio.ensure_future() instead. - """ - - warnings.warn("asyncio.async() function is deprecated, use ensure_future()", - DeprecationWarning) - - return ensure_future(coro_or_future, loop=loop) - - -def ensure_future(coro_or_future, *, loop=None): - """Wrap a coroutine or an awaitable in a future. - - If the argument is a Future, it is returned directly. - """ - if isinstance(coro_or_future, futures.Future): - if loop is not None and loop is not coro_or_future._loop: - raise ValueError('loop argument must agree with Future') - return coro_or_future - elif coroutines.iscoroutine(coro_or_future): - if loop is None: - loop = events.get_event_loop() - task = loop.create_task(coro_or_future) - if task._source_traceback: - del task._source_traceback[-1] - return task - elif compat.PY35 and inspect.isawaitable(coro_or_future): - return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) - else: - raise TypeError('A Future, a coroutine or an awaitable is required') - - -@coroutine -def _wrap_awaitable(awaitable): - """Helper for asyncio.ensure_future(). - - Wraps awaitable (an object with __await__) into a coroutine - that will later be wrapped in a Task by ensure_future(). - """ - return (yield from awaitable.__await__()) - - -class _GatheringFuture(futures.Future): - """Helper for gather(). - - This overrides cancel() to cancel all the children and act more - like Task.cancel(), which doesn't immediately mark itself as - cancelled. - """ - - def __init__(self, children, *, loop=None): - super().__init__(loop=loop) - self._children = children - - def cancel(self): - if self.done(): - return False - for child in self._children: - child.cancel() - return True - - -def gather(*coros_or_futures, loop=None, return_exceptions=False): - """Return a future aggregating results from the given coroutines - or futures. - - All futures must share the same event loop. If all the tasks are - done successfully, the returned future's result is the list of - results (in the order of the original sequence, not necessarily - the order of results arrival). If *return_exceptions* is True, - exceptions in the tasks are treated the same as successful - results, and gathered in the result list; otherwise, the first - raised exception will be immediately propagated to the returned - future. - - Cancellation: if the outer Future is cancelled, all children (that - have not completed yet) are also cancelled. If any child is - cancelled, this is treated as if it raised CancelledError -- - the outer Future is *not* cancelled in this case. (This is to - prevent the cancellation of one child to cause other children to - be cancelled.) - """ - if not coros_or_futures: - outer = futures.Future(loop=loop) - outer.set_result([]) - return outer - - arg_to_fut = {} - for arg in set(coros_or_futures): - if not isinstance(arg, futures.Future): - fut = ensure_future(arg, loop=loop) - if loop is None: - loop = fut._loop - # The caller cannot control this future, the "destroy pending task" - # warning should not be emitted. - fut._log_destroy_pending = False - else: - fut = arg - if loop is None: - loop = fut._loop - elif fut._loop is not loop: - raise ValueError("futures are tied to different event loops") - arg_to_fut[arg] = fut - - children = [arg_to_fut[arg] for arg in coros_or_futures] - nchildren = len(children) - outer = _GatheringFuture(children, loop=loop) - nfinished = 0 - results = [None] * nchildren - - def _done_callback(i, fut): - nonlocal nfinished - if outer.done(): - if not fut.cancelled(): - # Mark exception retrieved. - fut.exception() - return - - if fut.cancelled(): - res = futures.CancelledError() - if not return_exceptions: - outer.set_exception(res) - return - elif fut._exception is not None: - res = fut.exception() # Mark exception retrieved. - if not return_exceptions: - outer.set_exception(res) - return - else: - res = fut._result - results[i] = res - nfinished += 1 - if nfinished == nchildren: - outer.set_result(results) - - for i, fut in enumerate(children): - fut.add_done_callback(functools.partial(_done_callback, i)) - return outer - - -def shield(arg, *, loop=None): - """Wait for a future, shielding it from cancellation. - - The statement - - res = yield from shield(something()) - - is exactly equivalent to the statement - - res = yield from something() - - *except* that if the coroutine containing it is cancelled, the - task running in something() is not cancelled. From the POV of - something(), the cancellation did not happen. But its caller is - still cancelled, so the yield-from expression still raises - CancelledError. Note: If something() is cancelled by other means - this will still cancel shield(). - - If you want to completely ignore cancellation (not recommended) - you can combine shield() with a try/except clause, as follows: - - try: - res = yield from shield(something()) - except CancelledError: - res = None - """ - inner = ensure_future(arg, loop=loop) - if inner.done(): - # Shortcut. - return inner - loop = inner._loop - outer = futures.Future(loop=loop) - - def _done_callback(inner): - if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() - return - - if inner.cancelled(): - outer.cancel() - else: - exc = inner.exception() - if exc is not None: - outer.set_exception(exc) - else: - outer.set_result(inner.result()) - - inner.add_done_callback(_done_callback) - return outer - - -def run_coroutine_threadsafe(coro, loop): - """Submit a coroutine object to a given event loop. - - Return a concurrent.futures.Future to access the result. - """ - if not coroutines.iscoroutine(coro): - raise TypeError('A coroutine object is required') - future = concurrent.futures.Future() - - def callback(): - try: - futures._chain_future(ensure_future(coro, loop=loop), future) - except Exception as exc: - if future.set_running_or_notify_cancel(): - future.set_exception(exc) - raise - - loop.call_soon_threadsafe(callback) - return future - - -def timeout(timeout, *, loop=None): - """A factory which produce a context manager with timeout. - - Useful in cases when you want to apply timeout logic around block - of code or in cases when asyncio.wait_for is not suitable. - - For example: - - >>> with asyncio.timeout(0.001): - ... yield from coro() - - - timeout: timeout value in seconds - loop: asyncio compatible event loop - """ - if loop is None: - loop = events.get_event_loop() - return _Timeout(timeout, loop=loop) - - -class _Timeout: - def __init__(self, timeout, *, loop): - self._timeout = timeout - self._loop = loop - self._task = None - self._cancelled = False - self._cancel_handler = None - - def __enter__(self): - self._task = Task.current_task(loop=self._loop) - if self._task is None: - raise RuntimeError('Timeout context manager should be used ' - 'inside a task') - self._cancel_handler = self._loop.call_later( - self._timeout, self._cancel_task) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is futures.CancelledError and self._cancelled: - self._cancel_handler = None - self._task = None - raise futures.TimeoutError - self._cancel_handler.cancel() - self._cancel_handler = None - self._task = None - - def _cancel_task(self): - self._cancelled = self._task.cancel() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/test_utils.py --- a/Lib/asyncio/test_utils.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,463 +0,0 @@ -"""Utilities shared by tests.""" - -import collections -import contextlib -import io -import logging -import os -import re -import socket -import socketserver -import sys -import tempfile -import threading -import time -import unittest -from unittest import mock - -from http.server import HTTPServer -from wsgiref.simple_server import WSGIRequestHandler, WSGIServer - -try: - import ssl -except ImportError: # pragma: no cover - ssl = None - -from . import base_events -from . import compat -from . import events -from . import futures -from . import selectors -from . import tasks -from .coroutines import coroutine -from .log import logger - - -if sys.platform == 'win32': # pragma: no cover - from .windows_utils import socketpair -else: - from socket import socketpair # pragma: no cover - - -def dummy_ssl_context(): - if ssl is None: - return None - else: - return ssl.SSLContext(ssl.PROTOCOL_SSLv23) - - -def run_briefly(loop): - @coroutine - def once(): - pass - gen = once() - t = loop.create_task(gen) - # Don't log a warning if the task is not done after run_until_complete(). - # It occurs if the loop is stopped or if a task raises a BaseException. - t._log_destroy_pending = False - try: - loop.run_until_complete(t) - finally: - gen.close() - - -def run_until(loop, pred, timeout=30): - deadline = time.time() + timeout - while not pred(): - if timeout is not None: - timeout = deadline - time.time() - if timeout <= 0: - raise futures.TimeoutError() - loop.run_until_complete(tasks.sleep(0.001, loop=loop)) - - -def run_once(loop): - """Legacy API to run once through the event loop. - - This is the recommended pattern for test code. It will poll the - selector once and run all callbacks scheduled in response to I/O - events. - """ - loop.call_soon(loop.stop) - loop.run_forever() - - -class SilentWSGIRequestHandler(WSGIRequestHandler): - - def get_stderr(self): - return io.StringIO() - - def log_message(self, format, *args): - pass - - -class SilentWSGIServer(WSGIServer): - - request_timeout = 2 - - def get_request(self): - request, client_addr = super().get_request() - request.settimeout(self.request_timeout) - return request, client_addr - - def handle_error(self, request, client_address): - pass - - -class SSLWSGIServerMixin: - - def finish_request(self, request, client_address): - # The relative location of our test directory (which - # contains the ssl key and certificate files) differs - # between the stdlib and stand-alone asyncio. - # Prefer our own if we can find it. - here = os.path.join(os.path.dirname(__file__), '..', 'tests') - if not os.path.isdir(here): - here = os.path.join(os.path.dirname(os.__file__), - 'test', 'test_asyncio') - keyfile = os.path.join(here, 'ssl_key.pem') - certfile = os.path.join(here, 'ssl_cert.pem') - ssock = ssl.wrap_socket(request, - keyfile=keyfile, - certfile=certfile, - server_side=True) - try: - self.RequestHandlerClass(ssock, client_address, self) - ssock.close() - except OSError: - # maybe socket has been closed by peer - pass - - -class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer): - pass - - -def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls): - - def app(environ, start_response): - status = '200 OK' - headers = [('Content-type', 'text/plain')] - start_response(status, headers) - return [b'Test message'] - - # Run the test WSGI server in a separate thread in order not to - # interfere with event handling in the main thread - server_class = server_ssl_cls if use_ssl else server_cls - httpd = server_class(address, SilentWSGIRequestHandler) - httpd.set_app(app) - httpd.address = httpd.server_address - server_thread = threading.Thread( - target=lambda: httpd.serve_forever(poll_interval=0.05)) - server_thread.start() - try: - yield httpd - finally: - httpd.shutdown() - httpd.server_close() - server_thread.join() - - -if hasattr(socket, 'AF_UNIX'): - - class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer): - - def server_bind(self): - socketserver.UnixStreamServer.server_bind(self) - self.server_name = '127.0.0.1' - self.server_port = 80 - - - class UnixWSGIServer(UnixHTTPServer, WSGIServer): - - request_timeout = 2 - - def server_bind(self): - UnixHTTPServer.server_bind(self) - self.setup_environ() - - def get_request(self): - request, client_addr = super().get_request() - request.settimeout(self.request_timeout) - # Code in the stdlib expects that get_request - # will return a socket and a tuple (host, port). - # However, this isn't true for UNIX sockets, - # as the second return value will be a path; - # hence we return some fake data sufficient - # to get the tests going - return request, ('127.0.0.1', '') - - - class SilentUnixWSGIServer(UnixWSGIServer): - - def handle_error(self, request, client_address): - pass - - - class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer): - pass - - - def gen_unix_socket_path(): - with tempfile.NamedTemporaryFile() as file: - return file.name - - - @contextlib.contextmanager - def unix_socket_path(): - path = gen_unix_socket_path() - try: - yield path - finally: - try: - os.unlink(path) - except OSError: - pass - - - @contextlib.contextmanager - def run_test_unix_server(*, use_ssl=False): - with unix_socket_path() as path: - yield from _run_test_server(address=path, use_ssl=use_ssl, - server_cls=SilentUnixWSGIServer, - server_ssl_cls=UnixSSLWSGIServer) - - -@contextlib.contextmanager -def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False): - yield from _run_test_server(address=(host, port), use_ssl=use_ssl, - server_cls=SilentWSGIServer, - server_ssl_cls=SSLWSGIServer) - - -def make_test_protocol(base): - dct = {} - for name in dir(base): - if name.startswith('__') and name.endswith('__'): - # skip magic names - continue - dct[name] = MockCallback(return_value=None) - return type('TestProtocol', (base,) + base.__bases__, dct)() - - -class TestSelector(selectors.BaseSelector): - - def __init__(self): - self.keys = {} - - def register(self, fileobj, events, data=None): - key = selectors.SelectorKey(fileobj, 0, events, data) - self.keys[fileobj] = key - return key - - def unregister(self, fileobj): - return self.keys.pop(fileobj) - - def select(self, timeout): - return [] - - def get_map(self): - return self.keys - - -class TestLoop(base_events.BaseEventLoop): - """Loop for unittests. - - It manages self time directly. - If something scheduled to be executed later then - on next loop iteration after all ready handlers done - generator passed to __init__ is calling. - - Generator should be like this: - - def gen(): - ... - when = yield ... - ... = yield time_advance - - Value returned by yield is absolute time of next scheduled handler. - Value passed to yield is time advance to move loop's time forward. - """ - - def __init__(self, gen=None): - super().__init__() - - if gen is None: - def gen(): - yield - self._check_on_close = False - else: - self._check_on_close = True - - self._gen = gen() - next(self._gen) - self._time = 0 - self._clock_resolution = 1e-9 - self._timers = [] - self._selector = TestSelector() - - self.readers = {} - self.writers = {} - self.reset_counters() - - def time(self): - return self._time - - def advance_time(self, advance): - """Move test time forward.""" - if advance: - self._time += advance - - def close(self): - super().close() - if self._check_on_close: - try: - self._gen.send(0) - except StopIteration: - pass - else: # pragma: no cover - raise AssertionError("Time generator is not finished") - - def add_reader(self, fd, callback, *args): - self.readers[fd] = events.Handle(callback, args, self) - - def remove_reader(self, fd): - self.remove_reader_count[fd] += 1 - if fd in self.readers: - del self.readers[fd] - return True - else: - return False - - def assert_reader(self, fd, callback, *args): - assert fd in self.readers, 'fd {} is not registered'.format(fd) - handle = self.readers[fd] - assert handle._callback == callback, '{!r} != {!r}'.format( - handle._callback, callback) - assert handle._args == args, '{!r} != {!r}'.format( - handle._args, args) - - def add_writer(self, fd, callback, *args): - self.writers[fd] = events.Handle(callback, args, self) - - def remove_writer(self, fd): - self.remove_writer_count[fd] += 1 - if fd in self.writers: - del self.writers[fd] - return True - else: - return False - - def assert_writer(self, fd, callback, *args): - assert fd in self.writers, 'fd {} is not registered'.format(fd) - handle = self.writers[fd] - assert handle._callback == callback, '{!r} != {!r}'.format( - handle._callback, callback) - assert handle._args == args, '{!r} != {!r}'.format( - handle._args, args) - - def reset_counters(self): - self.remove_reader_count = collections.defaultdict(int) - self.remove_writer_count = collections.defaultdict(int) - - def _run_once(self): - super()._run_once() - for when in self._timers: - advance = self._gen.send(when) - self.advance_time(advance) - self._timers = [] - - def call_at(self, when, callback, *args): - self._timers.append(when) - return super().call_at(when, callback, *args) - - def _process_events(self, event_list): - return - - def _write_to_self(self): - pass - - -def MockCallback(**kwargs): - return mock.Mock(spec=['__call__'], **kwargs) - - -class MockPattern(str): - """A regex based str with a fuzzy __eq__. - - Use this helper with 'mock.assert_called_with', or anywhere - where a regex comparison between strings is needed. - - For instance: - mock_call.assert_called_with(MockPattern('spam.*ham')) - """ - def __eq__(self, other): - return bool(re.search(str(self), other, re.S)) - - -def get_function_source(func): - source = events._get_function_source(func) - if source is None: - raise ValueError("unable to get the source of %r" % (func,)) - return source - - -class TestCase(unittest.TestCase): - def set_event_loop(self, loop, *, cleanup=True): - assert loop is not None - # ensure that the event loop is passed explicitly in asyncio - events.set_event_loop(None) - if cleanup: - self.addCleanup(loop.close) - - def new_test_loop(self, gen=None): - loop = TestLoop(gen) - self.set_event_loop(loop) - return loop - - def tearDown(self): - events.set_event_loop(None) - - # Detect CPython bug #23353: ensure that yield/yield-from is not used - # in an except block of a generator - self.assertEqual(sys.exc_info(), (None, None, None)) - - if not compat.PY34: - # Python 3.3 compatibility - def subTest(self, *args, **kwargs): - class EmptyCM: - def __enter__(self): - pass - def __exit__(self, *exc): - pass - return EmptyCM() - - -@contextlib.contextmanager -def disable_logger(): - """Context manager to disable asyncio logger. - - For example, it can be used to ignore warnings in debug mode. - """ - old_level = logger.level - try: - logger.setLevel(logging.CRITICAL+1) - yield - finally: - logger.setLevel(old_level) - - -def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, - family=socket.AF_INET): - """Create a mock of a non-blocking socket.""" - sock = mock.MagicMock(socket.socket) - sock.proto = proto - sock.type = type - sock.family = family - sock.gettimeout.return_value = 0.0 - return sock - - -def force_legacy_ssl_support(): - return mock.patch('asyncio.sslproto._is_sslproto_available', - return_value=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/transports.py --- a/Lib/asyncio/transports.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,298 +0,0 @@ -"""Abstract Transport class.""" - -from asyncio import compat - -__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport', - 'Transport', 'DatagramTransport', 'SubprocessTransport', - ] - - -class BaseTransport: - """Base class for transports.""" - - def __init__(self, extra=None): - if extra is None: - extra = {} - self._extra = extra - - def get_extra_info(self, name, default=None): - """Get optional transport information.""" - return self._extra.get(name, default) - - def is_closing(self): - """Return True if the transport is closing or closed.""" - raise NotImplementedError - - def close(self): - """Close the transport. - - Buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's connection_lost() method will (eventually) called - with None as its argument. - """ - raise NotImplementedError - - -class ReadTransport(BaseTransport): - """Interface for read-only transports.""" - - def pause_reading(self): - """Pause the receiving end. - - No data will be passed to the protocol's data_received() - method until resume_reading() is called. - """ - raise NotImplementedError - - def resume_reading(self): - """Resume the receiving end. - - Data received will once again be passed to the protocol's - data_received() method. - """ - raise NotImplementedError - - -class WriteTransport(BaseTransport): - """Interface for write-only transports.""" - - def set_write_buffer_limits(self, high=None, low=None): - """Set the high- and low-water limits for write flow control. - - These two values control when to call the protocol's - pause_writing() and resume_writing() methods. If specified, - the low-water limit must be less than or equal to the - high-water limit. Neither value can be negative. - - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to an - implementation-specific value less than or equal to the - high-water limit. Setting high to zero forces low to zero as - well, and causes pause_writing() to be called whenever the - buffer becomes non-empty. Setting low to zero causes - resume_writing() to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. - """ - raise NotImplementedError - - def get_write_buffer_size(self): - """Return the current size of the write buffer.""" - raise NotImplementedError - - def write(self, data): - """Write some data bytes to the transport. - - This does not block; it buffers the data and arranges for it - to be sent out asynchronously. - """ - raise NotImplementedError - - def writelines(self, list_of_data): - """Write a list (or any iterable) of data bytes to the transport. - - The default implementation concatenates the arguments and - calls write() on the result. - """ - data = compat.flatten_list_bytes(list_of_data) - self.write(data) - - def write_eof(self): - """Close the write end after flushing buffered data. - - (This is like typing ^D into a UNIX program reading from stdin.) - - Data may still be received. - """ - raise NotImplementedError - - def can_write_eof(self): - """Return True if this transport supports write_eof(), False if not.""" - raise NotImplementedError - - def abort(self): - """Close the transport immediately. - - Buffered data will be lost. No more data will be received. - The protocol's connection_lost() method will (eventually) be - called with None as its argument. - """ - raise NotImplementedError - - -class Transport(ReadTransport, WriteTransport): - """Interface representing a bidirectional transport. - - There may be several implementations, but typically, the user does - not implement new transports; rather, the platform provides some - useful transports that are implemented using the platform's best - practices. - - The user never instantiates a transport directly; they call a - utility function, passing it a protocol factory and other - information necessary to create the transport and protocol. (E.g. - EventLoop.create_connection() or EventLoop.create_server().) - - The utility function will asynchronously create a transport and a - protocol and hook them up by calling the protocol's - connection_made() method, passing it the transport. - - The implementation here raises NotImplemented for every method - except writelines(), which calls write() in a loop. - """ - - -class DatagramTransport(BaseTransport): - """Interface for datagram (UDP) transports.""" - - def sendto(self, data, addr=None): - """Send data to the transport. - - This does not block; it buffers the data and arranges for it - to be sent out asynchronously. - addr is target socket address. - If addr is None use target address pointed on transport creation. - """ - raise NotImplementedError - - def abort(self): - """Close the transport immediately. - - Buffered data will be lost. No more data will be received. - The protocol's connection_lost() method will (eventually) be - called with None as its argument. - """ - raise NotImplementedError - - -class SubprocessTransport(BaseTransport): - - def get_pid(self): - """Get subprocess id.""" - raise NotImplementedError - - def get_returncode(self): - """Get subprocess returncode. - - See also - http://docs.python.org/3/library/subprocess#subprocess.Popen.returncode - """ - raise NotImplementedError - - def get_pipe_transport(self, fd): - """Get transport for pipe with number fd.""" - raise NotImplementedError - - def send_signal(self, signal): - """Send signal to subprocess. - - See also: - docs.python.org/3/library/subprocess#subprocess.Popen.send_signal - """ - raise NotImplementedError - - def terminate(self): - """Stop the subprocess. - - Alias for close() method. - - On Posix OSs the method sends SIGTERM to the subprocess. - On Windows the Win32 API function TerminateProcess() - is called to stop the subprocess. - - See also: - http://docs.python.org/3/library/subprocess#subprocess.Popen.terminate - """ - raise NotImplementedError - - def kill(self): - """Kill the subprocess. - - On Posix OSs the function sends SIGKILL to the subprocess. - On Windows kill() is an alias for terminate(). - - See also: - http://docs.python.org/3/library/subprocess#subprocess.Popen.kill - """ - raise NotImplementedError - - -class _FlowControlMixin(Transport): - """All the logic for (write) flow control in a mix-in base class. - - The subclass must implement get_write_buffer_size(). It must call - _maybe_pause_protocol() whenever the write buffer size increases, - and _maybe_resume_protocol() whenever it decreases. It may also - override set_write_buffer_limits() (e.g. to specify different - defaults). - - The subclass constructor must call super().__init__(extra). This - will call set_write_buffer_limits(). - - The user may call set_write_buffer_limits() and - get_write_buffer_size(), and their protocol's pause_writing() and - resume_writing() may be called. - """ - - def __init__(self, extra=None, loop=None): - super().__init__(extra) - assert loop is not None - self._loop = loop - self._protocol_paused = False - self._set_write_buffer_limits() - - def _maybe_pause_protocol(self): - size = self.get_write_buffer_size() - if size <= self._high_water: - return - if not self._protocol_paused: - self._protocol_paused = True - try: - self._protocol.pause_writing() - except Exception as exc: - self._loop.call_exception_handler({ - 'message': 'protocol.pause_writing() failed', - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - - def _maybe_resume_protocol(self): - if (self._protocol_paused and - self.get_write_buffer_size() <= self._low_water): - self._protocol_paused = False - try: - self._protocol.resume_writing() - except Exception as exc: - self._loop.call_exception_handler({ - 'message': 'protocol.resume_writing() failed', - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - - def get_write_buffer_limits(self): - return (self._low_water, self._high_water) - - def _set_write_buffer_limits(self, high=None, low=None): - if high is None: - if low is None: - high = 64*1024 - else: - high = 4*low - if low is None: - low = high // 4 - if not high >= low >= 0: - raise ValueError('high (%r) must be >= low (%r) must be >= 0' % - (high, low)) - self._high_water = high - self._low_water = low - - def set_write_buffer_limits(self, high=None, low=None): - self._set_write_buffer_limits(high=high, low=low) - self._maybe_pause_protocol() - - def get_write_buffer_size(self): - raise NotImplementedError diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/unix_events.py --- a/Lib/asyncio/unix_events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1007 +0,0 @@ -"""Selector event loop for Unix with signal handling.""" - -import errno -import os -import signal -import socket -import stat -import subprocess -import sys -import threading -import warnings - - -from . import base_events -from . import base_subprocess -from . import compat -from . import constants -from . import coroutines -from . import events -from . import futures -from . import selector_events -from . import selectors -from . import transports -from .coroutines import coroutine -from .log import logger - - -__all__ = ['SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'DefaultEventLoopPolicy', - ] - -if sys.platform == 'win32': # pragma: no cover - raise ImportError('Signals are not really supported on Windows') - - -def _sighandler_noop(signum, frame): - """Dummy signal handler.""" - pass - - -class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): - """Unix event loop. - - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. - """ - - def __init__(self, selector=None): - super().__init__(selector) - self._signal_handlers = {} - - def _socketpair(self): - return socket.socketpair() - - def close(self): - super().close() - for sig in list(self._signal_handlers): - self.remove_signal_handler(sig) - - def _process_self_data(self, data): - for signum in data: - if not signum: - # ignore null bytes written by _write_to_self() - continue - self._handle_signal(signum) - - def add_signal_handler(self, sig, callback, *args): - """Add a handler for a signal. UNIX only. - - Raise ValueError if the signal number is invalid or uncatchable. - Raise RuntimeError if there is a problem setting up the handler. - """ - if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): - raise TypeError("coroutines cannot be used " - "with add_signal_handler()") - self._check_signal(sig) - self._check_closed() - try: - # set_wakeup_fd() raises ValueError if this is not the - # main thread. By calling it early we ensure that an - # event loop running in another thread cannot add a signal - # handler. - signal.set_wakeup_fd(self._csock.fileno()) - except (ValueError, OSError) as exc: - raise RuntimeError(str(exc)) - - handle = events.Handle(callback, args, self) - self._signal_handlers[sig] = handle - - try: - # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will - # read signal numbers from this file descriptor to handle signals. - signal.signal(sig, _sighandler_noop) - - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(sig, False) - except OSError as exc: - del self._signal_handlers[sig] - if not self._signal_handlers: - try: - signal.set_wakeup_fd(-1) - except (ValueError, OSError) as nexc: - logger.info('set_wakeup_fd(-1) failed: %s', nexc) - - if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) - else: - raise - - def _handle_signal(self, sig): - """Internal helper that is the actual signal handler.""" - handle = self._signal_handlers.get(sig) - if handle is None: - return # Assume it's some race condition. - if handle._cancelled: - self.remove_signal_handler(sig) # Remove it properly. - else: - self._add_callback_signalsafe(handle) - - def remove_signal_handler(self, sig): - """Remove a handler for a signal. UNIX only. - - Return True if a signal handler was removed, False if not. - """ - self._check_signal(sig) - try: - del self._signal_handlers[sig] - except KeyError: - return False - - if sig == signal.SIGINT: - handler = signal.default_int_handler - else: - handler = signal.SIG_DFL - - try: - signal.signal(sig, handler) - except OSError as exc: - if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) - else: - raise - - if not self._signal_handlers: - try: - signal.set_wakeup_fd(-1) - except (ValueError, OSError) as exc: - logger.info('set_wakeup_fd(-1) failed: %s', exc) - - return True - - def _check_signal(self, sig): - """Internal helper to validate a signal. - - Raise ValueError if the signal number is invalid or uncatchable. - Raise RuntimeError if there is a problem setting up the handler. - """ - if not isinstance(sig, int): - raise TypeError('sig must be an int, not {!r}'.format(sig)) - - if not (1 <= sig < signal.NSIG): - raise ValueError( - 'sig {} out of range(1, {})'.format(sig, signal.NSIG)) - - def _make_read_pipe_transport(self, pipe, protocol, waiter=None, - extra=None): - return _UnixReadPipeTransport(self, pipe, protocol, waiter, extra) - - def _make_write_pipe_transport(self, pipe, protocol, waiter=None, - extra=None): - return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) - - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): - with events.get_child_watcher() as watcher: - waiter = futures.Future(loop=self) - transp = _UnixSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - - watcher.add_child_handler(transp.get_pid(), - self._child_watcher_callback, transp) - try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly - # sys.exc_info() - err = exc - else: - err = None - - if err is not None: - transp.close() - yield from transp._wait() - raise err - - return transp - - def _child_watcher_callback(self, pid, returncode, transp): - self.call_soon_threadsafe(transp._process_exited, returncode) - - @coroutine - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): - assert server_hostname is None or isinstance(server_hostname, str) - if ssl: - if server_hostname is None: - raise ValueError( - 'you have to pass server_hostname when using ssl') - else: - if server_hostname is not None: - raise ValueError('server_hostname is only meaningful with ssl') - - if path is not None: - if sock is not None: - raise ValueError( - 'path and sock can not be specified at the same time') - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) - try: - sock.setblocking(False) - yield from self.sock_connect(sock, path) - except: - sock.close() - raise - - else: - if sock is None: - raise ValueError('no path and sock were specified') - sock.setblocking(False) - - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) - return transport, protocol - - @coroutine - def create_unix_server(self, protocol_factory, path=None, *, - sock=None, backlog=100, ssl=None): - if isinstance(ssl, bool): - raise TypeError('ssl argument must be an SSLContext or None') - - if path is not None: - if sock is not None: - raise ValueError( - 'path and sock can not be specified at the same time') - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - - try: - sock.bind(path) - except OSError as exc: - sock.close() - if exc.errno == errno.EADDRINUSE: - # Let's improve the error message by adding - # with what exact address it occurs. - msg = 'Address {!r} is already in use'.format(path) - raise OSError(errno.EADDRINUSE, msg) from None - else: - raise - except: - sock.close() - raise - else: - if sock is None: - raise ValueError( - 'path was not specified, and no sock specified') - - if sock.family != socket.AF_UNIX: - raise ValueError( - 'A UNIX Domain Socket was expected, got {!r}'.format(sock)) - - server = base_events.Server(self, [sock]) - sock.listen(backlog) - sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server) - return server - - -if hasattr(os, 'set_blocking'): - def _set_nonblocking(fd): - os.set_blocking(fd, False) -else: - import fcntl - - def _set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) - - -class _UnixReadPipeTransport(transports.ReadTransport): - - max_size = 256 * 1024 # max bytes we read in one event loop iteration - - def __init__(self, loop, pipe, protocol, waiter=None, extra=None): - super().__init__(extra) - self._extra['pipe'] = pipe - self._loop = loop - self._pipe = pipe - self._fileno = pipe.fileno() - mode = os.fstat(self._fileno).st_mode - if not (stat.S_ISFIFO(mode) or - stat.S_ISSOCK(mode) or - stat.S_ISCHR(mode)): - raise ValueError("Pipe transport is for pipes/sockets only.") - _set_nonblocking(self._fileno) - self._protocol = protocol - self._closing = False - self._loop.call_soon(self._protocol.connection_made, self) - # only start reading when connection_made() has been called - self._loop.call_soon(self._loop.add_reader, - self._fileno, self._read_ready) - if waiter is not None: - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) - - def __repr__(self): - info = [self.__class__.__name__] - if self._pipe is None: - info.append('closed') - elif self._closing: - info.append('closing') - info.append('fd=%s' % self._fileno) - if self._pipe is not None: - polling = selector_events._test_selector_event( - self._loop._selector, - self._fileno, selectors.EVENT_READ) - if polling: - info.append('polling') - else: - info.append('idle') - else: - info.append('closed') - return '<%s>' % ' '.join(info) - - def _read_ready(self): - try: - data = os.read(self._fileno, self.max_size) - except (BlockingIOError, InterruptedError): - pass - except OSError as exc: - self._fatal_error(exc, 'Fatal read error on pipe transport') - else: - if data: - self._protocol.data_received(data) - else: - if self._loop.get_debug(): - logger.info("%r was closed by peer", self) - self._closing = True - self._loop.remove_reader(self._fileno) - self._loop.call_soon(self._protocol.eof_received) - self._loop.call_soon(self._call_connection_lost, None) - - def pause_reading(self): - self._loop.remove_reader(self._fileno) - - def resume_reading(self): - self._loop.add_reader(self._fileno, self._read_ready) - - def is_closing(self): - return self._closing - - def close(self): - if not self._closing: - self._close(None) - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self._pipe.close() - - def _fatal_error(self, exc, message='Fatal error on pipe transport'): - # should be called by exception handler only - if (isinstance(exc, OSError) and exc.errno == errno.EIO): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._close(exc) - - def _close(self, exc): - self._closing = True - self._loop.remove_reader(self._fileno) - self._loop.call_soon(self._call_connection_lost, exc) - - def _call_connection_lost(self, exc): - try: - self._protocol.connection_lost(exc) - finally: - self._pipe.close() - self._pipe = None - self._protocol = None - self._loop = None - - -class _UnixWritePipeTransport(transports._FlowControlMixin, - transports.WriteTransport): - - def __init__(self, loop, pipe, protocol, waiter=None, extra=None): - super().__init__(extra, loop) - self._extra['pipe'] = pipe - self._pipe = pipe - self._fileno = pipe.fileno() - mode = os.fstat(self._fileno).st_mode - is_socket = stat.S_ISSOCK(mode) - if not (is_socket or - stat.S_ISFIFO(mode) or - stat.S_ISCHR(mode)): - raise ValueError("Pipe transport is only for " - "pipes, sockets and character devices") - _set_nonblocking(self._fileno) - self._protocol = protocol - self._buffer = [] - self._conn_lost = 0 - self._closing = False # Set when close() or write_eof() called. - - self._loop.call_soon(self._protocol.connection_made, self) - - # On AIX, the reader trick (to be notified when the read end of the - # socket is closed) only works for sockets. On other platforms it - # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) - if is_socket or not sys.platform.startswith("aix"): - # only start reading when connection_made() has been called - self._loop.call_soon(self._loop.add_reader, - self._fileno, self._read_ready) - - if waiter is not None: - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) - - def __repr__(self): - info = [self.__class__.__name__] - if self._pipe is None: - info.append('closed') - elif self._closing: - info.append('closing') - info.append('fd=%s' % self._fileno) - if self._pipe is not None: - polling = selector_events._test_selector_event( - self._loop._selector, - self._fileno, selectors.EVENT_WRITE) - if polling: - info.append('polling') - else: - info.append('idle') - - bufsize = self.get_write_buffer_size() - info.append('bufsize=%s' % bufsize) - else: - info.append('closed') - return '<%s>' % ' '.join(info) - - def get_write_buffer_size(self): - return sum(len(data) for data in self._buffer) - - def _read_ready(self): - # Pipe was closed by peer. - if self._loop.get_debug(): - logger.info("%r was closed by peer", self) - if self._buffer: - self._close(BrokenPipeError()) - else: - self._close() - - def write(self, data): - assert isinstance(data, (bytes, bytearray, memoryview)), repr(data) - if isinstance(data, bytearray): - data = memoryview(data) - if not data: - return - - if self._conn_lost or self._closing: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('pipe closed by peer or ' - 'os.write(pipe, data) raised exception.') - self._conn_lost += 1 - return - - if not self._buffer: - # Attempt to send it right away first. - try: - n = os.write(self._fileno, data) - except (BlockingIOError, InterruptedError): - n = 0 - except Exception as exc: - self._conn_lost += 1 - self._fatal_error(exc, 'Fatal write error on pipe transport') - return - if n == len(data): - return - elif n > 0: - data = data[n:] - self._loop.add_writer(self._fileno, self._write_ready) - - self._buffer.append(data) - self._maybe_pause_protocol() - - def _write_ready(self): - data = b''.join(self._buffer) - assert data, 'Data should not be empty' - - self._buffer.clear() - try: - n = os.write(self._fileno, data) - except (BlockingIOError, InterruptedError): - self._buffer.append(data) - except Exception as exc: - self._conn_lost += 1 - # Remove writer here, _fatal_error() doesn't it - # because _buffer is empty. - self._loop.remove_writer(self._fileno) - self._fatal_error(exc, 'Fatal write error on pipe transport') - else: - if n == len(data): - self._loop.remove_writer(self._fileno) - self._maybe_resume_protocol() # May append to buffer. - if not self._buffer and self._closing: - self._loop.remove_reader(self._fileno) - self._call_connection_lost(None) - return - elif n > 0: - data = data[n:] - - self._buffer.append(data) # Try again later. - - def can_write_eof(self): - return True - - def write_eof(self): - if self._closing: - return - assert self._pipe - self._closing = True - if not self._buffer: - self._loop.remove_reader(self._fileno) - self._loop.call_soon(self._call_connection_lost, None) - - def is_closing(self): - return self._closing - - def close(self): - if self._pipe is not None and not self._closing: - # write_eof is all what we needed to close the write pipe - self.write_eof() - - # On Python 3.3 and older, objects with a destructor part of a reference - # cycle are never destroyed. It's not more the case on Python 3.4 thanks - # to the PEP 442. - if compat.PY34: - def __del__(self): - if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) - self._pipe.close() - - def abort(self): - self._close(None) - - def _fatal_error(self, exc, message='Fatal error on pipe transport'): - # should be called by exception handler only - if isinstance(exc, (BrokenPipeError, ConnectionResetError)): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._close(exc) - - def _close(self, exc=None): - self._closing = True - if self._buffer: - self._loop.remove_writer(self._fileno) - self._buffer.clear() - self._loop.remove_reader(self._fileno) - self._loop.call_soon(self._call_connection_lost, exc) - - def _call_connection_lost(self, exc): - try: - self._protocol.connection_lost(exc) - finally: - self._pipe.close() - self._pipe = None - self._protocol = None - self._loop = None - - -if hasattr(os, 'set_inheritable'): - # Python 3.4 and newer - _set_inheritable = os.set_inheritable -else: - import fcntl - - def _set_inheritable(fd, inheritable): - cloexec_flag = getattr(fcntl, 'FD_CLOEXEC', 1) - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if not inheritable: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - -class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): - - def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): - stdin_w = None - if stdin == subprocess.PIPE: - # Use a socket pair for stdin, since not all platforms - # support selecting read events on the write end of a - # socket (which we use in order to detect closing of the - # other end). Notably this is needed on AIX, and works - # just fine on other platforms. - stdin, stdin_w = self._loop._socketpair() - - # Mark the write end of the stdin pipe as non-inheritable, - # needed by close_fds=False on Python 3.3 and older - # (Python 3.4 implements the PEP 446, socketpair returns - # non-inheritable sockets) - _set_inheritable(stdin_w.fileno(), False) - self._proc = subprocess.Popen( - args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, - universal_newlines=False, bufsize=bufsize, **kwargs) - if stdin_w is not None: - stdin.close() - self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) - - -class AbstractChildWatcher: - """Abstract base class for monitoring child processes. - - Objects derived from this class monitor a collection of subprocesses and - report their termination or interruption by a signal. - - New callbacks are registered with .add_child_handler(). Starting a new - process must be done within a 'with' block to allow the watcher to suspend - its activity until the new process if fully registered (this is needed to - prevent a race condition in some implementations). - - Example: - with watcher: - proc = subprocess.Popen("sleep 1") - watcher.add_child_handler(proc.pid, callback) - - Notes: - Implementations of this class must be thread-safe. - - Since child watcher objects may catch the SIGCHLD signal and call - waitpid(-1), there should be only one active object per process. - """ - - def add_child_handler(self, pid, callback, *args): - """Register a new child handler. - - Arrange for callback(pid, returncode, *args) to be called when - process 'pid' terminates. Specifying another callback for the same - process replaces the previous handler. - - Note: callback() must be thread-safe. - """ - raise NotImplementedError() - - def remove_child_handler(self, pid): - """Removes the handler for process 'pid'. - - The function returns True if the handler was successfully removed, - False if there was nothing to remove.""" - - raise NotImplementedError() - - def attach_loop(self, loop): - """Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then it is - first detached before attaching to the new loop. - - Note: loop may be None. - """ - raise NotImplementedError() - - def close(self): - """Close the watcher. - - This must be called to make sure that any underlying resource is freed. - """ - raise NotImplementedError() - - def __enter__(self): - """Enter the watcher's context and allow starting new processes - - This function must return self""" - raise NotImplementedError() - - def __exit__(self, a, b, c): - """Exit the watcher's context""" - raise NotImplementedError() - - -class BaseChildWatcher(AbstractChildWatcher): - - def __init__(self): - self._loop = None - - def close(self): - self.attach_loop(None) - - def _do_waitpid(self, expected_pid): - raise NotImplementedError() - - def _do_waitpid_all(self): - raise NotImplementedError() - - def attach_loop(self, loop): - assert loop is None or isinstance(loop, events.AbstractEventLoop) - - if self._loop is not None: - self._loop.remove_signal_handler(signal.SIGCHLD) - - self._loop = loop - if loop is not None: - loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) - - # Prevent a race condition in case a child terminated - # during the switch. - self._do_waitpid_all() - - def _sig_chld(self): - try: - self._do_waitpid_all() - except Exception as exc: - # self._loop should always be available here - # as '_sig_chld' is added as a signal handler - # in 'attach_loop' - self._loop.call_exception_handler({ - 'message': 'Unknown exception in SIGCHLD handler', - 'exception': exc, - }) - - def _compute_returncode(self, status): - if os.WIFSIGNALED(status): - # The child process died because of a signal. - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - # The child process exited (e.g sys.exit()). - return os.WEXITSTATUS(status) - else: - # The child exited, but we don't understand its status. - # This shouldn't happen, but if it does, let's just - # return that status; perhaps that helps debug it. - return status - - -class SafeChildWatcher(BaseChildWatcher): - """'Safe' child watcher implementation. - - This implementation avoids disrupting other code spawning processes by - polling explicitly each process in the SIGCHLD handler instead of calling - os.waitpid(-1). - - This is a safe solution but it has a significant overhead when handling a - big number of children (O(n) each time SIGCHLD is raised) - """ - - def __init__(self): - super().__init__() - self._callbacks = {} - - def close(self): - self._callbacks.clear() - super().close() - - def __enter__(self): - return self - - def __exit__(self, a, b, c): - pass - - def add_child_handler(self, pid, callback, *args): - self._callbacks[pid] = (callback, args) - - # Prevent a race condition in case the child is already terminated. - self._do_waitpid(pid) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - - for pid in list(self._callbacks): - self._do_waitpid(pid) - - def _do_waitpid(self, expected_pid): - assert expected_pid > 0 - - try: - pid, status = os.waitpid(expected_pid, os.WNOHANG) - except ChildProcessError: - # The child process is already reaped - # (may happen if waitpid() is called elsewhere). - pid = expected_pid - returncode = 255 - logger.warning( - "Unknown child process pid %d, will report returncode 255", - pid) - else: - if pid == 0: - # The child process is still alive. - return - - returncode = self._compute_returncode(status) - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - expected_pid, returncode) - - try: - callback, args = self._callbacks.pop(pid) - except KeyError: # pragma: no cover - # May happen if .remove_child_handler() is called - # after os.waitpid() returns. - if self._loop.get_debug(): - logger.warning("Child watcher got an unexpected pid: %r", - pid, exc_info=True) - else: - callback(pid, returncode, *args) - - -class FastChildWatcher(BaseChildWatcher): - """'Fast' child watcher implementation. - - This implementation reaps every terminated processes by calling - os.waitpid(-1) directly, possibly breaking other code spawning processes - and waiting for their termination. - - There is no noticeable overhead when handling a big number of children - (O(1) each time a child terminates). - """ - def __init__(self): - super().__init__() - self._callbacks = {} - self._lock = threading.Lock() - self._zombies = {} - self._forks = 0 - - def close(self): - self._callbacks.clear() - self._zombies.clear() - super().close() - - def __enter__(self): - with self._lock: - self._forks += 1 - - return self - - def __exit__(self, a, b, c): - with self._lock: - self._forks -= 1 - - if self._forks or not self._zombies: - return - - collateral_victims = str(self._zombies) - self._zombies.clear() - - logger.warning( - "Caught subprocesses termination from unknown pids: %s", - collateral_victims) - - def add_child_handler(self, pid, callback, *args): - assert self._forks, "Must use the context manager" - with self._lock: - try: - returncode = self._zombies.pop(pid) - except KeyError: - # The child is running. - self._callbacks[pid] = callback, args - return - - # The child is dead already. We can fire the callback. - callback(pid, returncode, *args) - - def remove_child_handler(self, pid): - try: - del self._callbacks[pid] - return True - except KeyError: - return False - - def _do_waitpid_all(self): - # Because of signal coalescing, we must keep calling waitpid() as - # long as we're able to reap a child. - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG) - except ChildProcessError: - # No more child processes exist. - return - else: - if pid == 0: - # A child process is still alive. - return - - returncode = self._compute_returncode(status) - - with self._lock: - try: - callback, args = self._callbacks.pop(pid) - except KeyError: - # unknown child - if self._forks: - # It may not be registered yet. - self._zombies[pid] = returncode - if self._loop.get_debug(): - logger.debug('unknown process %s exited ' - 'with returncode %s', - pid, returncode) - continue - callback = None - else: - if self._loop.get_debug(): - logger.debug('process %s exited with returncode %s', - pid, returncode) - - if callback is None: - logger.warning( - "Caught subprocess termination from unknown pid: " - "%d -> %d", pid, returncode) - else: - callback(pid, returncode, *args) - - -class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - """UNIX event loop policy with a watcher for child processes.""" - _loop_factory = _UnixSelectorEventLoop - - def __init__(self): - super().__init__() - self._watcher = None - - def _init_watcher(self): - with events._lock: - if self._watcher is None: # pragma: no branch - self._watcher = SafeChildWatcher() - if isinstance(threading.current_thread(), - threading._MainThread): - self._watcher.attach_loop(self._local._loop) - - def set_event_loop(self, loop): - """Set the event loop. - - As a side effect, if a child watcher was set before, then calling - .set_event_loop() from the main thread will call .attach_loop(loop) on - the child watcher. - """ - - super().set_event_loop(loop) - - if self._watcher is not None and \ - isinstance(threading.current_thread(), threading._MainThread): - self._watcher.attach_loop(loop) - - def get_child_watcher(self): - """Get the watcher for child processes. - - If not yet set, a SafeChildWatcher object is automatically created. - """ - if self._watcher is None: - self._init_watcher() - - return self._watcher - - def set_child_watcher(self, watcher): - """Set the watcher for child processes.""" - - assert watcher is None or isinstance(watcher, AbstractChildWatcher) - - if self._watcher is not None: - self._watcher.close() - - self._watcher = watcher - -SelectorEventLoop = _UnixSelectorEventLoop -DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/windows_events.py --- a/Lib/asyncio/windows_events.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,774 +0,0 @@ -"""Selector and proactor event loops for Windows.""" - -import _winapi -import errno -import math -import socket -import struct -import weakref - -from . import events -from . import base_subprocess -from . import futures -from . import proactor_events -from . import selector_events -from . import tasks -from . import windows_utils -from . import _overlapped -from .coroutines import coroutine -from .log import logger - - -__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', - ] - - -NULL = 0 -INFINITE = 0xffffffff -ERROR_CONNECTION_REFUSED = 1225 -ERROR_CONNECTION_ABORTED = 1236 - -# Initial delay in seconds for connect_pipe() before retrying to connect -CONNECT_PIPE_INIT_DELAY = 0.001 - -# Maximum delay in seconds for connect_pipe() before retrying to connect -CONNECT_PIPE_MAX_DELAY = 0.100 - - -class _OverlappedFuture(futures.Future): - """Subclass of Future which represents an overlapped operation. - - Cancelling it will immediately cancel the overlapped operation. - """ - - def __init__(self, ov, *, loop=None): - super().__init__(loop=loop) - if self._source_traceback: - del self._source_traceback[-1] - self._ov = ov - - def _repr_info(self): - info = super()._repr_info() - if self._ov is not None: - state = 'pending' if self._ov.pending else 'completed' - info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address)) - return info - - def _cancel_overlapped(self): - if self._ov is None: - return - try: - self._ov.cancel() - except OSError as exc: - context = { - 'message': 'Cancelling an overlapped future failed', - 'exception': exc, - 'future': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - self._ov = None - - def cancel(self): - self._cancel_overlapped() - return super().cancel() - - def set_exception(self, exception): - super().set_exception(exception) - self._cancel_overlapped() - - def set_result(self, result): - super().set_result(result) - self._ov = None - - -class _BaseWaitHandleFuture(futures.Future): - """Subclass of Future which represents a wait handle.""" - - def __init__(self, ov, handle, wait_handle, *, loop=None): - super().__init__(loop=loop) - if self._source_traceback: - del self._source_traceback[-1] - # Keep a reference to the Overlapped object to keep it alive until the - # wait is unregistered - self._ov = ov - self._handle = handle - self._wait_handle = wait_handle - - # Should we call UnregisterWaitEx() if the wait completes - # or is cancelled? - self._registered = True - - def _poll(self): - # non-blocking wait: use a timeout of 0 millisecond - return (_winapi.WaitForSingleObject(self._handle, 0) == - _winapi.WAIT_OBJECT_0) - - def _repr_info(self): - info = super()._repr_info() - info.append('handle=%#x' % self._handle) - if self._handle is not None: - state = 'signaled' if self._poll() else 'waiting' - info.append(state) - if self._wait_handle is not None: - info.append('wait_handle=%#x' % self._wait_handle) - return info - - def _unregister_wait_cb(self, fut): - # The wait was unregistered: it's not safe to destroy the Overlapped - # object - self._ov = None - - def _unregister_wait(self): - if not self._registered: - return - self._registered = False - - wait_handle = self._wait_handle - self._wait_handle = None - try: - _overlapped.UnregisterWait(wait_handle) - except OSError as exc: - if exc.winerror != _overlapped.ERROR_IO_PENDING: - context = { - 'message': 'Failed to unregister the wait handle', - 'exception': exc, - 'future': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - return - # ERROR_IO_PENDING means that the unregister is pending - - self._unregister_wait_cb(None) - - def cancel(self): - self._unregister_wait() - return super().cancel() - - def set_exception(self, exception): - self._unregister_wait() - super().set_exception(exception) - - def set_result(self, result): - self._unregister_wait() - super().set_result(result) - - -class _WaitCancelFuture(_BaseWaitHandleFuture): - """Subclass of Future which represents a wait for the cancellation of a - _WaitHandleFuture using an event. - """ - - def __init__(self, ov, event, wait_handle, *, loop=None): - super().__init__(ov, event, wait_handle, loop=loop) - - self._done_callback = None - - def cancel(self): - raise RuntimeError("_WaitCancelFuture must not be cancelled") - - def _schedule_callbacks(self): - super(_WaitCancelFuture, self)._schedule_callbacks() - if self._done_callback is not None: - self._done_callback(self) - - -class _WaitHandleFuture(_BaseWaitHandleFuture): - def __init__(self, ov, handle, wait_handle, proactor, *, loop=None): - super().__init__(ov, handle, wait_handle, loop=loop) - self._proactor = proactor - self._unregister_proactor = True - self._event = _overlapped.CreateEvent(None, True, False, None) - self._event_fut = None - - def _unregister_wait_cb(self, fut): - if self._event is not None: - _winapi.CloseHandle(self._event) - self._event = None - self._event_fut = None - - # If the wait was cancelled, the wait may never be signalled, so - # it's required to unregister it. Otherwise, IocpProactor.close() will - # wait forever for an event which will never come. - # - # If the IocpProactor already received the event, it's safe to call - # _unregister() because we kept a reference to the Overlapped object - # which is used as an unique key. - self._proactor._unregister(self._ov) - self._proactor = None - - super()._unregister_wait_cb(fut) - - def _unregister_wait(self): - if not self._registered: - return - self._registered = False - - wait_handle = self._wait_handle - self._wait_handle = None - try: - _overlapped.UnregisterWaitEx(wait_handle, self._event) - except OSError as exc: - if exc.winerror != _overlapped.ERROR_IO_PENDING: - context = { - 'message': 'Failed to unregister the wait handle', - 'exception': exc, - 'future': self, - } - if self._source_traceback: - context['source_traceback'] = self._source_traceback - self._loop.call_exception_handler(context) - return - # ERROR_IO_PENDING is not an error, the wait was unregistered - - self._event_fut = self._proactor._wait_cancel(self._event, - self._unregister_wait_cb) - - -class PipeServer(object): - """Class representing a pipe server. - - This is much like a bound, listening socket. - """ - def __init__(self, address): - self._address = address - self._free_instances = weakref.WeakSet() - # initialize the pipe attribute before calling _server_pipe_handle() - # because this function can raise an exception and the destructor calls - # the close() method - self._pipe = None - self._accept_pipe_future = None - self._pipe = self._server_pipe_handle(True) - - def _get_unconnected_pipe(self): - # Create new instance and return previous one. This ensures - # that (until the server is closed) there is always at least - # one pipe handle for address. Therefore if a client attempt - # to connect it will not fail with FileNotFoundError. - tmp, self._pipe = self._pipe, self._server_pipe_handle(False) - return tmp - - def _server_pipe_handle(self, first): - # Return a wrapper for a new pipe handle. - if self.closed(): - return None - flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED - if first: - flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE - h = _winapi.CreateNamedPipe( - self._address, flags, - _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | - _winapi.PIPE_WAIT, - _winapi.PIPE_UNLIMITED_INSTANCES, - windows_utils.BUFSIZE, windows_utils.BUFSIZE, - _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) - pipe = windows_utils.PipeHandle(h) - self._free_instances.add(pipe) - return pipe - - def closed(self): - return (self._address is None) - - def close(self): - if self._accept_pipe_future is not None: - self._accept_pipe_future.cancel() - self._accept_pipe_future = None - # Close all instances which have not been connected to by a client. - if self._address is not None: - for pipe in self._free_instances: - pipe.close() - self._pipe = None - self._address = None - self._free_instances.clear() - - __del__ = close - - -class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): - """Windows version of selector event loop.""" - - def _socketpair(self): - return windows_utils.socketpair() - - -class ProactorEventLoop(proactor_events.BaseProactorEventLoop): - """Windows version of proactor event loop using IOCP.""" - - def __init__(self, proactor=None): - if proactor is None: - proactor = IocpProactor() - super().__init__(proactor) - - def _socketpair(self): - return windows_utils.socketpair() - - @coroutine - def create_pipe_connection(self, protocol_factory, address): - f = self._proactor.connect_pipe(address) - pipe = yield from f - protocol = protocol_factory() - trans = self._make_duplex_pipe_transport(pipe, protocol, - extra={'addr': address}) - return trans, protocol - - @coroutine - def start_serving_pipe(self, protocol_factory, address): - server = PipeServer(address) - - def loop_accept_pipe(f=None): - pipe = None - try: - if f: - pipe = f.result() - server._free_instances.discard(pipe) - - if server.closed(): - # A client connected before the server was closed: - # drop the client (close the pipe) and exit - pipe.close() - return - - protocol = protocol_factory() - self._make_duplex_pipe_transport( - pipe, protocol, extra={'addr': address}) - - pipe = server._get_unconnected_pipe() - if pipe is None: - return - - f = self._proactor.accept_pipe(pipe) - except OSError as exc: - if pipe and pipe.fileno() != -1: - self.call_exception_handler({ - 'message': 'Pipe accept failed', - 'exception': exc, - 'pipe': pipe, - }) - pipe.close() - elif self._debug: - logger.warning("Accept pipe failed on pipe %r", - pipe, exc_info=True) - except futures.CancelledError: - if pipe: - pipe.close() - else: - server._accept_pipe_future = f - f.add_done_callback(loop_accept_pipe) - - self.call_soon(loop_accept_pipe) - return [server] - - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): - waiter = futures.Future(loop=self) - transp = _WindowsSubprocessTransport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - waiter=waiter, extra=extra, - **kwargs) - try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly sys.exc_info() - err = exc - else: - err = None - - if err is not None: - transp.close() - yield from transp._wait() - raise err - - return transp - - -class IocpProactor: - """Proactor implementation using IOCP.""" - - def __init__(self, concurrency=0xffffffff): - self._loop = None - self._results = [] - self._iocp = _overlapped.CreateIoCompletionPort( - _overlapped.INVALID_HANDLE_VALUE, NULL, 0, concurrency) - self._cache = {} - self._registered = weakref.WeakSet() - self._unregistered = [] - self._stopped_serving = weakref.WeakSet() - - def __repr__(self): - return ('<%s overlapped#=%s result#=%s>' - % (self.__class__.__name__, len(self._cache), - len(self._results))) - - def set_loop(self, loop): - self._loop = loop - - def select(self, timeout=None): - if not self._results: - self._poll(timeout) - tmp = self._results - self._results = [] - return tmp - - def _result(self, value): - fut = futures.Future(loop=self._loop) - fut.set_result(value) - return fut - - def recv(self, conn, nbytes, flags=0): - self._register_with_iocp(conn) - ov = _overlapped.Overlapped(NULL) - try: - if isinstance(conn, socket.socket): - ov.WSARecv(conn.fileno(), nbytes, flags) - else: - ov.ReadFile(conn.fileno(), nbytes) - except BrokenPipeError: - return self._result(b'') - - def finish_recv(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_recv) - - def send(self, conn, buf, flags=0): - self._register_with_iocp(conn) - ov = _overlapped.Overlapped(NULL) - if isinstance(conn, socket.socket): - ov.WSASend(conn.fileno(), buf, flags) - else: - ov.WriteFile(conn.fileno(), buf) - - def finish_send(trans, key, ov): - try: - return ov.getresult() - except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: - raise ConnectionResetError(*exc.args) - else: - raise - - return self._register(ov, conn, finish_send) - - def accept(self, listener): - self._register_with_iocp(listener) - conn = self._get_accept_socket(listener.family) - ov = _overlapped.Overlapped(NULL) - ov.AcceptEx(listener.fileno(), conn.fileno()) - - def finish_accept(trans, key, ov): - ov.getresult() - # Use SO_UPDATE_ACCEPT_CONTEXT so getsockname() etc work. - buf = struct.pack('@P', listener.fileno()) - conn.setsockopt(socket.SOL_SOCKET, - _overlapped.SO_UPDATE_ACCEPT_CONTEXT, buf) - conn.settimeout(listener.gettimeout()) - return conn, conn.getpeername() - - @coroutine - def accept_coro(future, conn): - # Coroutine closing the accept socket if the future is cancelled - try: - yield from future - except futures.CancelledError: - conn.close() - raise - - future = self._register(ov, listener, finish_accept) - coro = accept_coro(future, conn) - tasks.ensure_future(coro, loop=self._loop) - return future - - def connect(self, conn, address): - self._register_with_iocp(conn) - # The socket needs to be locally bound before we call ConnectEx(). - try: - _overlapped.BindLocal(conn.fileno(), conn.family) - except OSError as e: - if e.winerror != errno.WSAEINVAL: - raise - # Probably already locally bound; check using getsockname(). - if conn.getsockname()[1] == 0: - raise - ov = _overlapped.Overlapped(NULL) - ov.ConnectEx(conn.fileno(), address) - - def finish_connect(trans, key, ov): - ov.getresult() - # Use SO_UPDATE_CONNECT_CONTEXT so getsockname() etc work. - conn.setsockopt(socket.SOL_SOCKET, - _overlapped.SO_UPDATE_CONNECT_CONTEXT, 0) - return conn - - return self._register(ov, conn, finish_connect) - - def accept_pipe(self, pipe): - self._register_with_iocp(pipe) - ov = _overlapped.Overlapped(NULL) - connected = ov.ConnectNamedPipe(pipe.fileno()) - - if connected: - # ConnectNamePipe() failed with ERROR_PIPE_CONNECTED which means - # that the pipe is connected. There is no need to wait for the - # completion of the connection. - return self._result(pipe) - - def finish_accept_pipe(trans, key, ov): - ov.getresult() - return pipe - - return self._register(ov, pipe, finish_accept_pipe) - - @coroutine - def connect_pipe(self, address): - delay = CONNECT_PIPE_INIT_DELAY - while True: - # Unfortunately there is no way to do an overlapped connect to a pipe. - # Call CreateFile() in a loop until it doesn't fail with - # ERROR_PIPE_BUSY - try: - handle = _overlapped.ConnectPipe(address) - break - except OSError as exc: - if exc.winerror != _overlapped.ERROR_PIPE_BUSY: - raise - - # ConnectPipe() failed with ERROR_PIPE_BUSY: retry later - delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY) - yield from tasks.sleep(delay, loop=self._loop) - - return windows_utils.PipeHandle(handle) - - def wait_for_handle(self, handle, timeout=None): - """Wait for a handle. - - Return a Future object. The result of the future is True if the wait - completed, or False if the wait did not complete (on timeout). - """ - return self._wait_for_handle(handle, timeout, False) - - def _wait_cancel(self, event, done_callback): - fut = self._wait_for_handle(event, None, True) - # add_done_callback() cannot be used because the wait may only complete - # in IocpProactor.close(), while the event loop is not running. - fut._done_callback = done_callback - return fut - - def _wait_for_handle(self, handle, timeout, _is_cancel): - if timeout is None: - ms = _winapi.INFINITE - else: - # RegisterWaitForSingleObject() has a resolution of 1 millisecond, - # round away from zero to wait *at least* timeout seconds. - ms = math.ceil(timeout * 1e3) - - # We only create ov so we can use ov.address as a key for the cache. - ov = _overlapped.Overlapped(NULL) - wait_handle = _overlapped.RegisterWaitWithQueue( - handle, self._iocp, ov.address, ms) - if _is_cancel: - f = _WaitCancelFuture(ov, handle, wait_handle, loop=self._loop) - else: - f = _WaitHandleFuture(ov, handle, wait_handle, self, - loop=self._loop) - if f._source_traceback: - del f._source_traceback[-1] - - def finish_wait_for_handle(trans, key, ov): - # Note that this second wait means that we should only use - # this with handles types where a successful wait has no - # effect. So events or processes are all right, but locks - # or semaphores are not. Also note if the handle is - # signalled and then quickly reset, then we may return - # False even though we have not timed out. - return f._poll() - - self._cache[ov.address] = (f, ov, 0, finish_wait_for_handle) - return f - - def _register_with_iocp(self, obj): - # To get notifications of finished ops on this objects sent to the - # completion port, were must register the handle. - if obj not in self._registered: - self._registered.add(obj) - _overlapped.CreateIoCompletionPort(obj.fileno(), self._iocp, 0, 0) - # XXX We could also use SetFileCompletionNotificationModes() - # to avoid sending notifications to completion port of ops - # that succeed immediately. - - def _register(self, ov, obj, callback): - # Return a future which will be set with the result of the - # operation when it completes. The future's value is actually - # the value returned by callback(). - f = _OverlappedFuture(ov, loop=self._loop) - if f._source_traceback: - del f._source_traceback[-1] - if not ov.pending: - # The operation has completed, so no need to postpone the - # work. We cannot take this short cut if we need the - # NumberOfBytes, CompletionKey values returned by - # PostQueuedCompletionStatus(). - try: - value = callback(None, None, ov) - except OSError as e: - f.set_exception(e) - else: - f.set_result(value) - # Even if GetOverlappedResult() was called, we have to wait for the - # notification of the completion in GetQueuedCompletionStatus(). - # Register the overlapped operation to keep a reference to the - # OVERLAPPED object, otherwise the memory is freed and Windows may - # read uninitialized memory. - - # Register the overlapped operation for later. Note that - # we only store obj to prevent it from being garbage - # collected too early. - self._cache[ov.address] = (f, ov, obj, callback) - return f - - def _unregister(self, ov): - """Unregister an overlapped object. - - Call this method when its future has been cancelled. The event can - already be signalled (pending in the proactor event queue). It is also - safe if the event is never signalled (because it was cancelled). - """ - self._unregistered.append(ov) - - def _get_accept_socket(self, family): - s = socket.socket(family) - s.settimeout(0) - return s - - def _poll(self, timeout=None): - if timeout is None: - ms = INFINITE - elif timeout < 0: - raise ValueError("negative timeout") - else: - # GetQueuedCompletionStatus() has a resolution of 1 millisecond, - # round away from zero to wait *at least* timeout seconds. - ms = math.ceil(timeout * 1e3) - if ms >= INFINITE: - raise ValueError("timeout too big") - - while True: - status = _overlapped.GetQueuedCompletionStatus(self._iocp, ms) - if status is None: - break - ms = 0 - - err, transferred, key, address = status - try: - f, ov, obj, callback = self._cache.pop(address) - except KeyError: - if self._loop.get_debug(): - self._loop.call_exception_handler({ - 'message': ('GetQueuedCompletionStatus() returned an ' - 'unexpected event'), - 'status': ('err=%s transferred=%s key=%#x address=%#x' - % (err, transferred, key, address)), - }) - - # key is either zero, or it is used to return a pipe - # handle which should be closed to avoid a leak. - if key not in (0, _overlapped.INVALID_HANDLE_VALUE): - _winapi.CloseHandle(key) - continue - - if obj in self._stopped_serving: - f.cancel() - # Don't call the callback if _register() already read the result or - # if the overlapped has been cancelled - elif not f.done(): - try: - value = callback(transferred, key, ov) - except OSError as e: - f.set_exception(e) - self._results.append(f) - else: - f.set_result(value) - self._results.append(f) - - # Remove unregisted futures - for ov in self._unregistered: - self._cache.pop(ov.address, None) - self._unregistered.clear() - - def _stop_serving(self, obj): - # obj is a socket or pipe handle. It will be closed in - # BaseProactorEventLoop._stop_serving() which will make any - # pending operations fail quickly. - self._stopped_serving.add(obj) - - def close(self): - # Cancel remaining registered operations. - for address, (fut, ov, obj, callback) in list(self._cache.items()): - if fut.cancelled(): - # Nothing to do with cancelled futures - pass - elif isinstance(fut, _WaitCancelFuture): - # _WaitCancelFuture must not be cancelled - pass - else: - try: - fut.cancel() - except OSError as exc: - if self._loop is not None: - context = { - 'message': 'Cancelling a future failed', - 'exception': exc, - 'future': fut, - } - if fut._source_traceback: - context['source_traceback'] = fut._source_traceback - self._loop.call_exception_handler(context) - - while self._cache: - if not self._poll(1): - logger.debug('taking long time to close proactor') - - self._results = [] - if self._iocp is not None: - _winapi.CloseHandle(self._iocp) - self._iocp = None - - def __del__(self): - self.close() - - -class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport): - - def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): - self._proc = windows_utils.Popen( - args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, - bufsize=bufsize, **kwargs) - - def callback(f): - returncode = self._proc.poll() - self._process_exited(returncode) - - f = self._loop._proactor.wait_for_handle(int(self._proc._handle)) - f.add_done_callback(callback) - - -SelectorEventLoop = _WindowsSelectorEventLoop - - -class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): - _loop_factory = SelectorEventLoop - - -DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncio/windows_utils.py --- a/Lib/asyncio/windows_utils.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,223 +0,0 @@ -""" -Various Windows specific bits and pieces -""" - -import sys - -if sys.platform != 'win32': # pragma: no cover - raise ImportError('win32 only') - -import _winapi -import itertools -import msvcrt -import os -import socket -import subprocess -import tempfile -import warnings - - -__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle'] - - -# Constants/globals - - -BUFSIZE = 8192 -PIPE = subprocess.PIPE -STDOUT = subprocess.STDOUT -_mmap_counter = itertools.count() - - -if hasattr(socket, 'socketpair'): - # Since Python 3.5, socket.socketpair() is now also available on Windows - socketpair = socket.socketpair -else: - # Replacement for socket.socketpair() - def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): - """A socket pair usable as a self-pipe, for Windows. - - Origin: https://gist.github.com/4325783, by Geert Jansen. - Public domain. - """ - if family == socket.AF_INET: - host = '127.0.0.1' - elif family == socket.AF_INET6: - host = '::1' - else: - raise ValueError("Only AF_INET and AF_INET6 socket address " - "families are supported") - if type != socket.SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with setblocking(0) - # that prevents us from having to create a thread. - lsock = socket.socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen(1) - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket.socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) - - -# Replacement for os.pipe() using handles instead of fds - - -def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): - """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' % - (os.getpid(), next(_mmap_counter))) - - if duplex: - openmode = _winapi.PIPE_ACCESS_DUPLEX - access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE - obsize, ibsize = bufsize, bufsize - else: - openmode = _winapi.PIPE_ACCESS_INBOUND - access = _winapi.GENERIC_WRITE - obsize, ibsize = 0, bufsize - - openmode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE - - if overlapped[0]: - openmode |= _winapi.FILE_FLAG_OVERLAPPED - - if overlapped[1]: - flags_and_attribs = _winapi.FILE_FLAG_OVERLAPPED - else: - flags_and_attribs = 0 - - h1 = h2 = None - try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) - - h2 = _winapi.CreateFile( - address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, - flags_and_attribs, _winapi.NULL) - - ov = _winapi.ConnectNamedPipe(h1, overlapped=True) - ov.GetOverlappedResult(True) - return h1, h2 - except: - if h1 is not None: - _winapi.CloseHandle(h1) - if h2 is not None: - _winapi.CloseHandle(h2) - raise - - -# Wrapper for a pipe handle - - -class PipeHandle: - """Wrapper for an overlapped pipe handle which is vaguely file-object like. - - The IOCP event loop can use these instead of socket objects. - """ - def __init__(self, handle): - self._handle = handle - - def __repr__(self): - if self._handle is not None: - handle = 'handle=%r' % self._handle - else: - handle = 'closed' - return '<%s %s>' % (self.__class__.__name__, handle) - - @property - def handle(self): - return self._handle - - def fileno(self): - if self._handle is None: - raise ValueError("I/O operatioon on closed pipe") - return self._handle - - def close(self, *, CloseHandle=_winapi.CloseHandle): - if self._handle is not None: - CloseHandle(self._handle) - self._handle = None - - def __del__(self): - if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning) - self.close() - - def __enter__(self): - return self - - def __exit__(self, t, v, tb): - self.close() - - -# Replacement for subprocess.Popen using overlapped pipe handles - - -class Popen(subprocess.Popen): - """Replacement for subprocess.Popen using overlapped pipe handles. - - The stdin, stdout, stderr are None or instances of PipeHandle. - """ - def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwds): - assert not kwds.get('universal_newlines') - assert kwds.get('bufsize', 0) == 0 - stdin_rfd = stdout_wfd = stderr_wfd = None - stdin_wh = stdout_rh = stderr_rh = None - if stdin == PIPE: - stdin_rh, stdin_wh = pipe(overlapped=(False, True), duplex=True) - stdin_rfd = msvcrt.open_osfhandle(stdin_rh, os.O_RDONLY) - else: - stdin_rfd = stdin - if stdout == PIPE: - stdout_rh, stdout_wh = pipe(overlapped=(True, False)) - stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0) - else: - stdout_wfd = stdout - if stderr == PIPE: - stderr_rh, stderr_wh = pipe(overlapped=(True, False)) - stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0) - elif stderr == STDOUT: - stderr_wfd = stdout_wfd - else: - stderr_wfd = stderr - try: - super().__init__(args, stdin=stdin_rfd, stdout=stdout_wfd, - stderr=stderr_wfd, **kwds) - except: - for h in (stdin_wh, stdout_rh, stderr_rh): - if h is not None: - _winapi.CloseHandle(h) - raise - else: - if stdin_wh is not None: - self.stdin = PipeHandle(stdin_wh) - if stdout_rh is not None: - self.stdout = PipeHandle(stdout_rh) - if stderr_rh is not None: - self.stderr = PipeHandle(stderr_rh) - finally: - if stdin == PIPE: - os.close(stdin_rfd) - if stdout == PIPE: - os.close(stdout_wfd) - if stderr == PIPE: - os.close(stderr_wfd) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/asyncore.py --- a/Lib/asyncore.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/asyncore.py Mon Jan 25 17:05:13 2016 +0100 @@ -57,8 +57,8 @@ ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ errorcode -_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, - EBADF}) +_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF)) try: socket_map @@ -112,7 +112,7 @@ obj.handle_expt_event() if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() - except OSError as e: + except socket.error as e: if e.args[0] not in _DISCONNECTED: obj.handle_error() else: @@ -141,7 +141,10 @@ time.sleep(timeout) return - r, w, e = select.select(r, w, e, timeout) + try: + r, w, e = select.select(r, w, e, timeout) + except InterruptedError: + return for fd in r: obj = map.get(fd) @@ -179,8 +182,10 @@ flags |= select.POLLOUT if flags: pollster.register(fd, flags) - - r = pollster.poll(timeout) + try: + r = pollster.poll(timeout) + except InterruptedError: + r = [] for fd, flags in r: obj = map.get(fd) if obj is None: @@ -215,7 +220,7 @@ connecting = False closing = False addr = None - ignore_log_types = frozenset({'warning'}) + ignore_log_types = frozenset(['warning']) def __init__(self, sock=None, map=None): if map is None: @@ -235,7 +240,7 @@ # passed be connected. try: self.addr = sock.getpeername() - except OSError as err: + except socket.error as err: if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. @@ -250,7 +255,7 @@ self.socket = None def __repr__(self): - status = [self.__class__.__module__+"."+self.__class__.__qualname__] + status = [self.__class__.__module__+"."+self.__class__.__name__] if self.accepting and self.addr: status.append('listening') elif self.connected: @@ -299,7 +304,7 @@ self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1 ) - except OSError: + except socket.error: pass # ================================================== @@ -340,7 +345,7 @@ self.addr = address self.handle_connect_event() else: - raise OSError(err, errorcode[err]) + raise socket.error(err, errorcode[err]) def accept(self): # XXX can return either an address pair or None @@ -348,7 +353,7 @@ conn, addr = self.socket.accept() except TypeError: return None - except OSError as why: + except socket.error as why: if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: @@ -360,7 +365,7 @@ try: result = self.socket.send(data) return result - except OSError as why: + except socket.error as why: if why.args[0] == EWOULDBLOCK: return 0 elif why.args[0] in _DISCONNECTED: @@ -379,8 +384,8 @@ return b'' else: return data - except OSError as why: - # winsock sometimes raises ENOTCONN + except socket.error as why: + # winsock sometimes throws ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() return b'' @@ -392,12 +397,25 @@ self.accepting = False self.connecting = False self.del_channel() - if self.socket is not None: - try: - self.socket.close() - except OSError as why: - if why.args[0] not in (ENOTCONN, EBADF): - raise + try: + self.socket.close() + except socket.error as why: + if why.args[0] not in (ENOTCONN, EBADF): + raise + + # cheap inheritance, used to pass all other attribute + # references to the underlying socket object. + def __getattr__(self, attr): + try: + retattr = getattr(self.socket, attr) + except AttributeError: + raise AttributeError("%s instance has no attribute '%s'" + %(self.__class__.__name__, attr)) + else: + msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s " \ + "instead" % {'me' : self.__class__.__name__, 'attr' : attr} + warnings.warn(msg, DeprecationWarning, stacklevel=2) + return retattr # log and log_info may be overridden to provide more sophisticated # logging and warning methods. In general, log is for 'hit' logging @@ -425,7 +443,7 @@ def handle_connect_event(self): err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: - raise OSError(err, _strerror(err)) + raise socket.error(err, _strerror(err)) self.handle_connect() self.connected = True self.connecting = False @@ -514,7 +532,7 @@ def initiate_send(self): num_sent = 0 - num_sent = dispatcher.send(self, self.out_buffer[:65536]) + num_sent = dispatcher.send(self, self.out_buffer[:512]) self.out_buffer = self.out_buffer[num_sent:] def handle_write(self): @@ -585,6 +603,8 @@ # Regardless, this is useful for pipes, and stdin/stdout... if os.name == 'posix': + import fcntl + class file_wrapper: # Here we override just enough to make a file # look like a socket for the purposes of asyncore. @@ -593,11 +613,6 @@ def __init__(self, fd): self.fd = os.dup(fd) - def __del__(self): - if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning) - self.close() - def recv(self, *args): return os.read(self.fd, *args) @@ -616,10 +631,7 @@ write = send def close(self): - if self.fd < 0: - return os.close(self.fd) - self.fd = -1 def fileno(self): return self.fd @@ -635,7 +647,9 @@ pass self.set_file(fd) # set it to non-blocking mode - os.set_blocking(fd, False) + flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) def set_file(self, fd): self.socket = file_wrapper(fd) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/base64.py --- a/Lib/base64.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/base64.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,6 @@ #! /usr/bin/env python3 -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""RFC 3548: Base16, Base32, Base64 Data Encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -17,8 +17,6 @@ # Generalized interface for other encodings 'b64encode', 'b64decode', 'b32encode', 'b32decode', 'b16encode', 'b16decode', - # Base85 and Ascii85 encodings - 'b85encode', 'b85decode', 'a85encode', 'a85decode', # Standard Base64 encoding 'standard_b64encode', 'standard_b64decode', # Some common Base64 alternatives. As referenced by RFC 3458, see thread @@ -37,13 +35,19 @@ return s.encode('ascii') except UnicodeEncodeError: raise ValueError('string argument should contain only ASCII characters') - if isinstance(s, bytes_types): + elif isinstance(s, bytes_types): return s - try: - return memoryview(s).tobytes() - except TypeError: - raise TypeError("argument should be a bytes-like object or ASCII " - "string, not %r" % s.__class__.__name__) from None + else: + raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__) + +def _translate(s, altchars): + if not isinstance(s, bytes_types): + raise TypeError("expected bytes, not %s" % s.__class__.__name__) + translation = bytearray(range(256)) + for k, v in altchars.items(): + translation[ord(k)] = v[0] + return s.translate(translation) + # Base64 encoding/decoding uses binascii @@ -58,10 +62,16 @@ The encoded byte string is returned. """ - encoded = binascii.b2a_base64(s, newline=False) + if not isinstance(s, bytes_types): + raise TypeError("expected bytes, not %s" % s.__class__.__name__) + # Strip off the trailing newline + encoded = binascii.b2a_base64(s)[:-1] if altchars is not None: + if not isinstance(altchars, bytes_types): + raise TypeError("expected bytes, not %s" + % altchars.__class__.__name__) assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) + return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]}) return encoded @@ -83,7 +93,7 @@ if altchars is not None: altchars = _bytes_from_decode_data(altchars) assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) + s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'}) if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s): raise binascii.Error('Non-base64 digit found') return binascii.a2b_base64(s) @@ -106,10 +116,6 @@ """ return b64decode(s) - -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') -_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') - def urlsafe_b64encode(s): """Encode a byte string using a url-safe Base64 alphabet. @@ -117,7 +123,7 @@ returned. The alphabet uses '-' instead of '+' and '_' instead of '/'. """ - return b64encode(s).translate(_urlsafe_encode_translation) + return b64encode(s, b'-_') def urlsafe_b64decode(s): """Decode a byte string encoded with the standard Base64 alphabet. @@ -129,56 +135,69 @@ The alphabet uses '-' instead of '+' and '_' instead of '/'. """ - s = _bytes_from_decode_data(s) - s = s.translate(_urlsafe_decode_translation) - return b64decode(s) + return b64decode(s, b'-_') # Base32 encoding/decoding must be done in Python -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32tab2 = None -_b32rev = None +_b32alphabet = { + 0: b'A', 9: b'J', 18: b'S', 27: b'3', + 1: b'B', 10: b'K', 19: b'T', 28: b'4', + 2: b'C', 11: b'L', 20: b'U', 29: b'5', + 3: b'D', 12: b'M', 21: b'V', 30: b'6', + 4: b'E', 13: b'N', 22: b'W', 31: b'7', + 5: b'F', 14: b'O', 23: b'X', + 6: b'G', 15: b'P', 24: b'Y', + 7: b'H', 16: b'Q', 25: b'Z', + 8: b'I', 17: b'R', 26: b'2', + } + +_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())] +_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()]) + def b32encode(s): """Encode a byte string using Base32. s is the byte string to encode. The encoded byte string is returned. """ - global _b32tab2 - # Delay the initialization of the table to not waste memory - # if the function is never called - if _b32tab2 is None: - b32tab = [bytes((i,)) for i in _b32alphabet] - _b32tab2 = [a + b for a in b32tab for b in b32tab] - b32tab = None - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 + raise TypeError("expected bytes, not %s" % s.__class__.__name__) + quanta, leftover = divmod(len(s), 5) # Pad the last quantum with zero bits if necessary if leftover: s = s + bytes(5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2 - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5], 'big') - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) + quanta += 1 + encoded = bytes() + for i in range(quanta): + # c1 and c2 are 16 bits wide, c3 is 8 bits wide. The intent of this + # code is to process the 40 bits in units of 5 bits. So we take the 1 + # leftover bit of c1 and tack it onto c2. Then we take the 2 leftover + # bits of c2 and tack them onto c3. The shifts and masks are intended + # to give us values of exactly 5 bits in width. + c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5]) + c2 += (c1 & 1) << 16 # 17 bits wide + c3 += (c2 & 3) << 8 # 10 bits wide + encoded += bytes([_b32tab[c1 >> 11], # bits 1 - 5 + _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10 + _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15 + _b32tab[c2 >> 12], # bits 16 - 20 (1 - 5) + _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10) + _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15) + _b32tab[c3 >> 5], # bits 31 - 35 (1 - 5) + _b32tab[c3 & 0x1f], # bits 36 - 40 (1 - 5) + ]) # Adjust for any leftover partial quanta if leftover == 1: - encoded[-6:] = b'======' + return encoded[:-6] + b'======' elif leftover == 2: - encoded[-4:] = b'====' + return encoded[:-4] + b'====' elif leftover == 3: - encoded[-3:] = b'===' + return encoded[:-3] + b'===' elif leftover == 4: - encoded[-1:] = b'=' - return bytes(encoded) + return encoded[:-1] + b'=' + return encoded + def b32decode(s, casefold=False, map01=None): """Decode a Base32 encoded byte string. @@ -199,13 +218,9 @@ the input is incorrectly padded or if there are non-alphabet characters present in the input. """ - global _b32rev - # Delay the initialization of the table to not waste memory - # if the function is never called - if _b32rev is None: - _b32rev = {v: k for k, v in enumerate(_b32alphabet)} s = _bytes_from_decode_data(s) - if len(s) % 8: + quanta, leftover = divmod(len(s), 8) + if leftover: raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be @@ -213,42 +228,48 @@ if map01 is not None: map01 = _bytes_from_decode_data(map01) assert len(map01) == 1, repr(map01) - s = s.translate(bytes.maketrans(b'01', b'O' + map01)) + s = _translate(s, {b'0': b'O', b'1': map01}) if casefold: s = s.upper() # Strip off pad characters from the right. We need to count the pad # characters because this will tell us how many null bytes to remove from # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) + padchars = 0 + mo = re.search(b'(?P[=]*)$', s) + if mo: + padchars = len(mo.group('pad')) + if padchars > 0: + s = s[:-padchars] # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5, 'big') + parts = [] + acc = 0 + shift = 35 + for c in s: + val = _b32rev.get(c) + if val is None: + raise TypeError('Non-base32 digit found') + acc += _b32rev[c] << shift + shift -= 5 + if shift < 0: + parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii"))) + acc = 0 + shift = 35 # Process the last, partial quanta - if padchars: - acc <<= 5 * padchars - last = acc.to_bytes(5, 'big') - if padchars == 1: - decoded[-5:] = last[:-1] - elif padchars == 3: - decoded[-5:] = last[:-2] - elif padchars == 4: - decoded[-5:] = last[:-3] - elif padchars == 6: - decoded[-5:] = last[:-4] - else: - raise binascii.Error('Incorrect padding') - return bytes(decoded) + last = binascii.unhexlify(bytes('%010x' % acc, "ascii")) + if padchars == 0: + last = b'' # No characters + elif padchars == 1: + last = last[:-1] + elif padchars == 3: + last = last[:-2] + elif padchars == 4: + last = last[:-3] + elif padchars == 6: + last = last[:-4] + else: + raise binascii.Error('Incorrect padding') + parts.append(last) + return b''.join(parts) @@ -260,6 +281,8 @@ s is the byte string to encode. The encoded byte string is returned. """ + if not isinstance(s, bytes_types): + raise TypeError("expected bytes, not %s" % s.__class__.__name__) return binascii.hexlify(s).upper() @@ -281,206 +304,7 @@ raise binascii.Error('Non-base16 digit found') return binascii.unhexlify(s) -# -# Ascii85 encoding/decoding -# -_a85chars = None -_a85chars2 = None -_A85START = b"<~" -_A85END = b"~>" - -def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): - # Helper function for a85encode and b85encode - if not isinstance(b, bytes_types): - b = memoryview(b).tobytes() - - padding = (-len(b)) % 4 - if padding: - b = b + b'\0' * padding - words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) - - chunks = [b'z' if foldnuls and not word else - b'y' if foldspaces and word == 0x20202020 else - (chars2[word // 614125] + - chars2[word // 85 % 7225] + - chars[word % 85]) - for word in words] - - if padding and not pad: - if chunks[-1] == b'z': - chunks[-1] = chars[0] * 5 - chunks[-1] = chunks[-1][:-padding] - - return b''.join(chunks) - -def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): - """Encode a byte string using Ascii85. - - b is the byte string to encode. The encoded byte string is returned. - - foldspaces is an optional flag that uses the special short sequence 'y' - instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. - - wrapcol controls whether the output should have newline ('\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long. - - pad controls whether the input string is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. - - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. - """ - global _a85chars, _a85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _a85chars is None: - _a85chars = [bytes((i,)) for i in range(33, 118)] - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - - result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) - - if adobe: - result = _A85START + result - if wrapcol: - wrapcol = max(2 if adobe else 1, wrapcol) - chunks = [result[i: i + wrapcol] - for i in range(0, len(result), wrapcol)] - if adobe: - if len(chunks[-1]) + 2 > wrapcol: - chunks.append(b'') - result = b'\n'.join(chunks) - if adobe: - result += _A85END - - return result - -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): - """Decode an Ascii85 encoded byte string. - - s is the byte string to decode. - - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. - - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). - - ignorechars should be a byte string containing characters to ignore from the - input. This should only contain whitespace characters, and by default - contains all whitespace characters in ASCII. - """ - b = _bytes_from_decode_data(b) - if adobe: - if not (b.startswith(_A85START) and b.endswith(_A85END)): - raise ValueError("Ascii85 encoded byte sequences must be bracketed " - "by {!r} and {!r}".format(_A85START, _A85END)) - b = b[2:-2] # Strip off start/end markers - # - # We have to go through this stepwise, so as to ignore spaces and handle - # special short sequences - # - packI = struct.Struct('!I').pack - decoded = [] - decoded_append = decoded.append - curr = [] - curr_append = curr.append - curr_clear = curr.clear - for x in b + b'u' * 4: - if b'!'[0] <= x <= b'u'[0]: - curr_append(x) - if len(curr) == 5: - acc = 0 - for x in curr: - acc = 85 * acc + (x - 33) - try: - decoded_append(packI(acc)) - except struct.error: - raise ValueError('Ascii85 overflow') from None - curr_clear() - elif x == b'z'[0]: - if curr: - raise ValueError('z inside Ascii85 5-tuple') - decoded_append(b'\0\0\0\0') - elif foldspaces and x == b'y'[0]: - if curr: - raise ValueError('y inside Ascii85 5-tuple') - decoded_append(b'\x20\x20\x20\x20') - elif x in ignorechars: - # Skip whitespace - continue - else: - raise ValueError('Non-Ascii85 digit found: %c' % x) - - result = b''.join(decoded) - padding = 4 - len(curr) - if padding: - # Throw away the extra padding - result = result[:-padding] - return result - -# The following code is originally taken (with permission) from Mercurial - -_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") -_b85chars = None -_b85chars2 = None -_b85dec = None - -def b85encode(b, pad=False): - """Encode an ASCII-encoded byte array in base85 format. - - If pad is true, the input is padded with "\\0" so its length is a multiple of - 4 characters before encoding. - """ - global _b85chars, _b85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85chars is None: - _b85chars = [bytes((i,)) for i in _b85alphabet] - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - return _85encode(b, _b85chars, _b85chars2, pad) - -def b85decode(b): - """Decode base85-encoded byte array""" - global _b85dec - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85dec is None: - _b85dec = [None] * 256 - for i, c in enumerate(_b85alphabet): - _b85dec[c] = i - - b = _bytes_from_decode_data(b) - padding = (-len(b)) % 5 - b = b + b'~' * padding - out = [] - packI = struct.Struct('!I').pack - for i in range(0, len(b), 5): - chunk = b[i:i + 5] - acc = 0 - try: - for c in chunk: - acc = acc * 85 + _b85dec[c] - except TypeError: - for j, c in enumerate(chunk): - if _b85dec[c] is None: - raise ValueError('bad base85 character at position %d' - % (i + j)) from None - raise - try: - out.append(packI(acc)) - except struct.error: - raise ValueError('base85 overflow in hunk starting at byte %d' - % i) from None - - result = b''.join(out) - if padding: - result = result[:-padding] - return result # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it @@ -513,26 +337,12 @@ s = binascii.a2b_base64(line) output.write(s) -def _input_type_check(s): - try: - m = memoryview(s) - except TypeError as err: - msg = "expected bytes-like object, not %s" % s.__class__.__name__ - raise TypeError(msg) from err - if m.format not in ('c', 'b', 'B'): - msg = ("expected single byte elements, not %r from %s" % - (m.format, s.__class__.__name__)) - raise TypeError(msg) - if m.ndim != 1: - msg = ("expected 1-D data, not %d-D data from %s" % - (m.ndim, s.__class__.__name__)) - raise TypeError(msg) - def encodebytes(s): """Encode a bytestring into a bytestring containing multiple lines of base-64 data.""" - _input_type_check(s) + if not isinstance(s, bytes_types): + raise TypeError("expected bytes, not %s" % s.__class__.__name__) pieces = [] for i in range(0, len(s), MAXBINSIZE): chunk = s[i : i + MAXBINSIZE] @@ -549,7 +359,8 @@ def decodebytes(s): """Decode a bytestring of base-64 data into a bytestring.""" - _input_type_check(s) + if not isinstance(s, bytes_types): + raise TypeError("expected bytes, not %s" % s.__class__.__name__) return binascii.a2b_base64(s) def decodestring(s): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/bdb.py --- a/Lib/bdb.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/bdb.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,6 @@ import fnmatch import sys import os -from inspect import CO_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] @@ -23,7 +22,6 @@ self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} - self.frame_returning = None def canonic(self, filename): if filename == "<" + filename[1:-1] + ">": @@ -76,48 +74,20 @@ if not (self.stop_here(frame) or self.break_anywhere(frame)): # No need to trace this function return # None - # Ignore call events in generator except when stepping. - if self.stopframe and frame.f_code.co_flags & CO_GENERATOR: - return self.trace_dispatch self.user_call(frame, arg) if self.quitting: raise BdbQuit return self.trace_dispatch def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: - # Ignore return events in generator except when stepping. - if self.stopframe and frame.f_code.co_flags & CO_GENERATOR: - return self.trace_dispatch - try: - self.frame_returning = frame - self.user_return(frame, arg) - finally: - self.frame_returning = None + self.user_return(frame, arg) if self.quitting: raise BdbQuit - # The user issued a 'next' or 'until' command. - if self.stopframe is frame and self.stoplineno != -1: - self._set_stopinfo(None, None) return self.trace_dispatch def dispatch_exception(self, frame, arg): if self.stop_here(frame): - # When stepping with next/until/return in a generator frame, skip - # the internal StopIteration exception (with no traceback) - # triggered by a subiterator run with the 'yield from' statement. - if not (frame.f_code.co_flags & CO_GENERATOR - and arg[0] is StopIteration and arg[2] is None): - self.user_exception(frame, arg) - if self.quitting: raise BdbQuit - # Stop at the StopIteration or GeneratorExit exception when the user - # has set stopframe in a generator by issuing a return command, or a - # next/until command at the last statement in the generator before the - # exception. - elif (self.stopframe and frame is not self.stopframe - and self.stopframe.f_code.co_flags & CO_GENERATOR - and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) if self.quitting: raise BdbQuit - return self.trace_dispatch # Normally derived classes don't override the following @@ -140,8 +110,10 @@ if self.stoplineno == -1: return False return frame.f_lineno >= self.stoplineno - if not self.stopframe: - return True + while frame is not None and frame is not self.stopframe: + if frame is self.botframe: + return True + frame = frame.f_back return False def break_here(self, frame): @@ -214,14 +186,6 @@ def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) def set_next(self, frame): @@ -230,10 +194,7 @@ def set_return(self, frame): """Stop when returning from the given frame.""" - if frame.f_code.co_flags & CO_GENERATOR: - self._set_stopinfo(frame, None, -1) - else: - self._set_stopinfo(frame.f_back, frame) + self._set_stopinfo(frame.f_back, frame) def set_trace(self, frame=None): """Start debugging from `frame`. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/binhex.py --- a/Lib/binhex.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/binhex.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,8 +32,7 @@ pass # States (what have we written) -_DID_HEADER = 0 -_DID_DATA = 1 +[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3) # Various constants REASONABLY_LARGE = 32768 # Minimal amount we pass the rle-coder @@ -214,34 +213,30 @@ self._write(data) def close(self): - if self.state is None: - return - try: - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error('Close at the wrong time') - if self.rlen != 0: - raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,)) - self._writecrc() - finally: - self.state = None - ofp = self.ofp - del self.ofp - ofp.close() + if self.state < _DID_DATA: + self.close_data() + if self.state != _DID_DATA: + raise Error('Close at the wrong time') + if self.rlen != 0: + raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,)) + self._writecrc() + self.ofp.close() + self.state = None + del self.ofp def binhex(inp, out): """binhex(infilename, outfilename): create binhex-encoded copy of a file""" finfo = getfileinfo(inp) ofp = BinHex(finfo, out) - with io.open(inp, 'rb') as ifp: - # XXXX Do textfile translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) - ofp.close_data() + ifp = io.open(inp, 'rb') + # XXXX Do textfile translation on non-mac systems + while True: + d = ifp.read(128000) + if not d: break + ofp.write(d) + ofp.close_data() + ifp.close() ifp = openrsrc(inp, 'rb') while True: @@ -441,15 +436,11 @@ return self._read(n) def close(self): - if self.state is None: - return - try: - if self.rlen: - dummy = self.read_rsrc(self.rlen) - self._checkcrc() - finally: - self.state = None - self.ifp.close() + if self.rlen: + dummy = self.read_rsrc(self.rlen) + self._checkcrc() + self.state = _DID_RSRC + self.ifp.close() def hexbin(inp, out): """hexbin(infilename, outfilename) - Decode binhexed file""" @@ -458,12 +449,13 @@ if not out: out = ifp.FName - with io.open(out, 'wb') as ofp: - # XXXX Do translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) + ofp = io.open(out, 'wb') + # XXXX Do translation on non-mac systems + while True: + d = ifp.read(128000) + if not d: break + ofp.write(d) + ofp.close() ifp.close_data() d = ifp.read_rsrc(128000) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/bz2.py --- a/Lib/bz2.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/bz2.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,15 +4,13 @@ (de)compression, and functions for one-shot (de)compression. """ -__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", - "open", "compress", "decompress"] +__all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor", "compress", + "decompress"] __author__ = "Nadeem Vawda " -from builtins import open as _builtin_open import io import warnings -import _compression try: from threading import RLock @@ -24,11 +22,13 @@ _MODE_CLOSED = 0 _MODE_READ = 1 -# Value 2 no longer used +_MODE_READ_EOF = 2 _MODE_WRITE = 3 +_BUFFER_SIZE = 8192 -class BZ2File(_compression.BaseStream): + +class BZ2File(io.BufferedIOBase): """A file object providing transparent bzip2 (de)compression. @@ -39,20 +39,20 @@ returned as bytes, and data to be written should be given as bytes. """ - def __init__(self, filename, mode="r", buffering=None, compresslevel=9): + def __init__(self, filename=None, mode="r", buffering=None, + compresslevel=9, *, fileobj=None): """Open a bzip2-compressed file. - If filename is a str or bytes object, it gives the name - of the file to be opened. Otherwise, it should be a file object, - which will be used to read or write the compressed data. + If filename is given, open the named file. Otherwise, operate on + the file object given by fileobj. Exactly one of these two + parameters should be provided. - mode can be 'r' for reading (default), 'w' for (over)writing, - 'x' for creating exclusively, or 'a' for appending. These can - equivalently be given as 'rb', 'wb', 'xb', and 'ab'. + mode can be 'r' for reading (default), 'w' for (over)writing, or + 'a' for appending. buffering is ignored. Its use is deprecated. - If mode is 'w', 'x' or 'a', compresslevel can be a number between 1 + If mode is 'w' or 'a', compresslevel can be a number between 1 and 9 specifying the level of compression: 1 produces the least compression, and 9 (default) produces the most compression. @@ -60,11 +60,13 @@ multiple compressed streams. """ # This lock must be recursive, so that BufferedIOBase's - # writelines() does not deadlock. + # readline(), readlines() and writelines() don't deadlock. self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED + self._pos = 0 + self._size = -1 if buffering is not None: warnings.warn("Use of 'buffering' argument is deprecated", @@ -76,37 +78,28 @@ if mode in ("", "r", "rb"): mode = "rb" mode_code = _MODE_READ + self._decompressor = BZ2Decompressor() + self._buffer = None elif mode in ("w", "wb"): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) - elif mode in ("x", "xb"): - mode = "xb" - mode_code = _MODE_WRITE - self._compressor = BZ2Compressor(compresslevel) elif mode in ("a", "ab"): mode = "ab" mode_code = _MODE_WRITE self._compressor = BZ2Compressor(compresslevel) else: - raise ValueError("Invalid mode: %r" % (mode,)) + raise ValueError("Invalid mode: {!r}".format(mode)) - if isinstance(filename, (str, bytes)): - self._fp = _builtin_open(filename, mode) + if filename is not None and fileobj is None: + self._fp = open(filename, mode) self._closefp = True self._mode = mode_code - elif hasattr(filename, "read") or hasattr(filename, "write"): - self._fp = filename + elif fileobj is not None and filename is None: + self._fp = fileobj self._mode = mode_code else: - raise TypeError("filename must be a str or bytes object, or a file") - - if self._mode == _MODE_READ: - raw = _compression.DecompressReader(self._fp, - BZ2Decompressor, trailing_error=OSError) - self._buffer = io.BufferedReader(raw) - else: - self._pos = 0 + raise ValueError("Must give exactly one of filename and fileobj") def close(self): """Flush and close the file. @@ -118,8 +111,8 @@ if self._mode == _MODE_CLOSED: return try: - if self._mode == _MODE_READ: - self._buffer.close() + if self._mode in (_MODE_READ, _MODE_READ_EOF): + self._decompressor = None elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) self._compressor = None @@ -145,18 +138,96 @@ def seekable(self): """Return whether the file supports seeking.""" - return self.readable() and self._buffer.seekable() + return self.readable() and self._fp.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() - return self._mode == _MODE_READ + return self._mode in (_MODE_READ, _MODE_READ_EOF) def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE + # Mode-checking helper functions. + + def _check_not_closed(self): + if self.closed: + raise ValueError("I/O operation on closed file") + + def _check_can_read(self): + if not self.readable(): + raise io.UnsupportedOperation("File not open for reading") + + def _check_can_write(self): + if not self.writable(): + raise io.UnsupportedOperation("File not open for writing") + + def _check_can_seek(self): + if not self.readable(): + raise io.UnsupportedOperation("Seeking is only supported " + "on files open for reading") + if not self._fp.seekable(): + raise io.UnsupportedOperation("The underlying file object " + "does not support seeking") + + # Fill the readahead buffer if it is empty. Returns False on EOF. + def _fill_buffer(self): + if self._buffer: + return True + + if self._decompressor.unused_data: + rawblock = self._decompressor.unused_data + else: + rawblock = self._fp.read(_BUFFER_SIZE) + + if not rawblock: + if self._decompressor.eof: + self._mode = _MODE_READ_EOF + self._size = self._pos + return False + else: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + + # Continue to next stream. + if self._decompressor.eof: + self._decompressor = BZ2Decompressor() + + self._buffer = self._decompressor.decompress(rawblock) + return True + + # Read data until EOF. + # If return_data is false, consume the data without returning it. + def _read_all(self, return_data=True): + blocks = [] + while self._fill_buffer(): + if return_data: + blocks.append(self._buffer) + self._pos += len(self._buffer) + self._buffer = None + if return_data: + return b"".join(blocks) + + # Read a block of up to n bytes. + # If return_data is false, consume the data without returning it. + def _read_block(self, n, return_data=True): + blocks = [] + while n > 0 and self._fill_buffer(): + if n < len(self._buffer): + data = self._buffer[:n] + self._buffer = self._buffer[n:] + else: + data = self._buffer + self._buffer = None + if return_data: + blocks.append(data) + self._pos += len(data) + n -= len(data) + if return_data: + return b"".join(blocks) + def peek(self, n=0): """Return buffered data without advancing the file position. @@ -165,10 +236,9 @@ """ with self._lock: self._check_can_read() - # Relies on the undocumented fact that BufferedReader.peek() - # always returns at least one byte (except at EOF), independent - # of the value of n - return self._buffer.peek(n) + if self._mode == _MODE_READ_EOF or not self._fill_buffer(): + return b"" + return self._buffer def read(self, size=-1): """Read up to size uncompressed bytes from the file. @@ -178,29 +248,40 @@ """ with self._lock: self._check_can_read() - return self._buffer.read(size) + if self._mode == _MODE_READ_EOF or size == 0: + return b"" + elif size < 0: + return self._read_all() + else: + return self._read_block(size) def read1(self, size=-1): - """Read up to size uncompressed bytes, while trying to avoid - making multiple reads from the underlying stream. Reads up to a - buffer's worth of data if size is negative. + """Read up to size uncompressed bytes with at most one read + from the underlying stream. Returns b'' if the file is at EOF. """ with self._lock: self._check_can_read() - if size < 0: - size = io.DEFAULT_BUFFER_SIZE - return self._buffer.read1(size) + if (size == 0 or self._mode == _MODE_READ_EOF or + not self._fill_buffer()): + return b"" + if 0 < size < len(self._buffer): + data = self._buffer[:size] + self._buffer = self._buffer[size:] + else: + data = self._buffer + self._buffer = None + self._pos += len(data) + return data def readinto(self, b): - """Read bytes into b. + """Read up to len(b) bytes into b. Returns the number of bytes read (0 for EOF). """ with self._lock: - self._check_can_read() - return self._buffer.readinto(b) + return io.BufferedIOBase.readinto(self, b) def readline(self, size=-1): """Read a line of uncompressed bytes from the file. @@ -209,13 +290,11 @@ non-negative, no more than size bytes will be read (in which case the line may be incomplete). Returns b'' if already at EOF. """ - if not isinstance(size, int): - if not hasattr(size, "__index__"): - raise TypeError("Integer argument expected") - size = size.__index__() + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() with self._lock: - self._check_can_read() - return self._buffer.readline(size) + return io.BufferedIOBase.readline(self, size) def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. @@ -224,13 +303,11 @@ further lines will be read once the total size of the lines read so far equals or exceeds size. """ - if not isinstance(size, int): - if not hasattr(size, "__index__"): - raise TypeError("Integer argument expected") - size = size.__index__() + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() with self._lock: - self._check_can_read() - return self._buffer.readlines(size) + return io.BufferedIOBase.readlines(self, size) def write(self, data): """Write a byte string to the file. @@ -255,9 +332,17 @@ Line separators are not added between the written byte strings. """ with self._lock: - return _compression.BaseStream.writelines(self, seq) + return io.BufferedIOBase.writelines(self, seq) - def seek(self, offset, whence=io.SEEK_SET): + # Rewind the file to the beginning of the data stream. + def _rewind(self): + self._fp.seek(0, 0) + self._mode = _MODE_READ + self._pos = 0 + self._decompressor = BZ2Decompressor() + self._buffer = None + + def seek(self, offset, whence=0): """Change the file position. The new position is specified by offset, relative to the @@ -274,57 +359,39 @@ """ with self._lock: self._check_can_seek() - return self._buffer.seek(offset, whence) + + # Recalculate offset as an absolute file position. + if whence == 0: + pass + elif whence == 1: + offset = self._pos + offset + elif whence == 2: + # Seeking relative to EOF - we need to know the file's size. + if self._size < 0: + self._read_all(return_data=False) + offset = self._size + offset + else: + raise ValueError("Invalid value for whence: {}".format(whence)) + + # Make it so that offset is the number of bytes to skip forward. + if offset < self._pos: + self._rewind() + else: + offset -= self._pos + + # Read and discard data until we reach the desired position. + if self._mode != _MODE_READ_EOF: + self._read_block(offset, return_data=False) + + return self._pos def tell(self): """Return the current file position.""" with self._lock: self._check_not_closed() - if self._mode == _MODE_READ: - return self._buffer.tell() return self._pos -def open(filename, mode="rb", compresslevel=9, - encoding=None, errors=None, newline=None): - """Open a bzip2-compressed file in binary or text mode. - - The filename argument can be an actual filename (a str or bytes - object), or an existing file object to read from or write to. - - The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or - "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. - The default mode is "rb", and the default compresslevel is 9. - - For binary mode, this function is equivalent to the BZ2File - constructor: BZ2File(filename, mode, compresslevel). In this case, - the encoding, errors and newline arguments must not be provided. - - For text mode, a BZ2File object is created, and wrapped in an - io.TextIOWrapper instance with the specified encoding, error - handling behavior, and line ending(s). - - """ - if "t" in mode: - if "b" in mode: - raise ValueError("Invalid mode: %r" % (mode,)) - else: - if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") - if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") - if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") - - bz_mode = mode.replace("t", "") - binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel) - - if "t" in mode: - return io.TextIOWrapper(binary_file, encoding, errors, newline) - else: - return binary_file - - def compress(data, compresslevel=9): """Compress a block of data. @@ -341,19 +408,17 @@ For incremental decompression, use a BZ2Decompressor object instead. """ + if len(data) == 0: + return b"" + results = [] - while data: + while True: decomp = BZ2Decompressor() - try: - res = decomp.decompress(data) - except OSError: - if results: - break # Leftover data is not a valid bzip2 stream; ignore it. - else: - raise # Error on the first iteration; bail out. - results.append(res) + results.append(decomp.decompress(data)) if not decomp.eof: raise ValueError("Compressed data ended before the " "end-of-stream marker was reached") + if not decomp.unused_data: + return b"".join(results) + # There is unused data left over. Proceed to next stream. data = decomp.unused_data - return b"".join(results) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/cProfile.py --- a/Lib/cProfile.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/cProfile.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,20 +7,54 @@ __all__ = ["run", "runctx", "Profile"] import _lsprof -import profile as _pyprofile # ____________________________________________________________ # Simple interface def run(statement, filename=None, sort=-1): - return _pyprofile._Utils(Profile).run(statement, filename, sort) + """Run statement under profiler optionally saving results in filename + + This function takes a single argument that can be passed to the + "exec" statement, and an optional file name. In all cases this + routine attempts to "exec" its first argument and gather profiling + statistics from the execution. If no file name is present, then this + function automatically prints a simple profiling report, sorted by the + standard name string (file/line/function-name) that is presented in + each line. + """ + prof = Profile() + result = None + try: + try: + prof = prof.run(statement) + except SystemExit: + pass + finally: + if filename is not None: + prof.dump_stats(filename) + else: + result = prof.print_stats(sort) + return result def runctx(statement, globals, locals, filename=None, sort=-1): - return _pyprofile._Utils(Profile).runctx(statement, globals, locals, - filename, sort) + """Run statement under profiler, supplying your own globals and locals, + optionally saving results in filename. -run.__doc__ = _pyprofile.run.__doc__ -runctx.__doc__ = _pyprofile.runctx.__doc__ + statement and filename have the same semantics as profile.run + """ + prof = Profile() + result = None + try: + try: + prof = prof.runctx(statement, globals, locals) + except SystemExit: + pass + finally: + if filename is not None: + prof.dump_stats(filename) + else: + result = prof.print_stats(sort) + return result # ____________________________________________________________ @@ -43,9 +77,10 @@ def dump_stats(self, file): import marshal - with open(file, 'wb') as f: - self.create_stats() - marshal.dump(self.stats, f) + f = open(file, 'wb') + self.create_stats() + marshal.dump(self.stats, f) + f.close() def create_stats(self): self.disable() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/calendar.py --- a/Lib/calendar.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/calendar.py Mon Jan 25 17:05:13 2016 +0100 @@ -12,9 +12,7 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr", - "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", - "LocaleHTMLCalendar", "weekheader"] + "timegm", "month_name", "month_abbr", "day_name", "day_abbr"] # Exception raised for bad input (with string parameter for details) error = ValueError @@ -144,7 +142,7 @@ def iterweekdays(self): """ - Return an iterator for one week of weekday numbers starting with the + Return a iterator for one week of weekday numbers starting with the configured first one. """ for i in range(self.firstweekday, self.firstweekday + 7): @@ -163,11 +161,7 @@ oneday = datetime.timedelta(days=1) while True: yield date - try: - date += oneday - except OverflowError: - # Adding one day could fail after datetime.MAXYEAR - break + date += oneday if date.month != month and date.weekday() == self.firstweekday: break @@ -222,7 +216,7 @@ def yeardatescalendar(self, year, width=3): """ Return the data for the specified year ready for formatting. The return - value is a list of month rows. Each month row contains up to width months. + value is a list of month rows. Each month row contains upto width months. Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ @@ -607,63 +601,51 @@ def main(args): - import argparse - parser = argparse.ArgumentParser() - textgroup = parser.add_argument_group('text only arguments') - htmlgroup = parser.add_argument_group('html only arguments') - textgroup.add_argument( + import optparse + parser = optparse.OptionParser(usage="usage: %prog [options] [year [month]]") + parser.add_option( "-w", "--width", - type=int, default=2, - help="width of date column (default 2)" + dest="width", type="int", default=2, + help="width of date column (default 2, text only)" ) - textgroup.add_argument( + parser.add_option( "-l", "--lines", - type=int, default=1, - help="number of lines for each week (default 1)" + dest="lines", type="int", default=1, + help="number of lines for each week (default 1, text only)" ) - textgroup.add_argument( + parser.add_option( "-s", "--spacing", - type=int, default=6, - help="spacing between months (default 6)" + dest="spacing", type="int", default=6, + help="spacing between months (default 6, text only)" ) - textgroup.add_argument( + parser.add_option( "-m", "--months", - type=int, default=3, - help="months per row (default 3)" + dest="months", type="int", default=3, + help="months per row (default 3, text only)" ) - htmlgroup.add_argument( + parser.add_option( "-c", "--css", - default="calendar.css", - help="CSS to use for page" + dest="css", default="calendar.css", + help="CSS to use for page (html only)" ) - parser.add_argument( + parser.add_option( "-L", "--locale", - default=None, + dest="locale", default=None, help="locale to be used from month and weekday names" ) - parser.add_argument( + parser.add_option( "-e", "--encoding", - default=None, - help="encoding to use for output" + dest="encoding", default=None, + help="Encoding to use for output." ) - parser.add_argument( + parser.add_option( "-t", "--type", - default="text", + dest="type", default="text", choices=("text", "html"), help="output type (text or html)" ) - parser.add_argument( - "year", - nargs='?', type=int, - help="year number (1-9999)" - ) - parser.add_argument( - "month", - nargs='?', type=int, - help="month number (1-12, text only)" - ) - options = parser.parse_args(args[1:]) + (options, args) = parser.parse_args(args) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -681,10 +663,10 @@ encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write - if options.year is None: + if len(args) == 1: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif options.month is None: - write(cal.formatyearpage(options.year, **optdict)) + elif len(args) == 2: + write(cal.formatyearpage(int(args[1]), **optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) @@ -694,15 +676,18 @@ else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) - if options.month is None: + if len(args) != 3: optdict["c"] = options.spacing optdict["m"] = options.months - if options.year is None: + if len(args) == 1: result = cal.formatyear(datetime.date.today().year, **optdict) - elif options.month is None: - result = cal.formatyear(options.year, **optdict) + elif len(args) == 2: + result = cal.formatyear(int(args[1]), **optdict) + elif len(args) == 3: + result = cal.formatmonth(int(args[1]), int(args[2]), **optdict) else: - result = cal.formatmonth(options.year, options.month, **optdict) + parser.error("incorrect number of arguments") + sys.exit(1) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/cgi.py --- a/Lib/cgi.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/cgi.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,12 +32,10 @@ # ======= from io import StringIO, BytesIO, TextIOWrapper -from collections import Mapping import sys import os import urllib.parse from email.parser import FeedParser -from email.message import Message from warnings import warn import html import locale @@ -82,7 +80,7 @@ if logfile and not logfp: try: logfp = open(logfile, "a") - except OSError: + except IOError: pass if not logfp: log = nolog @@ -225,17 +223,17 @@ """ import http.client - boundary = b"" + boundary = "" if 'boundary' in pdict: boundary = pdict['boundary'] if not valid_boundary(boundary): raise ValueError('Invalid boundary in multipart form: %r' % (boundary,)) - nextpart = b"--" + boundary - lastpart = b"--" + boundary + b"--" + nextpart = "--" + boundary + lastpart = "--" + boundary + "--" partdict = {} - terminator = b"" + terminator = "" while terminator != lastpart: bytes = -1 @@ -254,7 +252,7 @@ raise ValueError('Maximum content length exceeded') data = fp.read(bytes) else: - data = b"" + data = "" # Read lines until end of part. lines = [] while 1: @@ -262,7 +260,7 @@ if not line: terminator = lastpart # End outer loop break - if line.startswith(b"--"): + if line.startswith("--"): terminator = line.rstrip() if terminator in (nextpart, lastpart): break @@ -274,12 +272,12 @@ if lines: # Strip final line terminator line = lines[-1] - if line[-2:] == b"\r\n": + if line[-2:] == "\r\n": line = line[:-2] - elif line[-1:] == b"\n": + elif line[-1:] == "\n": line = line[:-1] lines[-1] = line - data = b"".join(lines) + data = "".join(lines) line = headers['content-disposition'] if not line: continue @@ -474,24 +472,18 @@ self.qs_on_post = environ['QUERY_STRING'] if 'CONTENT_LENGTH' in environ: headers['content-length'] = environ['CONTENT_LENGTH'] - else: - if not (isinstance(headers, (Mapping, Message))): - raise TypeError("headers must be mapping or an instance of " - "email.message.Message") - self.headers = headers if fp is None: self.fp = sys.stdin.buffer # self.fp.read() must return bytes elif isinstance(fp, TextIOWrapper): self.fp = fp.buffer else: - if not (hasattr(fp, 'read') and hasattr(fp, 'readline')): - raise TypeError("fp must be file pointer") self.fp = fp self.encoding = encoding self.errors = errors + self.headers = headers if not isinstance(outerboundary, bytes): raise TypeError('outerboundary must be bytes, not %s' % type(outerboundary).__name__) @@ -560,18 +552,6 @@ else: self.read_single() - def __del__(self): - try: - self.file.close() - except AttributeError: - pass - - def __enter__(self): - return self - - def __exit__(self, *args): - self.file.close() - def __repr__(self): """Return a printable representation.""" return "FieldStorage(%r, %r, %r)" % ( @@ -656,9 +636,7 @@ """Dictionary style len(x) support.""" return len(self.keys()) - def __bool__(self): - if self.list is None: - raise TypeError("Cannot be converted to bool.") + def __nonzero__(self): return bool(self.list) def read_urlencoded(self): @@ -692,6 +670,7 @@ encoding=self.encoding, errors=self.errors) for key, value in query: self.list.append(MiniFieldStorage(key, value)) + FieldStorageClass = None klass = self.FieldStorageClass or self.__class__ first_line = self.fp.readline() # bytes @@ -699,13 +678,8 @@ raise ValueError("%s should return bytes, got %s" \ % (self.fp, type(first_line).__name__)) self.bytes_read += len(first_line) - - # Ensure that we consume the file until we've hit our inner boundary - while (first_line.strip() != (b"--" + self.innerboundary) and - first_line): - first_line = self.fp.readline() - self.bytes_read += len(first_line) - + # first line holds boundary ; ignore it, or check that + # b"--" + ib == first_line.strip() ? while True: parser = FeedParser() hdr_text = b"" @@ -720,17 +694,12 @@ self.bytes_read += len(hdr_text) parser.feed(hdr_text.decode(self.encoding, self.errors)) headers = parser.close() - - # Some clients add Content-Length for part headers, ignore them - if 'content-length' in headers: - del headers['content-length'] - part = klass(self.fp, headers, ib, environ, keep_blank_values, strict_parsing,self.limit-self.bytes_read, self.encoding, self.errors) self.bytes_read += part.bytes_read self.list.append(part) - if part.done or self.bytes_read >= self.length > 0: + if self.bytes_read >= self.length: break self.skip_lines() @@ -817,9 +786,6 @@ if not line: self.done = -1 break - if delim == b"\r": - line = delim + line - delim = b"" if line.startswith(b"--") and last_line_lfend: strippedline = line.rstrip() if strippedline == next_boundary: @@ -836,12 +802,6 @@ delim = b"\n" line = line[:-1] last_line_lfend = True - elif line.endswith(b"\r"): - # We may interrupt \r\n sequences if they span the 2**16 - # byte boundary - delim = b"\r" - line = line[:-1] - last_line_lfend = False else: delim = b"" last_line_lfend = False @@ -989,8 +949,8 @@ print("

    Current Working Directory:

    ") try: pwd = os.getcwd() - except OSError as msg: - print("OSError:", html.escape(str(msg))) + except os.error as msg: + print("os.error:", html.escape(str(msg))) else: print(html.escape(pwd)) print() @@ -1061,7 +1021,7 @@ return s -def valid_boundary(s): +def valid_boundary(s, _vb_pattern=None): import re if isinstance(s, bytes): _vb_pattern = b"^[ -~]{0,200}[!-~]$" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/cgitb.py --- a/Lib/cgitb.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/cgitb.py Mon Jan 25 17:05:13 2016 +0100 @@ -292,18 +292,14 @@ if self.logdir is not None: suffix = ['.txt', '.html'][self.format=="html"] (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) - try: - with os.fdopen(fd, 'w') as file: - file.write(doc) - msg = '%s contains the description of this error.' % path + file = os.fdopen(fd, 'w') + file.write(doc) + file.close() + msg = '

    %s contains the description of this error.' % path except: - msg = 'Tried to save traceback to %s, but failed.' % path - - if self.format == 'html': - self.file.write('

    %s

    \n' % msg) - else: - self.file.write(msg + '\n') + msg = '

    Tried to save traceback to %s, but failed.' % path + self.file.write(msg + '\n') try: self.file.flush() except: pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/chunk.py --- a/Lib/chunk.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/chunk.py Mon Jan 25 17:05:13 2016 +0100 @@ -21,7 +21,7 @@ usage of the Chunk class defined here is to instantiate an instance at the start of each chunk and read from the instance until it reaches the end, after which a new instance can be instantiated. At the end -of the file, creating a new instance will fail with an EOFError +of the file, creating a new instance will fail with a EOFError exception. Usage: @@ -70,7 +70,7 @@ self.size_read = 0 try: self.offset = self.file.tell() - except (AttributeError, OSError): + except (AttributeError, IOError): self.seekable = False else: self.seekable = True @@ -85,10 +85,8 @@ def close(self): if not self.closed: - try: - self.skip() - finally: - self.closed = True + self.skip() + self.closed = True def isatty(self): if self.closed: @@ -104,7 +102,7 @@ if self.closed: raise ValueError("I/O operation on closed file") if not self.seekable: - raise OSError("cannot seek") + raise IOError("cannot seek") if whence == 1: pos = pos + self.size_read elif whence == 2: @@ -128,7 +126,7 @@ if self.closed: raise ValueError("I/O operation on closed file") if self.size_read >= self.chunksize: - return b'' + return '' if size < 0: size = self.chunksize - self.size_read if size > self.chunksize - self.size_read: @@ -160,7 +158,7 @@ self.file.seek(n, 1) self.size_read = self.size_read + n return - except OSError: + except IOError: pass while self.size_read < self.chunksize: n = min(8192, self.chunksize - self.size_read) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/code.py --- a/Lib/code.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/code.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,7 +7,6 @@ import sys import traceback -import argparse from codeop import CommandCompiler, compile_command __all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact", @@ -106,10 +105,9 @@ The output is written by self.write(), below. """ - type, value, tb = sys.exc_info() + type, value, sys.last_traceback = sys.exc_info() sys.last_type = type sys.last_value = value - sys.last_traceback = tb if filename and type is SyntaxError: # Work hard to stuff the correct filename in the exception try: @@ -121,13 +119,8 @@ # Stuff in the right filename value = SyntaxError(msg, (filename, lineno, offset, line)) sys.last_value = value - if sys.excepthook is sys.__excepthook__: - lines = traceback.format_exception_only(type, value) - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(type, value, tb) + lines = traceback.format_exception_only(type, value) + self.write(''.join(lines)) def showtraceback(self): """Display the exception that just occurred. @@ -137,18 +130,20 @@ The output is written by self.write(), below. """ - sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() - sys.last_traceback = last_tb try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) - if sys.excepthook is sys.__excepthook__: - self.write(''.join(lines)) - else: - # If someone has set sys.excepthook, we let that take precedence - # over self.write - sys.excepthook(ei[0], ei[1], last_tb) + type, value, tb = sys.exc_info() + sys.last_type = type + sys.last_value = value + sys.last_traceback = tb + tblist = traceback.extract_tb(tb) + del tblist[:1] + lines = traceback.format_list(tblist) + if lines: + lines.insert(0, "Traceback (most recent call last):\n") + lines.extend(traceback.format_exception_only(type, value)) finally: - last_tb = ei = None + tblist = tb = None + self.write(''.join(lines)) def write(self, data): """Write a string. @@ -210,7 +205,7 @@ self.write("Python %s on %s\n%s\n(%s)\n" % (sys.version, sys.platform, cprt, self.__class__.__name__)) - elif banner: + else: self.write("%s\n" % str(banner)) more = 0 while 1: @@ -293,12 +288,4 @@ if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-q', action='store_true', - help="don't print version and copyright messages") - args = parser.parse_args() - if args.q or sys.flags.quiet: - banner = '' - else: - banner = None - interact(banner) + interact() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/codecs.py --- a/Lib/codecs.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/codecs.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,15 +20,8 @@ "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE", "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE", "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE", - "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder", - "StreamReader", "StreamWriter", - "StreamReaderWriter", "StreamRecoder", - "getencoder", "getdecoder", "getincrementalencoder", - "getincrementaldecoder", "getreader", "getwriter", - "encode", "decode", "iterencode", "iterdecode", "strict_errors", "ignore_errors", "replace_errors", "xmlcharrefreplace_errors", - "backslashreplace_errors", "namereplace_errors", "register_error", "lookup_error"] ### Constants @@ -80,19 +73,9 @@ ### Codec base classes (defining the API) class CodecInfo(tuple): - """Codec details when looking up the codec registry""" - - # Private API to allow Python 3.4 to blacklist the known non-Unicode - # codecs in the standard library. A more general mechanism to - # reliably distinguish test encodings from other codecs will hopefully - # be defined for Python 3.5 - # - # See http://bugs.python.org/issue19619 - _is_text_encoding = True # Assume codecs are text encodings by default def __new__(cls, encode, decode, streamreader=None, streamwriter=None, - incrementalencoder=None, incrementaldecoder=None, name=None, - *, _is_text_encoding=None): + incrementalencoder=None, incrementaldecoder=None, name=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode @@ -101,13 +84,11 @@ self.incrementaldecoder = incrementaldecoder self.streamwriter = streamwriter self.streamreader = streamreader - if _is_text_encoding is not None: - self._is_text_encoding = _is_text_encoding return self def __repr__(self): - return "<%s.%s object for encoding %s at %#x>" % \ - (self.__class__.__module__, self.__class__.__qualname__, + return "<%s.%s object for encoding %s at 0x%x>" % \ + (self.__class__.__module__, self.__class__.__name__, self.name, id(self)) class Codec: @@ -124,11 +105,9 @@ Python will use the official U+FFFD REPLACEMENT CHARACTER for the builtin Unicode codecs on decoding and '?' on encoding. - 'surrogateescape' - replace with private code points U+DCnn. 'xmlcharrefreplace' - Replace with the appropriate XML character reference (only for encoding). - 'backslashreplace' - Replace with backslashed escape sequences. - 'namereplace' - Replace with \\N{...} escape sequences + 'backslashreplace' - Replace with backslashed escape sequences (only for encoding). The set of allowed values can be extended via register_error. @@ -143,8 +122,8 @@ 'strict' handling. The method may not store state in the Codec instance. Use - StreamWriter for codecs which have to keep state in order to - make encoding efficient. + StreamCodec for codecs which have to keep state in order to + make encoding/decoding efficient. The encoder must be able to handle zero length input and return an empty object of the output object type in this @@ -166,8 +145,8 @@ 'strict' handling. The method may not store state in the Codec instance. Use - StreamReader for codecs which have to keep state in order to - make decoding efficient. + StreamCodec for codecs which have to keep state in order to + make encoding/decoding efficient. The decoder must be able to handle zero length input and return an empty object of the output object type in this @@ -258,7 +237,7 @@ """ def __init__(self, errors='strict'): """ - Create an IncrementalDecoder instance. + Create a IncrementalDecoder instance. The IncrementalDecoder may use different error handling schemes by providing the errors keyword argument. See the module docstring @@ -348,7 +327,8 @@ """ Creates a StreamWriter instance. - stream must be a file-like object open for writing. + stream must be a file-like object open for writing + (binary) data. The StreamWriter may use different error handling schemes by providing the errors keyword argument. These @@ -360,8 +340,7 @@ 'xmlcharrefreplace' - Replace with the appropriate XML character reference. 'backslashreplace' - Replace with backslashed escape - sequences. - 'namereplace' - Replace with \\N{...} escape sequences. + sequences (only for encoding). The set of allowed parameter values can be extended via register_error. @@ -423,7 +402,8 @@ """ Creates a StreamReader instance. - stream must be a file-like object open for reading. + stream must be a file-like object open for reading + (binary) data. The StreamReader may use different error handling schemes by providing the errors keyword argument. These @@ -431,8 +411,7 @@ 'strict' - raise a ValueError (or a subclass) 'ignore' - ignore the character and continue with the next - 'replace'- replace with a suitable replacement character - 'backslashreplace' - Replace with backslashed escape sequences; + 'replace'- replace with a suitable replacement character; The set of allowed parameter values can be extended via register_error. @@ -452,12 +431,13 @@ """ Decodes data from the stream self.stream and returns the resulting object. - chars indicates the number of decoded code points or bytes to - return. read() will never return more data than requested, - but it might return less, if there is not enough available. + chars indicates the number of characters to read from the + stream. read() will never return more than chars + characters, but it might return less, if there are not enough + characters available. - size indicates the approximate maximum number of decoded - bytes or code points to read for decoding. The decoder + size indicates the approximate maximum number of bytes to + read from the stream for decoding purposes. The decoder can modify this setting as appropriate. The default value -1 indicates to read and decode as much as possible. size is intended to prevent having to decode huge files in one @@ -468,7 +448,7 @@ will be returned, the rest of the input will be kept until the next call to read(). - The method should use a greedy read strategy, meaning that + The method should use a greedy read strategy meaning that it should read as much data as is allowed within the definition of the encoding and the given size, e.g. if optional encoding endings or state markers are available @@ -481,13 +461,16 @@ # read until we get the required number of characters (if available) while True: - # can the request be satisfied from the character buffer? - if chars >= 0: + # can the request can be satisfied from the character buffer? + if chars < 0: + if size < 0: + if self.charbuffer: + break + elif len(self.charbuffer) >= size: + break + else: if len(self.charbuffer) >= chars: break - elif size >= 0: - if len(self.charbuffer) >= size: - break # we need more data if size < 0: newdata = self.stream.read() @@ -495,8 +478,6 @@ newdata = self.stream.read(size) # decode bytes (those remaining from the last call included) data = self.bytebuffer + newdata - if not data: - break try: newchars, decodedbytes = self.decode(data, self.errors) except UnicodeDecodeError as exc: @@ -603,7 +584,7 @@ def readlines(self, sizehint=None, keepends=True): """ Read all lines available on the input stream - and return them as a list. + and return them as list of lines. Line breaks are implemented using the codec's decoder method and are included in the list entries. @@ -751,18 +732,19 @@ class StreamRecoder: - """ StreamRecoder instances translate data from one encoding to another. + """ StreamRecoder instances provide a frontend - backend + view of encoding data. They use the complete set of APIs returned by the codecs.lookup() function to implement their task. - Data written to the StreamRecoder is first decoded into an - intermediate format (depending on the "decode" codec) and then - written to the underlying stream using an instance of the provided - Writer class. + Data written to the stream is first decoded into an + intermediate format (which is dependent on the given codec + combination) and then written to the stream using an instance + of the provided Writer class. - In the other direction, data is read from the underlying stream using - a Reader instance and then encoded and returned to the caller. + In the other direction, data is read from the stream using a + Reader instance and then return encoded data to the caller. """ # Optional attributes set by the file wrappers below @@ -774,17 +756,22 @@ """ Creates a StreamRecoder instance which implements a two-way conversion: encode and decode work on the frontend (the - data visible to .read() and .write()) while Reader and Writer - work on the backend (the data in stream). + input to .read() and output of .write()) while + Reader and Writer work on the backend (reading and + writing to the stream). - You can use these objects to do transparent - transcodings from e.g. latin-1 to utf-8 and back. + You can use these objects to do transparent direct + recodings from e.g. latin-1 to utf-8 and back. stream must be a file-like object. - encode and decode must adhere to the Codec interface; Reader and + encode, decode must adhere to the Codec interface, Reader, Writer must be factory functions or classes providing the - StreamReader and StreamWriter interfaces resp. + StreamReader, StreamWriter interface resp. + + encode and decode are needed for the frontend translation, + Reader and Writer for the backend translation. Unicode is + used as intermediate encoding. Error handling is done in the same way as defined for the StreamWriter/Readers. @@ -859,7 +846,7 @@ ### Shortcuts -def open(filename, mode='r', encoding=None, errors='strict', buffering=1): +def open(filename, mode='rb', encoding=None, errors='strict', buffering=1): """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -869,8 +856,10 @@ codecs. Output is also codec dependent and will usually be Unicode as well. - Underlying encoded files are always opened in binary mode. - The default file mode is 'r', meaning to open the file in read mode. + Files are always opened in binary mode, even if no binary mode + was specified. This is done to avoid data loss due to encodings + using 8-bit values. The default file mode is 'rb' meaning to + open the file in binary read mode. encoding specifies the encoding which is to be used for the file. @@ -906,13 +895,13 @@ """ Return a wrapped version of file which provides transparent encoding translation. - Data written to the wrapped file is decoded according - to the given data_encoding and then encoded to the underlying - file using file_encoding. The intermediate data type + Strings written to the wrapped file are interpreted according + to the given data_encoding and then written to the original + file as string using file_encoding. The intermediate encoding will usually be Unicode but depends on the specified codecs. - Bytes read from the file are decoded using file_encoding and then - passed back to the caller encoded using data_encoding. + Strings are read from the file using file_encoding and then + passed back to the caller as string using data_encoding. If file_encoding is not given, it defaults to data_encoding. @@ -1011,7 +1000,7 @@ """ Encoding iterator. - Encodes the input strings from the iterator using an IncrementalEncoder. + Encodes the input strings from the iterator using a IncrementalEncoder. errors and kwargs are passed through to the IncrementalEncoder constructor. @@ -1029,7 +1018,7 @@ """ Decoding iterator. - Decodes the input strings from the iterator using an IncrementalDecoder. + Decodes the input strings from the iterator using a IncrementalDecoder. errors and kwargs are passed through to the IncrementalDecoder constructor. @@ -1053,7 +1042,10 @@ mapped to themselves. """ - return {i:i for i in rng} + res = {} + for i in rng: + res[i]=i + return res def make_encoding_map(decoding_map): @@ -1065,7 +1057,7 @@ during translation. One example where this happens is cp875.py which decodes - multiple character to \\u001a. + multiple character to \u001a. """ m = {} @@ -1084,7 +1076,6 @@ replace_errors = lookup_error("replace") xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace") backslashreplace_errors = lookup_error("backslashreplace") - namereplace_errors = lookup_error("namereplace") except LookupError: # In --disable-unicode builds, these error handler are missing strict_errors = None @@ -1092,7 +1083,6 @@ replace_errors = None xmlcharrefreplace_errors = None backslashreplace_errors = None - namereplace_errors = None # Tell modulefinder that using codecs probably needs the encodings # package diff -r 6db40a9955dc -r 0d413f60cc23 Lib/collections/__init__.py --- a/Lib/collections/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/collections/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,70 +1,25 @@ -'''This module implements specialized container datatypes providing -alternatives to Python's general purpose built-in containers, dict, -list, set, and tuple. - -* namedtuple factory function for creating tuple subclasses with named fields -* deque list-like container with fast appends and pops on either end -* ChainMap dict-like class for creating a single view of multiple mappings -* Counter dict subclass for counting hashable objects -* OrderedDict dict subclass that remembers the order entries were added -* defaultdict dict subclass that calls a factory function to supply missing values -* UserDict wrapper around dictionary objects for easier dict subclassing -* UserList wrapper around list objects for easier list subclassing -* UserString wrapper around string objects for easier string subclassing - -''' - __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', 'UserString', 'Counter', 'OrderedDict', 'ChainMap'] # For backwards compatibility, continue to make the collections ABCs # available through the collections module. -from _collections_abc import * -import _collections_abc -__all__ += _collections_abc.__all__ +from collections.abc import * +import collections.abc +__all__ += collections.abc.__all__ -from operator import itemgetter as _itemgetter, eq as _eq +from _collections import deque, defaultdict +from operator import itemgetter as _itemgetter from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq -from _weakref import proxy as _proxy +from weakref import proxy as _proxy from itertools import repeat as _repeat, chain as _chain, starmap as _starmap from reprlib import recursive_repr as _recursive_repr -try: - from _collections import deque -except ImportError: - pass -else: - MutableSequence.register(deque) - -try: - from _collections import defaultdict -except ImportError: - pass - - ################################################################################ ### OrderedDict ################################################################################ -class _OrderedDictKeysView(KeysView): - - def __reversed__(self): - yield from reversed(self._mapping) - -class _OrderedDictItemsView(ItemsView): - - def __reversed__(self): - for key in reversed(self._mapping): - yield (key, self._mapping[key]) - -class _OrderedDictValuesView(ValuesView): - - def __reversed__(self): - for key in reversed(self._mapping): - yield self._mapping[key] - class _Link(object): __slots__ = 'prev', 'next', 'key', '__weakref__' @@ -83,16 +38,12 @@ # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. - def __init__(*args, **kwds): + def __init__(self, *args, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries, but keyword arguments are not recommended because their insertion order is arbitrary. ''' - if not args: - raise TypeError("descriptor '__init__' of 'OrderedDict' object " - "needs an argument") - self, *args = args if len(args) > 1: raise TypeError('expected at most 1 arguments, got %d' % len(args)) try: @@ -128,8 +79,6 @@ link_next = link.next link_prev.next = link_next link_next.prev = link_prev - link.prev = None - link.next = None def __iter__(self): 'od.__iter__() <==> iter(od)' @@ -213,19 +162,9 @@ return size update = __update = MutableMapping.update - - def keys(self): - "D.keys() -> a set-like object providing a view on D's keys" - return _OrderedDictKeysView(self) - - def items(self): - "D.items() -> a set-like object providing a view on D's items" - return _OrderedDictItemsView(self) - - def values(self): - "D.values() -> an object providing a view on D's values" - return _OrderedDictValuesView(self) - + keys = MutableMapping.keys + values = MutableMapping.values + items = MutableMapping.items __ne__ = MutableMapping.__ne__ __marker = object() @@ -260,10 +199,13 @@ def __reduce__(self): 'Return state information for pickling' + items = [[k, self[k]] for k in self] inst_dict = vars(self).copy() for k in vars(OrderedDict()): inst_dict.pop(k, None) - return self.__class__, (), inst_dict or None, None, iter(self.items()) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) def copy(self): 'od.copy() -> a shallow copy of od' @@ -286,22 +228,16 @@ ''' if isinstance(other, OrderedDict): - return dict.__eq__(self, other) and all(map(_eq, self, other)) + return len(self)==len(other) and \ + all(p==q for p, q in zip(self.items(), other.items())) return dict.__eq__(self, other) -try: - from _collections import OrderedDict -except ImportError: - # Leave the pure Python version in place. - pass - - ################################################################################ ### namedtuple ################################################################################ -_class_template = """\ +_class_template = '''\ from builtins import property as _property, tuple as _tuple from operator import itemgetter as _itemgetter from collections import OrderedDict @@ -325,6 +261,16 @@ raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result)) return result + def __repr__(self): + 'Return a nicely formatted representation string' + return self.__class__.__name__ + '({repr_fmt})' % self + + def _asdict(self): + 'Return a new OrderedDict which maps field names to their values' + return OrderedDict(zip(self._fields, self)) + + __dict__ = property(_asdict) + def _replace(_self, **kwds): 'Return a new {typename} object replacing specified fields with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) @@ -332,20 +278,12 @@ raise ValueError('Got unexpected field names: %r' % list(kwds)) return result - def __repr__(self): - 'Return a nicely formatted representation string' - return self.__class__.__name__ + '({repr_fmt})' % self - - def _asdict(self): - 'Return a new OrderedDict which maps field names to their values.' - return OrderedDict(zip(self._fields, self)) - def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) {field_defs} -""" +''' _repr_template = '{name}=%r' @@ -377,35 +315,33 @@ """ - # Validate the field names. At the user's option, either generate an error - # message or automatically replace the field name with a valid name. + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. if isinstance(field_names, str): - field_names = field_names.replace(',', ' ').split() + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas field_names = list(map(str, field_names)) - typename = str(typename) if rename: seen = set() for index, name in enumerate(field_names): - if (not name.isidentifier() + if (not all(c.isalnum() or c=='_' for c in name) or _iskeyword(name) + or not name + or name[0].isdigit() or name.startswith('_') or name in seen): field_names[index] = '_%d' % index seen.add(name) for name in [typename] + field_names: - if type(name) != str: - raise TypeError('Type names and field names must be strings') - if not name.isidentifier(): - raise ValueError('Type names and field names must be valid ' - 'identifiers: %r' % name) + if not all(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) if _iskeyword(name): - raise ValueError('Type names and field names cannot be a ' - 'keyword: %r' % name) + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) seen = set() for name in field_names: if name.startswith('_') and not rename: - raise ValueError('Field names cannot start with an underscore: ' - '%r' % name) + raise ValueError('Field names cannot start with an underscore: %r' % name) if name in seen: raise ValueError('Encountered duplicate field name: %r' % name) seen.add(name) @@ -416,23 +352,25 @@ field_names = tuple(field_names), num_fields = len(field_names), arg_list = repr(tuple(field_names)).replace("'", "")[1:-1], - repr_fmt = ', '.join(_repr_template.format(name=name) - for name in field_names), + repr_fmt = ', '.join(_repr_template.format(name=name) for name in field_names), field_defs = '\n'.join(_field_template.format(index=index, name=name) for index, name in enumerate(field_names)) ) - # Execute the template string in a temporary namespace and support - # tracing utilities by setting a value for frame.f_globals['__name__'] + # Execute the template string in a temporary namespace and + # support tracing utilities by setting a value for frame.f_globals['__name__'] namespace = dict(__name__='namedtuple_%s' % typename) - exec(class_definition, namespace) + try: + exec(class_definition, namespace) + except SyntaxError as e: + raise SyntaxError(e.msg + ':\n\n' + class_definition) result = namespace[typename] result._source = class_definition if verbose: print(result._source) # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in environments where + # where the named tuple is created. Bypass this step in enviroments where # sys._getframe is not defined (Jython for example) or sys._getframe is not # defined for arguments greater than 0 (IronPython). try: @@ -509,7 +447,7 @@ # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 - def __init__(*args, **kwds): + def __init__(self, iterable=None, **kwds): '''Create a new, empty Counter object. And if given, count elements from an input iterable. Or, initialize the count from another mapping of elements to their counts. @@ -520,14 +458,8 @@ >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' - if not args: - raise TypeError("descriptor '__init__' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - super(Counter, self).__init__() - self.update(*args, **kwds) + super().__init__() + self.update(iterable, **kwds) def __missing__(self, key): 'The count of elements not in the Counter is zero.' @@ -578,7 +510,7 @@ raise NotImplementedError( 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - def update(*args, **kwds): + def update(self, iterable=None, **kwds): '''Like dict.update() but add counts instead of replacing them. Source can be an iterable, a dictionary, or another Counter instance. @@ -598,13 +530,6 @@ # contexts. Instead, we implement straight-addition. Both the inputs # and outputs are allowed to contain zero and negative counts. - if not args: - raise TypeError("descriptor 'update' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - iterable = args[0] if args else None if iterable is not None: if isinstance(iterable, Mapping): if self: @@ -612,13 +537,13 @@ for elem, count in iterable.items(): self[elem] = count + self_get(elem, 0) else: - super(Counter, self).update(iterable) # fast path when counter is empty + super().update(iterable) # fast path when counter is empty else: _count_elements(self, iterable) if kwds: self.update(kwds) - def subtract(*args, **kwds): + def subtract(self, iterable=None, **kwds): '''Like dict.update() but subtracts counts instead of replacing them. Counts can be reduced below zero. Both the inputs and outputs are allowed to contain zero and negative counts. @@ -634,13 +559,6 @@ -1 ''' - if not args: - raise TypeError("descriptor 'subtract' of 'Counter' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - iterable = args[0] if args else None if iterable is not None: self_get = self.get if isinstance(iterable, Mapping): @@ -760,22 +678,14 @@ def __pos__(self): 'Adds an empty counter, effectively stripping negative and zero counts' - result = Counter() - for elem, count in self.items(): - if count > 0: - result[elem] = count - return result + return self + Counter() def __neg__(self): '''Subtracts from an empty counter. Strips positive and zero counts, and flips the sign on negative counts. ''' - result = Counter() - for elem, count in self.items(): - if count < 0: - result[elem] = 0 - count - return result + return Counter() - self def _keep_positive(self): '''Internal method to strip elements with a negative or zero count''' @@ -907,13 +817,9 @@ __copy__ = copy - def new_child(self, m=None): # like Django's Context.push() - '''New ChainMap with a new map followed by all previous maps. - If no map is provided, an empty dict is used. - ''' - if m is None: - m = {} - return self.__class__(m, *self.maps) + def new_child(self): # like Django's Context.push() + 'New ChainMap with a new dict followed by all previous maps.' + return self.__class__({}, *self.maps) @property def parents(self): # like Django's Context.pop() @@ -955,22 +861,7 @@ class UserDict(MutableMapping): # Start by filling-out the abstract methods - def __init__(*args, **kwargs): - if not args: - raise TypeError("descriptor '__init__' of 'UserDict' object " - "needs an argument") - self, *args = args - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - if args: - dict = args[0] - elif 'dict' in kwargs: - dict = kwargs.pop('dict') - import warnings - warnings.warn("Passing 'dict' as keyword argument is deprecated", - DeprecationWarning, stacklevel=2) - else: - dict = None + def __init__(self, dict=None, **kwargs): self.data = {} if dict is not None: self.update(dict) @@ -1035,6 +926,7 @@ def __lt__(self, other): return self.data < self.__cast(other) def __le__(self, other): return self.data <= self.__cast(other) def __eq__(self, other): return self.data == self.__cast(other) + def __ne__(self, other): return self.data != self.__cast(other) def __gt__(self, other): return self.data > self.__cast(other) def __ge__(self, other): return self.data >= self.__cast(other) def __cast(self, other): @@ -1106,13 +998,15 @@ def __float__(self): return float(self.data) def __complex__(self): return complex(self.data) def __hash__(self): return hash(self.data) - def __getnewargs__(self): - return (self.data[:],) def __eq__(self, string): if isinstance(string, UserString): return self.data == string.data return self.data == string + def __ne__(self, string): + if isinstance(string, UserString): + return self.data != string.data + return self.data != string def __lt__(self, string): if isinstance(string, UserString): return self.data < string.data @@ -1152,13 +1046,9 @@ __rmul__ = __mul__ def __mod__(self, args): return self.__class__(self.data % args) - def __rmod__(self, format): - return self.__class__(format % args) # the following methods are defined in alphabetical order: def capitalize(self): return self.__class__(self.data.capitalize()) - def casefold(self): - return self.__class__(self.data.casefold()) def center(self, width, *args): return self.__class__(self.data.center(width, *args)) def count(self, sub, start=0, end=_sys.maxsize): @@ -1181,8 +1071,6 @@ return self.data.find(sub, start, end) def format(self, *args, **kwds): return self.data.format(*args, **kwds) - def format_map(self, mapping): - return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): return self.data.index(sub, start, end) def isalpha(self): return self.data.isalpha() @@ -1192,7 +1080,6 @@ def isidentifier(self): return self.data.isidentifier() def islower(self): return self.data.islower() def isnumeric(self): return self.data.isnumeric() - def isprintable(self): return self.data.isprintable() def isspace(self): return self.data.isspace() def istitle(self): return self.data.istitle() def isupper(self): return self.data.isupper() @@ -1201,7 +1088,6 @@ return self.__class__(self.data.ljust(width, *args)) def lower(self): return self.__class__(self.data.lower()) def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars)) - maketrans = str.maketrans def partition(self, sep): return self.data.partition(sep) def replace(self, old, new, maxsplit=-1): @@ -1236,3 +1122,44 @@ return self.__class__(self.data.translate(*args)) def upper(self): return self.__class__(self.data.upper()) def zfill(self, width): return self.__class__(self.data.zfill(width)) + + + +################################################################################ +### Simple tests +################################################################################ + +if __name__ == '__main__': + # verify that instances can be pickled + from pickle import loads, dumps + Point = namedtuple('Point', 'x, y', True) + p = Point(x=10, y=20) + assert p == loads(dumps(p)) + + # test and demonstrate ability to override methods + class Point(namedtuple('Point', 'x y')): + __slots__ = () + @property + def hypot(self): + return (self.x ** 2 + self.y ** 2) ** 0.5 + def __str__(self): + return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) + + for p in Point(3, 4), Point(14, 5/7.): + print (p) + + class Point(namedtuple('Point', 'x y')): + 'Point class with optimized _make() and _replace() without error-checking' + __slots__ = () + _make = classmethod(tuple.__new__) + def _replace(self, _map=map, **kwds): + return self._make(_map(kwds.get, ('x', 'y'), self)) + + print(Point(11, 22)._replace(x=100)) + + Point3D = namedtuple('Point3D', Point._fields + ('z',)) + print(Point3D.__doc__) + + import doctest + TestResults = namedtuple('TestResults', 'failed attempted') + print(TestResults(*doctest.testmod())) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/collections/__main__.py --- a/Lib/collections/__main__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,38 +0,0 @@ -################################################################################ -### Simple tests -################################################################################ - -# verify that instances can be pickled -from collections import namedtuple -from pickle import loads, dumps -Point = namedtuple('Point', 'x, y', True) -p = Point(x=10, y=20) -assert p == loads(dumps(p)) - -# test and demonstrate ability to override methods -class Point(namedtuple('Point', 'x y')): - __slots__ = () - @property - def hypot(self): - return (self.x ** 2 + self.y ** 2) ** 0.5 - def __str__(self): - return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - -for p in Point(3, 4), Point(14, 5/7.): - print (p) - -class Point(namedtuple('Point', 'x y')): - 'Point class with optimized _make() and _replace() without error-checking' - __slots__ = () - _make = classmethod(tuple.__new__) - def _replace(self, _map=map, **kwds): - return self._make(_map(kwds.get, ('x', 'y'), self)) - -print(Point(11, 22)._replace(x=100)) - -Point3D = namedtuple('Point3D', Point._fields + ('z',)) -print(Point3D.__doc__) - -import doctest, collections -TestResults = namedtuple('TestResults', 'failed attempted') -print(TestResults(*doctest.testmod(collections))) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/collections/abc.py --- a/Lib/collections/abc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/collections/abc.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,2 +1,654 @@ -from _collections_abc import * -from _collections_abc import __all__ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +from abc import ABCMeta, abstractmethod +import sys + +__all__ = ["Hashable", "Iterable", "Iterator", + "Sized", "Container", "Callable", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", + ] + + +### collection related types which are not exposed through builtin ### +## iterators ## +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +dict_proxy = type(type.__dict__) + + +### ONE-TRICK PONIES ### + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + for B in C.__mro__: + if "__hash__" in B.__dict__: + if B.__dict__["__hash__"]: + return True + break + return NotImplemented + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + if any("__iter__" in B.__dict__ for B in C.__mro__): + return True + return NotImplemented + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + if (any("__next__" in B.__dict__ for B in C.__mro__) and + any("__iter__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented + +Iterator.register(bytes_iterator) +Iterator.register(bytearray_iterator) +#Iterator.register(callable_iterator) +Iterator.register(dict_keyiterator) +Iterator.register(dict_valueiterator) +Iterator.register(dict_itemiterator) +Iterator.register(list_iterator) +Iterator.register(list_reverseiterator) +Iterator.register(range_iterator) +Iterator.register(set_iterator) +Iterator.register(str_iterator) +Iterator.register(tuple_iterator) +Iterator.register(zip_iterator) + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + if any("__len__" in B.__dict__ for B in C.__mro__): + return True + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + if any("__contains__" in B.__dict__ for B in C.__mro__): + return True + return NotImplemented + + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + if any("__call__" in B.__dict__ for B in C.__mro__): + return True + return NotImplemented + + +### SETS ### + + +class Set(Sized, Iterable, Container): + + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return other < self + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + return other <= self + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + def __ne__(self, other): + return not (self == other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + def isdisjoint(self, other): + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Sized, Iterable, Container): + + __slots__ = () + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + return KeysView(self) + + def items(self): + return ItemsView(self) + + def values(self): + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + def __ne__(self, other): + return not (self == other) + + +class MappingView(Sized): + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + for key in self._mapping: + yield key + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + +ItemsView.register(dict_items) + + +class ValuesView(MappingView): + + def __contains__(self, value): + for key in self._mapping: + if value == self._mapping[key]: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + + __slots__ = () + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + try: + key = next(iter(self)) + except StopIteration: + raise KeyError + value = self[key] + del self[key] + return key, value + + def clear(self): + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + if len(args) > 2: + raise TypeError("update() takes at most 2 positional " + "arguments ({} given)".format(len(args))) + elif not args: + raise TypeError("update() takes at least 1 argument (0 given)") + self = args[0] + other = args[1] if len(args) >= 2 else () + + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Sized, Iterable, Container): + + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value): + for i, v in enumerate(self): + if v == value: + return i + raise ValueError + + def count(self, value): + return sum(1 for v in self if v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) + + +class ByteString(Sequence): + + """This unifies bytes and bytearray. + + XXX Should add all their methods. + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + + __slots__ = () + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + raise IndexError + + def append(self, value): + self.insert(len(self), value) + + def clear(self): + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + for v in values: + self.append(v) + + def pop(self, index=-1): + v = self[index] + del self[index] + return v + + def remove(self, value): + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff -r 6db40a9955dc -r 0d413f60cc23 Lib/colorsys.py --- a/Lib/colorsys.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/colorsys.py Mon Jan 25 17:05:13 2016 +0100 @@ -33,25 +33,17 @@ # YIQ: used by composite video signals (linear combinations of RGB) # Y: perceived grey level (0.0 == black, 1.0 == white) # I, Q: color components -# -# There are a great many versions of the constants used in these formulae. -# The ones in this library uses constants from the FCC version of NTSC. def rgb_to_yiq(r, g, b): y = 0.30*r + 0.59*g + 0.11*b - i = 0.74*(r-y) - 0.27*(b-y) - q = 0.48*(r-y) + 0.41*(b-y) + i = 0.60*r - 0.28*g - 0.32*b + q = 0.21*r - 0.52*g + 0.31*b return (y, i, q) def yiq_to_rgb(y, i, q): - # r = y + (0.27*q + 0.41*i) / (0.74*0.41 + 0.27*0.48) - # b = y + (0.74*q - 0.48*i) / (0.74*0.41 + 0.27*0.48) - # g = y - (0.30*(r-y) + 0.11*(b-y)) / 0.59 - - r = y + 0.9468822170900693*i + 0.6235565819861433*q - g = y - 0.27478764629897834*i - 0.6356910791873801*q - b = y - 1.1085450346420322*i + 1.7090069284064666*q - + r = y + 0.948262*i + 0.624013*q + g = y - 0.276066*i - 0.639810*q + b = y - 1.105450*i + 1.729860*q if r < 0.0: r = 0.0 if g < 0.0: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/compileall.py --- a/Lib/compileall.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/compileall.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -"""Module/script to byte-compile all .py files to .pyc files. +"""Module/script to byte-compile all .py files to .pyc (or .pyo) files. When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from @@ -12,28 +12,37 @@ """ import os import sys -import importlib.util +import errno +import imp import py_compile import struct -try: - from concurrent.futures import ProcessPoolExecutor -except ImportError: - ProcessPoolExecutor = None -from functools import partial - __all__ = ["compile_dir","compile_file","compile_path"] -def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): +def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, + quiet=False, legacy=False, optimize=-1): + """Byte-compile all modules in the given directory tree. + + Arguments (only dir is required): + + dir: the directory to byte-compile + maxlevels: maximum recursion level (default 10) + ddir: the directory that will be prepended to the path to the + file as it is compiled into each byte-code file. + force: if True, force compilation, even if timestamps are up-to-date + quiet: if True, be quiet during compilation + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: optimization level or -1 for level of the interpreter + """ if not quiet: print('Listing {!r}...'.format(dir)) try: names = os.listdir(dir) - except OSError: - if quiet < 2: - print("Can't list {!r}".format(dir)) + except os.error: + print("Can't list {!r}".format(dir)) names = [] names.sort() + success = 1 for name in names: if name == '__pycache__': continue @@ -43,53 +52,17 @@ else: dfile = None if not os.path.isdir(fullname): - yield fullname + if not compile_file(fullname, ddir, force, rx, quiet, + legacy, optimize): + success = 0 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): - yield from _walk_dir(fullname, ddir=dfile, - maxlevels=maxlevels - 1, quiet=quiet) - -def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=0, legacy=False, optimize=-1, workers=1): - """Byte-compile all modules in the given directory tree. - - Arguments (only dir is required): - - dir: the directory to byte-compile - maxlevels: maximum recursion level (default 10) - ddir: the directory that will be prepended to the path to the - file as it is compiled into each byte-code file. - force: if True, force compilation, even if timestamps are up-to-date - quiet: full output with False or 0, errors only with 1, - no output with 2 - legacy: if True, produce legacy pyc paths instead of PEP 3147 paths - optimize: optimization level or -1 for level of the interpreter - workers: maximum number of parallel workers - """ - files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, - ddir=ddir) - success = True - if workers is not None and workers != 1 and ProcessPoolExecutor is not None: - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - - workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: - results = executor.map(partial(compile_file, - ddir=ddir, force=force, - rx=rx, quiet=quiet, - legacy=legacy, - optimize=optimize), - files) - success = min(results, default=True) - else: - for file in files: - if not compile_file(file, ddir, force, rx, quiet, - legacy, optimize): - success = False + if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, + quiet, legacy, optimize): + success = 0 return success -def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, +def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False, legacy=False, optimize=-1): """Byte-compile one file. @@ -99,12 +72,11 @@ ddir: if given, the directory name compiled in to the byte-code file. force: if True, force compilation, even if timestamps are up-to-date - quiet: full output with False or 0, errors only with 1, - no output with 2 + quiet: if True, be quiet during compilation legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter """ - success = True + success = 1 name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) @@ -116,27 +88,25 @@ return success if os.path.isfile(fullname): if legacy: - cfile = fullname + 'c' + cfile = fullname + ('c' if __debug__ else 'o') else: if optimize >= 0: - opt = optimize if optimize >= 1 else '' - cfile = importlib.util.cache_from_source( - fullname, optimization=opt) + cfile = imp.cache_from_source(fullname, + debug_override=not optimize) else: - cfile = importlib.util.cache_from_source(fullname) + cfile = imp.cache_from_source(fullname) cache_dir = os.path.dirname(cfile) head, tail = name[:-3], name[-3:] if tail == '.py': if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, - mtime) + expect = struct.pack('<4sl', imp.get_magic(), mtime) with open(cfile, 'rb') as chandle: actual = chandle.read(8) if expect == actual: return success - except OSError: + except IOError: pass if not quiet: print('Compiling {!r}...'.format(fullname)) @@ -144,10 +114,7 @@ ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize) except py_compile.PyCompileError as err: - success = False - if quiet >= 2: - return success - elif quiet: + if quiet: print('*** Error compiling {!r}...'.format(fullname)) else: print('*** ', end='') @@ -156,21 +123,20 @@ errors='backslashreplace') msg = msg.decode(sys.stdout.encoding) print(msg) - except (SyntaxError, UnicodeError, OSError) as e: - success = False - if quiet >= 2: - return success - elif quiet: + success = 0 + except (SyntaxError, UnicodeError, IOError) as e: + if quiet: print('*** Error compiling {!r}...'.format(fullname)) else: print('*** ', end='') print(e.__class__.__name__ + ':', e) + success = 0 else: if ok == 0: - success = False + success = 0 return success -def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, +def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, legacy=False, optimize=-1): """Byte-compile all module on sys.path. @@ -179,15 +145,14 @@ skip_curdir: if true, skip current directory (default True) maxlevels: max recursion level (default 0) force: as for compile_dir() (default False) - quiet: as for compile_dir() (default 0) + quiet: as for compile_dir() (default False) legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) """ - success = True + success = 1 for dir in sys.path: if (not dir or dir == os.curdir) and skip_curdir: - if quiet < 2: - print('Skipping current directory') + print('Skipping current directory') else: success = success and compile_dir(dir, maxlevels, None, force, quiet=quiet, @@ -204,15 +169,10 @@ parser.add_argument('-l', action='store_const', const=0, default=10, dest='maxlevels', help="don't recurse into subdirectories") - parser.add_argument('-r', type=int, dest='recursion', - help=('control the maximum recursion level. ' - 'if `-l` and `-r` options are specified, ' - 'then `-r` takes precedence.')) parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') - parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' - 'the error messages as well.') + parser.add_argument('-q', action='store_true', dest='quiet', + help='output only error messages') parser.add_argument('-b', action='store_true', dest='legacy', help='use legacy (pre-PEP3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, @@ -232,36 +192,27 @@ help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' 'to the equivalent of -l sys.path')) - parser.add_argument('-j', '--workers', default=1, - type=int, help='Run compileall concurrently') + args = parser.parse_args() - args = parser.parse_args() compile_dests = args.compile_dest + if (args.ddir and (len(compile_dests) != 1 + or not os.path.isdir(compile_dests[0]))): + parser.exit('-d destdir requires exactly one directory argument') if args.rx: import re args.rx = re.compile(args.rx) - - if args.recursion is not None: - maxlevels = args.recursion - else: - maxlevels = args.maxlevels - # if flist is provided then load it if args.flist: try: with (sys.stdin if args.flist=='-' else open(args.flist)) as f: for line in f: compile_dests.append(line.strip()) - except OSError: - if args.quiet < 2: - print("Error reading file list {}".format(args.flist)) + except EnvironmentError: + print("Error reading file list {}".format(args.flist)) return False - if args.workers is not None: - args.workers = args.workers or None - success = True try: if compile_dests: @@ -271,17 +222,15 @@ args.quiet, args.legacy): success = False else: - if not compile_dir(dest, maxlevels, args.ddir, + if not compile_dir(dest, args.maxlevels, args.ddir, args.force, args.rx, args.quiet, - args.legacy, workers=args.workers): + args.legacy): success = False return success else: - return compile_path(legacy=args.legacy, force=args.force, - quiet=args.quiet) + return compile_path(legacy=args.legacy) except KeyboardInterrupt: - if args.quiet < 2: - print("\n[interrupted]") + print("\n[interrupted]") return False return True diff -r 6db40a9955dc -r 0d413f60cc23 Lib/concurrent/futures/_base.py --- a/Lib/concurrent/futures/_base.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/concurrent/futures/_base.py Mon Jan 25 17:05:13 2016 +0100 @@ -111,14 +111,12 @@ def __init__(self, num_pending_calls, stop_on_exception): self.num_pending_calls = num_pending_calls self.stop_on_exception = stop_on_exception - self.lock = threading.Lock() super().__init__() def _decrement_pending_calls(self): - with self.lock: - self.num_pending_calls -= 1 - if not self.num_pending_calls: - self.event.set() + self.num_pending_calls -= 1 + if not self.num_pending_calls: + self.event.set() def add_result(self, future): super().add_result(future) @@ -181,8 +179,7 @@ Returns: An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + cancelled). Raises: TimeoutError: If the entire result iterator could not be generated @@ -191,16 +188,16 @@ if timeout is not None: end_time = timeout + time.time() - fs = set(fs) with _AcquireFutures(fs): finished = set( f for f in fs if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) - pending = fs - finished + pending = set(fs) - finished waiter = _create_and_install_waiters(fs, _AS_COMPLETED) try: - yield from finished + for future in finished: + yield future while pending: if timeout is None: @@ -225,8 +222,7 @@ finally: for f in fs: - with f._condition: - f._waiters.remove(waiter) + f._waiters.remove(waiter) DoneAndNotDoneFutures = collections.namedtuple( 'DoneAndNotDoneFutures', 'done not_done') @@ -273,8 +269,7 @@ waiter.event.wait(timeout) for f in fs: - with f._condition: - f._waiters.remove(waiter) + f._waiters.remove(waiter) done.update(waiter.finished_futures) return DoneAndNotDoneFutures(done, set(fs) - done) @@ -302,20 +297,17 @@ with self._condition: if self._state == FINISHED: if self._exception: - return '<%s at %#x state=%s raised %s>' % ( - self.__class__.__name__, - id(self), + return '' % ( + hex(id(self)), _STATE_TO_DESCRIPTION_MAP[self._state], self._exception.__class__.__name__) else: - return '<%s at %#x state=%s returned %s>' % ( - self.__class__.__name__, - id(self), + return '' % ( + hex(id(self)), _STATE_TO_DESCRIPTION_MAP[self._state], self._result.__class__.__name__) - return '<%s at %#x state=%s>' % ( - self.__class__.__name__, - id(self), + return '' % ( + hex(id(self)), _STATE_TO_DESCRIPTION_MAP[self._state]) def cancel(self): @@ -338,7 +330,7 @@ return True def cancelled(self): - """Return True if the future was cancelled.""" + """Return True if the future has cancelled.""" with self._condition: return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED] @@ -476,8 +468,8 @@ return True else: LOGGER.critical('Future %s in unexpected state: %s', - id(self), - self._state) + id(self.future), + self.future._state) raise RuntimeError('Future in unexpected state') def set_result(self, result): @@ -520,18 +512,14 @@ """ raise NotImplementedError() - def map(self, fn, *iterables, timeout=None, chunksize=1): - """Returns an iterator equivalent to map(fn, iter). + def map(self, fn, *iterables, timeout=None): + """Returns a iterator equivalent to map(fn, iter). Args: - fn: A callable that will take as many arguments as there are + fn: A callable that will take take as many arguments as there are passed iterables. timeout: The maximum number of seconds to wait. If None, then there is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by - ThreadPoolExecutor. Returns: An iterator equivalent to: map(func, *iterables) but the calls may diff -r 6db40a9955dc -r 0d413f60cc23 Lib/concurrent/futures/process.py --- a/Lib/concurrent/futures/process.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/concurrent/futures/process.py Mon Jan 25 17:05:13 2016 +0100 @@ -40,7 +40,7 @@ Process #1..n: - reads _CallItems from "Call Q", executes the calls, and puts the resulting - _ResultItems in "Result Q" + _ResultItems in "Request Q" """ __author__ = 'Brian Quinlan (brian@sweetapp.com)' @@ -49,15 +49,11 @@ import os from concurrent.futures import _base import queue -from queue import Full import multiprocessing -from multiprocessing import SimpleQueue +from multiprocessing.queues import SimpleQueue, Full from multiprocessing.connection import wait import threading import weakref -from functools import partial -import itertools -import traceback # Workers are created as daemon threads and processes. This is done to allow the # interpreter to exit when there are still idle processes in a @@ -91,27 +87,6 @@ # (Futures in the call queue cannot be cancelled). EXTRA_QUEUED_CALLS = 1 -# Hack to embed stringification of remote traceback in local traceback - -class _RemoteTraceback(Exception): - def __init__(self, tb): - self.tb = tb - def __str__(self): - return self.tb - -class _ExceptionWithTraceback: - def __init__(self, exc, tb): - tb = traceback.format_exception(type(exc), exc, tb) - tb = ''.join(tb) - self.exc = exc - self.tb = '\n"""\n%s"""' % tb - def __reduce__(self): - return _rebuild_exc, (self.exc, self.tb) - -def _rebuild_exc(exc, tb): - exc.__cause__ = _RemoteTraceback(tb) - return exc - class _WorkItem(object): def __init__(self, future, fn, args, kwargs): self.future = future @@ -132,26 +107,6 @@ self.args = args self.kwargs = kwargs -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk - -def _process_chunk(fn, chunk): - """ Processes a chunk of an iterable passed to map. - - Runs the function passed to map() on a chunk of the - iterable passed to map. - - This function is run in a separate process. - - """ - return [fn(*args) for args in chunk] - def _process_worker(call_queue, result_queue): """Evaluates calls from call_queue and places the results in result_queue. @@ -174,8 +129,8 @@ try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: - exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(call_item.work_id, exception=exc)) + result_queue.put(_ResultItem(call_item.work_id, + exception=e)) else: result_queue.put(_ResultItem(call_item.work_id, result=r)) @@ -285,8 +240,6 @@ "terminated abruptly while the future was " "running or pending." )) - # Delete references to object. See issue16284 - del work_item pending_work_items.clear() # Terminate remaining workers forcibly: the queues or their # locks may be in a dirty state and block forever. @@ -311,8 +264,6 @@ work_item.future.set_exception(result_item.exception) else: work_item.future.set_result(result_item.result) - # Delete references to object. See issue16284 - del work_item # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: @@ -346,7 +297,7 @@ # sysconf not available or setting not available return if nsems_max == -1: - # indetermined limit, assume that limit is determined + # indetermine limit, assume that limit is determined # by available memory only return if nsems_max >= 256: @@ -376,11 +327,8 @@ _check_system_limits() if max_workers is None: - self._max_workers = os.cpu_count() or 1 + self._max_workers = multiprocessing.cpu_count() else: - if max_workers <= 0: - raise ValueError("max_workers must be greater than 0") - self._max_workers = max_workers # Make the call queue slightly larger than the number of processes to @@ -455,35 +403,6 @@ return f submit.__doc__ = _base.Executor.submit.__doc__ - def map(self, fn, *iterables, timeout=None, chunksize=1): - """Returns an iterator equivalent to map(fn, iter). - - Args: - fn: A callable that will take as many arguments as there are - passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. - - Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. - - Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. - Exception: If fn(*args) raises for any values. - """ - if chunksize < 1: - raise ValueError("chunksize must be >= 1.") - - results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), - timeout=timeout) - return itertools.chain.from_iterable(results) - def shutdown(self, wait=True): with self._shutdown_lock: self._shutdown_thread = True @@ -492,7 +411,7 @@ self._result_queue.put(None) if wait: self._queue_management_thread.join() - # To reduce the risk of opening too many files, remove references to + # To reduce the risk of openning too many files, remove references to # objects that use file descriptors. self._queue_management_thread = None self._call_queue = None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/concurrent/futures/thread.py --- a/Lib/concurrent/futures/thread.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/concurrent/futures/thread.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,6 @@ import queue import threading import weakref -import os # Workers are created as daemon threads. This is done to allow the interpreter # to exit when there are still idle threads in a ThreadPoolExecutor's thread @@ -64,8 +63,6 @@ work_item = work_queue.get(block=True) if work_item is not None: work_item.run() - # Delete references to object. See issue16284 - del work_item continue executor = executor_reference() # Exit if: @@ -81,20 +78,13 @@ _base.LOGGER.critical('Exception in worker', exc_info=True) class ThreadPoolExecutor(_base.Executor): - def __init__(self, max_workers=None): + def __init__(self, max_workers): """Initializes a new ThreadPoolExecutor instance. Args: max_workers: The maximum number of threads that can be used to execute the given calls. """ - if max_workers is None: - # Use this number because ThreadPoolExecutor is often - # used to overlap I/O instead of CPU work. - max_workers = (os.cpu_count() or 1) * 5 - if max_workers <= 0: - raise ValueError("max_workers must be greater than 0") - self._max_workers = max_workers self._work_queue = queue.Queue() self._threads = set() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/configparser.py --- a/Lib/configparser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/configparser.py Mon Jan 25 17:05:13 2016 +0100 @@ -17,8 +17,7 @@ __init__(defaults=None, dict_type=_default_dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, - empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): + empty_lines_in_values=True): Create the parser. When `defaults' is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -48,25 +47,6 @@ When `allow_no_value' is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is - named accordingly. By default it is called ``"DEFAULT"`` but this can - be customized to point to any other valid section name. Its current - value can be retrieved using the ``parser_instance.default_section`` - attribute and may be modified at runtime. - - When `interpolation` is given, it should be an Interpolation subclass - instance. It will be used as the handler for option value - pre-processing when using getters. RawConfigParser object s don't do - any sort of interpolation, whereas ConfigParser uses an instance of - BasicInterpolation. The library also provides a ``zc.buildbot`` - inspired ExtendedInterpolation implementation. - - When `converters` is given, it should be a dictionary where each key - represents the name of a type converter and each value is a callable - implementing the conversion from string to the desired datatype. Every - converter gets its corresponding get*() method on the parser object and - section proxies. - sections() Return all the configuration section names, sans DEFAULT. @@ -119,9 +99,10 @@ yes, on for True). Returns False or True. items(section=_UNSET, raw=False, vars=None) - If section is given, return a list of tuples with (name, value) for - each option in the section. Otherwise, return a list of tuples with - (section_name, section_proxy) for each section, including DEFAULTSECT. + If section is given, return a list of tuples with (section_name, + section_proxy) for each section, including DEFAULTSECT. Otherwise, + return a list of tuples with (name, value) for each option + in the section. remove_section(section) Remove the given file section and all its options. @@ -149,11 +130,9 @@ __all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", - "InterpolationMissingOptionError", "InterpolationSyntaxError", - "ParsingError", "MissingSectionHeaderError", + "InterpolationSyntaxError", "ParsingError", + "MissingSectionHeaderError", "ConfigParser", "SafeConfigParser", "RawConfigParser", - "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] DEFAULTSECT = "DEFAULT" @@ -166,6 +145,23 @@ class Error(Exception): """Base class for ConfigParser exceptions.""" + def _get_message(self): + """Getter for 'message'; needed only to override deprecation in + BaseException. + """ + return self.__message + + def _set_message(self, value): + """Setter for 'message'; needed only to override deprecation in + BaseException. + """ + self.__message = value + + # BaseException.message has been deprecated since Python 2.6. To prevent + # DeprecationWarning from popping up over this pre-existing attribute, use + # a new property that takes lookup precedence. + message = property(_get_message, _set_message) + def __init__(self, msg=''): self.message = msg Exception.__init__(self, msg) @@ -196,7 +192,7 @@ def __init__(self, section, source=None, lineno=None): msg = [repr(section), " already exists"] if source is not None: - message = ["While reading from ", repr(source)] + message = ["While reading from ", source] if lineno is not None: message.append(" [line {0:2d}]".format(lineno)) message.append(": section ") @@ -222,7 +218,7 @@ msg = [repr(option), " in section ", repr(section), " already exists"] if source is not None: - message = ["While reading from ", repr(source)] + message = ["While reading from ", source] if lineno is not None: message.append(" [line {0:2d}]".format(lineno)) message.append(": option ") @@ -263,9 +259,12 @@ """A string substitution required a setting which was not available.""" def __init__(self, option, section, rawval, reference): - msg = ("Bad value substitution: option {!r} in section {!r} contains " - "an interpolation key {!r} which is not a valid option name. " - "Raw value: {!r}".format(option, section, reference, rawval)) + msg = ("Bad value substitution:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\tkey : %s\n" + "\trawval : %s\n" + % (section, option, reference, rawval)) InterpolationError.__init__(self, option, section, msg) self.reference = reference self.args = (option, section, rawval, reference) @@ -283,11 +282,11 @@ """Raised when substitutions are nested too deeply.""" def __init__(self, option, section, rawval): - msg = ("Recursion limit exceeded in value substitution: option {!r} " - "in section {!r} contains an interpolation key which " - "cannot be substituted in {} steps. Raw value: {!r}" - "".format(option, section, MAX_INTERPOLATION_DEPTH, - rawval)) + msg = ("Value interpolation too deeply recursive:\n" + "\tsection: [%s]\n" + "\toption : %s\n" + "\trawval : %s\n" + % (section, option, rawval)) InterpolationError.__init__(self, option, section, msg) self.args = (option, section, rawval) @@ -305,7 +304,7 @@ raise ValueError("Required argument `source' not given.") elif filename: source = filename - Error.__init__(self, 'Source contains parsing errors: %r' % source) + Error.__init__(self, 'Source contains parsing errors: %s' % source) self.source = source self.errors = [] self.args = (source, ) @@ -341,7 +340,7 @@ def __init__(self, filename, lineno, line): Error.__init__( self, - 'File contains no section headers.\nfile: %r, line: %d\n%r' % + 'File contains no section headers.\nfile: %s, line: %d\n%r' % (filename, lineno, line)) self.source = filename self.lineno = lineno @@ -403,9 +402,8 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, depth): - rawval = parser.get(section, option, raw=True, fallback=rest) if depth > MAX_INTERPOLATION_DEPTH: - raise InterpolationDepthError(option, section, rawval) + raise InterpolationDepthError(option, section, rest) while rest: p = rest.find("%") if p < 0: @@ -430,7 +428,7 @@ v = map[var] except KeyError: raise InterpolationMissingOptionError( - option, section, rawval, var) from None + option, section, rest, var) if "%" in v: self._interpolate_some(parser, option, accum, v, section, map, depth + 1) @@ -459,14 +457,13 @@ tmp_value = self._KEYCRE.sub('', tmp_value) # valid syntax if '$' in tmp_value: raise ValueError("invalid interpolation syntax in %r at " - "position %d" % (value, tmp_value.find('$'))) + "position %d" % (value, tmp_value.find('%'))) return value def _interpolate_some(self, parser, option, accum, rest, section, map, depth): - rawval = parser.get(section, option, raw=True, fallback=rest) if depth > MAX_INTERPOLATION_DEPTH: - raise InterpolationDepthError(option, section, rawval) + raise InterpolationDepthError(option, section, rest) while rest: p = rest.find("$") if p < 0: @@ -503,7 +500,7 @@ "More than one ':' found: %r" % (rest,)) except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( - option, section, rawval, ":".join(path)) from None + option, section, rest, ":".join(path)) if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -536,7 +533,7 @@ value = value % vars except KeyError as e: raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) from None + option, section, rawval, e.args[0]) else: break if value and "%(" in value: @@ -601,12 +598,11 @@ comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET): self._dict = dict_type self._sections = self._dict() self._defaults = self._dict() - self._converters = ConverterMapping(self) self._proxies = self._dict() self._proxies[default_section] = SectionProxy(self, default_section) if defaults: @@ -634,8 +630,6 @@ self._interpolation = self._DEFAULT_INTERPOLATION if self._interpolation is None: self._interpolation = Interpolation() - if converters is not _UNSET: - self._converters.update(converters) def defaults(self): return self._defaults @@ -671,7 +665,7 @@ try: opts = self._sections[section].copy() except KeyError: - raise NoSectionError(section) from None + raise NoSectionError(section) opts.update(self._defaults) return list(opts.keys()) @@ -694,7 +688,7 @@ try: with open(filename, encoding=encoding) as fp: self._read(fp, filename) - except OSError: + except IOError: continue read_ok.append(filename) return read_ok @@ -799,31 +793,36 @@ def _get(self, section, conv, option, **kwargs): return conv(self.get(section, option, **kwargs)) - def _get_conv(self, section, option, conv, *, raw=False, vars=None, - fallback=_UNSET, **kwargs): + def getint(self, section, option, *, raw=False, vars=None, + fallback=_UNSET): try: - return self._get(section, conv, option, raw=raw, vars=vars, - **kwargs) + return self._get(section, int, option, raw=raw, vars=vars) except (NoSectionError, NoOptionError): if fallback is _UNSET: raise - return fallback - - # getint, getfloat and getboolean provided directly for backwards compat - def getint(self, section, option, *, raw=False, vars=None, - fallback=_UNSET, **kwargs): - return self._get_conv(section, option, int, raw=raw, vars=vars, - fallback=fallback, **kwargs) + else: + return fallback def getfloat(self, section, option, *, raw=False, vars=None, - fallback=_UNSET, **kwargs): - return self._get_conv(section, option, float, raw=raw, vars=vars, - fallback=fallback, **kwargs) + fallback=_UNSET): + try: + return self._get(section, float, option, raw=raw, vars=vars) + except (NoSectionError, NoOptionError): + if fallback is _UNSET: + raise + else: + return fallback def getboolean(self, section, option, *, raw=False, vars=None, - fallback=_UNSET, **kwargs): - return self._get_conv(section, option, self._convert_to_boolean, - raw=raw, vars=vars, fallback=fallback, **kwargs) + fallback=_UNSET): + try: + return self._get(section, self._convert_to_boolean, option, + raw=raw, vars=vars) + except (NoSectionError, NoOptionError): + if fallback is _UNSET: + raise + else: + return fallback def items(self, section=_UNSET, raw=False, vars=None): """Return a list of (name, value) tuples for each option in a section. @@ -854,19 +853,6 @@ value_getter = lambda option: d[option] return [(option, value_getter(option)) for option in d.keys()] - def popitem(self): - """Remove a section from the parser and return it as - a (section_name, section_proxy) tuple. If no section is present, raise - KeyError. - - The section DEFAULT is never returned because it cannot be removed. - """ - for key in self.sections(): - value = self[key] - del self[key] - return key, value - raise KeyError - def optionxform(self, optionstr): return optionstr.lower() @@ -895,7 +881,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) from None + raise NoSectionError(section) sectdict[self.optionxform(option)] = value def write(self, fp, space_around_delimiters=True): @@ -936,7 +922,7 @@ try: sectdict = self._sections[section] except KeyError: - raise NoSectionError(section) from None + raise NoSectionError(section) option = self.optionxform(option) existed = option in sectdict if existed: @@ -962,10 +948,7 @@ # XXX this is not atomic if read_dict fails at any point. Then again, # no update method in configparser is atomic in this implementation. - if key == self.default_section: - self._defaults.clear() - elif key in self._sections: - self._sections[key].clear() + self.remove_section(key) self.read_dict({key: value}) def __delitem__(self, key): @@ -1010,26 +993,18 @@ indent_level = 0 e = None # None, or an exception for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize + comment_start = None # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes + for prefix in self._inline_comment_prefixes: + index = line.find(prefix) + if index == 0 or (index > 0 and line[index-1].isspace()): + comment_start = index + break # strip full line comments for prefix in self._comment_prefixes: if line.strip().startswith(prefix): comment_start = 0 break - if comment_start == sys.maxsize: - comment_start = None value = line[:comment_start].strip() if not value: if self._empty_lines_in_values: @@ -1173,10 +1148,6 @@ if not isinstance(value, str): raise TypeError("option values must be strings") - @property - def converters(self): - return self._converters - class ConfigParser(RawConfigParser): """ConfigParser implementing interpolation.""" @@ -1217,10 +1188,6 @@ """Creates a view on a section of the specified `name` in `parser`.""" self._parser = parser self._name = name - for conv in parser.converters: - key = 'get' + conv - getter = functools.partial(self.get, _impl=getattr(parser, key)) - setattr(self, key, getter) def __repr__(self): return ''.format(self._name) @@ -1254,6 +1221,22 @@ else: return self._parser.defaults() + def get(self, option, fallback=None, *, raw=False, vars=None): + return self._parser.get(self._name, option, raw=raw, vars=vars, + fallback=fallback) + + def getint(self, option, fallback=None, *, raw=False, vars=None): + return self._parser.getint(self._name, option, raw=raw, vars=vars, + fallback=fallback) + + def getfloat(self, option, fallback=None, *, raw=False, vars=None): + return self._parser.getfloat(self._name, option, raw=raw, vars=vars, + fallback=fallback) + + def getboolean(self, option, fallback=None, *, raw=False, vars=None): + return self._parser.getboolean(self._name, option, raw=raw, vars=vars, + fallback=fallback) + @property def parser(self): # The parser object of the proxy is read-only. @@ -1263,77 +1246,3 @@ def name(self): # The name of the section on a proxy is read-only. return self._name - - def get(self, option, fallback=None, *, raw=False, vars=None, - _impl=None, **kwargs): - """Get an option value. - - Unless `fallback` is provided, `None` will be returned if the option - is not found. - - """ - # If `_impl` is provided, it should be a getter method on the parser - # object that provides the desired type conversion. - if not _impl: - _impl = self._parser.get - return _impl(self._name, option, raw=raw, vars=vars, - fallback=fallback, **kwargs) - - -class ConverterMapping(MutableMapping): - """Enables reuse of get*() methods between the parser and section proxies. - - If a parser class implements a getter directly, the value for the given - key will be ``None``. The presence of the converter name here enables - section proxies to find and use the implementation on the parser class. - """ - - GETTERCRE = re.compile(r"^get(?P.+)$") - - def __init__(self, parser): - self._parser = parser - self._data = {} - for getter in dir(self._parser): - m = self.GETTERCRE.match(getter) - if not m or not callable(getattr(self._parser, getter)): - continue - self._data[m.group('name')] = None # See class docstring. - - def __getitem__(self, key): - return self._data[key] - - def __setitem__(self, key, value): - try: - k = 'get' + key - except TypeError: - raise ValueError('Incompatible key: {} (type: {})' - ''.format(key, type(key))) - if k == 'get': - raise ValueError('Incompatible key: cannot use "" as a name') - self._data[key] = value - func = functools.partial(self._parser._get_conv, conv=value) - func.converter = value - setattr(self._parser, k, func) - for proxy in self._parser.values(): - getter = functools.partial(proxy.get, _impl=func) - setattr(proxy, k, getter) - - def __delitem__(self, key): - try: - k = 'get' + (key or None) - except TypeError: - raise KeyError(key) - del self._data[key] - for inst in itertools.chain((self._parser,), self._parser.values()): - try: - delattr(inst, k) - except AttributeError: - # don't raise since the entry was present in _data, silently - # clean up - continue - - def __iter__(self): - return iter(self._data) - - def __len__(self): - return len(self._data) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/contextlib.py --- a/Lib/contextlib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/contextlib.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,9 @@ """Utilities for with-statement contexts. See PEP 343.""" import sys -from collections import deque from functools import wraps -__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress"] +__all__ = ["contextmanager", "closing", "ContextDecorator"] class ContextDecorator(object): @@ -14,12 +12,12 @@ def _recreate_cm(self): """Return a recreated instance of self. - Allows an otherwise one-shot context manager like + Allows otherwise one-shot context managers like _GeneratorContextManager to support use as - a decorator via implicit recreation. + decorators via implicit recreation. - This is a private interface just for _GeneratorContextManager. - See issue #11647 for details. + Note: this is a private interface just for _GCM in 3.2 but will be + renamed and documented for third party use in 3.3 """ return self @@ -34,31 +32,21 @@ class _GeneratorContextManager(ContextDecorator): """Helper for @contextmanager decorator.""" - def __init__(self, func, args, kwds): + def __init__(self, func, *args, **kwds): self.gen = func(*args, **kwds) self.func, self.args, self.kwds = func, args, kwds - # Issue 19330: ensure context manager instances have good docstrings - doc = getattr(func, "__doc__", None) - if doc is None: - doc = type(self).__doc__ - self.__doc__ = doc - # Unfortunately, this still doesn't provide good help output when - # inspecting the created context manager instances, since pydoc - # currently bypasses the instance docstring and shows the docstring - # for the class instead. - # See http://bugs.python.org/issue19404 for more details. def _recreate_cm(self): # _GCM instances are one-shot context managers, so the # CM must be recreated each time a decorated function is # called - return self.__class__(self.func, self.args, self.kwds) + return self.__class__(self.func, *self.args, **self.kwds) def __enter__(self): try: return next(self.gen) except StopIteration: - raise RuntimeError("generator didn't yield") from None + raise RuntimeError("generator didn't yield") def __exit__(self, type, value, traceback): if type is None: @@ -77,17 +65,10 @@ self.gen.throw(type, value, traceback) raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: - # Suppress StopIteration *unless* it's the same exception that + # Suppress the exception *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration - # raised inside the "with" statement from being suppressed. + # raised inside the "with" statement from being suppressed return exc is not value - except RuntimeError as exc: - # Likewise, avoid suppressing if a StopIteration exception - # was passed to throw() and later wrapped into a RuntimeError - # (see PEP 479). - if exc.__cause__ is value: - return False - raise except: # only re-raise if it's *not* the exception that was # passed to throw(), because __exit__() must not raise @@ -130,7 +111,7 @@ """ @wraps(func) def helper(*args, **kwds): - return _GeneratorContextManager(func, args, kwds) + return _GeneratorContextManager(func, *args, **kwds) return helper @@ -157,205 +138,3 @@ return self.thing def __exit__(self, *exc_info): self.thing.close() - - -class _RedirectStream: - - _stream = None - - def __init__(self, new_target): - self._new_target = new_target - # We use a list of old targets to make this CM re-entrant - self._old_targets = [] - - def __enter__(self): - self._old_targets.append(getattr(sys, self._stream)) - setattr(sys, self._stream, self._new_target) - return self._new_target - - def __exit__(self, exctype, excinst, exctb): - setattr(sys, self._stream, self._old_targets.pop()) - - -class redirect_stdout(_RedirectStream): - """Context manager for temporarily redirecting stdout to another file. - - # How to send help() to stderr - with redirect_stdout(sys.stderr): - help(dir) - - # How to write help() to a file - with open('help.txt', 'w') as f: - with redirect_stdout(f): - help(pow) - """ - - _stream = "stdout" - - -class redirect_stderr(_RedirectStream): - """Context manager for temporarily redirecting stderr to another file.""" - - _stream = "stderr" - - -class suppress: - """Context manager to suppress specified exceptions - - After the exception is suppressed, execution proceeds with the next - statement following the with statement. - - with suppress(FileNotFoundError): - os.remove(somefile) - # Execution still resumes here if the file was already removed - """ - - def __init__(self, *exceptions): - self._exceptions = exceptions - - def __enter__(self): - pass - - def __exit__(self, exctype, excinst, exctb): - # Unlike isinstance and issubclass, CPython exception handling - # currently only looks at the concrete type hierarchy (ignoring - # the instance and subclass checking hooks). While Guido considers - # that a bug rather than a feature, it's a fairly hard one to fix - # due to various internal implementation details. suppress provides - # the simpler issubclass based semantics, rather than trying to - # exactly reproduce the limitations of the CPython interpreter. - # - # See http://bugs.python.org/issue12029 for more details - return exctype is not None and issubclass(exctype, self._exceptions) - - -# Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(object): - """Context manager for dynamic management of a stack of exit callbacks - - For example: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception - - """ - def __init__(self): - self._exit_callbacks = deque() - - def pop_all(self): - """Preserve the context stack by transferring it to a new instance""" - new_stack = type(self)() - new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() - return new_stack - - def _push_cm_exit(self, cm, cm_exit): - """Helper to correctly register callbacks to __exit__ methods""" - def _exit_wrapper(*exc_details): - return cm_exit(cm, *exc_details) - _exit_wrapper.__self__ = cm - self.push(_exit_wrapper) - - def push(self, exit): - """Registers a callback with the standard __exit__ method signature - - Can suppress exceptions the same way __exit__ methods can. - - Also accepts any object with an __exit__ method (registering a call - to the method instead of the object itself) - """ - # We use an unbound method rather than a bound method to follow - # the standard lookup behaviour for special methods - _cb_type = type(exit) - try: - exit_method = _cb_type.__exit__ - except AttributeError: - # Not a context manager, so assume its a callable - self._exit_callbacks.append(exit) - else: - self._push_cm_exit(exit, exit_method) - return exit # Allow use as a decorator - - def callback(self, callback, *args, **kwds): - """Registers an arbitrary callback and arguments. - - Cannot suppress exceptions. - """ - def _exit_wrapper(exc_type, exc, tb): - callback(*args, **kwds) - # We changed the signature, so using @wraps is not appropriate, but - # setting __wrapped__ may still help with introspection - _exit_wrapper.__wrapped__ = callback - self.push(_exit_wrapper) - return callback # Allow use as a decorator - - def enter_context(self, cm): - """Enters the supplied context manager - - If successful, also pushes its __exit__ method as a callback and - returns the result of the __enter__ method. - """ - # We look up the special methods on the type to match the with statement - _cm_type = type(cm) - _exit = _cm_type.__exit__ - result = _cm_type.__enter__(cm) - self._push_cm_exit(cm, _exit) - return result - - def close(self): - """Immediately unwind the context stack""" - self.__exit__(None, None, None) - - def __enter__(self): - return self - - def __exit__(self, *exc_details): - received_exc = exc_details[0] is not None - - # We manipulate the exception state so it behaves as though - # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] - def _fix_exception_context(new_exc, old_exc): - # Context may not be correct, so find the end of the chain - while 1: - exc_context = new_exc.__context__ - if exc_context is old_exc: - # Context is already set correctly (see issue 20317) - return - if exc_context is None or exc_context is frame_exc: - break - new_exc = exc_context - # Change the end of the chain to point to the exception - # we expect it to reference - new_exc.__context__ = old_exc - - # Callbacks are invoked in LIFO order to match the behaviour of - # nested context managers - suppressed_exc = False - pending_raise = False - while self._exit_callbacks: - cb = self._exit_callbacks.pop() - try: - if cb(*exc_details): - suppressed_exc = True - pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() - # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) - pending_raise = True - exc_details = new_exc_details - if pending_raise: - try: - # bare "raise exc_details[1]" replaces our carefully - # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] - except BaseException: - exc_details[1].__context__ = fixed_ctx - raise - return received_exc and suppressed_exc diff -r 6db40a9955dc -r 0d413f60cc23 Lib/copy.py --- a/Lib/copy.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/copy.py Mon Jan 25 17:05:13 2016 +0100 @@ -76,14 +76,6 @@ if copier: return copier(x) - try: - issc = issubclass(cls, type) - except TypeError: # cls is not a class - issc = False - if issc: - # treat it as a regular class: - return _copy_immutable(x) - copier = getattr(cls, "__copy__", None) if copier: return copier(x) @@ -94,7 +86,7 @@ else: reductor = getattr(x, "__reduce_ex__", None) if reductor: - rv = reductor(4) + rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: @@ -110,7 +102,7 @@ def _copy_immutable(x): return x for t in (type(None), int, float, bool, str, tuple, - bytes, frozenset, type, range, + frozenset, type, range, types.BuiltinFunctionType, type(Ellipsis), types.FunctionType, weakref.ref): d[t] = _copy_immutable @@ -171,7 +163,7 @@ else: reductor = getattr(x, "__reduce_ex__", None) if reductor: - rv = reductor(4) + rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: @@ -221,15 +213,17 @@ d[list] = _deepcopy_list def _deepcopy_tuple(x, memo): - y = [deepcopy(a, memo) for a in x] + y = [] + for a in x: + y.append(deepcopy(a, memo)) # We're not going to put the tuple in the memo, but it's still important we # check for it, in case the tuple contains recursive mutable structures. try: return memo[id(x)] except KeyError: pass - for k, j in zip(x, y): - if k is not j: + for i in range(len(x)): + if x[i] is not y[i]: y = tuple(y) break else: @@ -279,7 +273,7 @@ if n > 2: state = info[2] else: - state = None + state = {} if n > 3: listiter = info[3] else: @@ -293,7 +287,7 @@ y = callable(*args) memo[id(x)] = y - if state is not None: + if state: if deep: state = deepcopy(state, memo) if hasattr(y, '__setstate__'): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/copyreg.py --- a/Lib/copyreg.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/copyreg.py Mon Jan 25 17:05:13 2016 +0100 @@ -87,12 +87,6 @@ def __newobj__(cls, *args): return cls.__new__(cls, *args) -def __newobj_ex__(cls, args, kwargs): - """Used by pickle protocol 4, instead of __newobj__ to allow classes with - keyword-only arguments to be pickled correctly. - """ - return cls.__new__(cls, *args, **kwargs) - def _slotnames(cls): """Return a list of slot names for a given class. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/crypt.py --- a/Lib/crypt.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/crypt.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,16 +1,15 @@ """Wrapper to the POSIX crypt library call and associated functionality.""" import _crypt -import string as _string -from random import SystemRandom as _SystemRandom -from collections import namedtuple as _namedtuple +import string +from random import choice +from collections import namedtuple -_saltchars = _string.ascii_letters + _string.digits + './' -_sr = _SystemRandom() +_saltchars = string.ascii_letters + string.digits + './' -class _Method(_namedtuple('_Method', 'name ident salt_chars total_size')): +class _Method(namedtuple('_Method', 'name ident salt_chars total_size')): """Class representing a salt method per the Modular Crypt Format or the legacy 2-character crypt method.""" @@ -19,6 +18,7 @@ return ''.format(self.name) + def mksalt(method=None): """Generate a salt for the specified method. @@ -28,7 +28,7 @@ if method is None: method = methods[0] s = '${}$'.format(method.ident) if method.ident else '' - s += ''.join(_sr.choice(_saltchars) for char in range(method.salt_chars)) + s += ''.join(choice(_saltchars) for _ in range(method.salt_chars)) return s @@ -54,8 +54,9 @@ METHOD_SHA512 = _Method('SHA512', '6', 16, 106) methods = [] -for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5, METHOD_CRYPT): +for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): _result = crypt('', _method) if _result and len(_result) == _method.total_size: methods.append(_method) +methods.append(METHOD_CRYPT) del _result, _method diff -r 6db40a9955dc -r 0d413f60cc23 Lib/csv.py --- a/Lib/csv.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/csv.py Mon Jan 25 17:05:13 2016 +0100 @@ -13,12 +13,11 @@ from io import StringIO -__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", - "Error", "Dialect", "__doc__", "excel", "excel_tab", - "field_size_limit", "reader", "writer", - "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter", - "unix_dialect"] +__all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", + "Error", "Dialect", "__doc__", "excel", "excel_tab", + "field_size_limit", "reader", "writer", + "register_dialect", "get_dialect", "list_dialects", "Sniffer", + "unregister_dialect", "__version__", "DictReader", "DictWriter" ] class Dialect: """Describe a CSV dialect. @@ -147,14 +146,17 @@ wrong_fields = [k for k in rowdict if k not in self.fieldnames] if wrong_fields: raise ValueError("dict contains fields not in fieldnames: " - + ", ".join([repr(x) for x in wrong_fields])) - return (rowdict.get(key, self.restval) for key in self.fieldnames) + + ", ".join(wrong_fields)) + return [rowdict.get(key, self.restval) for key in self.fieldnames] def writerow(self, rowdict): return self.writer.writerow(self._dict_to_list(rowdict)) def writerows(self, rowdicts): - return self.writer.writerows(map(self._dict_to_list, rowdicts)) + rows = [] + for rowdict in rowdicts: + rows.append(self._dict_to_list(rowdict)) + return self.writer.writerows(rows) # Guard Sniffer's type checking against builds that exclude complex() try: @@ -229,21 +231,20 @@ quotes = {} delims = {} spaces = 0 - groupindex = regexp.groupindex for m in matches: - n = groupindex['quote'] - 1 + n = regexp.groupindex['quote'] - 1 key = m[n] if key: quotes[key] = quotes.get(key, 0) + 1 try: - n = groupindex['delim'] - 1 + n = regexp.groupindex['delim'] - 1 key = m[n] except KeyError: continue if key and (delimiters is None or key in delimiters): delims[key] = delims.get(key, 0) + 1 try: - n = groupindex['space'] - 1 + n = regexp.groupindex['space'] - 1 except KeyError: continue if m[n]: @@ -263,9 +264,8 @@ # if we see an extra quote between delimiters, we've got a # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) + dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ + {'delim':delim, 'quote':quotechar}, re.MULTILINE) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/__init__.py --- a/Lib/ctypes/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -26,7 +26,7 @@ # libraries. OS X 10.3 is Darwin 7, so we check for # that. - if int(_os.uname().release.split('.')[0]) < 8: + if int(_os.uname()[2].split('.')[0]) < 8: DEFAULT_MODE = RTLD_GLOBAL from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \ @@ -34,22 +34,24 @@ FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR -# WINOLEAPI -> HRESULT -# WINOLEAPI_(type) -# -# STDMETHODCALLTYPE -# -# STDMETHOD(name) -# STDMETHOD_(type, name) -# -# STDAPICALLTYPE +""" +WINOLEAPI -> HRESULT +WINOLEAPI_(type) + +STDMETHODCALLTYPE + +STDMETHOD(name) +STDMETHOD_(type, name) + +STDAPICALLTYPE +""" def create_string_buffer(init, size=None): """create_string_buffer(aBytes) -> character array create_string_buffer(anInteger) -> character array - create_string_buffer(aBytes, anInteger) -> character array + create_string_buffer(aString, anInteger) -> character array """ - if isinstance(init, bytes): + if isinstance(init, (str, bytes)): if size is None: size = len(init)+1 buftype = c_char * size @@ -237,8 +239,14 @@ class c_char_p(_SimpleCData): _type_ = "z" - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) + if _os.name == "nt": + def __repr__(self): + if not windll.kernel32.IsBadStringPtrA(self, -1): + return "%s(%r)" % (self.__class__.__name__, self.value) + return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value) + else: + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, cast(self, c_void_p).value) _check_size(c_char_p, "P") class c_void_p(_SimpleCData): @@ -253,8 +261,6 @@ class c_wchar_p(_SimpleCData): _type_ = "Z" - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) class c_wchar(_SimpleCData): _type_ = "u" @@ -280,7 +286,7 @@ create_unicode_buffer(anInteger) -> character array create_unicode_buffer(aString, anInteger) -> character array """ - if isinstance(init, str): + if isinstance(init, (str, bytes)): if size is None: size = len(init)+1 buftype = c_wchar * size @@ -349,7 +355,7 @@ self._handle = handle def __repr__(self): - return "<%s '%s', handle %x at %#x>" % \ + return "<%s '%s', handle %x at %x>" % \ (self.__class__.__name__, self._name, (self._handle & (_sys.maxsize*2 + 1)), id(self) & (_sys.maxsize*2 + 1)) @@ -389,7 +395,7 @@ _type_ = "l" # _check_retval_ is called with the function's result when it # is used as restype. It checks for the FAILED bit, and - # raises an OSError if it is set. + # raises a WindowsError if it is set. # # The _check_retval_ method is implemented in C, so that the # method definition itself is not included in the traceback @@ -401,7 +407,7 @@ class OleDLL(CDLL): """This class represents a dll exporting functions using the Windows stdcall calling convention, and returning HRESULT. - HRESULT error values are automatically raised as OSError + HRESULT error values are automatically raised as WindowsError exceptions. """ _func_flags_ = _FUNCFLAG_STDCALL @@ -450,7 +456,7 @@ code = GetLastError() if descr is None: descr = FormatError(code).strip() - return OSError(None, descr, None, code) + return WindowsError(code, descr) if sizeof(c_uint) == sizeof(c_void_p): c_size_t = c_uint diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/_endian.py --- a/Lib/ctypes/_endian.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/_endian.py Mon Jan 25 17:05:13 2016 +0100 @@ -45,7 +45,6 @@ class BigEndianStructure(Structure, metaclass=_swapped_meta): """Structure with big endian byte order""" - __slots__ = () _swappedbytes_ = None elif sys.byteorder == "big": @@ -54,7 +53,6 @@ BigEndianStructure = Structure class LittleEndianStructure(Structure, metaclass=_swapped_meta): """Structure with little endian byte order""" - __slots__ = () _swappedbytes_ = None else: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/macholib/fetch_macholib.bat --- a/Lib/ctypes/macholib/fetch_macholib.bat Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/macholib/fetch_macholib.bat Mon Jan 25 17:05:13 2016 +0100 @@ -1,1 +1,1 @@ -svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . +svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/__init__.py --- a/Lib/ctypes/test/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,14 +1,208 @@ -import os -import unittest -from test import support +import os, sys, unittest, getopt, time -# skip tests if _ctypes was not built -ctypes = support.import_module('ctypes') -ctypes_symbols = dir(ctypes) +use_resources = [] -def need_symbol(name): - return unittest.skipUnless(name in ctypes_symbols, - '{!r} is required'.format(name)) +class ResourceDenied(Exception): + """Test skipped because it requested a disallowed resource. -def load_tests(*args): - return support.load_package_tests(os.path.dirname(__file__), *args) + This is raised when a test calls requires() for a resource that + has not be enabled. Resources are defined by test modules. + """ + +def is_resource_enabled(resource): + """Test whether a resource is enabled. + + If the caller's module is __main__ then automatically return True.""" + if sys._getframe().f_back.f_globals.get("__name__") == "__main__": + return True + result = use_resources is not None and \ + (resource in use_resources or "*" in use_resources) + if not result: + _unavail[resource] = None + return result + +_unavail = {} +def requires(resource, msg=None): + """Raise ResourceDenied if the specified resource is not available. + + If the caller's module is __main__ then automatically return True.""" + # see if the caller's module is __main__ - if so, treat as if + # the resource was set + if sys._getframe().f_back.f_globals.get("__name__") == "__main__": + return + if not is_resource_enabled(resource): + if msg is None: + msg = "Use of the `%s' resource not enabled" % resource + raise ResourceDenied(msg) + +def find_package_modules(package, mask): + import fnmatch + if (hasattr(package, "__loader__") and + hasattr(package.__loader__, '_files')): + path = package.__name__.replace(".", os.path.sep) + mask = os.path.join(path, mask) + for fnm in package.__loader__._files.keys(): + if fnmatch.fnmatchcase(fnm, mask): + yield os.path.splitext(fnm)[0].replace(os.path.sep, ".") + else: + path = package.__path__[0] + for fnm in os.listdir(path): + if fnmatch.fnmatchcase(fnm, mask): + yield "%s.%s" % (package.__name__, os.path.splitext(fnm)[0]) + +def get_tests(package, mask, verbosity, exclude=()): + """Return a list of skipped test modules, and a list of test cases.""" + tests = [] + skipped = [] + for modname in find_package_modules(package, mask): + if modname.split(".")[-1] in exclude: + skipped.append(modname) + if verbosity > 1: + print("Skipped %s: excluded" % modname, file=sys.stderr) + continue + try: + mod = __import__(modname, globals(), locals(), ['*']) + except ResourceDenied as detail: + skipped.append(modname) + if verbosity > 1: + print("Skipped %s: %s" % (modname, detail), file=sys.stderr) + continue + for name in dir(mod): + if name.startswith("_"): + continue + o = getattr(mod, name) + if type(o) is type(unittest.TestCase) and issubclass(o, unittest.TestCase): + tests.append(o) + return skipped, tests + +def usage(): + print(__doc__) + return 1 + +def test_with_refcounts(runner, verbosity, testcase): + """Run testcase several times, tracking reference counts.""" + import gc + import ctypes + ptc = ctypes._pointer_type_cache.copy() + cfc = ctypes._c_functype_cache.copy() + wfc = ctypes._win_functype_cache.copy() + + # when searching for refcount leaks, we have to manually reset any + # caches that ctypes has. + def cleanup(): + ctypes._pointer_type_cache = ptc.copy() + ctypes._c_functype_cache = cfc.copy() + ctypes._win_functype_cache = wfc.copy() + gc.collect() + + test = unittest.makeSuite(testcase) + for i in range(5): + rc = sys.gettotalrefcount() + runner.run(test) + cleanup() + COUNT = 5 + refcounts = [None] * COUNT + for i in range(COUNT): + rc = sys.gettotalrefcount() + runner.run(test) + cleanup() + refcounts[i] = sys.gettotalrefcount() - rc + if filter(None, refcounts): + print("%s leaks:\n\t" % testcase, refcounts) + elif verbosity: + print("%s: ok." % testcase) + +class TestRunner(unittest.TextTestRunner): + def run(self, test, skipped): + "Run the given test case or test suite." + # Same as unittest.TextTestRunner.run, except that it reports + # skipped tests. + result = self._makeResult() + startTime = time.time() + test(result) + stopTime = time.time() + timeTaken = stopTime - startTime + result.printErrors() + self.stream.writeln(result.separator2) + run = result.testsRun + if _unavail: #skipped: + requested = list(_unavail.keys()) + requested.sort() + self.stream.writeln("Ran %d test%s in %.3fs (%s module%s skipped)" % + (run, run != 1 and "s" or "", timeTaken, + len(skipped), + len(skipped) != 1 and "s" or "")) + self.stream.writeln("Unavailable resources: %s" % ", ".join(requested)) + else: + self.stream.writeln("Ran %d test%s in %.3fs" % + (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln() + if not result.wasSuccessful(): + self.stream.write("FAILED (") + failed, errored = map(len, (result.failures, result.errors)) + if failed: + self.stream.write("failures=%d" % failed) + if errored: + if failed: self.stream.write(", ") + self.stream.write("errors=%d" % errored) + self.stream.writeln(")") + else: + self.stream.writeln("OK") + return result + + +def main(*packages): + try: + opts, args = getopt.getopt(sys.argv[1:], "rqvu:x:") + except getopt.error: + return usage() + + verbosity = 1 + search_leaks = False + exclude = [] + for flag, value in opts: + if flag == "-q": + verbosity -= 1 + elif flag == "-v": + verbosity += 1 + elif flag == "-r": + try: + sys.gettotalrefcount + except AttributeError: + print("-r flag requires Python debug build", file=sys.stderr) + return -1 + search_leaks = True + elif flag == "-u": + use_resources.extend(value.split(",")) + elif flag == "-x": + exclude.extend(value.split(",")) + + mask = "test_*.py" + if args: + mask = args[0] + + for package in packages: + run_tests(package, mask, verbosity, search_leaks, exclude) + + +def run_tests(package, mask, verbosity, search_leaks, exclude): + skipped, testcases = get_tests(package, mask, verbosity, exclude) + runner = TestRunner(verbosity=verbosity) + + suites = [unittest.makeSuite(o) for o in testcases] + suite = unittest.TestSuite(suites) + result = runner.run(suite, skipped) + + if search_leaks: + # hunt for refcount leaks + runner = BasicTestRunner() + for t in testcases: + test_with_refcounts(runner, verbosity, t) + + return bool(result.errors) + +class BasicTestRunner: + def run(self, test): + result = unittest.TestResult() + test(result) + return result diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/__main__.py --- a/Lib/ctypes/test/__main__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -from ctypes.test import load_tests -import unittest - -unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/runtests.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/ctypes/test/runtests.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,19 @@ +"""Usage: runtests.py [-q] [-r] [-v] [-u resources] [mask] + +Run all tests found in this directory, and print a summary of the results. +Command line flags: + -q quiet mode: don't prnt anything while the tests are running + -r run tests repeatedly, look for refcount leaks + -u + Add resources to the lits of allowed resources. '*' allows all + resources. + -v verbose mode: print the test currently executed + -x + Exclude specified tests. + mask mask to select filenames containing testcases, wildcards allowed +""" +import sys +import ctypes.test + +if __name__ == "__main__": + sys.exit(ctypes.test.main(ctypes.test)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_arrays.py --- a/Lib/ctypes/test/test_arrays.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_arrays.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,8 +1,6 @@ import unittest from ctypes import * -from ctypes.test import need_symbol - formats = "bBhHiIlLqQfd" formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ @@ -86,8 +84,8 @@ self.assertEqual(values, [1, 2, 3, 4, 5]) def test_classcache(self): - self.assertIsNot(ARRAY(c_int, 3), ARRAY(c_int, 4)) - self.assertIs(ARRAY(c_int, 3), ARRAY(c_int, 3)) + self.assertTrue(not ARRAY(c_int, 3) is ARRAY(c_int, 4)) + self.assertTrue(ARRAY(c_int, 3) is ARRAY(c_int, 3)) def test_from_address(self): # Failed with 0.9.8, reported by JUrner @@ -100,16 +98,20 @@ self.assertEqual(sz[1:4:2], b"o") self.assertEqual(sz.value, b"foo") - @need_symbol('create_unicode_buffer') - def test_from_addressW(self): - p = create_unicode_buffer("foo") - sz = (c_wchar * 3).from_address(addressof(p)) - self.assertEqual(sz[:], "foo") - self.assertEqual(sz[::], "foo") - self.assertEqual(sz[::-1], "oof") - self.assertEqual(sz[::3], "f") - self.assertEqual(sz[1:4:2], "o") - self.assertEqual(sz.value, "foo") + try: + create_unicode_buffer + except NameError: + pass + else: + def test_from_addressW(self): + p = create_unicode_buffer("foo") + sz = (c_wchar * 3).from_address(addressof(p)) + self.assertEqual(sz[:], "foo") + self.assertEqual(sz[::], "foo") + self.assertEqual(sz[::-1], "oof") + self.assertEqual(sz[::3], "f") + self.assertEqual(sz[1:4:2], "o") + self.assertEqual(sz.value, "foo") def test_cache(self): # Array types are cached internally in the _ctypes extension, @@ -123,7 +125,7 @@ # Create a new array type based on it: t1 = my_int * 1 t2 = my_int * 1 - self.assertIs(t1, t2) + self.assertTrue(t1 is t2) def test_subclass(self): class T(Array): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_as_parameter.py --- a/Lib/ctypes/test/test_as_parameter.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_as_parameter.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import unittest from ctypes import * -from ctypes.test import need_symbol import _ctypes_test dll = CDLL(_ctypes_test.__file__) @@ -18,8 +17,11 @@ def wrap(self, param): return param - @need_symbol('c_wchar') def test_wchar_parm(self): + try: + c_wchar + except NameError: + return f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] result = f(self.wrap(1), self.wrap("x"), self.wrap(3), self.wrap(4), self.wrap(5.0), self.wrap(6.0)) @@ -132,7 +134,7 @@ f.argtypes = [c_longlong, MyCallback] def callback(value): - self.assertIsInstance(value, int) + self.assertTrue(isinstance(value, int)) return value & 0x7FFFFFFF cb = MyCallback(callback) @@ -194,7 +196,7 @@ a = A() a._as_parameter_ = a - with self.assertRaises(RecursionError): + with self.assertRaises(RuntimeError): c_int.from_param(a) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_bitfields.py --- a/Lib/ctypes/test/test_bitfields.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_bitfields.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ from ctypes import * -from ctypes.test import need_symbol import unittest import os @@ -128,18 +127,20 @@ result = self.fail_fields(("a", c_char, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_char')) + try: + c_wchar + except NameError: + pass + else: + result = self.fail_fields(("a", c_wchar, 1)) + self.assertEqual(result, (TypeError, 'bit fields not allowed for type c_wchar')) + class Dummy(Structure): _fields_ = [] result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) - @need_symbol('c_wchar') - def test_c_wchar(self): - result = self.fail_fields(("a", c_wchar, 1)) - self.assertEqual(result, - (TypeError, 'bit fields not allowed for type c_wchar')) - def test_single_bitfield_size(self): for c_typ in int_types: result = self.fail_fields(("a", c_typ, -1)) @@ -206,7 +207,7 @@ class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_int, 32)] - self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) + self.assertEqual(sizeof(X), sizeof(c_int)*2) def test_mixed_3(self): class X(Structure): @@ -239,53 +240,5 @@ _anonymous_ = ["_"] _fields_ = [("_", X)] - @need_symbol('c_uint32') - def test_uint32(self): - class X(Structure): - _fields_ = [("a", c_uint32, 32)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFDCBA987 - self.assertEqual(x.a, 0xFDCBA987) - - @need_symbol('c_uint64') - def test_uint64(self): - class X(Structure): - _fields_ = [("a", c_uint64, 64)] - x = X() - x.a = 10 - self.assertEqual(x.a, 10) - x.a = 0xFEDCBA9876543211 - self.assertEqual(x.a, 0xFEDCBA9876543211) - - @need_symbol('c_uint32') - def test_uint32_swap_little_endian(self): - # Issue #23319 - class Little(LittleEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Little.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xef\xcd\xab\x21') - - @need_symbol('c_uint32') - def test_uint32_swap_big_endian(self): - # Issue #23319 - class Big(BigEndianStructure): - _fields_ = [("a", c_uint32, 24), - ("b", c_uint32, 4), - ("c", c_uint32, 4)] - b = bytearray(4) - x = Big.from_buffer(b) - x.a = 0xabcdef - x.b = 1 - x.c = 2 - self.assertEqual(b, b'\xab\xcd\xef\x12') - if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_buffers.py --- a/Lib/ctypes/test/test_buffers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_buffers.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ from ctypes import * -from ctypes.test import need_symbol import unittest class StringBufferTestCase(unittest.TestCase): @@ -8,12 +7,12 @@ b = create_string_buffer(32) self.assertEqual(len(b), 32) self.assertEqual(sizeof(b), 32 * sizeof(c_char)) - self.assertIs(type(b[0]), bytes) + self.assertTrue(type(b[0]) is bytes) b = create_string_buffer(b"abc") self.assertEqual(len(b), 4) # trailing nul char self.assertEqual(sizeof(b), 4 * sizeof(c_char)) - self.assertIs(type(b[0]), bytes) + self.assertTrue(type(b[0]) is bytes) self.assertEqual(b[0], b"a") self.assertEqual(b[:], b"abc\0") self.assertEqual(b[::], b"abc\0") @@ -21,44 +20,43 @@ self.assertEqual(b[::2], b"ac") self.assertEqual(b[::5], b"a") - self.assertRaises(TypeError, create_string_buffer, "abc") - def test_buffer_interface(self): self.assertEqual(len(bytearray(create_string_buffer(0))), 0) self.assertEqual(len(bytearray(create_string_buffer(1))), 1) - @need_symbol('c_wchar') - def test_unicode_buffer(self): - b = create_unicode_buffer(32) - self.assertEqual(len(b), 32) - self.assertEqual(sizeof(b), 32 * sizeof(c_wchar)) - self.assertIs(type(b[0]), str) + try: + c_wchar + except NameError: + pass + else: + def test_unicode_buffer(self): + b = create_unicode_buffer(32) + self.assertEqual(len(b), 32) + self.assertEqual(sizeof(b), 32 * sizeof(c_wchar)) + self.assertTrue(type(b[0]) is str) - b = create_unicode_buffer("abc") - self.assertEqual(len(b), 4) # trailing nul char - self.assertEqual(sizeof(b), 4 * sizeof(c_wchar)) - self.assertIs(type(b[0]), str) - self.assertEqual(b[0], "a") - self.assertEqual(b[:], "abc\0") - self.assertEqual(b[::], "abc\0") - self.assertEqual(b[::-1], "\0cba") - self.assertEqual(b[::2], "ac") - self.assertEqual(b[::5], "a") + b = create_unicode_buffer("abc") + self.assertEqual(len(b), 4) # trailing nul char + self.assertEqual(sizeof(b), 4 * sizeof(c_wchar)) + self.assertTrue(type(b[0]) is str) + self.assertEqual(b[0], "a") + self.assertEqual(b[:], "abc\0") + self.assertEqual(b[::], "abc\0") + self.assertEqual(b[::-1], "\0cba") + self.assertEqual(b[::2], "ac") + self.assertEqual(b[::5], "a") - self.assertRaises(TypeError, create_unicode_buffer, b"abc") - - @need_symbol('c_wchar') - def test_unicode_conversion(self): - b = create_unicode_buffer("abc") - self.assertEqual(len(b), 4) # trailing nul char - self.assertEqual(sizeof(b), 4 * sizeof(c_wchar)) - self.assertIs(type(b[0]), str) - self.assertEqual(b[0], "a") - self.assertEqual(b[:], "abc\0") - self.assertEqual(b[::], "abc\0") - self.assertEqual(b[::-1], "\0cba") - self.assertEqual(b[::2], "ac") - self.assertEqual(b[::5], "a") + def test_unicode_conversion(self): + b = create_unicode_buffer("abc") + self.assertEqual(len(b), 4) # trailing nul char + self.assertEqual(sizeof(b), 4 * sizeof(c_wchar)) + self.assertTrue(type(b[0]) is str) + self.assertEqual(b[0], "a") + self.assertEqual(b[:], "abc\0") + self.assertEqual(b[::], "abc\0") + self.assertEqual(b[::-1], "\0cba") + self.assertEqual(b[::2], "ac") + self.assertEqual(b[::5], "a") if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_bytes.py --- a/Lib/ctypes/test/test_bytes.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_bytes.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,40 +6,27 @@ class BytesTest(unittest.TestCase): def test_c_char(self): x = c_char(b"x") - self.assertRaises(TypeError, c_char, "x") x.value = b"y" - with self.assertRaises(TypeError): - x.value = "y" c_char.from_param(b"x") - self.assertRaises(TypeError, c_char.from_param, "x") (c_char * 3)(b"a", b"b", b"c") - self.assertRaises(TypeError, c_char * 3, "a", "b", "c") def test_c_wchar(self): x = c_wchar("x") - self.assertRaises(TypeError, c_wchar, b"x") x.value = "y" - with self.assertRaises(TypeError): - x.value = b"y" c_wchar.from_param("x") - self.assertRaises(TypeError, c_wchar.from_param, b"x") (c_wchar * 3)("a", "b", "c") - self.assertRaises(TypeError, c_wchar * 3, b"a", b"b", b"c") def test_c_char_p(self): c_char_p(b"foo bar") - self.assertRaises(TypeError, c_char_p, "foo bar") def test_c_wchar_p(self): c_wchar_p("foo bar") - self.assertRaises(TypeError, c_wchar_p, b"foo bar") def test_struct(self): class X(Structure): _fields_ = [("a", c_char * 3)] x = X(b"abc") - self.assertRaises(TypeError, X, "abc") self.assertEqual(x.a, b"abc") self.assertEqual(type(x.a), bytes) @@ -48,18 +35,16 @@ _fields_ = [("a", c_wchar * 3)] x = X("abc") - self.assertRaises(TypeError, X, b"abc") self.assertEqual(x.a, "abc") self.assertEqual(type(x.a), str) - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') - def test_BSTR(self): - from _ctypes import _SimpleCData - class BSTR(_SimpleCData): - _type_ = "X" + if sys.platform == "win32": + def test_BSTR(self): + from _ctypes import _SimpleCData + class BSTR(_SimpleCData): + _type_ = "X" - BSTR("abc") - + BSTR("abc") if __name__ == '__main__': unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_byteswap.py --- a/Lib/ctypes/test/test_byteswap.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_byteswap.py Mon Jan 25 17:05:13 2016 +0100 @@ -14,41 +14,20 @@ # For Structures and Unions, these types are created on demand. class Test(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): + def X_test(self): print(sys.byteorder, file=sys.stderr) for i in range(32): bits = BITS() setattr(bits, "i%s" % i, 1) dump(bits) - def test_slots(self): - class BigPoint(BigEndianStructure): - __slots__ = () - _fields_ = [("x", c_int), ("y", c_int)] - - class LowPoint(LittleEndianStructure): - __slots__ = () - _fields_ = [("x", c_int), ("y", c_int)] - - big = BigPoint() - little = LowPoint() - big.x = 4 - big.y = 2 - little.x = 2 - little.y = 4 - with self.assertRaises(AttributeError): - big.z = 42 - with self.assertRaises(AttributeError): - little.z = 24 - def test_endian_short(self): if sys.byteorder == "little": - self.assertIs(c_short.__ctype_le__, c_short) - self.assertIs(c_short.__ctype_be__.__ctype_le__, c_short) + self.assertTrue(c_short.__ctype_le__ is c_short) + self.assertTrue(c_short.__ctype_be__.__ctype_le__ is c_short) else: - self.assertIs(c_short.__ctype_be__, c_short) - self.assertIs(c_short.__ctype_le__.__ctype_be__, c_short) + self.assertTrue(c_short.__ctype_be__ is c_short) + self.assertTrue(c_short.__ctype_le__.__ctype_be__ is c_short) s = c_short.__ctype_be__(0x1234) self.assertEqual(bin(struct.pack(">h", 0x1234)), "1234") self.assertEqual(bin(s), "1234") @@ -71,11 +50,11 @@ def test_endian_int(self): if sys.byteorder == "little": - self.assertIs(c_int.__ctype_le__, c_int) - self.assertIs(c_int.__ctype_be__.__ctype_le__, c_int) + self.assertTrue(c_int.__ctype_le__ is c_int) + self.assertTrue(c_int.__ctype_be__.__ctype_le__ is c_int) else: - self.assertIs(c_int.__ctype_be__, c_int) - self.assertIs(c_int.__ctype_le__.__ctype_be__, c_int) + self.assertTrue(c_int.__ctype_be__ is c_int) + self.assertTrue(c_int.__ctype_le__.__ctype_be__ is c_int) s = c_int.__ctype_be__(0x12345678) self.assertEqual(bin(struct.pack(">i", 0x12345678)), "12345678") @@ -99,11 +78,11 @@ def test_endian_longlong(self): if sys.byteorder == "little": - self.assertIs(c_longlong.__ctype_le__, c_longlong) - self.assertIs(c_longlong.__ctype_be__.__ctype_le__, c_longlong) + self.assertTrue(c_longlong.__ctype_le__ is c_longlong) + self.assertTrue(c_longlong.__ctype_be__.__ctype_le__ is c_longlong) else: - self.assertIs(c_longlong.__ctype_be__, c_longlong) - self.assertIs(c_longlong.__ctype_le__.__ctype_be__, c_longlong) + self.assertTrue(c_longlong.__ctype_be__ is c_longlong) + self.assertTrue(c_longlong.__ctype_le__.__ctype_be__ is c_longlong) s = c_longlong.__ctype_be__(0x1234567890ABCDEF) self.assertEqual(bin(struct.pack(">q", 0x1234567890ABCDEF)), "1234567890ABCDEF") @@ -127,11 +106,11 @@ def test_endian_float(self): if sys.byteorder == "little": - self.assertIs(c_float.__ctype_le__, c_float) - self.assertIs(c_float.__ctype_be__.__ctype_le__, c_float) + self.assertTrue(c_float.__ctype_le__ is c_float) + self.assertTrue(c_float.__ctype_be__.__ctype_le__ is c_float) else: - self.assertIs(c_float.__ctype_be__, c_float) - self.assertIs(c_float.__ctype_le__.__ctype_be__, c_float) + self.assertTrue(c_float.__ctype_be__ is c_float) + self.assertTrue(c_float.__ctype_le__.__ctype_be__ is c_float) s = c_float(math.pi) self.assertEqual(bin(struct.pack("f", math.pi)), bin(s)) # Hm, what's the precision of a float compared to a double? @@ -145,11 +124,11 @@ def test_endian_double(self): if sys.byteorder == "little": - self.assertIs(c_double.__ctype_le__, c_double) - self.assertIs(c_double.__ctype_be__.__ctype_le__, c_double) + self.assertTrue(c_double.__ctype_le__ is c_double) + self.assertTrue(c_double.__ctype_be__.__ctype_le__ is c_double) else: - self.assertIs(c_double.__ctype_be__, c_double) - self.assertIs(c_double.__ctype_le__.__ctype_be__, c_double) + self.assertTrue(c_double.__ctype_be__ is c_double) + self.assertTrue(c_double.__ctype_le__.__ctype_be__ is c_double) s = c_double(math.pi) self.assertEqual(s.value, math.pi) self.assertEqual(bin(struct.pack("d", math.pi)), bin(s)) @@ -161,14 +140,14 @@ self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s)) def test_endian_other(self): - self.assertIs(c_byte.__ctype_le__, c_byte) - self.assertIs(c_byte.__ctype_be__, c_byte) + self.assertTrue(c_byte.__ctype_le__ is c_byte) + self.assertTrue(c_byte.__ctype_be__ is c_byte) - self.assertIs(c_ubyte.__ctype_le__, c_ubyte) - self.assertIs(c_ubyte.__ctype_be__, c_ubyte) + self.assertTrue(c_ubyte.__ctype_le__ is c_ubyte) + self.assertTrue(c_ubyte.__ctype_be__ is c_ubyte) - self.assertIs(c_char.__ctype_le__, c_char) - self.assertIs(c_char.__ctype_be__, c_char) + self.assertTrue(c_char.__ctype_le__ is c_char) + self.assertTrue(c_char.__ctype_be__ is c_char) def test_struct_fields_1(self): if sys.byteorder == "little": diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_callbacks.py --- a/Lib/ctypes/test/test_callbacks.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_callbacks.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import unittest from ctypes import * -from ctypes.test import need_symbol import _ctypes_test class Callbacks(unittest.TestCase): @@ -89,10 +88,9 @@ # disabled: would now (correctly) raise a RuntimeWarning about # a memory leak. A callback function cannot return a non-integral # C type without causing a memory leak. - @unittest.skip('test disabled') - def test_char_p(self): - self.check_type(c_char_p, "abc") - self.check_type(c_char_p, "def") +## def test_char_p(self): +## self.check_type(c_char_p, "abc") +## self.check_type(c_char_p, "def") def test_pyobject(self): o = () @@ -144,12 +142,13 @@ CFUNCTYPE(None)(lambda x=Nasty(): None) -@need_symbol('WINFUNCTYPE') -class StdcallCallbacks(Callbacks): - try: +try: + WINFUNCTYPE +except NameError: + pass +else: + class StdcallCallbacks(Callbacks): functype = WINFUNCTYPE - except NameError: - pass ################################################################ @@ -179,7 +178,7 @@ from ctypes.util import find_library libc_path = find_library("c") if not libc_path: - self.skipTest('could not find libc') + return # cannot test libc = CDLL(libc_path) @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) @@ -191,19 +190,23 @@ libc.qsort(array, len(array), sizeof(c_int), cmp_func) self.assertEqual(array[:], [1, 5, 7, 33, 99]) - @need_symbol('WINFUNCTYPE') - def test_issue_8959_b(self): - from ctypes.wintypes import BOOL, HWND, LPARAM - global windowCount - windowCount = 0 + try: + WINFUNCTYPE + except NameError: + pass + else: + def test_issue_8959_b(self): + from ctypes.wintypes import BOOL, HWND, LPARAM + global windowCount + windowCount = 0 - @WINFUNCTYPE(BOOL, HWND, LPARAM) - def EnumWindowsCallbackFunc(hwnd, lParam): - global windowCount - windowCount += 1 - return True #Allow windows to keep enumerating + @WINFUNCTYPE(BOOL, HWND, LPARAM) + def EnumWindowsCallbackFunc(hwnd, lParam): + global windowCount + windowCount += 1 + return True #Allow windows to keep enumerating - windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) def test_callback_register_int(self): # Issue #8275: buggy handling of callback args under Win64 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_cast.py --- a/Lib/ctypes/test/test_cast.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_cast.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ from ctypes import * -from ctypes.test import need_symbol import unittest import sys @@ -39,14 +38,14 @@ p = cast(array, POINTER(c_char_p)) # array and p share a common _objects attribute - self.assertIs(p._objects, array._objects) + self.assertTrue(p._objects is array._objects) self.assertEqual(array._objects, {'0': b"foo bar", id(array): array}) p[0] = b"spam spam" self.assertEqual(p._objects, {'0': b"spam spam", id(array): array}) - self.assertIs(array._objects, p._objects) + self.assertTrue(array._objects is p._objects) p[1] = b"foo bar" self.assertEqual(p._objects, {'1': b'foo bar', '0': b"spam spam", id(array): array}) - self.assertIs(array._objects, p._objects) + self.assertTrue(array._objects is p._objects) def test_other(self): p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int)) @@ -76,11 +75,15 @@ self.assertEqual(cast(cast(s, c_void_p), c_char_p).value, b"hiho") - @need_symbol('c_wchar_p') - def test_wchar_p(self): - s = c_wchar_p("hiho") - self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value, - "hiho") + try: + c_wchar_p + except NameError: + pass + else: + def test_wchar_p(self): + s = c_wchar_p("hiho") + self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value, + "hiho") if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_cfuncs.py --- a/Lib/ctypes/test/test_cfuncs.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_cfuncs.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,6 @@ import unittest from ctypes import * -from ctypes.test import need_symbol import _ctypes_test @@ -189,12 +188,12 @@ self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) -# The following repeats the above tests with stdcall functions (where +# The following repeates the above tests with stdcall functions (where # they are available) try: WinDLL except NameError: - def stdcall_dll(*_): pass + pass else: class stdcall_dll(WinDLL): def __getattr__(self, name): @@ -204,9 +203,9 @@ setattr(self, name, func) return func -@need_symbol('WinDLL') -class stdcallCFunctions(CFunctions): - _dll = stdcall_dll(_ctypes_test.__file__) + class stdcallCFunctions(CFunctions): + _dll = stdcall_dll(_ctypes_test.__file__) + pass if __name__ == '__main__': unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_checkretval.py --- a/Lib/ctypes/test/test_checkretval.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_checkretval.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,6 @@ import unittest from ctypes import * -from ctypes.test import need_symbol class CHECKED(c_int): def _check_retval_(value): @@ -26,11 +25,15 @@ del dll._testfunc_p_p.restype self.assertEqual(42, dll._testfunc_p_p(42)) - @need_symbol('oledll') - def test_oledll(self): - self.assertRaises(OSError, - oledll.oleaut32.CreateTypeLib2, - 0, None, None) + try: + oledll + except NameError: + pass + else: + def test_oledll(self): + self.assertRaises(WindowsError, + oledll.oleaut32.CreateTypeLib2, + 0, None, None) if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_errcheck.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/ctypes/test/test_errcheck.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,19 @@ +import sys +from ctypes import * + +##class HMODULE(Structure): +## _fields_ = [("value", c_void_p)] + +## def __repr__(self): +## return "" % self.value + +##windll.kernel32.GetModuleHandleA.restype = HMODULE + +##print windll.kernel32.GetModuleHandleA("python23.dll") +##print hex(sys.dllhandle) + +##def nonzero(handle): +## return (GetLastError(), handle) + +##windll.kernel32.GetModuleHandleA.errcheck = nonzero +##print windll.kernel32.GetModuleHandleA("spam") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_find.py --- a/Lib/ctypes/test/test_find.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_find.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,91 +1,82 @@ import unittest -import os import sys -import test.support from ctypes import * from ctypes.util import find_library +from ctypes.test import is_resource_enabled + +if sys.platform == "win32": + lib_gl = find_library("OpenGL32") + lib_glu = find_library("Glu32") + lib_gle = None +elif sys.platform == "darwin": + lib_gl = lib_glu = find_library("OpenGL") + lib_gle = None +else: + lib_gl = find_library("GL") + lib_glu = find_library("GLU") + lib_gle = find_library("gle") + +## print, for debugging +if is_resource_enabled("printing"): + if lib_gl or lib_glu or lib_gle: + print("OpenGL libraries:") + for item in (("GL", lib_gl), + ("GLU", lib_glu), + ("gle", lib_gle)): + print("\t", item) + # On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. class Test_OpenGL_libs(unittest.TestCase): - @classmethod - def setUpClass(cls): - lib_gl = lib_glu = lib_gle = None - if sys.platform == "win32": - lib_gl = find_library("OpenGL32") - lib_glu = find_library("Glu32") - elif sys.platform == "darwin": - lib_gl = lib_glu = find_library("OpenGL") - else: - lib_gl = find_library("GL") - lib_glu = find_library("GLU") - lib_gle = find_library("gle") - - ## print, for debugging - if test.support.verbose: - print("OpenGL libraries:") - for item in (("GL", lib_gl), - ("GLU", lib_glu), - ("gle", lib_gle)): - print("\t", item) - - cls.gl = cls.glu = cls.gle = None + def setUp(self): + self.gl = self.glu = self.gle = None if lib_gl: - try: - cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) - except OSError: - pass + self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) if lib_glu: - try: - cls.glu = CDLL(lib_glu, RTLD_GLOBAL) - except OSError: - pass + self.glu = CDLL(lib_glu, RTLD_GLOBAL) if lib_gle: try: - cls.gle = CDLL(lib_gle) + self.gle = CDLL(lib_gle) except OSError: pass - @classmethod - def tearDownClass(cls): - cls.gl = cls.glu = cls.gle = None + if lib_gl: + def test_gl(self): + if self.gl: + self.gl.glClearIndex - def test_gl(self): - if self.gl is None: - self.skipTest('lib_gl not available') - self.gl.glClearIndex + if lib_glu: + def test_glu(self): + if self.glu: + self.glu.gluBeginCurve - def test_glu(self): - if self.glu is None: - self.skipTest('lib_glu not available') - self.glu.gluBeginCurve + if lib_gle: + def test_gle(self): + if self.gle: + self.gle.gleGetJoinStyle - def test_gle(self): - if self.gle is None: - self.skipTest('lib_gle not available') - self.gle.gleGetJoinStyle +##if os.name == "posix" and sys.platform != "darwin": -# On platforms where the default shared library suffix is '.so', -# at least some libraries can be loaded as attributes of the cdll -# object, since ctypes now tries loading the lib again -# with '.so' appended of the first try fails. -# -# Won't work for libc, unfortunately. OTOH, it isn't -# needed for libc since this is already mapped into the current -# process (?) -# -# On MAC OSX, it won't work either, because dlopen() needs a full path, -# and the default suffix is either none or '.dylib'. -@unittest.skip('test disabled') -@unittest.skipUnless(os.name=="posix" and sys.platform != "darwin", - 'test not suitable for this platform') -class LoadLibs(unittest.TestCase): - def test_libm(self): - import math - libm = cdll.libm - sqrt = libm.sqrt - sqrt.argtypes = (c_double,) - sqrt.restype = c_double - self.assertEqual(sqrt(2), math.sqrt(2)) +## # On platforms where the default shared library suffix is '.so', +## # at least some libraries can be loaded as attributes of the cdll +## # object, since ctypes now tries loading the lib again +## # with '.so' appended of the first try fails. +## # +## # Won't work for libc, unfortunately. OTOH, it isn't +## # needed for libc since this is already mapped into the current +## # process (?) +## # +## # On MAC OSX, it won't work either, because dlopen() needs a full path, +## # and the default suffix is either none or '.dylib'. + +## class LoadLibs(unittest.TestCase): +## def test_libm(self): +## import math +## libm = cdll.libm +## sqrt = libm.sqrt +## sqrt.argtypes = (c_double,) +## sqrt.restype = c_double +## self.assertEqual(sqrt(2), math.sqrt(2)) if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_frombuffer.py --- a/Lib/ctypes/test/test_frombuffer.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_frombuffer.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,7 @@ self._init_called = True class Test(unittest.TestCase): - def test_from_buffer(self): + def test_fom_buffer(self): a = array.array("i", range(16)) x = (c_int * 16).from_buffer(a) @@ -23,64 +23,25 @@ a[0], a[-1] = 200, -200 self.assertEqual(x[:], a.tolist()) - self.assertRaises(BufferError, a.append, 100) - self.assertRaises(BufferError, a.pop) + self.assertTrue(a in x._objects.values()) - del x; del y; gc.collect(); gc.collect(); gc.collect() - a.append(100) - a.pop() - x = (c_int * 16).from_buffer(a) - - self.assertIn(a, [obj.obj if isinstance(obj, memoryview) else obj - for obj in x._objects.values()]) + self.assertRaises(ValueError, + c_int.from_buffer, a, -1) expected = x[:] del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], expected) - with self.assertRaisesRegex(TypeError, "not writable"): - (c_char * 16).from_buffer(b"a" * 16) - with self.assertRaisesRegex(TypeError, "not writable"): - (c_char * 16).from_buffer(memoryview(b"a" * 16)) - with self.assertRaisesRegex(TypeError, "not C contiguous"): - (c_char * 16).from_buffer(memoryview(bytearray(b"a" * 16))[::-1]) - msg = "bytes-like object is required" - with self.assertRaisesRegex(TypeError, msg): - (c_char * 16).from_buffer("a" * 16) + self.assertRaises(TypeError, + (c_char * 16).from_buffer, "a" * 16) - def test_fortran_contiguous(self): - try: - import _testbuffer - except ImportError as err: - self.skipTest(str(err)) - flags = _testbuffer.ND_WRITABLE | _testbuffer.ND_FORTRAN - array = _testbuffer.ndarray( - [97] * 16, format="B", shape=[4, 4], flags=flags) - with self.assertRaisesRegex(TypeError, "not C contiguous"): - (c_char * 16).from_buffer(array) - array = memoryview(array) - self.assertTrue(array.f_contiguous) - self.assertFalse(array.c_contiguous) - with self.assertRaisesRegex(TypeError, "not C contiguous"): - (c_char * 16).from_buffer(array) - - def test_from_buffer_with_offset(self): + def test_fom_buffer_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer(a, sizeof(c_int)) self.assertEqual(x[:], a.tolist()[1:]) - with self.assertRaises(ValueError): - c_int.from_buffer(a, -1) - with self.assertRaises(ValueError): - (c_int * 16).from_buffer(a, sizeof(c_int)) - with self.assertRaises(ValueError): - (c_int * 1).from_buffer(a, 16 * sizeof(c_int)) - - def test_from_buffer_memoryview(self): - a = [c_char.from_buffer(memoryview(bytearray(b'a')))] - a.append(a) - del a - gc.collect() # Should not crash + self.assertRaises(ValueError, lambda: (c_int * 16).from_buffer(a, sizeof(c_int))) + self.assertRaises(ValueError, lambda: (c_int * 1).from_buffer(a, 16 * sizeof(c_int))) def test_from_buffer_copy(self): a = array.array("i", range(16)) @@ -95,30 +56,26 @@ a[0], a[-1] = 200, -200 self.assertEqual(x[:], list(range(16))) - a.append(100) - self.assertEqual(x[:], list(range(16))) + self.assertEqual(x._objects, None) - self.assertEqual(x._objects, None) + self.assertRaises(ValueError, + c_int.from_buffer, a, -1) del a; gc.collect(); gc.collect(); gc.collect() self.assertEqual(x[:], list(range(16))) x = (c_char * 16).from_buffer_copy(b"a" * 16) self.assertEqual(x[:], b"a" * 16) - with self.assertRaises(TypeError): - (c_char * 16).from_buffer_copy("a" * 16) - def test_from_buffer_copy_with_offset(self): + def test_fom_buffer_copy_with_offset(self): a = array.array("i", range(16)) x = (c_int * 15).from_buffer_copy(a, sizeof(c_int)) self.assertEqual(x[:], a.tolist()[1:]) - with self.assertRaises(ValueError): - c_int.from_buffer_copy(a, -1) - with self.assertRaises(ValueError): - (c_int * 16).from_buffer_copy(a, sizeof(c_int)) - with self.assertRaises(ValueError): - (c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int)) + self.assertRaises(ValueError, + (c_int * 16).from_buffer_copy, a, sizeof(c_int)) + self.assertRaises(ValueError, + (c_int * 1).from_buffer_copy, a, 16 * sizeof(c_int)) if __name__ == '__main__': unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_funcptr.py --- a/Lib/ctypes/test/test_funcptr.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_funcptr.py Mon Jan 25 17:05:13 2016 +0100 @@ -75,7 +75,7 @@ ## "lpfnWndProc", WNDPROC_2(wndproc)) # instead: - self.assertIs(WNDPROC, WNDPROC_2) + self.assertTrue(WNDPROC is WNDPROC_2) # 'wndclass.lpfnWndProc' leaks 94 references. Why? self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_functions.py --- a/Lib/ctypes/test/test_functions.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_functions.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,6 @@ """ from ctypes import * -from ctypes.test import need_symbol import sys, unittest try: @@ -64,16 +63,22 @@ pass - @need_symbol('c_wchar') def test_wchar_parm(self): + try: + c_wchar + except NameError: + return f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] result = f(1, "x", 3, 4, 5.0, 6.0) self.assertEqual(result, 139) self.assertEqual(type(result), int) - @need_symbol('c_wchar') def test_wchar_result(self): + try: + c_wchar + except NameError: + return f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] f.restype = c_wchar @@ -150,8 +155,11 @@ self.assertEqual(result, -21) self.assertEqual(type(result), float) - @need_symbol('c_longlong') def test_longlongresult(self): + try: + c_longlong + except NameError: + return f = dll._testfunc_q_bhilfd f.restype = c_longlong f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] @@ -288,7 +296,6 @@ result = f(-10, cb) self.assertEqual(result, -18) - @need_symbol('c_longlong') def test_longlong_callbacks(self): f = dll._testfunc_callback_q_qf @@ -299,7 +306,7 @@ f.argtypes = [c_longlong, MyCallback] def callback(value): - self.assertIsInstance(value, int) + self.assertTrue(isinstance(value, int)) return value & 0x7FFFFFFF cb = MyCallback(callback) @@ -341,16 +348,16 @@ s2h = dll.ret_2h_func(inp) self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') - def test_struct_return_2H_stdcall(self): - class S2H(Structure): - _fields_ = [("x", c_short), - ("y", c_short)] + if sys.platform == "win32": + def test_struct_return_2H_stdcall(self): + class S2H(Structure): + _fields_ = [("x", c_short), + ("y", c_short)] - windll.s_ret_2h_func.restype = S2H - windll.s_ret_2h_func.argtypes = [S2H] - s2h = windll.s_ret_2h_func(S2H(99, 88)) - self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) + windll.s_ret_2h_func.restype = S2H + windll.s_ret_2h_func.argtypes = [S2H] + s2h = windll.s_ret_2h_func(S2H(99, 88)) + self.assertEqual((s2h.x, s2h.y), (99*2, 88*3)) def test_struct_return_8H(self): class S8I(Structure): @@ -369,24 +376,23 @@ self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) - @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') - def test_struct_return_8H_stdcall(self): - class S8I(Structure): - _fields_ = [("a", c_int), - ("b", c_int), - ("c", c_int), - ("d", c_int), - ("e", c_int), - ("f", c_int), - ("g", c_int), - ("h", c_int)] - windll.s_ret_8i_func.restype = S8I - windll.s_ret_8i_func.argtypes = [S8I] - inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) - s8i = windll.s_ret_8i_func(inp) - self.assertEqual( - (s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), - (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) + if sys.platform == "win32": + def test_struct_return_8H_stdcall(self): + class S8I(Structure): + _fields_ = [("a", c_int), + ("b", c_int), + ("c", c_int), + ("d", c_int), + ("e", c_int), + ("f", c_int), + ("g", c_int), + ("h", c_int)] + windll.s_ret_8i_func.restype = S8I + windll.s_ret_8i_func.argtypes = [S8I] + inp = S8I(9, 8, 7, 6, 5, 4, 3, 2) + s8i = windll.s_ret_8i_func(inp) + self.assertEqual((s8i.a, s8i.b, s8i.c, s8i.d, s8i.e, s8i.f, s8i.g, s8i.h), + (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_sf1651235(self): # see http://www.python.org/sf/1651235 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_integers.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/ctypes/test/test_integers.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +# superseeded by test_numbers.py +import unittest + +if __name__ == '__main__': + unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_internals.py --- a/Lib/ctypes/test/test_internals.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_internals.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,14 +5,17 @@ # XXX This test must be reviewed for correctness!!! -# ctypes' types are container types. -# -# They have an internal memory block, which only consists of some bytes, -# but it has to keep references to other objects as well. This is not -# really needed for trivial C types like int or char, but it is important -# for aggregate types like strings or pointers in particular. -# -# What about pointers? +""" +ctypes' types are container types. + +They have an internal memory block, which only consists of some bytes, +but it has to keep references to other objects as well. This is not +really needed for trivial C types like int or char, but it is important +for aggregate types like strings or pointers in particular. + +What about pointers? + +""" class ObjectsTestCase(unittest.TestCase): def assertSame(self, a, b): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_keeprefs.py --- a/Lib/ctypes/test/test_keeprefs.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_keeprefs.py Mon Jan 25 17:05:13 2016 +0100 @@ -94,8 +94,7 @@ self.assertEqual(x._objects, {'1': i}) class DeletePointerTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): + def X_test(self): class X(Structure): _fields_ = [("p", POINTER(c_char_p))] x = X() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_loading.py --- a/Lib/ctypes/test/test_loading.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_loading.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,46 +1,38 @@ from ctypes import * +import sys, unittest import os -import sys -import unittest -import test.support from ctypes.util import find_library +from ctypes.test import is_resource_enabled libc_name = None +if os.name == "nt": + libc_name = find_library("c") +elif os.name == "ce": + libc_name = "coredll" +elif sys.platform == "cygwin": + libc_name = "cygwin1.dll" +else: + libc_name = find_library("c") -def setUpModule(): - global libc_name - if os.name == "nt": - libc_name = find_library("c") - elif os.name == "ce": - libc_name = "coredll" - elif sys.platform == "cygwin": - libc_name = "cygwin1.dll" - else: - libc_name = find_library("c") - - if test.support.verbose: - print("libc_name is", libc_name) +if is_resource_enabled("printing"): + print("libc_name is", libc_name) class LoaderTest(unittest.TestCase): unknowndll = "xxrandomnamexx" - def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) - self.assertRaises(OSError, CDLL, self.unknowndll) + if libc_name is not None: + def test_load(self): + CDLL(libc_name) + CDLL(os.path.basename(libc_name)) + self.assertRaises(OSError, CDLL, self.unknowndll) - def test_load_version(self): - if libc_name is None: - self.skipTest('could not find libc') - if os.path.basename(libc_name) != 'libc.so.6': - self.skipTest('wrong libc path for test') - cdll.LoadLibrary("libc.so.6") - # linux uses version, libc 9 should not exist - self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") - self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) + if libc_name is not None and os.path.basename(libc_name) == "libc.so.6": + def test_load_version(self): + cdll.LoadLibrary("libc.so.6") + # linux uses version, libc 9 should not exist + self.assertRaises(OSError, cdll.LoadLibrary, "libc.so.9") + self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): for name in ("c", "m"): @@ -49,73 +41,66 @@ cdll.LoadLibrary(lib) CDLL(lib) - @unittest.skipUnless(os.name in ("nt", "ce"), - 'test specific to Windows (NT/CE)') - def test_load_library(self): - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - #self.assertIsNotNone(libc_name) - if test.support.verbose: - print(find_library("kernel32")) - print(find_library("user32")) + if os.name in ("nt", "ce"): + def test_load_library(self): + self.assertFalse(libc_name is None) + if is_resource_enabled("printing"): + print(find_library("kernel32")) + print(find_library("user32")) - if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW - elif os.name == "ce": - windll.coredll.GetModuleHandleW - windll["coredll"].GetModuleHandleW - windll.LoadLibrary("coredll").GetModuleHandleW - WinDLL("coredll").GetModuleHandleW + if os.name == "nt": + windll.kernel32.GetModuleHandleW + windll["kernel32"].GetModuleHandleW + windll.LoadLibrary("kernel32").GetModuleHandleW + WinDLL("kernel32").GetModuleHandleW + elif os.name == "ce": + windll.coredll.GetModuleHandleW + windll["coredll"].GetModuleHandleW + windll.LoadLibrary("coredll").GetModuleHandleW + WinDLL("coredll").GetModuleHandleW - @unittest.skipUnless(os.name in ("nt", "ce"), - 'test specific to Windows (NT/CE)') - def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) - # We load the same function both via ordinal and name - func_ord = dll[2] - func_name = dll.GetString - # addressof gets the address where the function pointer is stored - a_ord = addressof(func_ord) - a_name = addressof(func_name) - f_ord_addr = c_void_p.from_address(a_ord).value - f_name_addr = c_void_p.from_address(a_name).value - self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) + def test_load_ordinal_functions(self): + import _ctypes_test + dll = WinDLL(_ctypes_test.__file__) + # We load the same function both via ordinal and name + func_ord = dll[2] + func_name = dll.GetString + # addressof gets the address where the function pointer is stored + a_ord = addressof(func_ord) + a_name = addressof(func_name) + f_ord_addr = c_void_p.from_address(a_ord).value + f_name_addr = c_void_p.from_address(a_name).value + self.assertEqual(hex(f_ord_addr), hex(f_name_addr)) - self.assertRaises(AttributeError, dll.__getitem__, 1234) + self.assertRaises(AttributeError, dll.__getitem__, 1234) - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary - # On winXP 64-bit, advapi32 loads at an address that does - # NOT fit into a 32-bit integer. FreeLibrary must be able - # to accept this address. + if os.name == "nt": + def test_1703286_A(self): + from _ctypes import LoadLibrary, FreeLibrary + # On winXP 64-bit, advapi32 loads at an address that does + # NOT fit into a 32-bit integer. FreeLibrary must be able + # to accept this address. - # These are tests for http://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) + # These are tests for http://www.python.org/sf/1703286 + handle = LoadLibrary("advapi32") + FreeLibrary(handle) - @unittest.skipUnless(os.name == "nt", 'Windows-specific test') - def test_1703286_B(self): - # Since on winXP 64-bit advapi32 loads like described - # above, the (arbitrarily selected) CloseEventLog function - # also has a high address. 'call_function' should accept - # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 - # Calling CloseEventLog with a NULL argument should fail, - # but the call should not segfault or so. - self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") - self.assertTrue(proc) - # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) + def test_1703286_B(self): + # Since on winXP 64-bit advapi32 loads like described + # above, the (arbitrarily selected) CloseEventLog function + # also has a high address. 'call_function' should accept + # addresses so large. + from _ctypes import call_function + advapi32 = windll.advapi32 + # Calling CloseEventLog with a NULL argument should fail, + # but the call should not segfault or so. + self.assertEqual(0, advapi32.CloseEventLog(None)) + windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + windll.kernel32.GetProcAddress.restype = c_void_p + proc = windll.kernel32.GetProcAddress(advapi32._handle, b"CloseEventLog") + self.assertTrue(proc) + # This is the real test: call the function via 'call_function' + self.assertEqual(0, call_function(proc, (None,))) if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_macholib.py --- a/Lib/ctypes/test/test_macholib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_macholib.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,33 +3,35 @@ import unittest # Bob Ippolito: -# -# Ok.. the code to find the filename for __getattr__ should look -# something like: -# -# import os -# from macholib.dyld import dyld_find -# -# def find_lib(name): -# possible = ['lib'+name+'.dylib', name+'.dylib', -# name+'.framework/'+name] -# for dylib in possible: -# try: -# return os.path.realpath(dyld_find(dylib)) -# except ValueError: -# pass -# raise ValueError, "%s not found" % (name,) -# -# It'll have output like this: -# -# >>> find_lib('pthread') -# '/usr/lib/libSystem.B.dylib' -# >>> find_lib('z') -# '/usr/lib/libz.1.dylib' -# >>> find_lib('IOKit') -# '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit' -# -# -bob +""" +Ok.. the code to find the filename for __getattr__ should look +something like: + +import os +from macholib.dyld import dyld_find + +def find_lib(name): + possible = ['lib'+name+'.dylib', name+'.dylib', + name+'.framework/'+name] + for dylib in possible: + try: + return os.path.realpath(dyld_find(dylib)) + except ValueError: + pass + raise ValueError, "%s not found" % (name,) + +It'll have output like this: + + >>> find_lib('pthread') +'/usr/lib/libSystem.B.dylib' + >>> find_lib('z') +'/usr/lib/libz.1.dylib' + >>> find_lib('IOKit') +'/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit' + +-bob + +""" from ctypes.macholib.dyld import dyld_find @@ -43,21 +45,18 @@ raise ValueError("%s not found" % (name,)) class MachOTest(unittest.TestCase): - @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') - def test_find(self): + if sys.platform == "darwin": + def test_find(self): - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + self.assertEqual(find_lib('pthread'), + '/usr/lib/libSystem.B.dylib') - result = find_lib('z') - # Issue #21093: dyld default search path includes $HOME/lib and - # /usr/local/lib before /usr/lib, which caused test failures if - # a local copy of libz exists in one of them. Now ignore the head - # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + result = find_lib('z') + self.assertTrue(result.startswith('/usr/lib/libz.1')) + self.assertTrue(result.endswith('.dylib')) - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertEqual(find_lib('IOKit'), + '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_memfunctions.py --- a/Lib/ctypes/test/test_memfunctions.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_memfunctions.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,19 +2,17 @@ from test import support import unittest from ctypes import * -from ctypes.test import need_symbol class MemFunctionsTest(unittest.TestCase): - @unittest.skip('test disabled') - def test_overflow(self): - # string_at and wstring_at must use the Python calling - # convention (which acquires the GIL and checks the Python - # error flag). Provoke an error and catch it; see also issue - # #3554: - self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: wstring_at(u"foo", sys.maxint - 1)) - self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: string_at("foo", sys.maxint - 1)) +## def test_overflow(self): +## # string_at and wstring_at must use the Python calling +## # convention (which acquires the GIL and checks the Python +## # error flag). Provoke an error and catch it; see also issue +## # #3554: +## self.assertRaises((OverflowError, MemoryError, SystemError), +## lambda: wstring_at(u"foo", sys.maxint - 1)) +## self.assertRaises((OverflowError, MemoryError, SystemError), +## lambda: string_at("foo", sys.maxint - 1)) def test_memmove(self): # large buffers apparently increase the chance that the memory @@ -63,17 +61,21 @@ self.assertEqual(string_at(b"foo bar", 7), b"foo bar") self.assertEqual(string_at(b"foo bar", 3), b"foo") - @need_symbol('create_unicode_buffer') - def test_wstring_at(self): - p = create_unicode_buffer("Hello, World") - a = create_unicode_buffer(1000000) - result = memmove(a, p, len(p) * sizeof(c_wchar)) - self.assertEqual(a.value, "Hello, World") + try: + create_unicode_buffer + except NameError: + pass + else: + def test_wstring_at(self): + p = create_unicode_buffer("Hello, World") + a = create_unicode_buffer(1000000) + result = memmove(a, p, len(p) * sizeof(c_wchar)) + self.assertEqual(a.value, "Hello, World") - self.assertEqual(wstring_at(a), "Hello, World") - self.assertEqual(wstring_at(a, 5), "Hello") - self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") - self.assertEqual(wstring_at(a, 0), "") + self.assertEqual(wstring_at(a), "Hello, World") + self.assertEqual(wstring_at(a, 5), "Hello") + self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") + self.assertEqual(wstring_at(a, 0), "") if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_numbers.py --- a/Lib/ctypes/test/test_numbers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_numbers.py Mon Jan 25 17:05:13 2016 +0100 @@ -82,13 +82,12 @@ self.assertRaises(TypeError, t, "") self.assertRaises(TypeError, t, None) - @unittest.skip('test disabled') - def test_valid_ranges(self): - # invalid values of the correct type - # raise ValueError (not OverflowError) - for t, (l, h) in zip(unsigned_types, unsigned_ranges): - self.assertRaises(ValueError, t, l-1) - self.assertRaises(ValueError, t, h+1) +## def test_valid_ranges(self): +## # invalid values of the correct type +## # raise ValueError (not OverflowError) +## for t, (l, h) in zip(unsigned_types, unsigned_ranges): +## self.assertRaises(ValueError, t, l-1) +## self.assertRaises(ValueError, t, h+1) def test_from_param(self): # the from_param class method attribute always @@ -105,7 +104,7 @@ def test_floats(self): # c_float and c_double can be created from - # Python int and float + # Python int, long and float class FloatLike(object): def __float__(self): return 2.0 @@ -182,10 +181,10 @@ a = array(t._type_, [3.14]) v = t.from_address(a.buffer_info()[0]) self.assertEqual(v.value, a[0]) - self.assertIs(type(v), t) + self.assertTrue(type(v) is t) a[0] = 2.3456e17 self.assertEqual(v.value, a[0]) - self.assertIs(type(v), t) + self.assertTrue(type(v) is t) def test_char_from_address(self): from ctypes import c_char @@ -195,43 +194,31 @@ a[0] = ord('x') v = c_char.from_address(a.buffer_info()[0]) self.assertEqual(v.value, b'x') - self.assertIs(type(v), c_char) + self.assertTrue(type(v) is c_char) a[0] = ord('?') self.assertEqual(v.value, b'?') # array does not support c_bool / 't' - @unittest.skip('test disabled') - def test_bool_from_address(self): - from ctypes import c_bool - from array import array - a = array(c_bool._type_, [True]) - v = t.from_address(a.buffer_info()[0]) - self.assertEqual(v.value, a[0]) - self.assertEqual(type(v) is t) - a[0] = False - self.assertEqual(v.value, a[0]) - self.assertEqual(type(v) is t) + # def test_bool_from_address(self): + # from ctypes import c_bool + # from array import array + # a = array(c_bool._type_, [True]) + # v = t.from_address(a.buffer_info()[0]) + # self.assertEqual(v.value, a[0]) + # self.assertEqual(type(v) is t) + # a[0] = False + # self.assertEqual(v.value, a[0]) + # self.assertEqual(type(v) is t) def test_init(self): # c_int() can be initialized from Python's int, and c_int. - # Not from c_long or so, which seems strange, abc should + # Not from c_long or so, which seems strange, abd should # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) - def test_float_overflow(self): - import sys - big_int = int(sys.float_info.max) * 2 - for t in float_types + [c_longdouble]: - self.assertRaises(OverflowError, t, big_int) - if (hasattr(t, "__ctype_be__")): - self.assertRaises(OverflowError, t.__ctype_be__, big_int) - if (hasattr(t, "__ctype_le__")): - self.assertRaises(OverflowError, t.__ctype_le__, big_int) - - @unittest.skip('test disabled') - def test_perf(self): - check_perf() +## def test_perf(self): +## check_perf() from ctypes import _SimpleCData class c_int_S(_SimpleCData): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_objects.py --- a/Lib/ctypes/test/test_objects.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_objects.py Mon Jan 25 17:05:13 2016 +0100 @@ -59,9 +59,12 @@ import ctypes.test.test_objects class TestCase(unittest.TestCase): - def test(self): - failures, tests = doctest.testmod(ctypes.test.test_objects) - self.assertFalse(failures, 'doctests failed, see output above') + if sys.hexversion > 0x02040000: + # Python 2.3 has no ELLIPSIS flag, so we don't test with this + # version: + def test(self): + doctest.testmod(ctypes.test.test_objects) if __name__ == '__main__': - doctest.testmod(ctypes.test.test_objects) + if sys.hexversion > 0x02040000: + doctest.testmod(ctypes.test.test_objects) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_parameters.py --- a/Lib/ctypes/test/test_parameters.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_parameters.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ import unittest, sys -from ctypes.test import need_symbol class SimpleTypesTestCase(unittest.TestCase): @@ -36,9 +35,10 @@ self.assertEqual(CVOIDP.from_param("abc"), "abcabc") self.assertEqual(CCHARP.from_param("abc"), "abcabcabcabc") - @need_symbol('c_wchar_p') - def test_subclasses_c_wchar_p(self): - from ctypes import c_wchar_p + try: + from ctypes import c_wchar_p + except ImportError: + return class CWCHARP(c_wchar_p): def from_param(cls, value): @@ -54,7 +54,7 @@ # c_char_p.from_param on a Python String packs the string # into a cparam object s = b"123" - self.assertIs(c_char_p.from_param(s)._obj, s) + self.assertTrue(c_char_p.from_param(s)._obj is s) # new in 0.9.1: convert (encode) unicode to ascii self.assertEqual(c_char_p.from_param(b"123")._obj, b"123") @@ -64,11 +64,15 @@ # calling c_char_p.from_param with a c_char_p instance # returns the argument itself: a = c_char_p(b"123") - self.assertIs(c_char_p.from_param(a), a) + self.assertTrue(c_char_p.from_param(a) is a) - @need_symbol('c_wchar_p') def test_cw_strings(self): - from ctypes import byref, c_wchar_p + from ctypes import byref + try: + from ctypes import c_wchar_p + except ImportError: +## print "(No c_wchar_p)" + return c_wchar_p.from_param("123") @@ -135,6 +139,9 @@ self.assertRaises(TypeError, LPINT.from_param, c_long*3) self.assertRaises(TypeError, LPINT.from_param, c_uint*3) +## def test_performance(self): +## check_perf() + def test_noctypes_argtype(self): import _ctypes_test from ctypes import CDLL, c_void_p, ArgumentError diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_pep3118.py --- a/Lib/ctypes/test/test_pep3118.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_pep3118.py Mon Jan 25 17:05:13 2016 +0100 @@ -96,9 +96,6 @@ class aUnion(Union): _fields_ = [("a", c_int)] -class StructWithArrays(Structure): - _fields_ = [("x", c_long * 3 * 2), ("y", Point * 4)] - class Incomplete(Structure): pass @@ -148,10 +145,10 @@ ## arrays and pointers - (c_double * 4, " 2) result = f(-10, cb) self.assertEqual(result, -18) cb = None @@ -44,29 +44,29 @@ # this is the standard refcount for func self.assertEqual(grc(func), 2) - # the CFuncPtr instance holds at least one refcount on func: + # the CFuncPtr instance holds atr least one refcount on func: f = OtherCallback(func) - self.assertGreater(grc(func), 2) + self.assertTrue(grc(func) > 2) # and may release it again del f - self.assertGreaterEqual(grc(func), 2) + self.assertTrue(grc(func) >= 2) # but now it must be gone gc.collect() - self.assertEqual(grc(func), 2) + self.assertTrue(grc(func) == 2) class X(ctypes.Structure): _fields_ = [("a", OtherCallback)] x = X() x.a = OtherCallback(func) - # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + # the CFuncPtr instance holds atr least one refcount on func: + self.assertTrue(grc(func) > 2) # and may release it again del x - self.assertGreaterEqual(grc(func), 2) + self.assertTrue(grc(func) >= 2) # and now it must be gone again gc.collect() @@ -74,8 +74,8 @@ f = OtherCallback(func) - # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + # the CFuncPtr instance holds atr least one refcount on func: + self.assertTrue(grc(func) > 2) # create a cycle f.cycle = f diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_returnfuncptrs.py --- a/Lib/ctypes/test/test_returnfuncptrs.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_returnfuncptrs.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import unittest from ctypes import * -import os import _ctypes_test @@ -34,34 +33,5 @@ self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) self.assertRaises(TypeError, strchr, b"abcdef") - def test_from_dll(self): - dll = CDLL(_ctypes_test.__file__) - # _CFuncPtr instances are now callable with a tuple argument - # which denotes a function name and a dll: - strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(("my_strchr", dll)) - self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") - self.assertEqual(strchr(b"abcdef", b"x"), None) - self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) - self.assertRaises(TypeError, strchr, b"abcdef") - - # Issue 6083: Reference counting bug - def test_from_dll_refcount(self): - class BadSequence(tuple): - def __getitem__(self, key): - if key == 0: - return "my_strchr" - if key == 1: - return CDLL(_ctypes_test.__file__) - raise IndexError - - # _CFuncPtr instances are now callable with a tuple argument - # which denotes a function name and a dll: - strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)( - BadSequence(("my_strchr", CDLL(_ctypes_test.__file__)))) - self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") - self.assertEqual(strchr(b"abcdef", b"x"), None) - self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) - self.assertRaises(TypeError, strchr, b"abcdef") - if __name__ == "__main__": unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_slicing.py --- a/Lib/ctypes/test/test_slicing.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_slicing.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import unittest from ctypes import * -from ctypes.test import need_symbol import _ctypes_test @@ -126,40 +125,44 @@ self.assertEqual(p[2:5:-3], s[2:5:-3]) - @need_symbol('c_wchar') - def test_wchar_ptr(self): - s = "abcdefghijklmnopqrstuvwxyz\0" + try: + c_wchar + except NameError: + pass + else: + def test_wchar_ptr(self): + s = "abcdefghijklmnopqrstuvwxyz\0" - dll = CDLL(_ctypes_test.__file__) - dll.my_wcsdup.restype = POINTER(c_wchar) - dll.my_wcsdup.argtypes = POINTER(c_wchar), - dll.my_free.restype = None - res = dll.my_wcsdup(s) - self.assertEqual(res[:len(s)], s) - self.assertEqual(res[:len(s):], s) - self.assertEqual(res[len(s)-1:-1:-1], s[::-1]) - self.assertEqual(res[len(s)-1:5:-7], s[:5:-7]) + dll = CDLL(_ctypes_test.__file__) + dll.my_wcsdup.restype = POINTER(c_wchar) + dll.my_wcsdup.argtypes = POINTER(c_wchar), + dll.my_free.restype = None + res = dll.my_wcsdup(s) + self.assertEqual(res[:len(s)], s) + self.assertEqual(res[:len(s):], s) + self.assertEqual(res[len(s)-1:-1:-1], s[::-1]) + self.assertEqual(res[len(s)-1:5:-7], s[:5:-7]) - import operator - self.assertRaises(TypeError, operator.setitem, - res, slice(0, 5), "abcde") - dll.my_free(res) + import operator + self.assertRaises(TypeError, operator.setitem, + res, slice(0, 5), "abcde") + dll.my_free(res) - if sizeof(c_wchar) == sizeof(c_short): - dll.my_wcsdup.restype = POINTER(c_short) - elif sizeof(c_wchar) == sizeof(c_int): - dll.my_wcsdup.restype = POINTER(c_int) - elif sizeof(c_wchar) == sizeof(c_long): - dll.my_wcsdup.restype = POINTER(c_long) - else: - self.skipTest('Pointers to c_wchar are not supported') - res = dll.my_wcsdup(s) - tmpl = list(range(ord("a"), ord("z")+1)) - self.assertEqual(res[:len(s)-1], tmpl) - self.assertEqual(res[:len(s)-1:], tmpl) - self.assertEqual(res[len(s)-2:-1:-1], tmpl[::-1]) - self.assertEqual(res[len(s)-2:5:-7], tmpl[:5:-7]) - dll.my_free(res) + if sizeof(c_wchar) == sizeof(c_short): + dll.my_wcsdup.restype = POINTER(c_short) + elif sizeof(c_wchar) == sizeof(c_int): + dll.my_wcsdup.restype = POINTER(c_int) + elif sizeof(c_wchar) == sizeof(c_long): + dll.my_wcsdup.restype = POINTER(c_long) + else: + return + res = dll.my_wcsdup(s) + tmpl = list(range(ord("a"), ord("z")+1)) + self.assertEqual(res[:len(s)-1], tmpl) + self.assertEqual(res[:len(s)-1:], tmpl) + self.assertEqual(res[len(s)-2:-1:-1], tmpl[::-1]) + self.assertEqual(res[len(s)-2:5:-7], tmpl[:5:-7]) + dll.my_free(res) ################################################################ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_strings.py --- a/Lib/ctypes/test/test_strings.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_strings.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import unittest from ctypes import * -from ctypes.test import need_symbol class StringArrayTestCase(unittest.TestCase): def test(self): @@ -54,33 +53,36 @@ ## print BUF.from_param(c_char_p("python")) ## print BUF.from_param(BUF(*"pyth")) -@need_symbol('c_wchar') -class WStringArrayTestCase(unittest.TestCase): - def test(self): - BUF = c_wchar * 4 +try: + c_wchar +except NameError: + pass +else: + class WStringArrayTestCase(unittest.TestCase): + def test(self): + BUF = c_wchar * 4 - buf = BUF("a", "b", "c") - self.assertEqual(buf.value, "abc") + buf = BUF("a", "b", "c") + self.assertEqual(buf.value, "abc") - buf.value = "ABCD" - self.assertEqual(buf.value, "ABCD") + buf.value = "ABCD" + self.assertEqual(buf.value, "ABCD") - buf.value = "x" - self.assertEqual(buf.value, "x") + buf.value = "x" + self.assertEqual(buf.value, "x") - buf[1] = "Z" - self.assertEqual(buf.value, "xZCD") + buf[1] = "Z" + self.assertEqual(buf.value, "xZCD") - @unittest.skipIf(sizeof(c_wchar) < 4, - "sizeof(wchar_t) is smaller than 4 bytes") - def test_nonbmp(self): - u = chr(0x10ffff) - w = c_wchar(u) - self.assertEqual(w.value, u) + @unittest.skipIf(sizeof(c_wchar) < 4, + "sizeof(wchar_t) is smaller than 4 bytes") + def test_nonbmp(self): + u = chr(0x10ffff) + w = c_wchar(u) + self.assertEqual(w.value, u) class StringTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_basic_strings(self): + def XX_test_basic_strings(self): cs = c_string("abcdef") # Cannot call len on a c_string any longer @@ -106,36 +108,33 @@ self.assertRaises(TypeError, c_string, "123") - @unittest.skip('test disabled') - def test_sized_strings(self): + def XX_test_sized_strings(self): # New in releases later than 0.4.0: self.assertRaises(TypeError, c_string, None) # New in releases later than 0.4.0: # c_string(number) returns an empty string of size number - self.assertEqual(len(c_string(32).raw), 32) + self.assertTrue(len(c_string(32).raw) == 32) self.assertRaises(ValueError, c_string, -1) self.assertRaises(ValueError, c_string, 0) # These tests fail, because it is no longer initialized -## self.assertEqual(c_string(2).value, "") -## self.assertEqual(c_string(2).raw, "\000\000") - self.assertEqual(c_string(2).raw[-1], "\000") - self.assertEqual(len(c_string(2).raw), 2) +## self.assertTrue(c_string(2).value == "") +## self.assertTrue(c_string(2).raw == "\000\000") + self.assertTrue(c_string(2).raw[-1] == "\000") + self.assertTrue(len(c_string(2).raw) == 2) - @unittest.skip('test disabled') - def test_initialized_strings(self): + def XX_test_initialized_strings(self): - self.assertEqual(c_string("ab", 4).raw[:2], "ab") - self.assertEqual(c_string("ab", 4).raw[:2:], "ab") - self.assertEqual(c_string("ab", 4).raw[:2:-1], "ba") - self.assertEqual(c_string("ab", 4).raw[:2:2], "a") - self.assertEqual(c_string("ab", 4).raw[-1], "\000") - self.assertEqual(c_string("ab", 2).raw, "a\000") + self.assertTrue(c_string("ab", 4).raw[:2] == "ab") + self.assertTrue(c_string("ab", 4).raw[:2:] == "ab") + self.assertTrue(c_string("ab", 4).raw[:2:-1] == "ba") + self.assertTrue(c_string("ab", 4).raw[:2:2] == "a") + self.assertTrue(c_string("ab", 4).raw[-1] == "\000") + self.assertTrue(c_string("ab", 2).raw == "a\000") - @unittest.skip('test disabled') - def test_toolong(self): + def XX_test_toolong(self): cs = c_string("abcdef") # Much too long string: self.assertRaises(ValueError, setattr, cs, "value", "123456789012345") @@ -143,53 +142,54 @@ # One char too long values: self.assertRaises(ValueError, setattr, cs, "value", "1234567") - @unittest.skip('test disabled') - def test_perf(self): - check_perf() +## def test_perf(self): +## check_perf() -@need_symbol('c_wchar') -class WStringTestCase(unittest.TestCase): - def test_wchar(self): - c_wchar("x") - repr(byref(c_wchar("x"))) - c_wchar("x") +try: + c_wchar +except NameError: + pass +else: + class WStringTestCase(unittest.TestCase): + def test_wchar(self): + c_wchar("x") + repr(byref(c_wchar("x"))) + c_wchar("x") - @unittest.skip('test disabled') - def test_basic_wstrings(self): - cs = c_wstring("abcdef") + def X_test_basic_wstrings(self): + cs = c_wstring("abcdef") - # XXX This behaviour is about to change: - # len returns the size of the internal buffer in bytes. - # This includes the terminating NUL character. - self.assertEqual(sizeof(cs), 14) + # XXX This behaviour is about to change: + # len returns the size of the internal buffer in bytes. + # This includes the terminating NUL character. + self.assertTrue(sizeof(cs) == 14) - # The value property is the string up to the first terminating NUL. - self.assertEqual(cs.value, "abcdef") - self.assertEqual(c_wstring("abc\000def").value, "abc") + # The value property is the string up to the first terminating NUL. + self.assertTrue(cs.value == "abcdef") + self.assertTrue(c_wstring("abc\000def").value == "abc") - self.assertEqual(c_wstring("abc\000def").value, "abc") + self.assertTrue(c_wstring("abc\000def").value == "abc") - # The raw property is the total buffer contents: - self.assertEqual(cs.raw, "abcdef\000") - self.assertEqual(c_wstring("abc\000def").raw, "abc\000def\000") + # The raw property is the total buffer contents: + self.assertTrue(cs.raw == "abcdef\000") + self.assertTrue(c_wstring("abc\000def").raw == "abc\000def\000") - # We can change the value: - cs.value = "ab" - self.assertEqual(cs.value, "ab") - self.assertEqual(cs.raw, "ab\000\000\000\000\000") + # We can change the value: + cs.value = "ab" + self.assertTrue(cs.value == "ab") + self.assertTrue(cs.raw == "ab\000\000\000\000\000") - self.assertRaises(TypeError, c_wstring, "123") - self.assertRaises(ValueError, c_wstring, 0) + self.assertRaises(TypeError, c_wstring, "123") + self.assertRaises(ValueError, c_wstring, 0) - @unittest.skip('test disabled') - def test_toolong(self): - cs = c_wstring("abcdef") - # Much too long string: - self.assertRaises(ValueError, setattr, cs, "value", "123456789012345") + def X_test_toolong(self): + cs = c_wstring("abcdef") + # Much too long string: + self.assertRaises(ValueError, setattr, cs, "value", "123456789012345") - # One char too long values: - self.assertRaises(ValueError, setattr, cs, "value", "1234567") + # One char too long values: + self.assertRaises(ValueError, setattr, cs, "value", "1234567") def run_test(rep, msg, func, arg): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_structures.py --- a/Lib/ctypes/test/test_structures.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_structures.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,8 +1,6 @@ import unittest from ctypes import * -from ctypes.test import need_symbol from struct import calcsize -import _testcapi class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -84,7 +82,7 @@ class Y(Structure): _fields_ = [("x", c_char * 3), ("y", c_int)] - self.assertEqual(alignment(Y), alignment(c_int)) + self.assertEqual(alignment(Y), calcsize("i")) self.assertEqual(sizeof(Y), calcsize("3si")) class SI(Structure): @@ -109,7 +107,7 @@ def test_emtpy(self): # I had problems with these # - # Although these are pathological cases: Empty Structures! + # Although these are patological cases: Empty Structures! class X(Structure): _fields_ = [] @@ -176,6 +174,13 @@ self.assertEqual(sizeof(X), 10) self.assertEqual(X.b.offset, 2) + class X(Structure): + _fields_ = [("a", c_byte), + ("b", c_longlong)] + _pack_ = 4 + self.assertEqual(sizeof(X), 12) + self.assertEqual(X.b.offset, 4) + import struct longlong_size = struct.calcsize("q") longlong_align = struct.calcsize("bq") - longlong_size @@ -183,16 +188,9 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] - _pack_ = 4 - self.assertEqual(sizeof(X), min(4, longlong_align) + longlong_size) - self.assertEqual(X.b.offset, min(4, longlong_align)) - - class X(Structure): - _fields_ = [("a", c_byte), - ("b", c_longlong)] _pack_ = 8 - self.assertEqual(sizeof(X), min(8, longlong_align) + longlong_size) + self.assertEqual(sizeof(X), longlong_align + longlong_size) self.assertEqual(X.b.offset, min(8, longlong_align)) @@ -201,14 +199,6 @@ "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) - # Issue 15989 - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.INT_MAX + 1} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) - d = {"_fields_": [("a", c_byte)], - "_pack_": _testcapi.UINT_MAX + 2} - self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) - def test_initializers(self): class Person(Structure): _fields_ = [("name", c_char*6), @@ -292,8 +282,12 @@ self.assertEqual(p.phone.number, b"5678") self.assertEqual(p.age, 5) - @need_symbol('c_wchar') def test_structures_with_wchar(self): + try: + c_wchar + except NameError: + return # no unicode + class PersonW(Structure): _fields_ = [("name", c_wchar * 12), ("age", c_int)] @@ -322,7 +316,7 @@ self.assertEqual(cls, RuntimeError) self.assertEqual(msg, "(Phone) : " - "expected bytes, int found") + "expected string, int found") cls, msg = self.get_except(Person, b"Someone", (b"a", b"b", b"c")) self.assertEqual(cls, RuntimeError) @@ -351,14 +345,14 @@ except Exception as detail: return detail.__class__, str(detail) - @unittest.skip('test disabled') - def test_subclass_creation(self): - meta = type(Structure) - # same as 'class X(Structure): pass' - # fails, since we need either a _fields_ or a _abstract_ attribute - cls, msg = self.get_except(meta, "X", (Structure,), {}) - self.assertEqual((cls, msg), - (AttributeError, "class must define a '_fields_' attribute")) + +## def test_subclass_creation(self): +## meta = type(Structure) +## # same as 'class X(Structure): pass' +## # fails, since we need either a _fields_ or a _abstract_ attribute +## cls, msg = self.get_except(meta, "X", (Structure,), {}) +## self.assertEqual((cls, msg), +## (AttributeError, "class must define a '_fields_' attribute")) def test_abstract_class(self): class X(Structure): @@ -371,9 +365,9 @@ ## class X(Structure): ## _fields_ = [] - self.assertIn("in_dll", dir(type(Structure))) - self.assertIn("from_address", dir(type(Structure))) - self.assertIn("in_dll", dir(type(Structure))) + self.assertTrue("in_dll" in dir(type(Structure))) + self.assertTrue("from_address" in dir(type(Structure))) + self.assertTrue("in_dll" in dir(type(Structure))) def test_positional_args(self): # see also http://bugs.python.org/issue5042 @@ -443,8 +437,8 @@ try: Recursive._fields_ = [("next", Recursive)] except AttributeError as details: - self.assertIn("Structure or union cannot contain itself", - str(details)) + self.assertTrue("Structure or union cannot contain itself" in + str(details)) else: self.fail("Structure or union cannot contain itself") @@ -460,7 +454,8 @@ try: Second._fields_ = [("first", First)] except AttributeError as details: - self.assertIn("_fields_ is final", str(details)) + self.assertTrue("_fields_ is final" in + str(details)) else: self.fail("AttributeError not raised") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_unicode.py --- a/Lib/ctypes/test/test_unicode.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_unicode.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,55 +1,58 @@ import unittest import ctypes -from ctypes.test import need_symbol -import _ctypes_test +try: + ctypes.c_wchar +except AttributeError: + pass +else: + import _ctypes_test -@need_symbol('c_wchar') -class UnicodeTestCase(unittest.TestCase): - def test_wcslen(self): - dll = ctypes.CDLL(_ctypes_test.__file__) - wcslen = dll.my_wcslen - wcslen.argtypes = [ctypes.c_wchar_p] + class UnicodeTestCase(unittest.TestCase): + def test_wcslen(self): + dll = ctypes.CDLL(_ctypes_test.__file__) + wcslen = dll.my_wcslen + wcslen.argtypes = [ctypes.c_wchar_p] - self.assertEqual(wcslen("abc"), 3) - self.assertEqual(wcslen("ab\u2070"), 3) - self.assertRaises(ctypes.ArgumentError, wcslen, b"ab\xe4") + self.assertEqual(wcslen("abc"), 3) + self.assertEqual(wcslen("ab\u2070"), 3) + self.assertRaises(ctypes.ArgumentError, wcslen, b"ab\xe4") - def test_buffers(self): - buf = ctypes.create_unicode_buffer("abc") - self.assertEqual(len(buf), 3+1) + def test_buffers(self): + buf = ctypes.create_unicode_buffer("abc") + self.assertEqual(len(buf), 3+1) - buf = ctypes.create_unicode_buffer("ab\xe4\xf6\xfc") - self.assertEqual(buf[:], "ab\xe4\xf6\xfc\0") - self.assertEqual(buf[::], "ab\xe4\xf6\xfc\0") - self.assertEqual(buf[::-1], '\x00\xfc\xf6\xe4ba') - self.assertEqual(buf[::2], 'a\xe4\xfc') - self.assertEqual(buf[6:5:-1], "") + buf = ctypes.create_unicode_buffer("ab\xe4\xf6\xfc") + self.assertEqual(buf[:], "ab\xe4\xf6\xfc\0") + self.assertEqual(buf[::], "ab\xe4\xf6\xfc\0") + self.assertEqual(buf[::-1], '\x00\xfc\xf6\xe4ba') + self.assertEqual(buf[::2], 'a\xe4\xfc') + self.assertEqual(buf[6:5:-1], "") -func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p + func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p -class StringTestCase(UnicodeTestCase): - def setUp(self): - func.argtypes = [ctypes.c_char_p] - func.restype = ctypes.c_char_p + class StringTestCase(UnicodeTestCase): + def setUp(self): + func.argtypes = [ctypes.c_char_p] + func.restype = ctypes.c_char_p - def tearDown(self): - func.argtypes = None - func.restype = ctypes.c_int + def tearDown(self): + func.argtypes = None + func.restype = ctypes.c_int - def test_func(self): - self.assertEqual(func(b"abc\xe4"), b"abc\xe4") + def test_func(self): + self.assertEqual(func(b"abc\xe4"), b"abc\xe4") - def test_buffers(self): - buf = ctypes.create_string_buffer(b"abc") - self.assertEqual(len(buf), 3+1) + def test_buffers(self): + buf = ctypes.create_string_buffer(b"abc") + self.assertEqual(len(buf), 3+1) - buf = ctypes.create_string_buffer(b"ab\xe4\xf6\xfc") - self.assertEqual(buf[:], b"ab\xe4\xf6\xfc\0") - self.assertEqual(buf[::], b"ab\xe4\xf6\xfc\0") - self.assertEqual(buf[::-1], b'\x00\xfc\xf6\xe4ba') - self.assertEqual(buf[::2], b'a\xe4\xfc') - self.assertEqual(buf[6:5:-1], b"") + buf = ctypes.create_string_buffer(b"ab\xe4\xf6\xfc") + self.assertEqual(buf[:], b"ab\xe4\xf6\xfc\0") + self.assertEqual(buf[::], b"ab\xe4\xf6\xfc\0") + self.assertEqual(buf[::-1], b'\x00\xfc\xf6\xe4ba') + self.assertEqual(buf[::2], b'a\xe4\xfc') + self.assertEqual(buf[6:5:-1], b"") if __name__ == '__main__': diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_values.py --- a/Lib/ctypes/test/test_values.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_values.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,6 @@ """ import unittest -import sys from ctypes import * import _ctypes_test @@ -28,72 +27,62 @@ ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") -class PythonValuesTestCase(unittest.TestCase): - """This test only works when python itself is a dll/shared library""" + class Win_ValuesTestCase(unittest.TestCase): + """This test only works when python itself is a dll/shared library""" - def test_optimizeflag(self): - # This test accesses the Py_OptimizeFlag integer, which is - # exported by the Python dll and should match the sys.flags value + def test_optimizeflag(self): + # This test accesses the Py_OptimizeFlag intger, which is + # exported by the Python dll. - opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value - self.assertEqual(opt, sys.flags.optimize) + # It's value is set depending on the -O and -OO flags: + # if not given, it is 0 and __debug__ is 1. + # If -O is given, the flag is 1, for -OO it is 2. + # docstrings are also removed in the latter case. + opt = c_int.in_dll(pydll, "Py_OptimizeFlag").value + if __debug__: + self.assertEqual(opt, 0) + elif ValuesTestCase.__doc__ is not None: + self.assertEqual(opt, 1) + else: + self.assertEqual(opt, 2) - def test_frozentable(self): - # Python exports a PyImport_FrozenModules symbol. This is a - # pointer to an array of struct _frozen entries. The end of the - # array is marked by an entry containing a NULL name and zero - # size. + def test_frozentable(self): + # Python exports a PyImport_FrozenModules symbol. This is a + # pointer to an array of struct _frozen entries. The end of the + # array is marked by an entry containing a NULL name and zero + # size. - # In standard Python, this table contains a __hello__ - # module, and a __phello__ package containing a spam - # module. - class struct_frozen(Structure): - _fields_ = [("name", c_char_p), - ("code", POINTER(c_ubyte)), - ("size", c_int)] - FrozenTable = POINTER(struct_frozen) + # In standard Python, this table contains a __hello__ + # module, and a __phello__ package containing a spam + # module. + class struct_frozen(Structure): + _fields_ = [("name", c_char_p), + ("code", POINTER(c_ubyte)), + ("size", c_int)] + FrozenTable = POINTER(struct_frozen) - ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules") - # ft is a pointer to the struct_frozen entries: - items = [] - # _frozen_importlib changes size whenever importlib._bootstrap - # changes, so it gets a special case. We should make sure it's - # found, but don't worry about its size too much. The same - # applies to _frozen_importlib_external. - bootstrap_seen = [] - bootstrap_expected = [ - b'_frozen_importlib', - b'_frozen_importlib_external', - ] - for entry in ft: - # This is dangerous. We *can* iterate over a pointer, but - # the loop will not terminate (maybe with an access - # violation;-) because the pointer instance has no size. - if entry.name is None: - break + ft = FrozenTable.in_dll(pydll, "PyImport_FrozenModules") + # ft is a pointer to the struct_frozen entries: + items = [] + for entry in ft: + # This is dangerous. We *can* iterate over a pointer, but + # the loop will not terminate (maybe with an access + # violation;-) because the pointer instance has no size. + if entry.name is None: + break + items.append((entry.name, entry.size)) + import sys + if sys.version_info[:2] >= (2, 3): + expected = [("__hello__", 104), ("__phello__", -104), ("__phello__.spam", 104)] + else: + expected = [("__hello__", 100), ("__phello__", -100), ("__phello__.spam", 100)] + self.assertEqual(items, expected) - if entry.name in bootstrap_expected: - bootstrap_seen.append(entry.name) - self.assertTrue(entry.size, - "{!r} was reported as having no size".format(entry.name)) - continue - items.append((entry.name, entry.size)) + from ctypes import _pointer_type_cache + del _pointer_type_cache[struct_frozen] - expected = [(b"__hello__", 161), - (b"__phello__", -161), - (b"__phello__.spam", 161), - ] - self.assertEqual(items, expected) - - self.assertEqual(sorted(bootstrap_seen), bootstrap_expected, - "frozen bootstrap modules did not match PyImport_FrozenModules") - - from ctypes import _pointer_type_cache - del _pointer_type_cache[struct_frozen] - - def test_undefined(self): - self.assertRaises(ValueError, c_int.in_dll, pythonapi, - "Undefined_Symbol") + def test_undefined(self): + self.assertRaises(ValueError, c_int.in_dll, pydll, "Undefined_Symbol") if __name__ == '__main__': unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_win32.py --- a/Lib/ctypes/test/test_win32.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/test/test_win32.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,103 +1,74 @@ # Windows specific tests from ctypes import * +from ctypes.test import is_resource_enabled import unittest, sys -from test import support import _ctypes_test -# Only windows 32-bit has different calling conventions. -@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') -@unittest.skipUnless(sizeof(c_void_p) == sizeof(c_int), - "sizeof c_void_p and c_int differ") -class WindowsTestCase(unittest.TestCase): - def test_callconv_1(self): - # Testing stdcall function +if sys.platform == "win32" and sizeof(c_void_p) == sizeof(c_int): + # Only windows 32-bit has different calling conventions. - IsWindow = windll.user32.IsWindow - # ValueError: Procedure probably called with not enough arguments - # (4 bytes missing) - self.assertRaises(ValueError, IsWindow) + class WindowsTestCase(unittest.TestCase): + def test_callconv_1(self): + # Testing stdcall function - # This one should succeed... - self.assertEqual(0, IsWindow(0)) + IsWindow = windll.user32.IsWindow + # ValueError: Procedure probably called with not enough arguments (4 bytes missing) + self.assertRaises(ValueError, IsWindow) - # ValueError: Procedure probably called with too many arguments - # (8 bytes in excess) - self.assertRaises(ValueError, IsWindow, 0, 0, 0) + # This one should succeed... + self.assertEqual(0, IsWindow(0)) - def test_callconv_2(self): - # Calling stdcall function as cdecl + # ValueError: Procedure probably called with too many arguments (8 bytes in excess) + self.assertRaises(ValueError, IsWindow, 0, 0, 0) - IsWindow = cdll.user32.IsWindow + def test_callconv_2(self): + # Calling stdcall function as cdecl - # ValueError: Procedure called with not enough arguments - # (4 bytes missing) or wrong calling convention - self.assertRaises(ValueError, IsWindow, None) + IsWindow = cdll.user32.IsWindow -@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') -class FunctionCallTestCase(unittest.TestCase): - @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") - @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), - "SEH not enabled in debug builds") - def test_SEH(self): - # Call functions with invalid arguments, and make sure - # that access violations are trapped and raise an - # exception. - self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32) + # ValueError: Procedure called with not enough arguments (4 bytes missing) + # or wrong calling convention + self.assertRaises(ValueError, IsWindow, None) - def test_noargs(self): - # This is a special case on win32 x64 - windll.user32.GetDesktopWindow() +if sys.platform == "win32": + class FunctionCallTestCase(unittest.TestCase): -@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') -class TestWintypes(unittest.TestCase): - def test_HWND(self): - from ctypes import wintypes - self.assertEqual(sizeof(wintypes.HWND), sizeof(c_void_p)) + if is_resource_enabled("SEH"): + def test_SEH(self): + # Call functions with invalid arguments, and make sure + # that access violations are trapped and raise an + # exception. + self.assertRaises(WindowsError, windll.kernel32.GetModuleHandleA, 32) - def test_PARAM(self): - from ctypes import wintypes - self.assertEqual(sizeof(wintypes.WPARAM), - sizeof(c_void_p)) - self.assertEqual(sizeof(wintypes.LPARAM), - sizeof(c_void_p)) + def test_noargs(self): + # This is a special case on win32 x64 + windll.user32.GetDesktopWindow() - def test_COMError(self): - from _ctypes import COMError - if support.HAVE_DOCSTRINGS: - self.assertEqual(COMError.__doc__, - "Raised when a COM method call failed.") + class TestWintypes(unittest.TestCase): + def test_HWND(self): + from ctypes import wintypes + self.assertEqual(sizeof(wintypes.HWND), sizeof(c_void_p)) - ex = COMError(-1, "text", ("details",)) - self.assertEqual(ex.hresult, -1) - self.assertEqual(ex.text, "text") - self.assertEqual(ex.details, ("details",)) + def test_PARAM(self): + from ctypes import wintypes + self.assertEqual(sizeof(wintypes.WPARAM), + sizeof(c_void_p)) + self.assertEqual(sizeof(wintypes.LPARAM), + sizeof(c_void_p)) -@unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') -class TestWinError(unittest.TestCase): - def test_winerror(self): - # see Issue 16169 - import errno - ERROR_INVALID_PARAMETER = 87 - msg = FormatError(ERROR_INVALID_PARAMETER).strip() - args = (errno.EINVAL, msg, None, ERROR_INVALID_PARAMETER) + def test_COMError(self): + from _ctypes import COMError + self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") - e = WinError(ERROR_INVALID_PARAMETER) - self.assertEqual(e.args, args) - self.assertEqual(e.errno, errno.EINVAL) - self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER) - - windll.kernel32.SetLastError(ERROR_INVALID_PARAMETER) - try: - raise WinError() - except OSError as exc: - e = exc - self.assertEqual(e.args, args) - self.assertEqual(e.errno, errno.EINVAL) - self.assertEqual(e.winerror, ERROR_INVALID_PARAMETER) + ex = COMError(-1, "text", ("details",)) + self.assertEqual(ex.hresult, -1) + self.assertEqual(ex.text, "text") + self.assertEqual(ex.details, ("details",)) class Structures(unittest.TestCase): + def test_struct_by_value(self): class POINT(Structure): _fields_ = [("x", c_long), @@ -111,33 +82,9 @@ dll = CDLL(_ctypes_test.__file__) - pt = POINT(15, 25) - left = c_long.in_dll(dll, 'left') - top = c_long.in_dll(dll, 'top') - right = c_long.in_dll(dll, 'right') - bottom = c_long.in_dll(dll, 'bottom') - rect = RECT(left, top, right, bottom) - PointInRect = dll.PointInRect - PointInRect.argtypes = [POINTER(RECT), POINT] - self.assertEqual(1, PointInRect(byref(rect), pt)) - - ReturnRect = dll.ReturnRect - ReturnRect.argtypes = [c_int, RECT, POINTER(RECT), POINT, RECT, - POINTER(RECT), POINT, RECT] - ReturnRect.restype = RECT - for i in range(4): - ret = ReturnRect(i, rect, pointer(rect), pt, rect, - byref(rect), pt, rect) - # the c function will check and modify ret if something is - # passed in improperly - self.assertEqual(ret.left, left.value) - self.assertEqual(ret.right, right.value) - self.assertEqual(ret.top, top.value) - self.assertEqual(ret.bottom, bottom.value) - - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[RECT] + pt = POINT(10, 10) + rect = RECT(0, 0, 20, 20) + self.assertEqual(1, dll.PointInRect(byref(rect), pt)) if __name__ == '__main__': unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/test/test_wintypes.py --- a/Lib/ctypes/test/test_wintypes.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,41 +0,0 @@ -import sys -import unittest - -from ctypes import * - -@unittest.skipUnless(sys.platform.startswith('win'), 'Windows-only test') -class WinTypesTest(unittest.TestCase): - def test_variant_bool(self): - from ctypes import wintypes - # reads 16-bits from memory, anything non-zero is True - for true_value in (1, 32767, 32768, 65535, 65537): - true = POINTER(c_int16)(c_int16(true_value)) - value = cast(true, POINTER(wintypes.VARIANT_BOOL)) - self.assertEqual(repr(value.contents), 'VARIANT_BOOL(True)') - - vb = wintypes.VARIANT_BOOL() - self.assertIs(vb.value, False) - vb.value = True - self.assertIs(vb.value, True) - vb.value = true_value - self.assertIs(vb.value, True) - - for false_value in (0, 65536, 262144, 2**33): - false = POINTER(c_int16)(c_int16(false_value)) - value = cast(false, POINTER(wintypes.VARIANT_BOOL)) - self.assertEqual(repr(value.contents), 'VARIANT_BOOL(False)') - - # allow any bool conversion on assignment to value - for set_value in (65536, 262144, 2**33): - vb = wintypes.VARIANT_BOOL() - vb.value = set_value - self.assertIs(vb.value, True) - - vb = wintypes.VARIANT_BOOL() - vb.value = [2, 3] - self.assertIs(vb.value, True) - vb.value = [] - self.assertIs(vb.value, False) - -if __name__ == "__main__": - unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ctypes/util.py --- a/Lib/ctypes/util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ctypes/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -19,8 +19,6 @@ i = i + len(prefix) s, rest = sys.version[i:].split(" ", 1) majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - majorVersion += 1 minorVersion = int(s[2:3]) / 10.0 # I don't think paths are affected by minor version in version 6 if majorVersion == 6: @@ -38,16 +36,12 @@ return None if version <= 6: clibname = 'msvcrt' - elif version <= 13: + else: clibname = 'msvcr%d' % (version * 10) - else: - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - return None # If python was built with in debug mode - import importlib.machinery - if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES: + import imp + if imp.get_suffixes()[0][0] == '_d.pyd': clibname += 'd' return clibname+'.dll' @@ -91,14 +85,14 @@ elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump - import re, tempfile + import re, tempfile, errno def _findLib_gcc(name): expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name) fdout, ccout = tempfile.mkstemp() os.close(fdout) cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \ - 'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name try: f = os.popen(cmd) try: @@ -108,8 +102,9 @@ finally: try: os.unlink(ccout) - except FileNotFoundError: - pass + except OSError as e: + if e.errno != errno.ENOENT: + raise if rv == 10: raise OSError('gcc or cc command not found') res = re.search(expr, trace) @@ -138,10 +133,8 @@ cmd = 'if ! type objdump >/dev/null 2>&1; then exit 10; fi;' \ "objdump -p -j .dynamic 2>/dev/null " + f f = os.popen(cmd) - try: - dump = f.read() - finally: - rv = f.close() + dump = f.read() + rv = f.close() if rv == 10: raise OSError('objdump command not found') res = re.search(r'\sSONAME\s+([^\s]+)', dump) @@ -173,44 +166,14 @@ res.sort(key=_num_version) return res[-1] - elif sys.platform == "sunos5": - - def _findLib_crle(name, is64): - if not os.path.exists('/usr/bin/crle'): - return None - - if is64: - cmd = 'env LC_ALL=C /usr/bin/crle -64 2>/dev/null' - else: - cmd = 'env LC_ALL=C /usr/bin/crle 2>/dev/null' - - with contextlib.closing(os.popen(cmd)) as f: - for line in f.readlines(): - line = line.strip() - if line.startswith('Default Library Path (ELF):'): - paths = line.split()[4] - - if not paths: - return None - - for dir in paths.split(":"): - libfile = os.path.join(dir, "lib%s.so" % name) - if os.path.exists(libfile): - return libfile - - return None - - def find_library(name, is64 = False): - return _get_soname(_findLib_crle(name, is64) or _findLib_gcc(name)) - else: def _findSoname_ldconfig(name): import struct if struct.calcsize('l') == 4: - machine = os.uname().machine + '-32' + machine = os.uname()[4] + '-32' else: - machine = os.uname().machine + '-64' + machine = os.uname()[4] + '-64' mach_map = { 'x86_64-64': 'libc6,x86-64', 'ppc64-64': 'libc6,64bit', diff -r 6db40a9955dc -r 0d413f60cc23 Lib/curses/__init__.py --- a/Lib/curses/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/curses/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,7 @@ import curses from curses import textpad - curses.initscr() + curses.initwin() ... """ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/datetime.py --- a/Lib/datetime.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/datetime.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,19 @@ -"""Concrete date/time and related types. +"""Concrete date/time and related types -- prototype implemented in Python. -See http://www.iana.org/time-zones/repository/tz-link.html for -time zone and DST data sources. +See http://www.zope.org/Members/fdrake/DateTimeWiki/FrontPage + +See also http://dir.yahoo.com/Reference/calendars/ + +For a primer on DST, including many current DST rules, see +http://webexhibits.org/daylightsaving/ + +For more about DST than you ever wanted to know, see +ftp://elsie.nci.nih.gov/pub/ + +Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm + +This was originally copied from the sandbox of the CPython CVS repository. +Thanks to Tim Peters for suggesting using it. """ import time as _time @@ -12,7 +24,7 @@ MINYEAR = 1 MAXYEAR = 9999 -_MAXORDINAL = 3652059 # date.max.toordinal() +_MAXORDINAL = 3652059 # date.max.toordinal() # Utility functions, adapted from Python's Demo/classes/Dates.py, which # also assumes the current Gregorian calendar indefinitely extended in @@ -23,10 +35,9 @@ # for all computations. See the book for algorithms for converting between # proleptic Gregorian ordinals and many other calendar systems. -# -1 is a placeholder for indexing purposes. -_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] +_DAYS_IN_MONTH = [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] -_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes. +_DAYS_BEFORE_MONTH = [None] dbm = 0 for dim in _DAYS_IN_MONTH[1:]: _DAYS_BEFORE_MONTH.append(dbm) @@ -50,7 +61,7 @@ return _DAYS_IN_MONTH[month] def _days_before_month(year, month): - "year, month -> number of days in year preceding first day of month." + "year, month -> number of days in year preceeding first day of month." assert 1 <= month <= 12, 'month must be in 1..12' return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year)) @@ -162,9 +173,9 @@ # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. - freplace = None # the string to use for %f - zreplace = None # the string to use for %z - Zreplace = None # the string to use for %Z + freplace = None # the string to use for %f + zreplace = None # the string to use for %z + Zreplace = None # the string to use for %Z # Scan format for %z and %Z escapes, replacing as needed. newformat = [] @@ -217,6 +228,11 @@ newformat = "".join(newformat) return _time.strftime(newformat, timetuple) +def _call_tzinfo_method(tzinfo, methname, tzinfoarg): + if tzinfo is None: + return None + return getattr(tzinfo, methname)(tzinfoarg) + # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): if name is not None and not isinstance(name, str): @@ -240,31 +256,13 @@ raise ValueError("tzinfo.%s() must return a whole number " "of minutes, got %s" % (name, offset)) if not -timedelta(1) < offset < timedelta(1): - raise ValueError("%s()=%s, must be must be strictly between " - "-timedelta(hours=24) and timedelta(hours=24)" % - (name, offset)) - -def _check_int_field(value): - if isinstance(value, int): - return value - if not isinstance(value, float): - try: - value = value.__int__() - except AttributeError: - pass - else: - if isinstance(value, int): - return value - raise TypeError('__int__ returned non-int (type %s)' % - type(value).__name__) - raise TypeError('an integer is required (got type %s)' % - type(value).__name__) - raise TypeError('integer argument expected, got float') + raise ValueError("%s()=%s, must be must be strictly between" + " -timedelta(hours=24) and timedelta(hours=24)" + % (name, offset)) def _check_date_fields(year, month, day): - year = _check_int_field(year) - month = _check_int_field(month) - day = _check_int_field(day) + if not isinstance(year, int): + raise TypeError('int expected') if not MINYEAR <= year <= MAXYEAR: raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year) if not 1 <= month <= 12: @@ -272,13 +270,10 @@ dim = _days_in_month(year, month) if not 1 <= day <= dim: raise ValueError('day must be in 1..%d' % dim, day) - return year, month, day def _check_time_fields(hour, minute, second, microsecond): - hour = _check_int_field(hour) - minute = _check_int_field(minute) - second = _check_int_field(second) - microsecond = _check_int_field(microsecond) + if not isinstance(hour, int): + raise TypeError('int expected') if not 0 <= hour <= 23: raise ValueError('hour must be in 0..23', hour) if not 0 <= minute <= 59: @@ -287,7 +282,6 @@ raise ValueError('second must be in 0..59', second) if not 0 <= microsecond <= 999999: raise ValueError('microsecond must be in 0..999999', microsecond) - return hour, minute, second, microsecond def _check_tzinfo_arg(tz): if tz is not None and not isinstance(tz, tzinfo): @@ -297,26 +291,6 @@ raise TypeError("can't compare '%s' to '%s'" % ( type(x).__name__, type(y).__name__)) -def _divide_and_round(a, b): - """divide a by b and round result to the nearest integer - - When the ratio is exactly half-way between two integers, - the even integer is returned. - """ - # Based on the reference implementation for divmod_near - # in Objects/longobject.c. - q, r = divmod(a, b) - # round up if either r / b > 0.5, or r / b == 0.5 and q is odd. - # The expression r / b > 0.5 is equivalent to 2 * r > b if b is - # positive, 2 * r < b if b negative. - r *= 2 - greater_than_half = r > b if b > 0 else r < b - if greater_than_half or r == b and q % 2 == 1: - q += 1 - - return q - - class timedelta: """Represent the difference between two datetime objects. @@ -325,7 +299,7 @@ - add, subtract timedelta - unary plus, minus, abs - compare to timedelta - - multiply, divide by int + - multiply, divide by int/long In addition, datetime supports subtraction of two datetime objects returning a timedelta, and addition or subtraction of a datetime @@ -334,7 +308,7 @@ Representation: (days, seconds, microseconds). Why? Because I felt like it. """ - __slots__ = '_days', '_seconds', '_microseconds', '_hashcode' + __slots__ = '_days', '_seconds', '_microseconds' def __new__(cls, days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0): @@ -400,26 +374,38 @@ # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = round(microseconds + usdouble) + microseconds += usdouble + microseconds = round(microseconds, 0) + seconds, microseconds = divmod(microseconds, 1e6) + assert microseconds == int(microseconds) + assert seconds == int(seconds) + days, seconds = divmod(seconds, 24.*3600.) + assert days == int(days) + assert seconds == int(seconds) + d += int(days) + s += int(seconds) # can't overflow + assert isinstance(s, int) + assert abs(s) <= 3 * 24 * 3600 + else: seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days - s += seconds - else: - microseconds = int(microseconds) - seconds, microseconds = divmod(microseconds, 1000000) - days, seconds = divmod(seconds, 24*3600) - d += days - s += seconds - microseconds = round(microseconds + usdouble) - assert isinstance(s, int) - assert isinstance(microseconds, int) + s += int(seconds) # can't overflow + assert isinstance(s, int) + assert abs(s) <= 3 * 24 * 3600 + microseconds = float(microseconds) + microseconds += usdouble + microseconds = round(microseconds, 0) assert abs(s) <= 3 * 24 * 3600 assert abs(microseconds) < 3.1e6 # Just a little bit of carrying possible for microseconds and seconds. - seconds, us = divmod(microseconds, 1000000) - s += seconds + assert isinstance(microseconds, float) + assert int(microseconds) == microseconds + us = int(microseconds) + seconds, us = divmod(us, 1000000) + s += seconds # cant't overflow + assert isinstance(s, int) days, s = divmod(s, 24*3600) d += days @@ -427,31 +413,27 @@ assert isinstance(s, int) and 0 <= s < 24*3600 assert isinstance(us, int) and 0 <= us < 1000000 + self = object.__new__(cls) + + self._days = d + self._seconds = s + self._microseconds = us if abs(d) > 999999999: raise OverflowError("timedelta # of days is too large: %d" % d) - self = object.__new__(cls) - self._days = d - self._seconds = s - self._microseconds = us - self._hashcode = -1 return self def __repr__(self): if self._microseconds: - return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._days, - self._seconds, - self._microseconds) + return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, + self._days, + self._seconds, + self._microseconds) if self._seconds: - return "%s.%s(%d, %d)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._days, - self._seconds) - return "%s.%s(%d)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._days) + return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__, + self._days, + self._seconds) + return "%s(%d)" % ('datetime.' + self.__class__.__name__, self._days) def __str__(self): mm, ss = divmod(self._seconds, 60) @@ -467,7 +449,7 @@ def total_seconds(self): """Total seconds in the duration.""" - return ((self.days * 86400 + self.seconds) * 10**6 + + return ((self.days * 86400 + self.seconds)*10**6 + self.microseconds) / 10**6 # Read-only field accessors @@ -535,9 +517,8 @@ self._seconds * other, self._microseconds * other) if isinstance(other, float): - usec = self._to_microseconds() a, b = other.as_integer_ratio() - return timedelta(0, 0, _divide_and_round(usec * a, b)) + return self * a / b return NotImplemented __rmul__ = __mul__ @@ -562,10 +543,10 @@ if isinstance(other, timedelta): return usec / other._to_microseconds() if isinstance(other, int): - return timedelta(0, 0, _divide_and_round(usec, other)) + return timedelta(0, 0, usec / other) if isinstance(other, float): a, b = other.as_integer_ratio() - return timedelta(0, 0, _divide_and_round(b * usec, a)) + return timedelta(0, 0, b * usec / a) def __mod__(self, other): if isinstance(other, timedelta): @@ -588,6 +569,12 @@ else: return False + def __ne__(self, other): + if isinstance(other, timedelta): + return self._cmp(other) != 0 + else: + return True + def __le__(self, other): if isinstance(other, timedelta): return self._cmp(other) <= 0 @@ -617,9 +604,7 @@ return _cmp(self._getstate(), other._getstate()) def __hash__(self): - if self._hashcode == -1: - self._hashcode = hash(self._getstate()) - return self._hashcode + return hash(self._getstate()) def __bool__(self): return (self._days != 0 or @@ -652,7 +637,7 @@ Operators: __repr__, __str__ - __eq__, __le__, __lt__, __ge__, __gt__, __hash__ + __cmp__, __hash__ __add__, __radd__, __sub__ (add/radd only with timedelta arg) Methods: @@ -667,7 +652,7 @@ Properties (readonly): year, month, day """ - __slots__ = '_year', '_month', '_day', '_hashcode' + __slots__ = '_year', '_month', '_day' def __new__(cls, year, month=None, day=None): """Constructor. @@ -676,19 +661,17 @@ year, month, day (required, base 1) """ - if month is None and isinstance(year, bytes) and len(year) == 4 and \ - 1 <= year[2] <= 12: + if (isinstance(year, bytes) and len(year) == 4 and + 1 <= year[2] <= 12 and month is None): # Month is sane # Pickle support self = object.__new__(cls) self.__setstate(year) - self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) + _check_date_fields(year, month, day) self = object.__new__(cls) self._year = year self._month = month self._day = day - self._hashcode = -1 return self # Additional constructors @@ -728,11 +711,10 @@ >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' """ - return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._year, - self._month, - self._day) + return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__, + self._year, + self._month, + self._day) # XXX These shouldn't depend on time.localtime(), because that # clips the usable dates to [1970 .. 2038). At least ctime() is # easily done without using strftime() -- that's better too because @@ -752,8 +734,6 @@ return _wrap_strftime(self, fmt, self.timetuple()) def __format__(self, fmt): - if not isinstance(fmt, str): - raise TypeError("must be str, not %s" % type(fmt).__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) @@ -787,8 +767,7 @@ """day (1-31)""" return self._day - # Standard conversions, __eq__, __le__, __lt__, __ge__, __gt__, - # __hash__ (and helpers) + # Standard conversions, __cmp__, __hash__ (and helpers) def timetuple(self): "Return local time tuple compatible with time.localtime()." @@ -811,6 +790,7 @@ month = self._month if day is None: day = self._day + _check_date_fields(year, month, day) return date(year, month, day) # Comparisons of date objects with other. @@ -820,6 +800,11 @@ return self._cmp(other) == 0 return NotImplemented + def __ne__(self, other): + if isinstance(other, date): + return self._cmp(other) != 0 + return NotImplemented + def __le__(self, other): if isinstance(other, date): return self._cmp(other) <= 0 @@ -848,9 +833,7 @@ def __hash__(self): "Hash." - if self._hashcode == -1: - self._hashcode = hash(self._getstate()) - return self._hashcode + return hash(self._getstate()) # Computations @@ -897,7 +880,6 @@ ISO calendar algorithm taken from http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm - (used with permission) """ year = self._year week1monday = _isoweek1monday(year) @@ -921,6 +903,8 @@ return bytes([yhi, ylo, self._month, self._day]), def __setstate(self, string): + if len(string) != 4 or not (1 <= string[2] <= 12): + raise TypeError("not enough arguments") yhi, ylo, self._month, self._day = string self._year = yhi * 256 + ylo @@ -939,7 +923,6 @@ Subclasses must override the name(), utcoffset() and dst() methods. """ __slots__ = () - def tzname(self, dt): "datetime -> string name of time zone." raise NotImplementedError("tzinfo subclass must override tzname()") @@ -1013,7 +996,7 @@ Operators: __repr__, __str__ - __eq__, __le__, __lt__, __ge__, __gt__, __hash__ + __cmp__, __hash__ Methods: @@ -1026,7 +1009,6 @@ Properties (readonly): hour, minute, second, microsecond, tzinfo """ - __slots__ = '_hour', '_minute', '_second', '_microsecond', '_tzinfo', '_hashcode' def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): """Constructor. @@ -1037,22 +1019,18 @@ second, microsecond (default to zero) tzinfo (default to None) """ - if isinstance(hour, bytes) and len(hour) == 6 and hour[0] < 24: + self = object.__new__(cls) + if isinstance(hour, bytes) and len(hour) == 6: # Pickle support - self = object.__new__(cls) self.__setstate(hour, minute or None) - self._hashcode = -1 return self - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + _check_time_fields(hour, minute, second, microsecond) self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo - self._hashcode = -1 return self # Read-only field accessors @@ -1087,10 +1065,16 @@ def __eq__(self, other): if isinstance(other, time): - return self._cmp(other, allow_mixed=True) == 0 + return self._cmp(other) == 0 else: return False + def __ne__(self, other): + if isinstance(other, time): + return self._cmp(other) != 0 + else: + return True + def __le__(self, other): if isinstance(other, time): return self._cmp(other) <= 0 @@ -1115,7 +1099,7 @@ else: _cmperror(self, other) - def _cmp(self, other, allow_mixed=False): + def _cmp(self, other): assert isinstance(other, time) mytz = self._tzinfo ottz = other._tzinfo @@ -1131,13 +1115,10 @@ if base_compare: return _cmp((self._hour, self._minute, self._second, self._microsecond), - (other._hour, other._minute, other._second, - other._microsecond)) + (other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: - if allow_mixed: - return 2 # arbitrary non-zero value - else: - raise TypeError("cannot compare naive and aware times") + raise TypeError("cannot compare naive and aware times") myhhmm = self._hour * 60 + self._minute - myoff//timedelta(minutes=1) othhmm = other._hour * 60 + other._minute - otoff//timedelta(minutes=1) return _cmp((myhhmm, self._second, self._microsecond), @@ -1145,20 +1126,16 @@ def __hash__(self): """Hash.""" - if self._hashcode == -1: - tzoff = self.utcoffset() - if not tzoff: # zero or None - self._hashcode = hash(self._getstate()[0]) - else: - h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff, - timedelta(hours=1)) - assert not m % timedelta(minutes=1), "whole minute" - m //= timedelta(minutes=1) - if 0 <= h < 24: - self._hashcode = hash(time(h, m, self.second, self.microsecond)) - else: - self._hashcode = hash((h, m, self.second, self.microsecond)) - return self._hashcode + tzoff = self.utcoffset() + if not tzoff: # zero or None + return hash(self._getstate()[0]) + h, m = divmod(timedelta(hours=self.hour, minutes=self.minute) - tzoff, + timedelta(hours=1)) + assert not m % timedelta(minutes=1), "whole minute" + m //= timedelta(minutes=1) + if 0 <= h < 24: + return hash(time(h, m, self.second, self.microsecond)) + return hash((h, m, self.second, self.microsecond)) # Conversion to string @@ -1186,9 +1163,8 @@ s = ", %d" % self._second else: s = "" - s= "%s.%s(%d, %d%s)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._hour, self._minute, s) + s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__, + self._hour, self._minute, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1221,8 +1197,6 @@ return _wrap_strftime(self, fmt, timetuple) def __format__(self, fmt): - if not isinstance(fmt, str): - raise TypeError("must be str, not %s" % type(fmt).__name__) if len(fmt) != 0: return self.strftime(fmt) return str(self) @@ -1279,8 +1253,16 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo + _check_time_fields(hour, minute, second, microsecond) + _check_tzinfo_arg(tzinfo) return time(hour, minute, second, microsecond, tzinfo) + def __bool__(self): + if self.second or self.microsecond: + return True + offset = self.utcoffset() or timedelta(0) + return timedelta(hours=self.hour, minutes=self.minute) != offset + # Pickle support. def _getstate(self): @@ -1294,11 +1276,15 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): - raise TypeError("bad tzinfo state arg") - self._hour, self._minute, self._second, us1, us2, us3 = string + if len(string) != 6 or string[0] >= 24: + raise TypeError("an integer is required") + (self._hour, self._minute, self._second, + us1, us2, us3) = string self._microsecond = (((us1 << 8) | us2) << 8) | us3 - self._tzinfo = tzinfo + if tzinfo is None or isinstance(tzinfo, _tzinfo_class): + self._tzinfo = tzinfo + else: + raise TypeError("bad tzinfo state arg %r" % tzinfo) def __reduce__(self): return (time, self._getstate()) @@ -1313,32 +1299,27 @@ """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) The year, month and day arguments are required. tzinfo may be None, or an - instance of a tzinfo subclass. The remaining arguments may be ints. + instance of a tzinfo subclass. The remaining arguments may be ints or longs. """ - __slots__ = date.__slots__ + time.__slots__ + __slots__ = date.__slots__ + ( + '_hour', '_minute', '_second', + '_microsecond', '_tzinfo') def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0, microsecond=0, tzinfo=None): - if isinstance(year, bytes) and len(year) == 10 and 1 <= year[2] <= 12: + if isinstance(year, bytes) and len(year) == 10: # Pickle support - self = object.__new__(cls) + self = date.__new__(cls, year[:4]) self.__setstate(year, month) - self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) - self._year = year - self._month = month - self._day = day + _check_time_fields(hour, minute, second, microsecond) + self = date.__new__(cls, year, month, day) self._hour = hour self._minute = minute self._second = second self._microsecond = microsecond self._tzinfo = tzinfo - self._hashcode = -1 return self # Read-only field accessors @@ -1368,43 +1349,55 @@ return self._tzinfo @classmethod - def _fromtimestamp(cls, t, utc, tz): - """Construct a datetime from a POSIX timestamp (like time.time()). - - A timezone info object may be passed in as well. - """ - frac, t = _math.modf(t) - us = round(frac * 1e6) - if us >= 1000000: - t += 1 - us -= 1000000 - elif us < 0: - t -= 1 - us += 1000000 - - converter = _time.gmtime if utc else _time.localtime - y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tz) - - @classmethod def fromtimestamp(cls, t, tz=None): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ + _check_tzinfo_arg(tz) - result = cls._fromtimestamp(t, tz is not None, tz) + converter = _time.localtime if tz is None else _time.gmtime + + t, frac = divmod(t, 1.0) + us = int(frac * 1e6) + + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 + y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) + ss = min(ss, 59) # clamp out leap seconds if the platform has them + result = cls(y, m, d, hh, mm, ss, us, tz) if tz is not None: result = tz.fromutc(result) return result @classmethod def utcfromtimestamp(cls, t): - """Construct a naive UTC datetime from a POSIX timestamp.""" - return cls._fromtimestamp(t, True, None) + "Construct a UTC datetime from a POSIX timestamp (like time.time())." + t, frac = divmod(t, 1.0) + us = int(frac * 1e6) + # If timestamp is less than one microsecond smaller than a + # full second, us can be rounded up to 1000000. In this case, + # roll over to seconds, otherwise, ValueError is raised + # by the constructor. + if us == 1000000: + t += 1 + us = 0 + y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) + ss = min(ss, 59) # clamp out leap seconds if the platform has them + return cls(y, m, d, hh, mm, ss, us) + + # XXX This is supposed to do better than we *can* do by using time.time(), + # XXX if the platform supports a more accurate way. The C implementation + # XXX uses gettimeofday on platforms that have it, but that isn't + # XXX available from Python. So now() may return different results + # XXX across the implementations. @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." @@ -1441,15 +1434,6 @@ self.hour, self.minute, self.second, dst) - def timestamp(self): - "Return POSIX timestamp as float" - if self._tzinfo is None: - return _time.mktime((self.year, self.month, self.day, - self.hour, self.minute, self.second, - -1, -1, -1)) + self.microsecond / 1e6 - else: - return (self - _EPOCH).total_seconds() - def utctimetuple(self): "Return UTC time tuple compatible with time.gmtime()." offset = self.utcoffset() @@ -1491,35 +1475,14 @@ microsecond = self.microsecond if tzinfo is True: tzinfo = self.tzinfo - return datetime(year, month, day, hour, minute, second, microsecond, - tzinfo) + _check_date_fields(year, month, day) + _check_time_fields(hour, minute, second, microsecond) + _check_tzinfo_arg(tzinfo) + return datetime(year, month, day, hour, minute, second, + microsecond, tzinfo) - def astimezone(self, tz=None): - if tz is None: - if self.tzinfo is None: - raise ValueError("astimezone() requires an aware datetime") - ts = (self - _EPOCH) // timedelta(seconds=1) - localtm = _time.localtime(ts) - local = datetime(*localtm[:6]) - try: - # Extract TZ data if available - gmtoff = localtm.tm_gmtoff - zone = localtm.tm_zone - except AttributeError: - # Compute UTC offset and compare with the value implied - # by tm_isdst. If the values match, use the zone name - # implied by tm_isdst. - delta = local - datetime(*_time.gmtime(ts)[:6]) - dst = _time.daylight and localtm.tm_isdst > 0 - gmtoff = -(_time.altzone if dst else _time.timezone) - if delta == timedelta(seconds=gmtoff): - tz = timezone(delta, _time.tzname[dst]) - else: - tz = timezone(delta) - else: - tz = timezone(timedelta(seconds=gmtoff), zone) - - elif not isinstance(tz, tzinfo): + def astimezone(self, tz): + if not isinstance(tz, tzinfo): raise TypeError("tz argument must be an instance of tzinfo") mytz = self.tzinfo @@ -1562,9 +1525,10 @@ Optional argument sep specifies the separator between date and time, default 'T'. """ - s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + - _format_time(self._hour, self._minute, self._second, - self._microsecond)) + s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, + sep) + + _format_time(self._hour, self._minute, self._second, + self._microsecond)) off = self.utcoffset() if off is not None: if off.days < 0: @@ -1580,15 +1544,14 @@ def __repr__(self): """Convert to formal string, for repr().""" - L = [self._year, self._month, self._day, # These are never zero + L = [self._year, self._month, self._day, # These are never zero self._hour, self._minute, self._second, self._microsecond] if L[-1] == 0: del L[-1] if L[-1] == 0: del L[-1] - s = "%s.%s(%s)" % (self.__class__.__module__, - self.__class__.__qualname__, - ", ".join(map(str, L))) + s = ", ".join(map(str, L)) + s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s) if self._tzinfo is not None: assert s[-1:] == ")" s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" @@ -1620,9 +1583,7 @@ it mean anything in particular. For example, "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies. """ - if self._tzinfo is None: - return None - name = self._tzinfo.tzname(self) + name = _call_tzinfo_method(self._tzinfo, "tzname", self) _check_tzname(name) return name @@ -1645,12 +1606,20 @@ def __eq__(self, other): if isinstance(other, datetime): - return self._cmp(other, allow_mixed=True) == 0 + return self._cmp(other) == 0 elif not isinstance(other, date): return NotImplemented else: return False + def __ne__(self, other): + if isinstance(other, datetime): + return self._cmp(other) != 0 + elif not isinstance(other, date): + return NotImplemented + else: + return True + def __le__(self, other): if isinstance(other, datetime): return self._cmp(other) <= 0 @@ -1683,7 +1652,7 @@ else: _cmperror(self, other) - def _cmp(self, other, allow_mixed=False): + def _cmp(self, other): assert isinstance(other, datetime) mytz = self._tzinfo ottz = other._tzinfo @@ -1692,22 +1661,21 @@ if mytz is ottz: base_compare = True else: - myoff = self.utcoffset() - otoff = other.utcoffset() + if mytz is not None: + myoff = self.utcoffset() + if ottz is not None: + otoff = other.utcoffset() base_compare = myoff == otoff if base_compare: return _cmp((self._year, self._month, self._day, self._hour, self._minute, self._second, self._microsecond), - (other._year, other._month, other._day, - other._hour, other._minute, other._second, - other._microsecond)) + (other._year, other._month, other._day, + other._hour, other._minute, other._second, + other._microsecond)) if myoff is None or otoff is None: - if allow_mixed: - return 2 # arbitrary non-zero value - else: - raise TypeError("cannot compare naive and aware datetimes") + raise TypeError("cannot compare naive and aware datetimes") # XXX What follows could be done more efficiently... diff = self - other # this will take offsets into account if diff.days < 0: @@ -1760,15 +1728,12 @@ return base + otoff - myoff def __hash__(self): - if self._hashcode == -1: - tzoff = self.utcoffset() - if tzoff is None: - self._hashcode = hash(self._getstate()[0]) - else: - days = _ymd2ord(self.year, self.month, self.day) - seconds = self.hour * 3600 + self.minute * 60 + self.second - self._hashcode = hash(timedelta(days, seconds, self.microsecond) - tzoff) - return self._hashcode + tzoff = self.utcoffset() + if tzoff is None: + return hash(self._getstate()[0]) + days = _ymd2ord(self.year, self.month, self.day) + seconds = self.hour * 3600 + self.minute * 60 + self.second + return hash(timedelta(days, seconds, self.microsecond) - tzoff) # Pickle support. @@ -1785,13 +1750,14 @@ return (basestate, self._tzinfo) def __setstate(self, string, tzinfo): - if tzinfo is not None and not isinstance(tzinfo, _tzinfo_class): - raise TypeError("bad tzinfo state arg") (yhi, ylo, self._month, self._day, self._hour, self._minute, self._second, us1, us2, us3) = string self._year = yhi * 256 + ylo self._microsecond = (((us1 << 8) | us2) << 8) | us3 - self._tzinfo = tzinfo + if tzinfo is None or isinstance(tzinfo, _tzinfo_class): + self._tzinfo = tzinfo + else: + raise TypeError("bad tzinfo state arg %r" % tzinfo) def __reduce__(self): return (self.__class__, self._getstate()) @@ -1807,7 +1773,7 @@ # XXX This could be done more efficiently THURSDAY = 3 firstday = _ymd2ord(year, 1, 1) - firstweekday = (firstday + 6) % 7 # See weekday() above + firstweekday = (firstday + 6) % 7 # See weekday() above week1monday = firstday - firstweekday if firstweekday > THURSDAY: week1monday += 7 @@ -1828,12 +1794,13 @@ elif not isinstance(name, str): raise TypeError("name must be a string") if not cls._minoffset <= offset <= cls._maxoffset: - raise ValueError("offset must be a timedelta " - "strictly between -timedelta(hours=24) and " - "timedelta(hours=24).") - if (offset.microseconds != 0 or offset.seconds % 60 != 0): - raise ValueError("offset must be a timedelta " - "representing a whole number of minutes") + raise ValueError("offset must be a timedelta" + " strictly between -timedelta(hours=24) and" + " timedelta(hours=24).") + if (offset.microseconds != 0 or + offset.seconds % 60 != 0): + raise ValueError("offset must be a timedelta" + " representing a whole number of minutes") return cls._create(offset, name) @classmethod @@ -1850,8 +1817,6 @@ return (self._offset, self._name) def __eq__(self, other): - if type(other) != timezone: - return False return self._offset == other._offset def __hash__(self): @@ -1870,12 +1835,10 @@ if self is self.utc: return 'datetime.timezone.utc' if self._name is None: - return "%s.%s(%r)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._offset) - return "%s.%s(%r, %r)" % (self.__class__.__module__, - self.__class__.__qualname__, - self._offset, self._name) + return "%s(%r)" % ('datetime.' + self.__class__.__name__, + self._offset) + return "%s(%r, %r)" % ('datetime.' + self.__class__.__name__, + self._offset, self._name) def __str__(self): return self.tzname(None) @@ -1914,8 +1877,6 @@ @staticmethod def _name_from_offset(delta): - if not delta: - return 'UTC' if delta < timedelta(0): sign = '-' delta = -delta @@ -1928,217 +1889,218 @@ timezone.utc = timezone._create(timedelta(0)) timezone.min = timezone._create(timezone._minoffset) timezone.max = timezone._create(timezone._maxoffset) -_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) -# Some time zone algebra. For a datetime x, let -# x.n = x stripped of its timezone -- its naive time. -# x.o = x.utcoffset(), and assuming that doesn't raise an exception or -# return None -# x.d = x.dst(), and assuming that doesn't raise an exception or -# return None -# x.s = x's standard offset, x.o - x.d -# -# Now some derived rules, where k is a duration (timedelta). -# -# 1. x.o = x.s + x.d -# This follows from the definition of x.s. -# -# 2. If x and y have the same tzinfo member, x.s = y.s. -# This is actually a requirement, an assumption we need to make about -# sane tzinfo classes. -# -# 3. The naive UTC time corresponding to x is x.n - x.o. -# This is again a requirement for a sane tzinfo class. -# -# 4. (x+k).s = x.s -# This follows from #2, and that datimetimetz+timedelta preserves tzinfo. -# -# 5. (x+k).n = x.n + k -# Again follows from how arithmetic is defined. -# -# Now we can explain tz.fromutc(x). Let's assume it's an interesting case -# (meaning that the various tzinfo methods exist, and don't blow up or return -# None when called). -# -# The function wants to return a datetime y with timezone tz, equivalent to x. -# x is already in UTC. -# -# By #3, we want -# -# y.n - y.o = x.n [1] -# -# The algorithm starts by attaching tz to x.n, and calling that y. So -# x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] -# becomes true; in effect, we want to solve [2] for k: -# -# (y+k).n - (y+k).o = x.n [2] -# -# By #1, this is the same as -# -# (y+k).n - ((y+k).s + (y+k).d) = x.n [3] -# -# By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. -# Substituting that into [3], -# -# x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving -# k - (y+k).s - (y+k).d = 0; rearranging, -# k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so -# k = y.s - (y+k).d -# -# On the RHS, (y+k).d can't be computed directly, but y.s can be, and we -# approximate k by ignoring the (y+k).d term at first. Note that k can't be -# very large, since all offset-returning methods return a duration of magnitude -# less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must -# be 0, so ignoring it has no consequence then. -# -# In any case, the new value is -# -# z = y + y.s [4] -# -# It's helpful to step back at look at [4] from a higher level: it's simply -# mapping from UTC to tz's standard time. -# -# At this point, if -# -# z.n - z.o = x.n [5] -# -# we have an equivalent time, and are almost done. The insecurity here is -# at the start of daylight time. Picture US Eastern for concreteness. The wall -# time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good -# sense then. The docs ask that an Eastern tzinfo class consider such a time to -# be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST -# on the day DST starts. We want to return the 1:MM EST spelling because that's -# the only spelling that makes sense on the local wall clock. -# -# In fact, if [5] holds at this point, we do have the standard-time spelling, -# but that takes a bit of proof. We first prove a stronger result. What's the -# difference between the LHS and RHS of [5]? Let -# -# diff = x.n - (z.n - z.o) [6] -# -# Now -# z.n = by [4] -# (y + y.s).n = by #5 -# y.n + y.s = since y.n = x.n -# x.n + y.s = since z and y are have the same tzinfo member, -# y.s = z.s by #2 -# x.n + z.s -# -# Plugging that back into [6] gives -# -# diff = -# x.n - ((x.n + z.s) - z.o) = expanding -# x.n - x.n - z.s + z.o = cancelling -# - z.s + z.o = by #2 -# z.d -# -# So diff = z.d. -# -# If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time -# spelling we wanted in the endcase described above. We're done. Contrarily, -# if z.d = 0, then we have a UTC equivalent, and are also done. -# -# If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to -# add to z (in effect, z is in tz's standard time, and we need to shift the -# local clock into tz's daylight time). -# -# Let -# -# z' = z + z.d = z + diff [7] -# -# and we can again ask whether -# -# z'.n - z'.o = x.n [8] -# -# If so, we're done. If not, the tzinfo class is insane, according to the -# assumptions we've made. This also requires a bit of proof. As before, let's -# compute the difference between the LHS and RHS of [8] (and skipping some of -# the justifications for the kinds of substitutions we've done several times -# already): -# -# diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] -# x.n - (z.n + diff - z'.o) = replacing diff via [6] -# x.n - (z.n + x.n - (z.n - z.o) - z'.o) = -# x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n -# - z.n + z.n - z.o + z'.o = cancel z.n -# - z.o + z'.o = #1 twice -# -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo -# z'.d - z.d -# -# So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, -# we've found the UTC-equivalent so are done. In fact, we stop with [7] and -# return z', not bothering to compute z'.d. -# -# How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by -# a dst() offset, and starting *from* a time already in DST (we know z.d != 0), -# would have to change the result dst() returns: we start in DST, and moving -# a little further into it takes us out of DST. -# -# There isn't a sane case where this can happen. The closest it gets is at -# the end of DST, where there's an hour in UTC with no spelling in a hybrid -# tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During -# that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM -# UTC) because the docs insist on that, but 0:MM is taken as being in daylight -# time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local -# clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in -# standard time. Since that's what the local clock *does*, we want to map both -# UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous -# in local time, but so it goes -- it's the way the local clock works. -# -# When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, -# so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. -# z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] -# (correctly) concludes that z' is not UTC-equivalent to x. -# -# Because we know z.d said z was in daylight time (else [5] would have held and -# we would have stopped then), and we know z.d != z'.d (else [8] would have held -# and we have stopped then), and there are only 2 possible values dst() can -# return in Eastern, it follows that z'.d must be 0 (which it is in the example, -# but the reasoning doesn't depend on the example -- it depends on there being -# two possible dst() outcomes, one zero and the other non-zero). Therefore -# z' must be in standard time, and is the spelling we want in this case. -# -# Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is -# concerned (because it takes z' as being in standard time rather than the -# daylight time we intend here), but returning it gives the real-life "local -# clock repeats an hour" behavior when mapping the "unspellable" UTC hour into -# tz. -# -# When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with -# the 1:MM standard time spelling we want. -# -# So how can this break? One of the assumptions must be violated. Two -# possibilities: -# -# 1) [2] effectively says that y.s is invariant across all y belong to a given -# time zone. This isn't true if, for political reasons or continental drift, -# a region decides to change its base offset from UTC. -# -# 2) There may be versions of "double daylight" time where the tail end of -# the analysis gives up a step too early. I haven't thought about that -# enough to say. -# -# In any case, it's clear that the default fromutc() is strong enough to handle -# "almost all" time zones: so long as the standard offset is invariant, it -# doesn't matter if daylight time transition points change from year to year, or -# if daylight time is skipped in some years; it doesn't matter how large or -# small dst() may get within its bounds; and it doesn't even matter if some -# perverse time zone returns a negative dst()). So a breaking case must be -# pretty bizarre, and a tzinfo subclass can override fromutc() if it is. +""" +Some time zone algebra. For a datetime x, let + x.n = x stripped of its timezone -- its naive time. + x.o = x.utcoffset(), and assuming that doesn't raise an exception or + return None + x.d = x.dst(), and assuming that doesn't raise an exception or + return None + x.s = x's standard offset, x.o - x.d +Now some derived rules, where k is a duration (timedelta). + +1. x.o = x.s + x.d + This follows from the definition of x.s. + +2. If x and y have the same tzinfo member, x.s = y.s. + This is actually a requirement, an assumption we need to make about + sane tzinfo classes. + +3. The naive UTC time corresponding to x is x.n - x.o. + This is again a requirement for a sane tzinfo class. + +4. (x+k).s = x.s + This follows from #2, and that datimetimetz+timedelta preserves tzinfo. + +5. (x+k).n = x.n + k + Again follows from how arithmetic is defined. + +Now we can explain tz.fromutc(x). Let's assume it's an interesting case +(meaning that the various tzinfo methods exist, and don't blow up or return +None when called). + +The function wants to return a datetime y with timezone tz, equivalent to x. +x is already in UTC. + +By #3, we want + + y.n - y.o = x.n [1] + +The algorithm starts by attaching tz to x.n, and calling that y. So +x.n = y.n at the start. Then it wants to add a duration k to y, so that [1] +becomes true; in effect, we want to solve [2] for k: + + (y+k).n - (y+k).o = x.n [2] + +By #1, this is the same as + + (y+k).n - ((y+k).s + (y+k).d) = x.n [3] + +By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start. +Substituting that into [3], + + x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving + k - (y+k).s - (y+k).d = 0; rearranging, + k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so + k = y.s - (y+k).d + +On the RHS, (y+k).d can't be computed directly, but y.s can be, and we +approximate k by ignoring the (y+k).d term at first. Note that k can't be +very large, since all offset-returning methods return a duration of magnitude +less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must +be 0, so ignoring it has no consequence then. + +In any case, the new value is + + z = y + y.s [4] + +It's helpful to step back at look at [4] from a higher level: it's simply +mapping from UTC to tz's standard time. + +At this point, if + + z.n - z.o = x.n [5] + +we have an equivalent time, and are almost done. The insecurity here is +at the start of daylight time. Picture US Eastern for concreteness. The wall +time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good +sense then. The docs ask that an Eastern tzinfo class consider such a time to +be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST +on the day DST starts. We want to return the 1:MM EST spelling because that's +the only spelling that makes sense on the local wall clock. + +In fact, if [5] holds at this point, we do have the standard-time spelling, +but that takes a bit of proof. We first prove a stronger result. What's the +difference between the LHS and RHS of [5]? Let + + diff = x.n - (z.n - z.o) [6] + +Now + z.n = by [4] + (y + y.s).n = by #5 + y.n + y.s = since y.n = x.n + x.n + y.s = since z and y are have the same tzinfo member, + y.s = z.s by #2 + x.n + z.s + +Plugging that back into [6] gives + + diff = + x.n - ((x.n + z.s) - z.o) = expanding + x.n - x.n - z.s + z.o = cancelling + - z.s + z.o = by #2 + z.d + +So diff = z.d. + +If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time +spelling we wanted in the endcase described above. We're done. Contrarily, +if z.d = 0, then we have a UTC equivalent, and are also done. + +If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to +add to z (in effect, z is in tz's standard time, and we need to shift the +local clock into tz's daylight time). + +Let + + z' = z + z.d = z + diff [7] + +and we can again ask whether + + z'.n - z'.o = x.n [8] + +If so, we're done. If not, the tzinfo class is insane, according to the +assumptions we've made. This also requires a bit of proof. As before, let's +compute the difference between the LHS and RHS of [8] (and skipping some of +the justifications for the kinds of substitutions we've done several times +already): + + diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7] + x.n - (z.n + diff - z'.o) = replacing diff via [6] + x.n - (z.n + x.n - (z.n - z.o) - z'.o) = + x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n + - z.n + z.n - z.o + z'.o = cancel z.n + - z.o + z'.o = #1 twice + -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo + z'.d - z.d + +So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal, +we've found the UTC-equivalent so are done. In fact, we stop with [7] and +return z', not bothering to compute z'.d. + +How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by +a dst() offset, and starting *from* a time already in DST (we know z.d != 0), +would have to change the result dst() returns: we start in DST, and moving +a little further into it takes us out of DST. + +There isn't a sane case where this can happen. The closest it gets is at +the end of DST, where there's an hour in UTC with no spelling in a hybrid +tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During +that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM +UTC) because the docs insist on that, but 0:MM is taken as being in daylight +time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local +clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in +standard time. Since that's what the local clock *does*, we want to map both +UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous +in local time, but so it goes -- it's the way the local clock works. + +When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0, +so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going. +z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8] +(correctly) concludes that z' is not UTC-equivalent to x. + +Because we know z.d said z was in daylight time (else [5] would have held and +we would have stopped then), and we know z.d != z'.d (else [8] would have held +and we have stopped then), and there are only 2 possible values dst() can +return in Eastern, it follows that z'.d must be 0 (which it is in the example, +but the reasoning doesn't depend on the example -- it depends on there being +two possible dst() outcomes, one zero and the other non-zero). Therefore +z' must be in standard time, and is the spelling we want in this case. + +Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is +concerned (because it takes z' as being in standard time rather than the +daylight time we intend here), but returning it gives the real-life "local +clock repeats an hour" behavior when mapping the "unspellable" UTC hour into +tz. + +When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with +the 1:MM standard time spelling we want. + +So how can this break? One of the assumptions must be violated. Two +possibilities: + +1) [2] effectively says that y.s is invariant across all y belong to a given + time zone. This isn't true if, for political reasons or continental drift, + a region decides to change its base offset from UTC. + +2) There may be versions of "double daylight" time where the tail end of + the analysis gives up a step too early. I haven't thought about that + enough to say. + +In any case, it's clear that the default fromutc() is strong enough to handle +"almost all" time zones: so long as the standard offset is invariant, it +doesn't matter if daylight time transition points change from year to year, or +if daylight time is skipped in some years; it doesn't matter how large or +small dst() may get within its bounds; and it doesn't even matter if some +perverse time zone returns a negative dst()). So a breaking case must be +pretty bizarre, and a tzinfo subclass can override fromutc() if it is. +""" try: from _datetime import * except ImportError: pass else: # Clean up unused names - del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, - _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, - _check_date_fields, _check_int_field, _check_time_fields, - _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, - _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _is_leap, _isoweek1monday, _math, _ord2ymd, - _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord) + del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, + _DI100Y, _DI400Y, _DI4Y, _MAXORDINAL, _MONTHNAMES, + _build_struct_time, _call_tzinfo_method, _check_date_fields, + _check_time_fields, _check_tzinfo_arg, _check_tzname, + _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month, + _days_before_year, _days_in_month, _format_time, _is_leap, + _isoweek1monday, _math, _ord2ymd, _time, _time_class, _tzinfo_class, + _wrap_strftime, _ymd2ord) # XXX Since import * above excludes names that start with _, # docstring does not get overwritten. In the future, it may be # appropriate to maintain a single module level docstring and diff -r 6db40a9955dc -r 0d413f60cc23 Lib/dbm/__init__.py --- a/Lib/dbm/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/dbm/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -42,12 +42,7 @@ _defaultmod = None _modules = {} -error = (error, OSError) - -try: - from dbm import ndbm -except ImportError: - ndbm = None +error = (error, IOError) def open(file, flag='r', mode=0o666): @@ -111,10 +106,12 @@ try: f = io.open(filename + ".pag", "rb") f.close() - f = io.open(filename + ".dir", "rb") - f.close() + # dbm linked with gdbm on OS/2 doesn't have .dir file + if not (ndbm.library == "GNU gdbm" and sys.platform == "os2emx"): + f = io.open(filename + ".dir", "rb") + f.close() return "dbm.ndbm" - except OSError: + except IOError: # some dbm emulations based on Berkeley DB generate a .db file # some do not, but they should be caught by the bsd checks try: @@ -127,7 +124,7 @@ d = ndbm.open(filename) d.close() return "dbm.ndbm" - except OSError: + except IOError: pass # Check for dumbdbm next -- this has a .dir and a .dat file @@ -144,18 +141,18 @@ return "dbm.dumb" finally: f.close() - except OSError: + except (OSError, IOError): pass # See if the file exists, return None if not try: f = io.open(filename, "rb") - except OSError: + except IOError: return None - with f: - # Read the start of the file -- the magic number - s16 = f.read(16) + # Read the start of the file -- the magic number + s16 = f.read(16) + f.close() s = s16[0:4] # Return "" if not at least 4 bytes diff -r 6db40a9955dc -r 0d413f60cc23 Lib/dbm/dumb.py --- a/Lib/dbm/dumb.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/dbm/dumb.py Mon Jan 25 17:05:13 2016 +0100 @@ -21,7 +21,6 @@ """ -import ast as _ast import io as _io import os as _os import collections @@ -30,7 +29,7 @@ _BLOCKSIZE = 512 -error = OSError +error = IOError class _Database(collections.MutableMapping): @@ -45,7 +44,7 @@ _os = _os # for _commit() _io = _io # for _commit() - def __init__(self, filebasename, mode, flag='c'): + def __init__(self, filebasename, mode): self._mode = mode # The directory file is a text file. Each line looks like @@ -65,40 +64,29 @@ # The index is an in-memory dict, mirroring the directory file. self._index = None # maps keys to (pos, siz) pairs - # Handle the creation - self._create(flag) - self._update() - - def _create(self, flag): - if flag == 'n': - for filename in (self._datfile, self._bakfile, self._dirfile): - try: - _os.remove(filename) - except OSError: - pass # Mod by Jack: create data file if needed try: f = _io.open(self._datfile, 'r', encoding="Latin-1") - except OSError: - with _io.open(self._datfile, 'w', encoding="Latin-1") as f: - self._chmod(self._datfile) - else: - f.close() + except IOError: + f = _io.open(self._datfile, 'w', encoding="Latin-1") + self._chmod(self._datfile) + f.close() + self._update() # Read directory file into the in-memory index dict. def _update(self): self._index = {} try: f = _io.open(self._dirfile, 'r', encoding="Latin-1") - except OSError: + except IOError: pass else: - with f: - for line in f: - line = line.rstrip() - key, pos_and_siz_pair = _ast.literal_eval(line) - key = key.encode('Latin-1') - self._index[key] = pos_and_siz_pair + for line in f: + line = line.rstrip() + key, pos_and_siz_pair = eval(line) + key = key.encode('Latin-1') + self._index[key] = pos_and_siz_pair + f.close() # Write the index dict to the directory file. The original directory # file (if any) is renamed with a .bak extension first. If a .bak @@ -112,36 +100,32 @@ try: self._os.unlink(self._bakfile) - except OSError: + except self._os.error: pass try: self._os.rename(self._dirfile, self._bakfile) - except OSError: + except self._os.error: pass - with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: - self._chmod(self._dirfile) - for key, pos_and_siz_pair in self._index.items(): - # Use Latin-1 since it has no qualms with any value in any - # position; UTF-8, though, does care sometimes. - entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) - f.write(entry) + f = self._io.open(self._dirfile, 'w', encoding="Latin-1") + self._chmod(self._dirfile) + for key, pos_and_siz_pair in self._index.items(): + # Use Latin-1 since it has no qualms with any value in any + # position; UTF-8, though, does care sometimes. + f.write("%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair)) + f.close() sync = _commit - def _verify_open(self): - if self._index is None: - raise error('DBM object has already been closed') - def __getitem__(self, key): if isinstance(key, str): key = key.encode('utf-8') - self._verify_open() pos, siz = self._index[key] # may raise KeyError - with _io.open(self._datfile, 'rb') as f: - f.seek(pos) - dat = f.read(siz) + f = _io.open(self._datfile, 'rb') + f.seek(pos) + dat = f.read(siz) + f.close() return dat # Append val to the data file, starting at a _BLOCKSIZE-aligned @@ -149,13 +133,14 @@ # to get to an aligned offset. Return pair # (starting offset of val, len(val)) def _addval(self, val): - with _io.open(self._datfile, 'rb+') as f: - f.seek(0, 2) - pos = int(f.tell()) - npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE - f.write(b'\0'*(npos-pos)) - pos = npos - f.write(val) + f = _io.open(self._datfile, 'rb+') + f.seek(0, 2) + pos = int(f.tell()) + npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE + f.write(b'\0'*(npos-pos)) + pos = npos + f.write(val) + f.close() return (pos, len(val)) # Write val to the data file, starting at offset pos. The caller @@ -163,9 +148,10 @@ # pos to hold val, without overwriting some other value. Return # pair (pos, len(val)). def _setval(self, pos, val): - with _io.open(self._datfile, 'rb+') as f: - f.seek(pos) - f.write(val) + f = _io.open(self._datfile, 'rb+') + f.seek(pos) + f.write(val) + f.close() return (pos, len(val)) # key is a new key whose associated value starts in the data file @@ -173,9 +159,10 @@ # the in-memory index dict, and append one to the directory file. def _addkey(self, key, pos_and_siz_pair): self._index[key] = pos_and_siz_pair - with _io.open(self._dirfile, 'a', encoding="Latin-1") as f: - self._chmod(self._dirfile) - f.write("%r, %r\n" % (key.decode("Latin-1"), pos_and_siz_pair)) + f = _io.open(self._dirfile, 'a', encoding="Latin-1") + self._chmod(self._dirfile) + f.write("%r, %r\n" % (key.decode("Latin-1"), pos_and_siz_pair)) + f.close() def __setitem__(self, key, val): if isinstance(key, str): @@ -186,7 +173,6 @@ val = val.encode('utf-8') elif not isinstance(val, (bytes, bytearray)): raise TypeError("values must be bytes or strings") - self._verify_open() if key not in self._index: self._addkey(key, self._addval(val)) else: @@ -214,7 +200,6 @@ def __delitem__(self, key): if isinstance(key, str): key = key.encode('utf-8') - self._verify_open() # The blocks used by the associated value are lost. del self._index[key] # XXX It's unclear why we do a _commit() here (the code always @@ -224,44 +209,26 @@ self._commit() def keys(self): - try: - return list(self._index) - except TypeError: - raise error('DBM object has already been closed') from None + return list(self._index.keys()) def items(self): - self._verify_open() return [(key, self[key]) for key in self._index.keys()] def __contains__(self, key): if isinstance(key, str): key = key.encode('utf-8') - try: - return key in self._index - except TypeError: - if self._index is None: - raise error('DBM object has already been closed') from None - else: - raise + return key in self._index def iterkeys(self): - try: - return iter(self._index) - except TypeError: - raise error('DBM object has already been closed') from None + return iter(self._index.keys()) __iter__ = iterkeys def __len__(self): - try: - return len(self._index) - except TypeError: - raise error('DBM object has already been closed') from None + return len(self._index) def close(self): - try: - self._commit() - finally: - self._index = self._datfile = self._dirfile = self._bakfile = None + self._commit() + self._index = self._datfile = self._dirfile = self._bakfile = None __del__ = close @@ -269,27 +236,21 @@ if hasattr(self._os, 'chmod'): self._os.chmod(file, self._mode) - def __enter__(self): - return self - def __exit__(self, *args): - self.close() - - -def open(file, flag='c', mode=0o666): +def open(file, flag=None, mode=0o666): """Open the database file, filename, and return corresponding object. The flag argument, used to control how the database is opened in the - other DBM implementations, supports only the semantics of 'c' and 'n' - values. Other values will default to the semantics of 'c' value: - the database will always opened for update and will be created if it - does not exist. + other DBM implementations, is ignored in the dbm.dumb module; the + database is always opened for update, and will be created if it does + not exist. The optional mode argument is the UNIX mode of the file, used only when the database has to be created. It defaults to octal code 0o666 (and will be modified by the prevailing umask). """ + # flag argument is currently ignored # Modify mode depending on the umask try: @@ -300,4 +261,5 @@ else: # Turn off any bits that are set in the umask mode = mode & (~um) - return _Database(file, mode, flag=flag) + + return _Database(file, mode) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/decimal.py --- a/Lib/decimal.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/decimal.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,6402 @@ +# Copyright (c) 2004 Python Software Foundation. +# All rights reserved. + +# Written by Eric Price +# and Facundo Batista +# and Raymond Hettinger +# and Aahz +# and Tim Peters + +# This module should be kept in sync with the latest updates of the +# IBM specification as it evolves. Those updates will be treated +# as bug fixes (deviation from the spec is a compatibility, usability +# bug) and will be backported. At this point the spec is stabilizing +# and the updates are becoming fewer, smaller, and less significant. + +""" +This is an implementation of decimal floating point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" + +__all__ = [ + # Two major classes + 'Decimal', 'Context', + + # Contexts + 'DefaultContext', 'BasicContext', 'ExtendedContext', + + # Exceptions + 'DecimalException', 'Clamped', 'InvalidOperation', 'DivisionByZero', + 'Inexact', 'Rounded', 'Subnormal', 'Overflow', 'Underflow', + 'FloatOperation', + + # Constants for use in setting up contexts + 'ROUND_DOWN', 'ROUND_HALF_UP', 'ROUND_HALF_EVEN', 'ROUND_CEILING', + 'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP', + + # Functions for manipulating contexts + 'setcontext', 'getcontext', 'localcontext', + + # Limits for the C version for compatibility + 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', + + # C version: compile time choice that enables the thread local context + 'HAVE_THREADS' +] + +__version__ = '1.70' # Highest version of the spec this complies with + # See http://speleotrove.com/decimal/ + +import copy as _copy +import math as _math +import numbers as _numbers +import sys try: + from collections import namedtuple as _namedtuple + DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') +except ImportError: + DecimalTuple = lambda *args: args + +# Rounding +ROUND_DOWN = 'ROUND_DOWN' +ROUND_HALF_UP = 'ROUND_HALF_UP' +ROUND_HALF_EVEN = 'ROUND_HALF_EVEN' +ROUND_CEILING = 'ROUND_CEILING' +ROUND_FLOOR = 'ROUND_FLOOR' +ROUND_UP = 'ROUND_UP' +ROUND_HALF_DOWN = 'ROUND_HALF_DOWN' +ROUND_05UP = 'ROUND_05UP' + +# Compatibility with the C version +HAVE_THREADS = True +if sys.maxsize == 2**63-1: + MAX_PREC = 999999999999999999 + MAX_EMAX = 999999999999999999 + MIN_EMIN = -999999999999999999 +else: + MAX_PREC = 425000000 + MAX_EMAX = 425000000 + MIN_EMIN = -425000000 + +MIN_ETINY = MIN_EMIN - (MAX_PREC-1) + +# Errors + +class DecimalException(ArithmeticError): + """Base exception class. + + Used exceptions derive from this. + If an exception derives from another exception besides this (such as + Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + called if the others are present. This isn't actually used for + anything, though. + + handle -- Called when context._raise_error is called and the + trap_enabler is not set. First argument is self, second is the + context. More arguments can be given, those being after + the explanation in _raise_error (For example, + context._raise_error(NewError, '(-x)!', self._sign) would + call NewError().handle(context, self._sign).) + + To define a new exception, it should be sufficient to have it derive + from DecimalException. + """ + def handle(self, context, *args): + pass + + +class Clamped(DecimalException): + """Exponent of a 0 changed to fit bounds. + + This occurs and signals clamped if the exponent of a result has been + altered in order to fit the constraints of a specific concrete + representation. This may occur when the exponent of a zero result would + be outside the bounds of a representation, or when a large normal + number would have an encoded exponent that cannot be represented. In + this latter case, the exponent is reduced to fit and the corresponding + number of zero digits are appended to the coefficient ("fold-down"). + """ + +class InvalidOperation(DecimalException): + """An invalid operation was performed. + + Various bad things cause this: + + Something creates a signaling NaN + -INF + INF + 0 * (+-)INF + (+-)INF / (+-)INF + x % 0 + (+-)INF % x + x._rescale( non-integer ) + sqrt(-x) , x > 0 + 0 ** 0 + x ** (non-integer) + x ** (+-)INF + An operand is invalid + + The result of the operation after these is a quiet positive NaN, + except when the cause is a signaling NaN, in which case the result is + also a quiet NaN, but with the original sign, and an optional + diagnostic information. + """ + def handle(self, context, *args): + if args: + ans = _dec_from_triple(args[0]._sign, args[0]._int, 'n', True) + return ans._fix_nan(context) + return _NaN + +class ConversionSyntax(InvalidOperation): + """Trying to convert badly formed string. + + This occurs and signals invalid-operation if an string is being + converted to a number and it does not conform to the numeric string + syntax. The result is [0,qNaN]. + """ + def handle(self, context, *args): + return _NaN + +class DivisionByZero(DecimalException, ZeroDivisionError): + """Division by 0. + + This occurs and signals division-by-zero if division of a finite number + by zero was attempted (during a divide-integer or divide operation, or a + power operation with negative right-hand operand), and the dividend was + not zero. + + The result of the operation is [sign,inf], where sign is the exclusive + or of the signs of the operands for divide, or is 1 for an odd power of + -0, for power. + """ + + def handle(self, context, sign, *args): + return _SignedInfinity[sign] + +class DivisionImpossible(InvalidOperation): + """Cannot perform the division adequately. + + This occurs and signals invalid-operation if the integer result of a + divide-integer or remainder operation had too many digits (would be + longer than precision). The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class DivisionUndefined(InvalidOperation, ZeroDivisionError): + """Undefined result of division. + + This occurs and signals invalid-operation if division by zero was + attempted (during a divide-integer, divide, or remainder operation), and + the dividend is also zero. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Inexact(DecimalException): + """Had to round, losing information. + + This occurs and signals inexact whenever the result of an operation is + not exact (that is, it needed to be rounded and any discarded digits + were non-zero), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The inexact signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) was inexact. + """ + +class InvalidContext(InvalidOperation): + """Invalid context. Unknown rounding, for example. + + This occurs and signals invalid-operation if an invalid context was + detected during an operation. This can occur if contexts are not checked + on creation and either the precision exceeds the capability of the + underlying concrete representation or an unknown or unsupported rounding + was specified. These aspects of the context need only be checked when + the values are required to be used. The result is [0,qNaN]. + """ + + def handle(self, context, *args): + return _NaN + +class Rounded(DecimalException): + """Number got rounded (not necessarily changed during rounding). + + This occurs and signals rounded whenever the result of an operation is + rounded (that is, some zero or non-zero digits were discarded from the + coefficient), or if an overflow or underflow condition occurs. The + result in all cases is unchanged. + + The rounded signal may be tested (or trapped) to determine if a given + operation (or sequence of operations) caused a loss of precision. + """ + +class Subnormal(DecimalException): + """Exponent < Emin before rounding. + + This occurs and signals subnormal whenever the result of a conversion or + operation is subnormal (that is, its adjusted exponent is less than + Emin, before any rounding). The result in all cases is unchanged. + + The subnormal signal may be tested (or trapped) to determine if a given + or operation (or sequence of operations) yielded a subnormal result. + """ + +class Overflow(Inexact, Rounded): + """Numerical overflow. + + This occurs and signals overflow if the adjusted exponent of a result + (from a conversion or from an operation that is not an attempt to divide + by zero), after rounding, would be greater than the largest value that + can be handled by the implementation (the value Emax). + + The result depends on the rounding mode: + + For round-half-up and round-half-even (and for round-half-down and + round-up, if implemented), the result of the operation is [sign,inf], + where sign is the sign of the intermediate result. For round-down, the + result is the largest finite number that can be represented in the + current precision, with the sign of the intermediate result. For + round-ceiling, the result is the same as for round-down if the sign of + the intermediate result is 1, or is [0,inf] otherwise. For round-floor, + the result is the same as for round-down if the sign of the intermediate + result is 0, or is [1,inf] otherwise. In all cases, Inexact and Rounded + will also be raised. + """ + + def handle(self, context, sign, *args): + if context.rounding in (ROUND_HALF_UP, ROUND_HALF_EVEN, + ROUND_HALF_DOWN, ROUND_UP): + return _SignedInfinity[sign] + if sign == 0: + if context.rounding == ROUND_CEILING: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + if sign == 1: + if context.rounding == ROUND_FLOOR: + return _SignedInfinity[sign] + return _dec_from_triple(sign, '9'*context.prec, + context.Emax-context.prec+1) + + +class Underflow(Inexact, Rounded, Subnormal): + """Numerical underflow with result rounded to 0. + + This occurs and signals underflow if a result is inexact and the + adjusted exponent of the result would be smaller (more negative) than + the smallest value that can be handled by the implementation (the value + Emin). That is, the result is both inexact and subnormal. + + The result after an underflow will be a subnormal number rounded, if + necessary, so that its exponent is not less than Etiny. This may result + in 0 with the sign of the intermediate result and an exponent of Etiny. + + In all cases, Inexact, Rounded, and Subnormal will also be raised. + """ + +class FloatOperation(DecimalException, TypeError): + """Enable stricter semantics for mixing floats and Decimals. + + If the signal is not trapped (default), mixing floats and Decimals is + permitted in the Decimal() constructor, context.create_decimal() and + all comparison operators. Both conversion and comparisons are exact. + Any occurrence of a mixed operation is silently recorded by setting + FloatOperation in the context flags. Explicit conversions with + Decimal.from_float() or context.create_decimal_from_float() do not + set the flag. + + Otherwise (the signal is trapped), only equality comparisons and explicit + conversions are silent. All other mixed operations raise FloatOperation. + """ + +# List of public traps and flags +_signals = [Clamped, DivisionByZero, Inexact, Overflow, Rounded, + Underflow, InvalidOperation, Subnormal, FloatOperation] + +# Map conditions (per the spec) to signals +_condition_map = {ConversionSyntax:InvalidOperation, + DivisionImpossible:InvalidOperation, + DivisionUndefined:InvalidOperation, + InvalidContext:InvalidOperation} + +# Valid rounding modes +_rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, + ROUND_FLOOR, ROUND_UP, ROUND_HALF_DOWN, ROUND_05UP) + +##### Context Functions ################################################## + +# The getcontext() and setcontext() function manage access to a thread-local +# current context. Py2.4 offers direct support for thread locals. If that +# is not available, use threading.current_thread() which is slower but will +# work for older Pythons. If threads are not part of the build, create a +# mock threading object with threading.local() returning the module namespace. + +try: + import threading +except ImportError: + # Python was compiled without threads; create a mock object instead + class MockThreading(object): + def local(self, sys=sys): + return sys.modules[__name__] + threading = MockThreading() + del MockThreading + +try: + threading.local + +except AttributeError: + + # To fix reloading, force it to create a new context + # Old contexts have different exceptions in their dicts, making problems. + if hasattr(threading.current_thread(), '__decimal_context__'): + del threading.current_thread().__decimal_context__ + + def setcontext(context): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + threading.current_thread().__decimal_context__ = context + + def getcontext(): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return threading.current_thread().__decimal_context__ + except AttributeError: + context = Context() + threading.current_thread().__decimal_context__ = context + return context + +else: + + local = threading.local() + if hasattr(local, '__decimal_context__'): + del local.__decimal_context__ + + def getcontext(_local=local): + """Returns this thread's context. + + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _local.__decimal_context__ + except AttributeError: + context = Context() + _local.__decimal_context__ = context + return context + + def setcontext(context, _local=local): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _local.__decimal_context__ = context + + del threading, local # Don't contaminate the namespace + +def localcontext(ctx=None): + """Return a context manager for a copy of the supplied context + + Uses a copy of the current context if no context is specified + The returned context manager creates a local decimal context + in a with statement: + def sin(x): + with localcontext() as ctx: + ctx.prec += 2 + # Rest of sin calculation algorithm + # uses a precision 2 greater than normal + return +s # Convert result to normal precision + + def sin(x): + with localcontext(ExtendedContext): + # Rest of sin calculation algorithm + # uses the Extended Context from the + # General Decimal Arithmetic Specification + return +s # Convert result to normal context + + >>> setcontext(DefaultContext) + >>> print(getcontext().prec) + 28 + >>> with localcontext(): + ... ctx = getcontext() + ... ctx.prec += 2 + ... print(ctx.prec) + ... + 30 + >>> with localcontext(ExtendedContext): + ... print(getcontext().prec) + ... + 9 + >>> print(getcontext().prec) + 28 + """ + if ctx is None: ctx = getcontext() + return _ContextManager(ctx) + + +##### Decimal class ####################################################### + +# Do not subclass Decimal from numbers.Real and do not register it as such +# (because Decimals are not interoperable with floats). See the notes in +# numbers.py for more detail. + +class Decimal(object): + """Floating point class for decimal arithmetic.""" + + __slots__ = ('_exp','_int','_sign', '_is_special') + # Generally, the value of the Decimal instance is given by + # (-1)**_sign * _int * 10**_exp + # Special values are signified by _is_special == True + + # We're immutable, so use __new__ not __init__ + def __new__(cls, value="0", context=None): + """Create a decimal point instance. + + >>> Decimal('3.14') # string input + Decimal('3.14') + >>> Decimal((0, (3, 1, 4), -2)) # tuple (sign, digit_tuple, exponent) + Decimal('3.14') + >>> Decimal(314) # int + Decimal('314') + >>> Decimal(Decimal(314)) # another decimal instance + Decimal('314') + >>> Decimal(' 3.14 \\n') # leading and trailing whitespace okay + Decimal('3.14') + """ + + # Note that the coefficient, self._int, is actually stored as + # a string rather than as a tuple of digits. This speeds up + # the "digits to integer" and "integer to digits" conversions + # that are used in almost every arithmetic operation on + # Decimals. This is an internal detail: the as_tuple function + # and the Decimal constructor still deal with tuples of + # digits. + + self = object.__new__(cls) + + # From a string + # REs insist on real strings, so we can too. + if isinstance(value, str): + m = _parser(value.strip()) + if m is None: + if context is None: + context = getcontext() + return context._raise_error(ConversionSyntax, + "Invalid literal for Decimal: %r" % value) + + if m.group('sign') == "-": + self._sign = 1 + else: + self._sign = 0 + intpart = m.group('int') + if intpart is not None: + # finite number + fracpart = m.group('frac') or '' + exp = int(m.group('exp') or '0') + self._int = str(int(intpart+fracpart)) + self._exp = exp - len(fracpart) + self._is_special = False + else: + diag = m.group('diag') + if diag is not None: + # NaN + self._int = str(int(diag or '0')).lstrip('0') + if m.group('signal'): + self._exp = 'N' + else: + self._exp = 'n' + else: + # infinity + self._int = '0' + self._exp = 'F' + self._is_special = True + return self + + # From an integer + if isinstance(value, int): + if value >= 0: + self._sign = 0 + else: + self._sign = 1 + self._exp = 0 + self._int = str(abs(value)) + self._is_special = False + return self + + # From another decimal + if isinstance(value, Decimal): + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + # From an internal working value + if isinstance(value, _WorkRep): + self._sign = value.sign + self._int = str(value.int) + self._exp = int(value.exp) + self._is_special = False + return self + + # tuple/list conversion (possibly from as_tuple()) + if isinstance(value, (list,tuple)): + if len(value) != 3: + raise ValueError('Invalid tuple size in creation of Decimal ' + 'from list or tuple. The list or tuple ' + 'should have exactly three elements.') + # process sign. The isinstance test rejects floats + if not (isinstance(value[0], int) and value[0] in (0,1)): + raise ValueError("Invalid sign. The first value in the tuple " + "should be an integer; either 0 for a " + "positive number or 1 for a negative number.") + self._sign = value[0] + if value[2] == 'F': + # infinity: value[1] is ignored + self._int = '0' + self._exp = value[2] + self._is_special = True + else: + # process and validate the digits in value[1] + digits = [] + for digit in value[1]: + if isinstance(digit, int) and 0 <= digit <= 9: + # skip leading zeros + if digits or digit != 0: + digits.append(digit) + else: + raise ValueError("The second value in the tuple must " + "be composed of integers in the range " + "0 through 9.") + if value[2] in ('n', 'N'): + # NaN: digits form the diagnostic + self._int = ''.join(map(str, digits)) + self._exp = value[2] + self._is_special = True + elif isinstance(value[2], int): + # finite number: digits give the coefficient + self._int = ''.join(map(str, digits or [0])) + self._exp = value[2] + self._is_special = False + else: + raise ValueError("The third value in the tuple must " + "be an integer, or one of the " + "strings 'F', 'n', 'N'.") + return self + + if isinstance(value, float): + if context is None: + context = getcontext() + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are " + "enabled") + value = Decimal.from_float(value) + self._exp = value._exp + self._sign = value._sign + self._int = value._int + self._is_special = value._is_special + return self + + raise TypeError("Cannot convert %r to Decimal" % value) + + # @classmethod, but @decorator is not valid Python 2.3 syntax, so + # don't use it (see notes on Py2.3 compatibility at top of file) + def from_float(cls, f): + """Converts a float to a decimal number, exactly. + + Note that Decimal.from_float(0.1) is not the same as Decimal('0.1'). + Since 0.1 is not exactly representable in binary floating point, the + value is stored as the nearest representable value which is + 0x1.999999999999ap-4. The exact equivalent of the value in decimal + is 0.1000000000000000055511151231257827021181583404541015625. + + >>> Decimal.from_float(0.1) + Decimal('0.1000000000000000055511151231257827021181583404541015625') + >>> Decimal.from_float(float('nan')) + Decimal('NaN') + >>> Decimal.from_float(float('inf')) + Decimal('Infinity') + >>> Decimal.from_float(-float('inf')) + Decimal('-Infinity') + >>> Decimal.from_float(-0.0) + Decimal('-0') + + """ + if isinstance(f, int): # handle integer inputs + return cls(f) + if not isinstance(f, float): + raise TypeError("argument must be int or float.") + if _math.isinf(f) or _math.isnan(f): + return cls(repr(f)) + if _math.copysign(1.0, f) == 1.0: + sign = 0 + else: + sign = 1 + n, d = abs(f).as_integer_ratio() + k = d.bit_length() - 1 + result = _dec_from_triple(sign, str(n*5**k), -k) + if cls is Decimal: + return result + else: + return cls(result) + from_float = classmethod(from_float) + + def _isnan(self): + """Returns whether the number is not actually one. + + 0 if a number + 1 if NaN + 2 if sNaN + """ + if self._is_special: + exp = self._exp + if exp == 'n': + return 1 + elif exp == 'N': + return 2 + return 0 + + def _isinfinity(self): + """Returns whether the number is infinite + + 0 if finite or not a number + 1 if +INF + -1 if -INF + """ + if self._exp == 'F': + if self._sign: + return -1 + return 1 + return 0 + + def _check_nans(self, other=None, context=None): + """Returns whether the number is not actually one. + + if self, other are sNaN, signal + if self, other are NaN return nan + return 0 + + Done before operations. + """ + + self_is_nan = self._isnan() + if other is None: + other_is_nan = False + else: + other_is_nan = other._isnan() + + if self_is_nan or other_is_nan: + if context is None: + context = getcontext() + + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if self_is_nan: + return self._fix_nan(context) + + return other._fix_nan(context) + return 0 + + def _compare_check_nans(self, other, context): + """Version of _check_nans used for the signaling comparisons + compare_signal, __le__, __lt__, __ge__, __gt__. + + Signal InvalidOperation if either self or other is a (quiet + or signaling) NaN. Signaling NaNs take precedence over quiet + NaNs. + + Return 0 if neither operand is a NaN. + + """ + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + if self.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + self) + elif other.is_snan(): + return context._raise_error(InvalidOperation, + 'comparison involving sNaN', + other) + elif self.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + self) + elif other.is_qnan(): + return context._raise_error(InvalidOperation, + 'comparison involving NaN', + other) + return 0 + + def __bool__(self): + """Return True if self is nonzero; otherwise return False. + + NaNs and infinities are considered nonzero. + """ + return self._is_special or self._int != '0' + + def _cmp(self, other): + """Compare the two non-NaN decimal instances self and other. + + Returns -1 if self < other, 0 if self == other and 1 + if self > other. This routine is for internal use only.""" + + if self._is_special or other._is_special: + self_inf = self._isinfinity() + other_inf = other._isinfinity() + if self_inf == other_inf: + return 0 + elif self_inf < other_inf: + return -1 + else: + return 1 + + # check for zeros; Decimal('0') == Decimal('-0') + if not self: + if not other: + return 0 + else: + return -((-1)**other._sign) + if not other: + return (-1)**self._sign + + # If different signs, neg one is less + if other._sign < self._sign: + return -1 + if self._sign < other._sign: + return 1 + + self_adjusted = self.adjusted() + other_adjusted = other.adjusted() + if self_adjusted == other_adjusted: + self_padded = self._int + '0'*(self._exp - other._exp) + other_padded = other._int + '0'*(other._exp - self._exp) + if self_padded == other_padded: + return 0 + elif self_padded < other_padded: + return -(-1)**self._sign + else: + return (-1)**self._sign + elif self_adjusted > other_adjusted: + return (-1)**self._sign + else: # self_adjusted < other_adjusted + return -((-1)**self._sign) + + # Note: The Decimal standard doesn't cover rich comparisons for + # Decimals. In particular, the specification is silent on the + # subject of what should happen for a comparison involving a NaN. + # We take the following approach: + # + # == comparisons involving a quiet NaN always return False + # != comparisons involving a quiet NaN always return True + # == or != comparisons involving a signaling NaN signal + # InvalidOperation, and return False or True as above if the + # InvalidOperation is not trapped. + # <, >, <= and >= comparisons involving a (quiet or signaling) + # NaN signal InvalidOperation, and return False if the + # InvalidOperation is not trapped. + # + # This behavior is designed to conform as closely as possible to + # that specified by IEEE 754. + + def __eq__(self, other, context=None): + self, other = _convert_for_comparison(self, other, equality_op=True) + if other is NotImplemented: + return other + if self._check_nans(other, context): + return False + return self._cmp(other) == 0 + + def __ne__(self, other, context=None): + self, other = _convert_for_comparison(self, other, equality_op=True) + if other is NotImplemented: + return other + if self._check_nans(other, context): + return True + return self._cmp(other) != 0 + + + def __lt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) < 0 + + def __le__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) <= 0 + + def __gt__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) > 0 + + def __ge__(self, other, context=None): + self, other = _convert_for_comparison(self, other) + if other is NotImplemented: + return other + ans = self._compare_check_nans(other, context) + if ans: + return False + return self._cmp(other) >= 0 + + def compare(self, other, context=None): + """Compares one to another. + + -1 => a < b + 0 => a = b + 1 => a > b + NaN => one is NaN + Like __cmp__, but returns Decimal instances. + """ + other = _convert_other(other, raiseit=True) + + # Compare(NaN, NaN) = NaN + if (self._is_special or other and other._is_special): + ans = self._check_nans(other, context) + if ans: + return ans + + return Decimal(self._cmp(other)) + + def __hash__(self): + """x.__hash__() <==> hash(x)""" + + # In order to make sure that the hash of a Decimal instance + # agrees with the hash of a numerically equal integer, float + # or Fraction, we follow the rules for numeric hashes outlined + # in the documentation. (See library docs, 'Built-in Types'). + if self._is_special: + if self.is_snan(): + raise TypeError('Cannot hash a signaling NaN value.') + elif self.is_nan(): + return _PyHASH_NAN + else: + if self._sign: + return -_PyHASH_INF + else: + return _PyHASH_INF + + if self._exp >= 0: + exp_hash = pow(10, self._exp, _PyHASH_MODULUS) + else: + exp_hash = pow(_PyHASH_10INV, -self._exp, _PyHASH_MODULUS) + hash_ = int(self._int) * exp_hash % _PyHASH_MODULUS + ans = hash_ if self >= 0 else -hash_ + return -2 if ans == -1 else ans + + def as_tuple(self): + """Represents the number as a triple tuple. + + To show the internals exactly as they are. + """ + return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + + def __repr__(self): + """Represents the number as an instance of Decimal.""" + # Invariant: eval(repr(d)) == d + return "Decimal('%s')" % str(self) + + def __str__(self, eng=False, context=None): + """Return string representation of the number in scientific notation. + + Captures all of the information in the underlying representation. + """ + + sign = ['', '-'][self._sign] + if self._is_special: + if self._exp == 'F': + return sign + 'Infinity' + elif self._exp == 'n': + return sign + 'NaN' + self._int + else: # self._exp == 'N' + return sign + 'sNaN' + self._int + + # number of digits of self._int to left of decimal point + leftdigits = self._exp + len(self._int) + + # dotplace is number of digits of self._int to the left of the + # decimal point in the mantissa of the output string (that is, + # after adjusting the exponent) + if self._exp <= 0 and leftdigits > -6: + # no exponent required + dotplace = leftdigits + elif not eng: + # usual scientific notation: 1 digit on left of the point + dotplace = 1 + elif self._int == '0': + # engineering notation, zero + dotplace = (leftdigits + 1) % 3 - 1 + else: + # engineering notation, nonzero + dotplace = (leftdigits - 1) % 3 + 1 + + if dotplace <= 0: + intpart = '0' + fracpart = '.' + '0'*(-dotplace) + self._int + elif dotplace >= len(self._int): + intpart = self._int+'0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] + fracpart = '.' + self._int[dotplace:] + if leftdigits == dotplace: + exp = '' + else: + if context is None: + context = getcontext() + exp = ['e', 'E'][context.capitals] + "%+d" % (leftdigits-dotplace) + + return sign + intpart + fracpart + exp + + def to_eng_string(self, context=None): + """Convert to engineering-type string. + + Engineering notation has an exponent which is a multiple of 3, so there + are up to 3 digits left of the decimal place. + + Same rules for when in exponential and when as a value as in __str__. + """ + return self.__str__(eng=True, context=context) + + def __neg__(self, context=None): + """Returns a copy with the sign switched. + + Rounds, if it has reason. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # -Decimal('0') is Decimal('0'), not Decimal('-0'), except + # in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = self.copy_negate() + + return ans._fix(context) + + def __pos__(self, context=None): + """Returns a copy, unless it is a sNaN. + + Rounds the number (if more then precision digits) + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + if not self and context.rounding != ROUND_FLOOR: + # + (-0) = 0, except in ROUND_FLOOR rounding mode. + ans = self.copy_abs() + else: + ans = Decimal(self) + + return ans._fix(context) + + def __abs__(self, round=True, context=None): + """Returns the absolute value of self. + + If the keyword argument 'round' is false, do not round. The + expression self.__abs__(round=False) is equivalent to + self.copy_abs(). + """ + if not round: + return self.copy_abs() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._sign: + ans = self.__neg__(context=context) + else: + ans = self.__pos__(context=context) + + return ans + + def __add__(self, other, context=None): + """Returns self + other. + + -INF + INF (or the reverse) cause InvalidOperation errors. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + # If both INF, same sign => same as both, opposite => error. + if self._sign != other._sign and other._isinfinity(): + return context._raise_error(InvalidOperation, '-INF + INF') + return Decimal(self) + if other._isinfinity(): + return Decimal(other) # Can't both be infinity here + + exp = min(self._exp, other._exp) + negativezero = 0 + if context.rounding == ROUND_FLOOR and self._sign != other._sign: + # If the answer is 0, the sign should be negative, in this case. + negativezero = 1 + + if not self and not other: + sign = min(self._sign, other._sign) + if negativezero: + sign = 1 + ans = _dec_from_triple(sign, '0', exp) + ans = ans._fix(context) + return ans + if not self: + exp = max(exp, other._exp - context.prec-1) + ans = other._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + if not other: + exp = max(exp, self._exp - context.prec-1) + ans = self._rescale(exp, context.rounding) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + op1, op2 = _normalize(op1, op2, context.prec) + + result = _WorkRep() + if op1.sign != op2.sign: + # Equal and opposite + if op1.int == op2.int: + ans = _dec_from_triple(negativezero, '0', exp) + ans = ans._fix(context) + return ans + if op1.int < op2.int: + op1, op2 = op2, op1 + # OK, now abs(op1) > abs(op2) + if op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = op2.sign, op1.sign + else: + result.sign = 0 + # So we know the sign, and op1 > 0. + elif op1.sign == 1: + result.sign = 1 + op1.sign, op2.sign = (0, 0) + else: + result.sign = 0 + # Now, op1 > abs(op2) > 0 + + if op2.sign == 0: + result.int = op1.int + op2.int + else: + result.int = op1.int - op2.int + + result.exp = op1.exp + ans = Decimal(result) + ans = ans._fix(context) + return ans + + __radd__ = __add__ + + def __sub__(self, other, context=None): + """Return self - other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if self._is_special or other._is_special: + ans = self._check_nans(other, context=context) + if ans: + return ans + + # self - other is computed as self + other.copy_negate() + return self.__add__(other.copy_negate(), context=context) + + def __rsub__(self, other, context=None): + """Return other - self""" + other = _convert_other(other) + if other is NotImplemented: + return other + + return other.__sub__(self, context=context) + + def __mul__(self, other, context=None): + """Return self * other. + + (+-) INF * 0 (or its reverse) raise InvalidOperation. + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + resultsign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if not other: + return context._raise_error(InvalidOperation, '(+-)INF * 0') + return _SignedInfinity[resultsign] + + if other._isinfinity(): + if not self: + return context._raise_error(InvalidOperation, '0 * (+-)INF') + return _SignedInfinity[resultsign] + + resultexp = self._exp + other._exp + + # Special case for multiplying by zero + if not self or not other: + ans = _dec_from_triple(resultsign, '0', resultexp) + # Fixing in case the exponent is out of bounds + ans = ans._fix(context) + return ans + + # Special case for multiplying by power of 10 + if self._int == '1': + ans = _dec_from_triple(resultsign, other._int, resultexp) + ans = ans._fix(context) + return ans + if other._int == '1': + ans = _dec_from_triple(resultsign, self._int, resultexp) + ans = ans._fix(context) + return ans + + op1 = _WorkRep(self) + op2 = _WorkRep(other) + + ans = _dec_from_triple(resultsign, str(op1.int * op2.int), resultexp) + ans = ans._fix(context) + + return ans + __rmul__ = __mul__ + + def __truediv__(self, other, context=None): + """Return self / other.""" + other = _convert_other(other) + if other is NotImplemented: + return NotImplemented + + if context is None: + context = getcontext() + + sign = self._sign ^ other._sign + + if self._is_special or other._is_special: + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity() and other._isinfinity(): + return context._raise_error(InvalidOperation, '(+-)INF/(+-)INF') + + if self._isinfinity(): + return _SignedInfinity[sign] + + if other._isinfinity(): + context._raise_error(Clamped, 'Division by infinity') + return _dec_from_triple(sign, '0', context.Etiny()) + + # Special cases for zeroes + if not other: + if not self: + return context._raise_error(DivisionUndefined, '0 / 0') + return context._raise_error(DivisionByZero, 'x / 0', sign) + + if not self: + exp = self._exp - other._exp + coeff = 0 + else: + # OK, so neither = 0, INF or NaN + shift = len(other._int) - len(self._int) + context.prec + 1 + exp = self._exp - other._exp - shift + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if shift >= 0: + coeff, remainder = divmod(op1.int * 10**shift, op2.int) + else: + coeff, remainder = divmod(op1.int, op2.int * 10**-shift) + if remainder: + # result is not exact; adjust to ensure correct rounding + if coeff % 5 == 0: + coeff += 1 + else: + # result is exact; get as close to ideal exponent as possible + ideal_exp = self._exp - other._exp + while exp < ideal_exp and coeff % 10 == 0: + coeff //= 10 + exp += 1 + + ans = _dec_from_triple(sign, str(coeff), exp) + return ans._fix(context) + + def _divide(self, other, context): + """Return (self // other, self % other), to context.prec precision. + + Assumes that neither self nor other is a NaN, that self is not + infinite and that other is nonzero. + """ + sign = self._sign ^ other._sign + if other._isinfinity(): + ideal_exp = self._exp + else: + ideal_exp = min(self._exp, other._exp) + + expdiff = self.adjusted() - other.adjusted() + if not self or other._isinfinity() or expdiff <= -2: + return (_dec_from_triple(sign, '0', 0), + self._rescale(ideal_exp, context.rounding)) + if expdiff <= context.prec: + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + if q < 10**context.prec: + return (_dec_from_triple(sign, str(q), 0), + _dec_from_triple(self._sign, str(r), ideal_exp)) + + # Here the quotient is too large to be representable + ans = context._raise_error(DivisionImpossible, + 'quotient too large in //, % or divmod') + return ans, ans + + def __rtruediv__(self, other, context=None): + """Swaps self/other and returns __truediv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__truediv__(self, context=context) + + def __divmod__(self, other, context=None): + """ + Return (self // other, self % other) + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return (ans, ans) + + sign = self._sign ^ other._sign + if self._isinfinity(): + if other._isinfinity(): + ans = context._raise_error(InvalidOperation, 'divmod(INF, INF)') + return ans, ans + else: + return (_SignedInfinity[sign], + context._raise_error(InvalidOperation, 'INF % x')) + + if not other: + if not self: + ans = context._raise_error(DivisionUndefined, 'divmod(0, 0)') + return ans, ans + else: + return (context._raise_error(DivisionByZero, 'x // 0', sign), + context._raise_error(InvalidOperation, 'x % 0')) + + quotient, remainder = self._divide(other, context) + remainder = remainder._fix(context) + return quotient, remainder + + def __rdivmod__(self, other, context=None): + """Swaps self/other and returns __divmod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__divmod__(self, context=context) + + def __mod__(self, other, context=None): + """ + self % other + """ + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + return context._raise_error(InvalidOperation, 'INF % x') + elif not other: + if self: + return context._raise_error(InvalidOperation, 'x % 0') + else: + return context._raise_error(DivisionUndefined, '0 % 0') + + remainder = self._divide(other, context)[1] + remainder = remainder._fix(context) + return remainder + + def __rmod__(self, other, context=None): + """Swaps self/other and returns __mod__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__mod__(self, context=context) + + def remainder_near(self, other, context=None): + """ + Remainder nearest to 0- abs(remainder-near) <= other/2 + """ + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + # self == +/-infinity -> InvalidOperation + if self._isinfinity(): + return context._raise_error(InvalidOperation, + 'remainder_near(infinity, x)') + + # other == 0 -> either InvalidOperation or DivisionUndefined + if not other: + if self: + return context._raise_error(InvalidOperation, + 'remainder_near(x, 0)') + else: + return context._raise_error(DivisionUndefined, + 'remainder_near(0, 0)') + + # other = +/-infinity -> remainder = self + if other._isinfinity(): + ans = Decimal(self) + return ans._fix(context) + + # self = 0 -> remainder = self, with ideal exponent + ideal_exponent = min(self._exp, other._exp) + if not self: + ans = _dec_from_triple(self._sign, '0', ideal_exponent) + return ans._fix(context) + + # catch most cases of large or small quotient + expdiff = self.adjusted() - other.adjusted() + if expdiff >= context.prec + 1: + # expdiff >= prec+1 => abs(self/other) > 10**prec + return context._raise_error(DivisionImpossible) + if expdiff <= -2: + # expdiff <= -2 => abs(self/other) < 0.1 + ans = self._rescale(ideal_exponent, context.rounding) + return ans._fix(context) + + # adjust both arguments to have the same exponent, then divide + op1 = _WorkRep(self) + op2 = _WorkRep(other) + if op1.exp >= op2.exp: + op1.int *= 10**(op1.exp - op2.exp) + else: + op2.int *= 10**(op2.exp - op1.exp) + q, r = divmod(op1.int, op2.int) + # remainder is r*10**ideal_exponent; other is +/-op2.int * + # 10**ideal_exponent. Apply correction to ensure that + # abs(remainder) <= abs(other)/2 + if 2*r + (q&1) > op2.int: + r -= op2.int + q += 1 + + if q >= 10**context.prec: + return context._raise_error(DivisionImpossible) + + # result has same sign as self unless r is negative + sign = self._sign + if r < 0: + sign = 1-sign + r = -r + + ans = _dec_from_triple(sign, str(r), ideal_exponent) + return ans._fix(context) + + def __floordiv__(self, other, context=None): + """self // other""" + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + if self._isinfinity(): + if other._isinfinity(): + return context._raise_error(InvalidOperation, 'INF // INF') + else: + return _SignedInfinity[self._sign ^ other._sign] + + if not other: + if self: + return context._raise_error(DivisionByZero, 'x // 0', + self._sign ^ other._sign) + else: + return context._raise_error(DivisionUndefined, '0 // 0') + + return self._divide(other, context)[0] + + def __rfloordiv__(self, other, context=None): + """Swaps self/other and returns __floordiv__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__floordiv__(self, context=context) + + def __float__(self): + """Float representation.""" + return float(str(self)) + + def __int__(self): + """Converts self to an int, truncating if necessary.""" + if self._is_special: + if self._isnan(): + raise ValueError("Cannot convert NaN to integer") + elif self._isinfinity(): + raise OverflowError("Cannot convert infinity to integer") + s = (-1)**self._sign + if self._exp >= 0: + return s*int(self._int)*10**self._exp + else: + return s*int(self._int[:self._exp] or '0') + + __trunc__ = __int__ + + def real(self): + return self + real = property(real) + + def imag(self): + return Decimal(0) + imag = property(imag) + + def conjugate(self): + return self + + def __complex__(self): + return complex(float(self)) + + def _fix_nan(self, context): + """Decapitate the payload of a NaN to fit the context""" + payload = self._int + + # maximum length of payload is precision if clamp=0, + # precision-1 if clamp=1. + max_payload_len = context.prec - context.clamp + if len(payload) > max_payload_len: + payload = payload[len(payload)-max_payload_len:].lstrip('0') + return _dec_from_triple(self._sign, payload, self._exp, True) + return Decimal(self) + + def _fix(self, context): + """Round if it is necessary to keep self within prec precision. + + Rounds and fixes the exponent. Does not raise on a sNaN. + + Arguments: + self - Decimal instance + context - context used. + """ + + if self._is_special: + if self._isnan(): + # decapitate payload if necessary + return self._fix_nan(context) + else: + # self is +/-Infinity; return unaltered + return Decimal(self) + + # if self is zero then exponent should be between Etiny and + # Emax if clamp==0, and between Etiny and Etop if clamp==1. + Etiny = context.Etiny() + Etop = context.Etop() + if not self: + exp_max = [context.Emax, Etop][context.clamp] + new_exp = min(max(self._exp, Etiny), exp_max) + if new_exp != self._exp: + context._raise_error(Clamped) + return _dec_from_triple(self._sign, '0', new_exp) + else: + return Decimal(self) + + # exp_min is the smallest allowable exponent of the result, + # equal to max(self.adjusted()-context.prec+1, Etiny) + exp_min = len(self._int) + self._exp - context.prec + if exp_min > Etop: + # overflow: exp_min > Etop iff self.adjusted() > Emax + ans = context._raise_error(Overflow, 'above Emax', self._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + self_is_subnormal = exp_min < Etiny + if self_is_subnormal: + exp_min = Etiny + + # round if self has too many digits + if self._exp < exp_min: + digits = len(self._int) + self._exp - exp_min + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp_min-1) + digits = 0 + rounding_method = self._pick_rounding_function[context.rounding] + changed = rounding_method(self, digits) + coeff = self._int[:digits] or '0' + if changed > 0: + coeff = str(int(coeff)+1) + if len(coeff) > context.prec: + coeff = coeff[:-1] + exp_min += 1 + + # check whether the rounding pushed the exponent out of range + if exp_min > Etop: + ans = context._raise_error(Overflow, 'above Emax', self._sign) + else: + ans = _dec_from_triple(self._sign, coeff, exp_min) + + # raise the appropriate signals, taking care to respect + # the precedence described in the specification + if changed and self_is_subnormal: + context._raise_error(Underflow) + if self_is_subnormal: + context._raise_error(Subnormal) + if changed: + context._raise_error(Inexact) + context._raise_error(Rounded) + if not ans: + # raise Clamped on underflow to 0 + context._raise_error(Clamped) + return ans + + if self_is_subnormal: + context._raise_error(Subnormal) + + # fold down if clamp == 1 and self has too few digits + if context.clamp == 1 and self._exp > Etop: + context._raise_error(Clamped) + self_padded = self._int + '0'*(self._exp - Etop) + return _dec_from_triple(self._sign, self_padded, Etop) + + # here self was representable to begin with; return unchanged + return Decimal(self) + + # for each of the rounding functions below: + # self is a finite, nonzero Decimal + # prec is an integer satisfying 0 <= prec < len(self._int) + # + # each function returns either -1, 0, or 1, as follows: + # 1 indicates that self should be rounded up (away from zero) + # 0 indicates that self should be truncated, and that all the + # digits to be truncated are zeros (so the value is unchanged) + # -1 indicates that there are nonzero digits to be truncated + + def _round_down(self, prec): + """Also known as round-towards-0, truncate.""" + if _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_up(self, prec): + """Rounds away from 0.""" + return -self._round_down(prec) + + def _round_half_up(self, prec): + """Rounds 5 up (away from 0)""" + if self._int[prec] in '56789': + return 1 + elif _all_zeros(self._int, prec): + return 0 + else: + return -1 + + def _round_half_down(self, prec): + """Round 5 down""" + if _exact_half(self._int, prec): + return -1 + else: + return self._round_half_up(prec) + + def _round_half_even(self, prec): + """Round 5 to even, rest to nearest.""" + if _exact_half(self._int, prec) and \ + (prec == 0 or self._int[prec-1] in '02468'): + return -1 + else: + return self._round_half_up(prec) + + def _round_ceiling(self, prec): + """Rounds up (not away from 0 if negative.)""" + if self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_floor(self, prec): + """Rounds down (not towards 0 if negative)""" + if not self._sign: + return self._round_down(prec) + else: + return -self._round_down(prec) + + def _round_05up(self, prec): + """Round down unless digit prec-1 is 0 or 5.""" + if prec and self._int[prec-1] not in '05': + return self._round_down(prec) + else: + return -self._round_down(prec) + + _pick_rounding_function = dict( + ROUND_DOWN = _round_down, + ROUND_UP = _round_up, + ROUND_HALF_UP = _round_half_up, + ROUND_HALF_DOWN = _round_half_down, + ROUND_HALF_EVEN = _round_half_even, + ROUND_CEILING = _round_ceiling, + ROUND_FLOOR = _round_floor, + ROUND_05UP = _round_05up, + ) + + def __round__(self, n=None): + """Round self to the nearest integer, or to a given precision. + + If only one argument is supplied, round a finite Decimal + instance self to the nearest integer. If self is infinite or + a NaN then a Python exception is raised. If self is finite + and lies exactly halfway between two integers then it is + rounded to the integer with even last digit. + + >>> round(Decimal('123.456')) + 123 + >>> round(Decimal('-456.789')) + -457 + >>> round(Decimal('-3.0')) + -3 + >>> round(Decimal('2.5')) + 2 + >>> round(Decimal('3.5')) + 4 + >>> round(Decimal('Inf')) + Traceback (most recent call last): + ... + OverflowError: cannot round an infinity + >>> round(Decimal('NaN')) + Traceback (most recent call last): + ... + ValueError: cannot round a NaN + + If a second argument n is supplied, self is rounded to n + decimal places using the rounding mode for the current + context. + + For an integer n, round(self, -n) is exactly equivalent to + self.quantize(Decimal('1En')). + + >>> round(Decimal('123.456'), 0) + Decimal('123') + >>> round(Decimal('123.456'), 2) + Decimal('123.46') + >>> round(Decimal('123.456'), -2) + Decimal('1E+2') + >>> round(Decimal('-Infinity'), 37) + Decimal('NaN') + >>> round(Decimal('sNaN123'), 0) + Decimal('NaN123') + + """ + if n is not None: + # two-argument form: use the equivalent quantize call + if not isinstance(n, int): + raise TypeError('Second argument to round should be integral') + exp = _dec_from_triple(0, '1', -n) + return self.quantize(exp) + + # one-argument form + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_HALF_EVEN)) + + def __floor__(self): + """Return the floor of self, as an integer. + + For a finite Decimal instance self, return the greatest + integer n such that n <= self. If self is infinite or a NaN + then a Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_FLOOR)) + + def __ceil__(self): + """Return the ceiling of self, as an integer. + + For a finite Decimal instance self, return the least integer n + such that n >= self. If self is infinite or a NaN then a + Python exception is raised. + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot round a NaN") + else: + raise OverflowError("cannot round an infinity") + return int(self._rescale(0, ROUND_CEILING)) + + def fma(self, other, third, context=None): + """Fused multiply-add. + + Returns self*other+third with no rounding of the intermediate + product self*other. + + self and other are multiplied together, with no rounding of + the result. The third operand is then added to the result, + and a single final rounding is performed. + """ + + other = _convert_other(other, raiseit=True) + third = _convert_other(third, raiseit=True) + + # compute product; raise InvalidOperation if either operand is + # a signaling NaN or if the product is zero times infinity. + if self._is_special or other._is_special: + if context is None: + context = getcontext() + if self._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', self) + if other._exp == 'N': + return context._raise_error(InvalidOperation, 'sNaN', other) + if self._exp == 'n': + product = self + elif other._exp == 'n': + product = other + elif self._exp == 'F': + if not other: + return context._raise_error(InvalidOperation, + 'INF * 0 in fma') + product = _SignedInfinity[self._sign ^ other._sign] + elif other._exp == 'F': + if not self: + return context._raise_error(InvalidOperation, + '0 * INF in fma') + product = _SignedInfinity[self._sign ^ other._sign] + else: + product = _dec_from_triple(self._sign ^ other._sign, + str(int(self._int) * int(other._int)), + self._exp + other._exp) + + return product.__add__(third, context) + + def _power_modulo(self, other, modulo, context=None): + """Three argument version of __pow__""" + + other = _convert_other(other) + if other is NotImplemented: + return other + modulo = _convert_other(modulo) + if modulo is NotImplemented: + return modulo + + if context is None: + context = getcontext() + + # deal with NaNs: if there are any sNaNs then first one wins, + # (i.e. behaviour for NaNs is identical to that of fma) + self_is_nan = self._isnan() + other_is_nan = other._isnan() + modulo_is_nan = modulo._isnan() + if self_is_nan or other_is_nan or modulo_is_nan: + if self_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + self) + if other_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + other) + if modulo_is_nan == 2: + return context._raise_error(InvalidOperation, 'sNaN', + modulo) + if self_is_nan: + return self._fix_nan(context) + if other_is_nan: + return other._fix_nan(context) + return modulo._fix_nan(context) + + # check inputs: we apply same restrictions as Python's pow() + if not (self._isinteger() and + other._isinteger() and + modulo._isinteger()): + return context._raise_error(InvalidOperation, + 'pow() 3rd argument not allowed ' + 'unless all arguments are integers') + if other < 0: + return context._raise_error(InvalidOperation, + 'pow() 2nd argument cannot be ' + 'negative when 3rd argument specified') + if not modulo: + return context._raise_error(InvalidOperation, + 'pow() 3rd argument cannot be 0') + + # additional restriction for decimal: the modulus must be less + # than 10**prec in absolute value + if modulo.adjusted() >= context.prec: + return context._raise_error(InvalidOperation, + 'insufficient precision: pow() 3rd ' + 'argument must not have more than ' + 'precision digits') + + # define 0**0 == NaN, for consistency with two-argument pow + # (even though it hurts!) + if not other and not self: + return context._raise_error(InvalidOperation, + 'at least one of pow() 1st argument ' + 'and 2nd argument must be nonzero ;' + '0**0 is not defined') + + # compute sign of result + if other._iseven(): + sign = 0 + else: + sign = self._sign + + # convert modulo to a Python integer, and self and other to + # Decimal integers (i.e. force their exponents to be >= 0) + modulo = abs(int(modulo)) + base = _WorkRep(self.to_integral_value()) + exponent = _WorkRep(other.to_integral_value()) + + # compute result using integer pow() + base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo + for i in range(exponent.exp): + base = pow(base, 10, modulo) + base = pow(base, exponent.int, modulo) + + return _dec_from_triple(sign, str(base), 0) + + def _power_exact(self, other, p): + """Attempt to compute self**other exactly. + + Given Decimals self and other and an integer p, attempt to + compute an exact result for the power self**other, with p + digits of precision. Return None if self**other is not + exactly representable in p digits. + + Assumes that elimination of special cases has already been + performed: self and other must both be nonspecial; self must + be positive and not numerically equal to 1; other must be + nonzero. For efficiency, other._exp should not be too large, + so that 10**abs(other._exp) is a feasible calculation.""" + + # In the comments below, we write x for the value of self and y for the + # value of other. Write x = xc*10**xe and abs(y) = yc*10**ye, with xc + # and yc positive integers not divisible by 10. + + # The main purpose of this method is to identify the *failure* + # of x**y to be exactly representable with as little effort as + # possible. So we look for cheap and easy tests that + # eliminate the possibility of x**y being exact. Only if all + # these tests are passed do we go on to actually compute x**y. + + # Here's the main idea. Express y as a rational number m/n, with m and + # n relatively prime and n>0. Then for x**y to be exactly + # representable (at *any* precision), xc must be the nth power of a + # positive integer and xe must be divisible by n. If y is negative + # then additionally xc must be a power of either 2 or 5, hence a power + # of 2**n or 5**n. + # + # There's a limit to how small |y| can be: if y=m/n as above + # then: + # + # (1) if xc != 1 then for the result to be representable we + # need xc**(1/n) >= 2, and hence also xc**|y| >= 2. So + # if |y| <= 1/nbits(xc) then xc < 2**nbits(xc) <= + # 2**(1/|y|), hence xc**|y| < 2 and the result is not + # representable. + # + # (2) if xe != 0, |xe|*(1/n) >= 1, so |xe|*|y| >= 1. Hence if + # |y| < 1/|xe| then the result is not representable. + # + # Note that since x is not equal to 1, at least one of (1) and + # (2) must apply. Now |y| < 1/nbits(xc) iff |yc|*nbits(xc) < + # 10**-ye iff len(str(|yc|*nbits(xc)) <= -ye. + # + # There's also a limit to how large y can be, at least if it's + # positive: the normalized result will have coefficient xc**y, + # so if it's representable then xc**y < 10**p, and y < + # p/log10(xc). Hence if y*log10(xc) >= p then the result is + # not exactly representable. + + # if len(str(abs(yc*xe)) <= -ye then abs(yc*xe) < 10**-ye, + # so |y| < 1/xe and the result is not representable. + # Similarly, len(str(abs(yc)*xc_bits)) <= -ye implies |y| + # < 1/nbits(xc). + + x = _WorkRep(self) + xc, xe = x.int, x.exp + while xc % 10 == 0: + xc //= 10 + xe += 1 + + y = _WorkRep(other) + yc, ye = y.int, y.exp + while yc % 10 == 0: + yc //= 10 + ye += 1 + + # case where xc == 1: result is 10**(xe*y), with xe*y + # required to be an integer + if xc == 1: + xe *= yc + # result is now 10**(xe * 10**ye); xe * 10**ye must be integral + while xe % 10 == 0: + xe //= 10 + ye += 1 + if ye < 0: + return None + exponent = xe * 10**ye + if y.sign == 1: + exponent = -exponent + # if other is a nonnegative integer, use ideal exponent + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(exponent-ideal_exponent, p-1) + else: + zeros = 0 + return _dec_from_triple(0, '1' + '0'*zeros, exponent-zeros) + + # case where y is negative: xc must be either a power + # of 2 or a power of 5. + if y.sign == 1: + last_digit = xc % 10 + if last_digit in (2,4,6,8): + # quick test for power of 2 + if xc & -xc != xc: + return None + # now xc is a power of 2; e is its exponent + e = _nbits(xc)-1 + + # We now have: + # + # x = 2**e * 10**xe, e > 0, and y < 0. + # + # The exact result is: + # + # x**y = 5**(-e*y) * 10**(e*y + xe*y) + # + # provided that both e*y and xe*y are integers. Note that if + # 5**(-e*y) >= 10**p, then the result can't be expressed + # exactly with p digits of precision. + # + # Using the above, we can guard against large values of ye. + # 93/65 is an upper bound for log(10)/log(5), so if + # + # ye >= len(str(93*p//65)) + # + # then + # + # -e*y >= -y >= 10**ye > 93*p/65 > p*log(10)/log(5), + # + # so 5**(-e*y) >= 10**p, and the coefficient of the result + # can't be expressed in p digits. + + # emax >= largest e such that 5**e < 10**p. + emax = p*93//65 + if ye >= len(str(emax)): + return None + + # Find -e*y and -xe*y; both must be integers + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 5**e + + elif last_digit == 5: + # e >= log_5(xc) if xc is a power of 5; we have + # equality all the way up to xc=5**2658 + e = _nbits(xc)*28//65 + xc, remainder = divmod(5**e, xc) + if remainder: + return None + while xc % 5 == 0: + xc //= 5 + e -= 1 + + # Guard against large values of ye, using the same logic as in + # the 'xc is a power of 2' branch. 10/3 is an upper bound for + # log(10)/log(2). + emax = p*10//3 + if ye >= len(str(emax)): + return None + + e = _decimal_lshift_exact(e * yc, ye) + xe = _decimal_lshift_exact(xe * yc, ye) + if e is None or xe is None: + return None + + if e > emax: + return None + xc = 2**e + else: + return None + + if xc >= 10**p: + return None + xe = -e-xe + return _dec_from_triple(0, str(xc), xe) + + # now y is positive; find m and n such that y = m/n + if ye >= 0: + m, n = yc*10**ye, 1 + else: + if xe != 0 and len(str(abs(yc*xe))) <= -ye: + return None + xc_bits = _nbits(xc) + if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: + return None + m, n = yc, 10**(-ye) + while m % 2 == n % 2 == 0: + m //= 2 + n //= 2 + while m % 5 == n % 5 == 0: + m //= 5 + n //= 5 + + # compute nth root of xc*10**xe + if n > 1: + # if 1 < xc < 2**n then xc isn't an nth power + if xc != 1 and xc_bits <= n: + return None + + xe, rem = divmod(xe, n) + if rem != 0: + return None + + # compute nth root of xc using Newton's method + a = 1 << -(-_nbits(xc)//n) # initial estimate + while True: + q, r = divmod(xc, a**(n-1)) + if a <= q: + break + else: + a = (a*(n-1) + q)//n + if not (a == q and r == 0): + return None + xc = a + + # now xc*10**xe is the nth root of the original xc*10**xe + # compute mth power of xc*10**xe + + # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > + # 10**p and the result is not representable. + if xc > 1 and m > p*100//_log10_lb(xc): + return None + xc = xc**m + xe *= m + if xc > 10**p: + return None + + # by this point the result *is* exactly representable + # adjust the exponent to get as close as possible to the ideal + # exponent, if necessary + str_xc = str(xc) + if other._isinteger() and other._sign == 0: + ideal_exponent = self._exp*int(other) + zeros = min(xe-ideal_exponent, p-len(str_xc)) + else: + zeros = 0 + return _dec_from_triple(0, str_xc+'0'*zeros, xe-zeros) + + def __pow__(self, other, modulo=None, context=None): + """Return self ** other [ % modulo]. + + With two arguments, compute self**other. + + With three arguments, compute (self**other) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - other must be nonnegative + - either self or other (or both) must be nonzero + - modulo must be nonzero and must have at most p digits, + where p is the context precision. + + If any of these restrictions is violated the InvalidOperation + flag is raised. + + The result of pow(self, other, modulo) is identical to the + result that would be obtained by computing (self**other) % + modulo with unbounded precision, but is computed more + efficiently. It is always exact. + """ + + if modulo is not None: + return self._power_modulo(other, modulo, context) + + other = _convert_other(other) + if other is NotImplemented: + return other + + if context is None: + context = getcontext() + + # either argument is a NaN => result is NaN + ans = self._check_nans(other, context) + if ans: + return ans + + # 0**0 = NaN (!), x**0 = 1 for nonzero x (including +/-Infinity) + if not other: + if not self: + return context._raise_error(InvalidOperation, '0 ** 0') + else: + return _One + + # result has sign 1 iff self._sign is 1 and other is an odd integer + result_sign = 0 + if self._sign == 1: + if other._isinteger(): + if not other._iseven(): + result_sign = 1 + else: + # -ve**noninteger = NaN + # (-0)**noninteger = 0**noninteger + if self: + return context._raise_error(InvalidOperation, + 'x ** y with x negative and y not an integer') + # negate self, without doing any unwanted rounding + self = self.copy_negate() + + # 0**(+ve or Inf)= 0; 0**(-ve or -Inf) = Infinity + if not self: + if other._sign == 0: + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # Inf**(+ve or Inf) = Inf; Inf**(-ve or -Inf) = 0 + if self._isinfinity(): + if other._sign == 0: + return _SignedInfinity[result_sign] + else: + return _dec_from_triple(result_sign, '0', 0) + + # 1**other = 1, but the choice of exponent and the flags + # depend on the exponent of self, and on whether other is a + # positive integer, a negative integer, or neither + if self == _One: + if other._isinteger(): + # exp = max(self._exp*max(int(other), 0), + # 1-context.prec) but evaluating int(other) directly + # is dangerous until we know other is small (other + # could be 1e999999999) + if other._sign == 1: + multiplier = 0 + elif other > context.prec: + multiplier = context.prec + else: + multiplier = int(other) + + exp = self._exp * multiplier + if exp < 1-context.prec: + exp = 1-context.prec + context._raise_error(Rounded) + else: + context._raise_error(Inexact) + context._raise_error(Rounded) + exp = 1-context.prec + + return _dec_from_triple(result_sign, '1'+'0'*-exp, exp) + + # compute adjusted exponent of self + self_adj = self.adjusted() + + # self ** infinity is infinity if self > 1, 0 if self < 1 + # self ** -infinity is infinity if self < 1, 0 if self > 1 + if other._isinfinity(): + if (other._sign == 0) == (self_adj < 0): + return _dec_from_triple(result_sign, '0', 0) + else: + return _SignedInfinity[result_sign] + + # from here on, the result always goes through the call + # to _fix at the end of this function. + ans = None + exact = False + + # crude test to catch cases of extreme overflow/underflow. If + # log10(self)*other >= 10**bound and bound >= len(str(Emax)) + # then 10**bound >= 10**len(str(Emax)) >= Emax+1 and hence + # self**other >= 10**(Emax+1), so overflow occurs. The test + # for underflow is similar. + bound = self._log10_exp_bound() + other.adjusted() + if (self_adj >= 0) == (other._sign == 0): + # self > 1 and other +ve, or self < 1 and other -ve + # possibility of overflow + if bound >= len(str(context.Emax)): + ans = _dec_from_triple(result_sign, '1', context.Emax+1) + else: + # self > 1 and other -ve, or self < 1 and other +ve + # possibility of underflow to 0 + Etiny = context.Etiny() + if bound >= len(str(-Etiny)): + ans = _dec_from_triple(result_sign, '1', Etiny-1) + + # try for an exact result with precision +1 + if ans is None: + ans = self._power_exact(other, context.prec + 1) + if ans is not None: + if result_sign == 1: + ans = _dec_from_triple(1, ans._int, ans._exp) + exact = True + + # usual case: inexact result, x**y computed directly as exp(y*log(x)) + if ans is None: + p = context.prec + x = _WorkRep(self) + xc, xe = x.int, x.exp + y = _WorkRep(other) + yc, ye = y.int, y.exp + if y.sign == 1: + yc = -yc + + # compute correctly rounded result: start with precision +3, + # then increase precision until result is unambiguously roundable + extra = 3 + while True: + coeff, exp = _dpower(xc, xe, yc, ye, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(result_sign, str(coeff), exp) + + # unlike exp, ln and log10, the power function respects the + # rounding mode; no need to switch to ROUND_HALF_EVEN here + + # There's a difficulty here when 'other' is not an integer and + # the result is exact. In this case, the specification + # requires that the Inexact flag be raised (in spite of + # exactness), but since the result is exact _fix won't do this + # for us. (Correspondingly, the Underflow signal should also + # be raised for subnormal results.) We can't directly raise + # these signals either before or after calling _fix, since + # that would violate the precedence for signals. So we wrap + # the ._fix call in a temporary context, and reraise + # afterwards. + if exact and not other._isinteger(): + # pad with zeros up to length context.prec+1 if necessary; this + # ensures that the Rounded signal will be raised. + if len(ans._int) <= context.prec: + expdiff = context.prec + 1 - len(ans._int) + ans = _dec_from_triple(ans._sign, ans._int+'0'*expdiff, + ans._exp-expdiff) + + # create a copy of the current context, with cleared flags/traps + newcontext = context.copy() + newcontext.clear_flags() + for exception in _signals: + newcontext.traps[exception] = 0 + + # round in the new context + ans = ans._fix(newcontext) + + # raise Inexact, and if necessary, Underflow + newcontext._raise_error(Inexact) + if newcontext.flags[Subnormal]: + newcontext._raise_error(Underflow) + + # propagate signals to the original context; _fix could + # have raised any of Overflow, Underflow, Subnormal, + # Inexact, Rounded, Clamped. Overflow needs the correct + # arguments. Note that the order of the exceptions is + # important here. + if newcontext.flags[Overflow]: + context._raise_error(Overflow, 'above Emax', ans._sign) + for exception in Underflow, Subnormal, Inexact, Rounded, Clamped: + if newcontext.flags[exception]: + context._raise_error(exception) + + else: + ans = ans._fix(context) + + return ans + + def __rpow__(self, other, context=None): + """Swaps self/other and returns __pow__.""" + other = _convert_other(other) + if other is NotImplemented: + return other + return other.__pow__(self, context=context) + + def normalize(self, context=None): + """Normalize- strip trailing 0s, change anything equal to 0 to 0e0""" + + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + dup = self._fix(context) + if dup._isinfinity(): + return dup + + if not dup: + return _dec_from_triple(dup._sign, '0', 0) + exp_max = [context.Emax, context.Etop()][context.clamp] + end = len(dup._int) + exp = dup._exp + while dup._int[end-1] == '0' and exp < exp_max: + exp += 1 + end -= 1 + return _dec_from_triple(dup._sign, dup._int[:end], exp) + + def quantize(self, exp, rounding=None, context=None, watchexp=True): + """Quantize self so its exponent is the same as that of exp. + + Similar to self._rescale(exp._exp) but with error checking. + """ + exp = _convert_other(exp, raiseit=True) + + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + + if self._is_special or exp._is_special: + ans = self._check_nans(exp, context) + if ans: + return ans + + if exp._isinfinity() or self._isinfinity(): + if exp._isinfinity() and self._isinfinity(): + return Decimal(self) # if both are inf, it is OK + return context._raise_error(InvalidOperation, + 'quantize with one INF') + + # if we're not watching exponents, do a simple rescale + if not watchexp: + ans = self._rescale(exp._exp, rounding) + # raise Inexact and Rounded where appropriate + if ans._exp > self._exp: + context._raise_error(Rounded) + if ans != self: + context._raise_error(Inexact) + return ans + + # exp._exp should be between Etiny and Emax + if not (context.Etiny() <= exp._exp <= context.Emax): + return context._raise_error(InvalidOperation, + 'target exponent out of bounds in quantize') + + if not self: + ans = _dec_from_triple(self._sign, '0', exp._exp) + return ans._fix(context) + + self_adjusted = self.adjusted() + if self_adjusted > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if self_adjusted - exp._exp + 1 > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + ans = self._rescale(exp._exp, rounding) + if ans.adjusted() > context.Emax: + return context._raise_error(InvalidOperation, + 'exponent of quantize result too large for current context') + if len(ans._int) > context.prec: + return context._raise_error(InvalidOperation, + 'quantize result has too many digits for current context') + + # raise appropriate flags + if ans and ans.adjusted() < context.Emin: + context._raise_error(Subnormal) + if ans._exp > self._exp: + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + + # call to fix takes care of any necessary folddown, and + # signals Clamped if necessary + ans = ans._fix(context) + return ans + + def same_quantum(self, other): + """Return True if self and other have the same exponent; otherwise + return False. + + If either operand is a special value, the following rules are used: + * return True if both operands are infinities + * return True if both operands are NaNs + * otherwise, return False. + """ + other = _convert_other(other, raiseit=True) + if self._is_special or other._is_special: + return (self.is_nan() and other.is_nan() or + self.is_infinite() and other.is_infinite()) + return self._exp == other._exp + + def _rescale(self, exp, rounding): + """Rescale self so that the exponent is exp, either by padding with zeros + or by truncating digits, using the given rounding mode. + + Specials are returned without change. This operation is + quiet: it raises no flags, and uses no information from the + context. + + exp = exp to scale to (an integer) + rounding = rounding mode + """ + if self._is_special: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', exp) + + if self._exp >= exp: + # pad answer with zeros if necessary + return _dec_from_triple(self._sign, + self._int + '0'*(self._exp - exp), exp) + + # too many digits; round and lose data. If self.adjusted() < + # exp-1, replace self by 10**(exp-1) before rounding + digits = len(self._int) + self._exp - exp + if digits < 0: + self = _dec_from_triple(self._sign, '1', exp-1) + digits = 0 + this_function = self._pick_rounding_function[rounding] + changed = this_function(self, digits) + coeff = self._int[:digits] or '0' + if changed == 1: + coeff = str(int(coeff)+1) + return _dec_from_triple(self._sign, coeff, exp) + + def _round(self, places, rounding): + """Round a nonzero, nonspecial Decimal to a fixed number of + significant figures, using the given rounding mode. + + Infinities, NaNs and zeros are returned unaltered. + + This operation is quiet: it raises no flags, and uses no + information from the context. + + """ + if places <= 0: + raise ValueError("argument should be at least 1 in _round") + if self._is_special or not self: + return Decimal(self) + ans = self._rescale(self.adjusted()+1-places, rounding) + # it can happen that the rescale alters the adjusted exponent; + # for example when rounding 99.97 to 3 significant figures. + # When this happens we end up with an extra 0 at the end of + # the number; a second rescale fixes this. + if ans.adjusted() != self.adjusted(): + ans = ans._rescale(ans.adjusted()+1-places, rounding) + return ans + + def to_integral_exact(self, rounding=None, context=None): + """Rounds to a nearby integer. + + If no rounding mode is specified, take the rounding mode from + the context. This method raises the Rounded and Inexact flags + when appropriate. + + See also: to_integral_value, which does exactly the same as + this method except that it doesn't raise Inexact or Rounded. + """ + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + if not self: + return _dec_from_triple(self._sign, '0', 0) + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + ans = self._rescale(0, rounding) + if ans != self: + context._raise_error(Inexact) + context._raise_error(Rounded) + return ans + + def to_integral_value(self, rounding=None, context=None): + """Rounds to the nearest integer, without raising inexact, rounded.""" + if context is None: + context = getcontext() + if rounding is None: + rounding = context.rounding + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + return Decimal(self) + if self._exp >= 0: + return Decimal(self) + else: + return self._rescale(0, rounding) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + + def sqrt(self, context=None): + """Return the square root of self.""" + if context is None: + context = getcontext() + + if self._is_special: + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() and self._sign == 0: + return Decimal(self) + + if not self: + # exponent = self._exp // 2. sqrt(-0) = -0 + ans = _dec_from_triple(self._sign, '0', self._exp // 2) + return ans._fix(context) + + if self._sign == 1: + return context._raise_error(InvalidOperation, 'sqrt(-x), x > 0') + + # At this point self represents a positive number. Let p be + # the desired precision and express self in the form c*100**e + # with c a positive real number and e an integer, c and e + # being chosen so that 100**(p-1) <= c < 100**p. Then the + # (exact) square root of self is sqrt(c)*10**e, and 10**(p-1) + # <= sqrt(c) < 10**p, so the closest representable Decimal at + # precision p is n*10**e where n = round_half_even(sqrt(c)), + # the closest integer to sqrt(c) with the even integer chosen + # in the case of a tie. + # + # To ensure correct rounding in all cases, we use the + # following trick: we compute the square root to an extra + # place (precision p+1 instead of precision p), rounding down. + # Then, if the result is inexact and its last digit is 0 or 5, + # we increase the last digit to 1 or 6 respectively; if it's + # exact we leave the last digit alone. Now the final round to + # p places (or fewer in the case of underflow) will round + # correctly and raise the appropriate flags. + + # use an extra digit of precision + prec = context.prec+1 + + # write argument in the form c*100**e where e = self._exp//2 + # is the 'ideal' exponent, to be used if the square root is + # exactly representable. l is the number of 'digits' of c in + # base 100, so that 100**(l-1) <= c < 100**l. + op = _WorkRep(self) + e = op.exp >> 1 + if op.exp & 1: + c = op.int * 10 + l = (len(self._int) >> 1) + 1 + else: + c = op.int + l = len(self._int)+1 >> 1 + + # rescale so that c has exactly prec base 100 'digits' + shift = prec-l + if shift >= 0: + c *= 100**shift + exact = True + else: + c, remainder = divmod(c, 100**-shift) + exact = not remainder + e -= shift + + # find n = floor(sqrt(c)) using Newton's method + n = 10**prec + while True: + q = c//n + if n <= q: + break + else: + n = n + q >> 1 + exact = exact and n*n == c + + if exact: + # result is exact; rescale to use ideal exponent e + if shift >= 0: + # assert n % 10**shift == 0 + n //= 10**shift + else: + n *= 10**-shift + e += shift + else: + # result is not exact; fix last digit as described above + if n % 5 == 0: + n += 1 + + ans = _dec_from_triple(0, str(n), e) + + # round, and fit to current context + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def max(self, other, context=None): + """Returns the larger value. + + Like max(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + # If both operands are finite and equal in numerical value + # then an ordering is applied: + # + # If the signs differ then max returns the operand with the + # positive sign and min returns the operand with the negative sign + # + # If the signs are the same then the exponent is used to select + # the result. This is exactly the ordering used in compare_total. + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min(self, other, context=None): + """Returns the smaller value. + + Like min(self, other) except if one is not a number, returns + NaN (and signals if one is sNaN). Also rounds. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self._cmp(other) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def _isinteger(self): + """Returns whether self is an integer""" + if self._is_special: + return False + if self._exp >= 0: + return True + rest = self._int[self._exp:] + return rest == '0'*len(rest) + + def _iseven(self): + """Returns True if self is even. Assumes self is an integer.""" + if not self or self._exp > 0: + return True + return self._int[-1+self._exp] in '02468' + + def adjusted(self): + """Return the adjusted exponent of self""" + try: + return self._exp + len(self._int) - 1 + # If NaN or Infinity, self._exp is string + except TypeError: + return 0 + + def canonical(self, context=None): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + """ + return self + + def compare_signal(self, other, context=None): + """Compares self to the other operand numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + """ + other = _convert_other(other, raiseit = True) + ans = self._compare_check_nans(other, context) + if ans: + return ans + return self.compare(other, context=context) + + def compare_total(self, other): + """Compares self to other using the abstract representations. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + """ + other = _convert_other(other, raiseit=True) + + # if one is negative and the other is positive, it's easy + if self._sign and not other._sign: + return _NegativeOne + if not self._sign and other._sign: + return _One + sign = self._sign + + # let's handle both NaN types + self_nan = self._isnan() + other_nan = other._isnan() + if self_nan or other_nan: + if self_nan == other_nan: + # compare payloads as though they're integers + self_key = len(self._int), self._int + other_key = len(other._int), other._int + if self_key < other_key: + if sign: + return _One + else: + return _NegativeOne + if self_key > other_key: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + if sign: + if self_nan == 1: + return _NegativeOne + if other_nan == 1: + return _One + if self_nan == 2: + return _NegativeOne + if other_nan == 2: + return _One + else: + if self_nan == 1: + return _One + if other_nan == 1: + return _NegativeOne + if self_nan == 2: + return _One + if other_nan == 2: + return _NegativeOne + + if self < other: + return _NegativeOne + if self > other: + return _One + + if self._exp < other._exp: + if sign: + return _One + else: + return _NegativeOne + if self._exp > other._exp: + if sign: + return _NegativeOne + else: + return _One + return _Zero + + + def compare_total_mag(self, other): + """Compares self to other using abstract repr., ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + other = _convert_other(other, raiseit=True) + + s = self.copy_abs() + o = other.copy_abs() + return s.compare_total(o) + + def copy_abs(self): + """Returns a copy with the sign set to 0. """ + return _dec_from_triple(0, self._int, self._exp, self._is_special) + + def copy_negate(self): + """Returns a copy with the sign inverted.""" + if self._sign: + return _dec_from_triple(0, self._int, self._exp, self._is_special) + else: + return _dec_from_triple(1, self._int, self._exp, self._is_special) + + def copy_sign(self, other): + """Returns self with the sign of other.""" + other = _convert_other(other, raiseit=True) + return _dec_from_triple(other._sign, self._int, + self._exp, self._is_special) + + def exp(self, context=None): + """Returns e ** self.""" + + if context is None: + context = getcontext() + + # exp(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # exp(-Infinity) = 0 + if self._isinfinity() == -1: + return _Zero + + # exp(0) = 1 + if not self: + return _One + + # exp(Infinity) = Infinity + if self._isinfinity() == 1: + return Decimal(self) + + # the result is now guaranteed to be inexact (the true + # mathematical result is transcendental). There's no need to + # raise Rounded and Inexact here---they'll always be raised as + # a result of the call to _fix. + p = context.prec + adj = self.adjusted() + + # we only need to do any computation for quite a small range + # of adjusted exponents---for example, -29 <= adj <= 10 for + # the default context. For smaller exponent the result is + # indistinguishable from 1 at the given precision, while for + # larger exponent the result either overflows or underflows. + if self._sign == 0 and adj > len(str((context.Emax+1)*3)): + # overflow + ans = _dec_from_triple(0, '1', context.Emax+1) + elif self._sign == 1 and adj > len(str((-context.Etiny()+1)*3)): + # underflow to 0 + ans = _dec_from_triple(0, '1', context.Etiny()-1) + elif self._sign == 0 and adj < -p: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '1' + '0'*(p-1) + '1', -p) + elif self._sign == 1 and adj < -p-1: + # p+1 digits; final round will raise correct flags + ans = _dec_from_triple(0, '9'*(p+1), -p-1) + # general case + else: + op = _WorkRep(self) + c, e = op.int, op.exp + if op.sign == 1: + c = -c + + # compute correctly rounded result: increase precision by + # 3 digits at a time until we get an unambiguously + # roundable result + extra = 3 + while True: + coeff, exp = _dexp(c, e, p+extra) + if coeff % (5*10**(len(str(coeff))-p-1)): + break + extra += 3 + + ans = _dec_from_triple(0, str(coeff), exp) + + # at this stage, ans should round correctly with *any* + # rounding mode, not just with ROUND_HALF_EVEN + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + + return ans + + def is_canonical(self): + """Return True if self is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + """ + return True + + def is_finite(self): + """Return True if self is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + """ + return not self._is_special + + def is_infinite(self): + """Return True if self is infinite; otherwise return False.""" + return self._exp == 'F' + + def is_nan(self): + """Return True if self is a qNaN or sNaN; otherwise return False.""" + return self._exp in ('n', 'N') + + def is_normal(self, context=None): + """Return True if self is a normal number; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return context.Emin <= self.adjusted() + + def is_qnan(self): + """Return True if self is a quiet NaN; otherwise return False.""" + return self._exp == 'n' + + def is_signed(self): + """Return True if self is negative; otherwise return False.""" + return self._sign == 1 + + def is_snan(self): + """Return True if self is a signaling NaN; otherwise return False.""" + return self._exp == 'N' + + def is_subnormal(self, context=None): + """Return True if self is subnormal; otherwise return False.""" + if self._is_special or not self: + return False + if context is None: + context = getcontext() + return self.adjusted() < context.Emin + + def is_zero(self): + """Return True if self is a zero; otherwise return False.""" + return not self._is_special and self._int == '0' + + def _ln_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.ln(). + In other words, compute r such that self.ln() >= 10**r. Assumes + that self is finite and positive and that self != 1. + """ + + # for 0.1 <= x <= 10 we use the inequalities 1-1/x <= ln(x) <= x-1 + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # argument >= 10; we use 23/10 = 2.3 as a lower bound for ln(10) + return len(str(adj*23//10)) - 1 + if adj <= -2: + # argument <= 0.1 + return len(str((-1-adj)*23//10)) - 1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(c) + return len(num) - len(den) - (num < den) + # adj == -1, 0.1 <= self < 1 + return e + len(str(10**-e - c)) - 1 + + + def ln(self, context=None): + """Returns the natural (base e) logarithm of self.""" + + if context is None: + context = getcontext() + + # ln(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # ln(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # ln(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # ln(1.0) == 0.0 + if self == _One: + return _Zero + + # ln(negative) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'ln of a negative value') + + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision by 3 + # until we get an unambiguously roundable result + places = p - self._ln_exp_bound() + 2 # at least p+3 places + while True: + coeff = _dlog(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def _log10_exp_bound(self): + """Compute a lower bound for the adjusted exponent of self.log10(). + In other words, find r such that self.log10() >= 10**r. + Assumes that self is finite and positive and that self != 1. + """ + + # For x >= 10 or x < 0.1 we only need a bound on the integer + # part of log10(self), and this comes directly from the + # exponent of x. For 0.1 <= x <= 10 we use the inequalities + # 1-1/x <= log(x) <= x-1. If x > 1 we have |log10(x)| > + # (1-1/x)/2.31 > 0. If x < 1 then |log10(x)| > (1-x)/2.31 > 0 + + adj = self._exp + len(self._int) - 1 + if adj >= 1: + # self >= 10 + return len(str(adj))-1 + if adj <= -2: + # self < 0.1 + return len(str(-1-adj))-1 + op = _WorkRep(self) + c, e = op.int, op.exp + if adj == 0: + # 1 < self < 10 + num = str(c-10**-e) + den = str(231*c) + return len(num) - len(den) - (num < den) + 2 + # adj == -1, 0.1 <= self < 1 + num = str(10**-e-c) + return len(num) + e - (num < "231") - 1 + + def log10(self, context=None): + """Returns the base 10 logarithm of self.""" + + if context is None: + context = getcontext() + + # log10(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + # log10(0.0) == -Infinity + if not self: + return _NegativeInfinity + + # log10(Infinity) = Infinity + if self._isinfinity() == 1: + return _Infinity + + # log10(negative or -Infinity) raises InvalidOperation + if self._sign == 1: + return context._raise_error(InvalidOperation, + 'log10 of a negative value') + + # log10(10**n) = n + if self._int[0] == '1' and self._int[1:] == '0'*(len(self._int) - 1): + # answer may need rounding + ans = Decimal(self._exp + len(self._int) - 1) + else: + # result is irrational, so necessarily inexact + op = _WorkRep(self) + c, e = op.int, op.exp + p = context.prec + + # correctly rounded result: repeatedly increase precision + # until result is unambiguously roundable + places = p-self._log10_exp_bound()+2 + while True: + coeff = _dlog10(c, e, places) + # assert len(str(abs(coeff)))-p >= 1 + if coeff % (5*10**(len(str(abs(coeff)))-p-1)): + break + places += 3 + ans = _dec_from_triple(int(coeff<0), str(abs(coeff)), -places) + + context = context._shallow_copy() + rounding = context._set_rounding(ROUND_HALF_EVEN) + ans = ans._fix(context) + context.rounding = rounding + return ans + + def logb(self, context=None): + """ Returns the exponent of the magnitude of self's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of self (as though it were truncated + to a single digit while maintaining the value of that digit and + without limiting the resulting exponent). + """ + # logb(NaN) = NaN + ans = self._check_nans(context=context) + if ans: + return ans + + if context is None: + context = getcontext() + + # logb(+/-Inf) = +Inf + if self._isinfinity(): + return _Infinity + + # logb(0) = -Inf, DivisionByZero + if not self: + return context._raise_error(DivisionByZero, 'logb(0)', 1) + + # otherwise, simply return the adjusted exponent of self, as a + # Decimal. Note that no attempt is made to fit the result + # into the current context. + ans = Decimal(self.adjusted()) + return ans._fix(context) + + def _islogical(self): + """Return True if self is a logical operand. + + For being logical, it must be a finite number with a sign of 0, + an exponent of 0, and a coefficient whose digits must all be + either 0 or 1. + """ + if self._sign != 0 or self._exp != 0: + return False + for dig in self._int: + if dig not in '01': + return False + return True + + def _fill_logical(self, context, opa, opb): + dif = context.prec - len(opa) + if dif > 0: + opa = '0'*dif + opa + elif dif < 0: + opa = opa[-context.prec:] + dif = context.prec - len(opb) + if dif > 0: + opb = '0'*dif + opb + elif dif < 0: + opb = opb[-context.prec:] + return opa, opb + + def logical_and(self, other, context=None): + """Applies an 'and' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)&int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_invert(self, context=None): + """Invert all its digits.""" + if context is None: + context = getcontext() + return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), + context) + + def logical_or(self, other, context=None): + """Applies an 'or' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)|int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def logical_xor(self, other, context=None): + """Applies an 'xor' operation between self and other's digits.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + if not self._islogical() or not other._islogical(): + return context._raise_error(InvalidOperation) + + # fill to context.prec + (opa, opb) = self._fill_logical(context, self._int, other._int) + + # make the operation, and clean starting zeroes + result = "".join([str(int(a)^int(b)) for a,b in zip(opa,opb)]) + return _dec_from_triple(0, result.lstrip('0') or '0', 0) + + def max_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = other + else: + ans = self + + return ans._fix(context) + + def min_mag(self, other, context=None): + """Compares the values numerically with their sign ignored.""" + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + if self._is_special or other._is_special: + # If one operand is a quiet NaN and the other is number, then the + # number is always returned + sn = self._isnan() + on = other._isnan() + if sn or on: + if on == 1 and sn == 0: + return self._fix(context) + if sn == 1 and on == 0: + return other._fix(context) + return self._check_nans(other, context) + + c = self.copy_abs()._cmp(other.copy_abs()) + if c == 0: + c = self.compare_total(other) + + if c == -1: + ans = self + else: + ans = other + + return ans._fix(context) + + def next_minus(self, context=None): + """Returns the largest representable number smaller than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == -1: + return _NegativeInfinity + if self._isinfinity() == 1: + return _dec_from_triple(0, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_FLOOR) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__sub__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_plus(self, context=None): + """Returns the smallest representable number larger than itself.""" + if context is None: + context = getcontext() + + ans = self._check_nans(context=context) + if ans: + return ans + + if self._isinfinity() == 1: + return _Infinity + if self._isinfinity() == -1: + return _dec_from_triple(1, '9'*context.prec, context.Etop()) + + context = context.copy() + context._set_rounding(ROUND_CEILING) + context._ignore_all_flags() + new_self = self._fix(context) + if new_self != self: + return new_self + return self.__add__(_dec_from_triple(0, '1', context.Etiny()-1), + context) + + def next_toward(self, other, context=None): + """Returns the number closest to self, in the direction towards other. + + The result is the closest representable number to self + (excluding self) that is in the direction towards other, + unless both have the same value. If the two operands are + numerically equal, then the result is a copy of self with the + sign set to be the same as the sign of other. + """ + other = _convert_other(other, raiseit=True) + + if context is None: + context = getcontext() + + ans = self._check_nans(other, context) + if ans: + return ans + + comparison = self._cmp(other) + if comparison == 0: + return self.copy_sign(other) + + if comparison == -1: + ans = self.next_plus(context) + else: # comparison == 1 + ans = self.next_minus(context) + + # decide which flags to raise using value of ans + if ans._isinfinity(): + context._raise_error(Overflow, + 'Infinite result from next_toward', + ans._sign) + context._raise_error(Inexact) + context._raise_error(Rounded) + elif ans.adjusted() < context.Emin: + context._raise_error(Underflow) + context._raise_error(Subnormal) + context._raise_error(Inexact) + context._raise_error(Rounded) + # if precision == 1 then we don't raise Clamped for a + # result 0E-Etiny. + if not ans: + context._raise_error(Clamped) + + return ans + + def number_class(self, context=None): + """Returns an indication of the class of self. + + The class is one of the following strings: + sNaN + NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + """ + if self.is_snan(): + return "sNaN" + if self.is_qnan(): + return "NaN" + inf = self._isinfinity() + if inf == 1: + return "+Infinity" + if inf == -1: + return "-Infinity" + if self.is_zero(): + if self._sign: + return "-Zero" + else: + return "+Zero" + if context is None: + context = getcontext() + if self.is_subnormal(context=context): + if self._sign: + return "-Subnormal" + else: + return "+Subnormal" + # just a normal, regular, boring number, :) + if self._sign: + return "-Normal" + else: + return "+Normal" + + def radix(self): + """Just returns 10, as this is Decimal, :)""" + return Decimal(10) + + def rotate(self, other, context=None): + """Returns a rotated copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's rotate! + rotated = rotdig[torot:] + rotdig[:torot] + return _dec_from_triple(self._sign, + rotated.lstrip('0') or '0', self._exp) + + def scaleb(self, other, context=None): + """Returns self operand after adding the second value to its exp.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + liminf = -2 * (context.Emax + context.prec) + limsup = 2 * (context.Emax + context.prec) + if not (liminf <= int(other) <= limsup): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + d = _dec_from_triple(self._sign, self._int, self._exp + int(other)) + d = d._fix(context) + return d + + def shift(self, other, context=None): + """Returns a shifted copy of self, value-of-other times.""" + if context is None: + context = getcontext() + + other = _convert_other(other, raiseit=True) + + ans = self._check_nans(other, context) + if ans: + return ans + + if other._exp != 0: + return context._raise_error(InvalidOperation) + if not (-context.prec <= int(other) <= context.prec): + return context._raise_error(InvalidOperation) + + if self._isinfinity(): + return Decimal(self) + + # get values, pad if necessary + torot = int(other) + rotdig = self._int + topad = context.prec - len(rotdig) + if topad > 0: + rotdig = '0'*topad + rotdig + elif topad < 0: + rotdig = rotdig[-topad:] + + # let's shift! + if torot < 0: + shifted = rotdig[:torot] + else: + shifted = rotdig + '0'*torot + shifted = shifted[-context.prec:] + + return _dec_from_triple(self._sign, + shifted.lstrip('0') or '0', self._exp) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + return (self.__class__, (str(self),)) + + def __copy__(self): + if type(self) is Decimal: + return self # I'm immutable; therefore I am my own clone + return self.__class__(str(self)) + + def __deepcopy__(self, memo): + if type(self) is Decimal: + return self # My components are also immutable + return self.__class__(str(self)) + + # PEP 3101 support. the _localeconv keyword argument should be + # considered private: it's provided for ease of testing only. + def __format__(self, specifier, context=None, _localeconv=None): + """Format a Decimal instance according to the given specifier. + + The specifier should be a standard format specifier, with the + form described in PEP 3101. Formatting types 'e', 'E', 'f', + 'F', 'g', 'G', 'n' and '%' are supported. If the formatting + type is omitted it defaults to 'g' or 'G', depending on the + value of context.capitals. + """ + + # Note: PEP 3101 says that if the type is not present then + # there should be at least one digit after the decimal point. + # We take the liberty of ignoring this requirement for + # Decimal---it's presumably there to make sure that + # format(float, '') behaves similarly to str(float). + if context is None: + context = getcontext() + + spec = _parse_format_specifier(specifier, _localeconv=_localeconv) + + # special values don't care about the type or precision + if self._is_special: + sign = _format_sign(self._sign, spec) + body = str(self.copy_abs()) + return _format_align(sign, body, spec) + + # a type of None defaults to 'g' or 'G', depending on context + if spec['type'] is None: + spec['type'] = ['g', 'G'][context.capitals] + + # if type is '%', adjust exponent of self accordingly + if spec['type'] == '%': + self = _dec_from_triple(self._sign, self._int, self._exp+2) + + # round if necessary, taking rounding mode from the context + rounding = context.rounding + precision = spec['precision'] + if precision is not None: + if spec['type'] in 'eE': + self = self._round(precision+1, rounding) + elif spec['type'] in 'fF%': + self = self._rescale(-precision, rounding) + elif spec['type'] in 'gG' and len(self._int) > precision: + self = self._round(precision, rounding) + # special case: zeros with a positive exponent can't be + # represented in fixed point; rescale them to 0e0. + if not self and self._exp > 0 and spec['type'] in 'fF%': + self = self._rescale(0, rounding) + + # figure out placement of the decimal point + leftdigits = self._exp + len(self._int) + if spec['type'] in 'eE': + if not self and precision is not None: + dotplace = 1 - precision + else: + dotplace = 1 + elif spec['type'] in 'fF%': + dotplace = leftdigits + elif spec['type'] in 'gG': + if self._exp <= 0 and leftdigits > -6: + dotplace = leftdigits + else: + dotplace = 1 + + # find digits before and after decimal point, and get exponent + if dotplace < 0: + intpart = '0' + fracpart = '0'*(-dotplace) + self._int + elif dotplace > len(self._int): + intpart = self._int + '0'*(dotplace-len(self._int)) + fracpart = '' + else: + intpart = self._int[:dotplace] or '0' + fracpart = self._int[dotplace:] + exp = leftdigits-dotplace + + # done with the decimal-specific stuff; hand over the rest + # of the formatting to the _format_number function + return _format_number(self._sign, intpart, fracpart, exp, spec) + +def _dec_from_triple(sign, coefficient, exponent, special=False): + """Create a decimal instance directly, without any validation, + normalization (e.g. removal of leading zeros) or argument + conversion. + + This function is for *internal use only*. + """ + + self = object.__new__(Decimal) + self._sign = sign + self._int = coefficient + self._exp = exponent + self._is_special = special + + return self + +# Register Decimal as a kind of Number (an abstract base class). +# However, do not register it as Real (because Decimals are not +# interoperable with floats). +_numbers.Number.register(Decimal) + + +##### Context class ####################################################### + +class _ContextManager(object): + """Context manager class to support localcontext(). + + Sets a copy of the supplied context in __enter__() and restores + the previous decimal context in __exit__() + """ + def __init__(self, new_context): + self.new_context = new_context.copy() + def __enter__(self): + self.saved_context = getcontext() + setcontext(self.new_context) + return self.new_context + def __exit__(self, t, v, tb): + setcontext(self.saved_context) + +class Context(object): + """Contains the context for a Decimal instance. + + Contains: + prec - precision (for use in rounding, division, square roots..) + rounding - rounding type (how you round) + traps - If traps[exception] = 1, then the exception is + raised when it is caused. Otherwise, a value is + substituted in. + flags - When an exception is caused, flags[exception] is set. + (Whether or not the trap_enabler is set) + Should be reset by user of Decimal instance. + Emin - Minimum exponent + Emax - Maximum exponent + capitals - If 1, 1*10^1 is printed as 1E+1. + If 0, printed as 1e1 + clamp - If 1, change exponents if too high (Default 0) + """ + + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, + capitals=None, clamp=None, flags=None, traps=None, + _ignored_flags=None): + # Set defaults; for everything except flags and _ignored_flags, + # inherit from DefaultContext. + try: + dc = DefaultContext + except NameError: + pass + + self.prec = prec if prec is not None else dc.prec + self.rounding = rounding if rounding is not None else dc.rounding + self.Emin = Emin if Emin is not None else dc.Emin + self.Emax = Emax if Emax is not None else dc.Emax + self.capitals = capitals if capitals is not None else dc.capitals + self.clamp = clamp if clamp is not None else dc.clamp + + if _ignored_flags is None: + self._ignored_flags = [] + else: + self._ignored_flags = _ignored_flags + + if traps is None: + self.traps = dc.traps.copy() + elif not isinstance(traps, dict): + self.traps = dict((s, int(s in traps)) for s in _signals + traps) + else: + self.traps = traps + + if flags is None: + self.flags = dict.fromkeys(_signals, 0) + elif not isinstance(flags, dict): + self.flags = dict((s, int(s in flags)) for s in _signals + flags) + else: + self.flags = flags + + def _set_integer_check(self, name, value, vmin, vmax): + if not isinstance(value, int): + raise TypeError("%s must be an integer" % name) + if vmin == '-inf': + if value > vmax: + raise ValueError("%s must be in [%s, %d]. got: %s" % (name, vmin, vmax, value)) + elif vmax == 'inf': + if value < vmin: + raise ValueError("%s must be in [%d, %s]. got: %s" % (name, vmin, vmax, value)) + else: + if value < vmin or value > vmax: + raise ValueError("%s must be in [%d, %d]. got %s" % (name, vmin, vmax, value)) + return object.__setattr__(self, name, value) + + def _set_signal_dict(self, name, d): + if not isinstance(d, dict): + raise TypeError("%s must be a signal dict" % d) + for key in d: + if not key in _signals: + raise KeyError("%s is not a valid signal dict" % d) + for key in _signals: + if not key in d: + raise KeyError("%s is not a valid signal dict" % d) + return object.__setattr__(self, name, d) + + def __setattr__(self, name, value): + if name == 'prec': + return self._set_integer_check(name, value, 1, 'inf') + elif name == 'Emin': + return self._set_integer_check(name, value, '-inf', 0) + elif name == 'Emax': + return self._set_integer_check(name, value, 0, 'inf') + elif name == 'capitals': + return self._set_integer_check(name, value, 0, 1) + elif name == 'clamp': + return self._set_integer_check(name, value, 0, 1) + elif name == 'rounding': + if not value in _rounding_modes: + # raise TypeError even for strings to have consistency + # among various implementations. + raise TypeError("%s: invalid rounding mode" % value) + return object.__setattr__(self, name, value) + elif name == 'flags' or name == 'traps': + return self._set_signal_dict(name, value) + elif name == '_ignored_flags': + return object.__setattr__(self, name, value) + else: + raise AttributeError( + "'decimal.Context' object has no attribute '%s'" % name) + + def __delattr__(self, name): + raise AttributeError("%s cannot be deleted" % name) + + # Support for pickling, copy, and deepcopy + def __reduce__(self): + flags = [sig for sig, v in self.flags.items() if v] + traps = [sig for sig, v in self.traps.items() if v] + return (self.__class__, + (self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, flags, traps)) + + def __repr__(self): + """Show the current context.""" + s = [] + s.append('Context(prec=%(prec)d, rounding=%(rounding)s, ' + 'Emin=%(Emin)d, Emax=%(Emax)d, capitals=%(capitals)d, ' + 'clamp=%(clamp)d' + % vars(self)) + names = [f.__name__ for f, v in self.flags.items() if v] + s.append('flags=[' + ', '.join(names) + ']') + names = [t.__name__ for t, v in self.traps.items() if v] + s.append('traps=[' + ', '.join(names) + ']') + return ', '.join(s) + ')' + + def clear_flags(self): + """Reset all flags to zero""" + for flag in self.flags: + self.flags[flag] = 0 + + def clear_traps(self): + """Reset all traps to zero""" + for flag in self.traps: + self.traps[flag] = 0 + + def _shallow_copy(self): + """Returns a shallow copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, self.flags, self.traps, + self._ignored_flags) + return nc + + def copy(self): + """Returns a deep copy from self.""" + nc = Context(self.prec, self.rounding, self.Emin, self.Emax, + self.capitals, self.clamp, + self.flags.copy(), self.traps.copy(), + self._ignored_flags) + return nc + __copy__ = copy + + def _raise_error(self, condition, explanation = None, *args): + """Handles an error + + If the flag is in _ignored_flags, returns the default response. + Otherwise, it sets the flag, then, if the corresponding + trap_enabler is set, it reraises the exception. Otherwise, it returns + the default value after setting the flag. + """ + error = _condition_map.get(condition, condition) + if error in self._ignored_flags: + # Don't touch the flag + return error().handle(self, *args) + + self.flags[error] = 1 + if not self.traps[error]: + # The errors define how to handle themselves. + return condition().handle(self, *args) + + # Errors should only be risked on copies of the context + # self._ignored_flags = [] + raise error(explanation) + + def _ignore_all_flags(self): + """Ignore all flags, if they are raised""" + return self._ignore_flags(*_signals) + + def _ignore_flags(self, *flags): + """Ignore the flags, if they are raised""" + # Do not mutate-- This way, copies of a context leave the original + # alone. + self._ignored_flags = (self._ignored_flags + list(flags)) + return list(flags) + + def _regard_flags(self, *flags): + """Stop ignoring the flags, if they are raised""" + if flags and isinstance(flags[0], (tuple,list)): + flags = flags[0] + for flag in flags: + self._ignored_flags.remove(flag) + + # We inherit object.__hash__, so we must deny this explicitly + __hash__ = None + + def Etiny(self): + """Returns Etiny (= Emin - prec + 1)""" + return int(self.Emin - self.prec + 1) + + def Etop(self): + """Returns maximum exponent (= Emax - prec + 1)""" + return int(self.Emax - self.prec + 1) + + def _set_rounding(self, type): + """Sets the rounding type. + + Sets the rounding type, and returns the current (previous) + rounding type. Often used like: + + context = context.copy() + # so you don't change the calling context + # if an error occurs in the middle. + rounding = context._set_rounding(ROUND_UP) + val = self.__sub__(other, context=context) + context._set_rounding(rounding) + + This will make it round up for that operation. + """ + rounding = self.rounding + self.rounding= type + return rounding + + def create_decimal(self, num='0'): + """Creates a new Decimal instance but using self as context. + + This method implements the to-number operation of the + IBM Decimal specification.""" + + if isinstance(num, str) and num != num.strip(): + return self._raise_error(ConversionSyntax, + "no trailing or leading whitespace is " + "permitted.") + + d = Decimal(num, context=self) + if d._isnan() and len(d._int) > self.prec - self.clamp: + return self._raise_error(ConversionSyntax, + "diagnostic info too long in NaN") + return d._fix(self) + + def create_decimal_from_float(self, f): + """Creates a new Decimal instance from a float but rounding using self + as the context. + + >>> context = Context(prec=5, rounding=ROUND_DOWN) + >>> context.create_decimal_from_float(3.1415926535897932) + Decimal('3.1415') + >>> context = Context(prec=5, traps=[Inexact]) + >>> context.create_decimal_from_float(3.1415926535897932) + Traceback (most recent call last): + ... + decimal.Inexact: None + + """ + d = Decimal.from_float(f) # An exact conversion + return d._fix(self) # Apply the context rounding + + # Methods + def abs(self, a): + """Returns the absolute value of the operand. + + If the operand is negative, the result is the same as using the minus + operation on the operand. Otherwise, the result is the same as using + the plus operation on the operand. + + >>> ExtendedContext.abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.abs(Decimal('101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.__abs__(context=self) + + def add(self, a, b): + """Return the sum of the two operands. + + >>> ExtendedContext.add(Decimal('12'), Decimal('7.00')) + Decimal('19.00') + >>> ExtendedContext.add(Decimal('1E+2'), Decimal('1.01E+4')) + Decimal('1.02E+4') + >>> ExtendedContext.add(1, Decimal(2)) + Decimal('3') + >>> ExtendedContext.add(Decimal(8), 5) + Decimal('13') + >>> ExtendedContext.add(5, 5) + Decimal('10') + """ + a = _convert_other(a, raiseit=True) + r = a.__add__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def _apply(self, a): + return str(a._fix(self)) + + def canonical(self, a): + """Returns the same Decimal object. + + As we do not have different encodings for the same number, the + received object already is in its canonical form. + + >>> ExtendedContext.canonical(Decimal('2.50')) + Decimal('2.50') + """ + if not isinstance(a, Decimal): + raise TypeError("canonical requires a Decimal as an argument.") + return a.canonical(context=self) + + def compare(self, a, b): + """Compares values numerically. + + If the signs of the operands differ, a value representing each operand + ('-1' if the operand is less than zero, '0' if the operand is zero or + negative zero, or '1' if the operand is greater than zero) is used in + place of that operand for the comparison instead of the actual + operand. + + The comparison is then effected by subtracting the second operand from + the first and then returning a value according to the result of the + subtraction: '-1' if the result is less than zero, '0' if the result is + zero or negative zero, or '1' if the result is greater than zero. + + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('2.10')) + Decimal('0') + >>> ExtendedContext.compare(Decimal('3'), Decimal('2.1')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('2.1'), Decimal('-3')) + Decimal('1') + >>> ExtendedContext.compare(Decimal('-3'), Decimal('2.1')) + Decimal('-1') + >>> ExtendedContext.compare(1, 2) + Decimal('-1') + >>> ExtendedContext.compare(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare(b, context=self) + + def compare_signal(self, a, b): + """Compares the values of the two operands numerically. + + It's pretty much like compare(), but all NaNs signal, with signaling + NaNs taking precedence over quiet NaNs. + + >>> c = ExtendedContext + >>> c.compare_signal(Decimal('2.1'), Decimal('3')) + Decimal('-1') + >>> c.compare_signal(Decimal('2.1'), Decimal('2.1')) + Decimal('0') + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('NaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.flags[InvalidOperation] = 0 + >>> print(c.flags[InvalidOperation]) + 0 + >>> c.compare_signal(Decimal('sNaN'), Decimal('2.1')) + Decimal('NaN') + >>> print(c.flags[InvalidOperation]) + 1 + >>> c.compare_signal(-1, 2) + Decimal('-1') + >>> c.compare_signal(Decimal(-1), 2) + Decimal('-1') + >>> c.compare_signal(-1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_signal(b, context=self) + + def compare_total(self, a, b): + """Compares two operands using their abstract representation. + + This is not like the standard compare, which use their numerical + value. Note that a total ordering is defined for all possible abstract + representations. + + >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('-127'), Decimal('12')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.3')) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal('12.30'), Decimal('12.30')) + Decimal('0') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('12.300')) + Decimal('1') + >>> ExtendedContext.compare_total(Decimal('12.3'), Decimal('NaN')) + Decimal('-1') + >>> ExtendedContext.compare_total(1, 2) + Decimal('-1') + >>> ExtendedContext.compare_total(Decimal(1), 2) + Decimal('-1') + >>> ExtendedContext.compare_total(1, Decimal(2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.compare_total(b) + + def compare_total_mag(self, a, b): + """Compares two operands using their abstract representation ignoring sign. + + Like compare_total, but with operand's sign ignored and assumed to be 0. + """ + a = _convert_other(a, raiseit=True) + return a.compare_total_mag(b) + + def copy_abs(self, a): + """Returns a copy of the operand with the sign set to 0. + + >>> ExtendedContext.copy_abs(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_abs(Decimal('-100')) + Decimal('100') + >>> ExtendedContext.copy_abs(-1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_abs() + + def copy_decimal(self, a): + """Returns a copy of the decimal object. + + >>> ExtendedContext.copy_decimal(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.copy_decimal(Decimal('-1.00')) + Decimal('-1.00') + >>> ExtendedContext.copy_decimal(1) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return Decimal(a) + + def copy_negate(self, a): + """Returns a copy of the operand with the sign inverted. + + >>> ExtendedContext.copy_negate(Decimal('101.5')) + Decimal('-101.5') + >>> ExtendedContext.copy_negate(Decimal('-101.5')) + Decimal('101.5') + >>> ExtendedContext.copy_negate(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_negate() + + def copy_sign(self, a, b): + """Copies the second operand's sign to the first one. + + In detail, it returns a copy of the first operand with the sign + equal to the sign of the second operand. + + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('7.33')) + Decimal('1.50') + >>> ExtendedContext.copy_sign(Decimal( '1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(Decimal('-1.50'), Decimal('-7.33')) + Decimal('-1.50') + >>> ExtendedContext.copy_sign(1, -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(Decimal(1), -2) + Decimal('-1') + >>> ExtendedContext.copy_sign(1, Decimal(-2)) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.copy_sign(b) + + def divide(self, a, b): + """Decimal division in a specified context. + + >>> ExtendedContext.divide(Decimal('1'), Decimal('3')) + Decimal('0.333333333') + >>> ExtendedContext.divide(Decimal('2'), Decimal('3')) + Decimal('0.666666667') + >>> ExtendedContext.divide(Decimal('5'), Decimal('2')) + Decimal('2.5') + >>> ExtendedContext.divide(Decimal('1'), Decimal('10')) + Decimal('0.1') + >>> ExtendedContext.divide(Decimal('12'), Decimal('12')) + Decimal('1') + >>> ExtendedContext.divide(Decimal('8.00'), Decimal('2')) + Decimal('4.00') + >>> ExtendedContext.divide(Decimal('2.400'), Decimal('2.0')) + Decimal('1.20') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('100')) + Decimal('10') + >>> ExtendedContext.divide(Decimal('1000'), Decimal('1')) + Decimal('1000') + >>> ExtendedContext.divide(Decimal('2.40E+6'), Decimal('2')) + Decimal('1.20E+6') + >>> ExtendedContext.divide(5, 5) + Decimal('1') + >>> ExtendedContext.divide(Decimal(5), 5) + Decimal('1') + >>> ExtendedContext.divide(5, Decimal(5)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__truediv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divide_int(self, a, b): + """Divides two numbers and returns the integer part of the result. + + >>> ExtendedContext.divide_int(Decimal('2'), Decimal('3')) + Decimal('0') + >>> ExtendedContext.divide_int(Decimal('10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal('1'), Decimal('0.3')) + Decimal('3') + >>> ExtendedContext.divide_int(10, 3) + Decimal('3') + >>> ExtendedContext.divide_int(Decimal(10), 3) + Decimal('3') + >>> ExtendedContext.divide_int(10, Decimal(3)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__floordiv__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def divmod(self, a, b): + """Return (a // b, a % b). + + >>> ExtendedContext.divmod(Decimal(8), Decimal(3)) + (Decimal('2'), Decimal('2')) + >>> ExtendedContext.divmod(Decimal(8), Decimal(4)) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(Decimal(8), 4) + (Decimal('2'), Decimal('0')) + >>> ExtendedContext.divmod(8, Decimal(4)) + (Decimal('2'), Decimal('0')) + """ + a = _convert_other(a, raiseit=True) + r = a.__divmod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def exp(self, a): + """Returns e ** a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.exp(Decimal('-Infinity')) + Decimal('0') + >>> c.exp(Decimal('-1')) + Decimal('0.367879441') + >>> c.exp(Decimal('0')) + Decimal('1') + >>> c.exp(Decimal('1')) + Decimal('2.71828183') + >>> c.exp(Decimal('0.693147181')) + Decimal('2.00000000') + >>> c.exp(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.exp(10) + Decimal('22026.4658') + """ + a =_convert_other(a, raiseit=True) + return a.exp(context=self) + + def fma(self, a, b, c): + """Returns a multiplied by b, plus c. + + The first two operands are multiplied together, using multiply, + the third operand is then added to the result of that + multiplication, using add, all with only one final rounding. + + >>> ExtendedContext.fma(Decimal('3'), Decimal('5'), Decimal('7')) + Decimal('22') + >>> ExtendedContext.fma(Decimal('3'), Decimal('-5'), Decimal('7')) + Decimal('-8') + >>> ExtendedContext.fma(Decimal('888565290'), Decimal('1557.96930'), Decimal('-86087.7578')) + Decimal('1.38435736E+12') + >>> ExtendedContext.fma(1, 3, 4) + Decimal('7') + >>> ExtendedContext.fma(1, Decimal(3), 4) + Decimal('7') + >>> ExtendedContext.fma(1, 3, Decimal(4)) + Decimal('7') + """ + a = _convert_other(a, raiseit=True) + return a.fma(b, c, context=self) + + def is_canonical(self, a): + """Return True if the operand is canonical; otherwise return False. + + Currently, the encoding of a Decimal instance is always + canonical, so this method returns True for any Decimal. + + >>> ExtendedContext.is_canonical(Decimal('2.50')) + True + """ + if not isinstance(a, Decimal): + raise TypeError("is_canonical requires a Decimal as an argument.") + return a.is_canonical() + + def is_finite(self, a): + """Return True if the operand is finite; otherwise return False. + + A Decimal instance is considered finite if it is neither + infinite nor a NaN. + + >>> ExtendedContext.is_finite(Decimal('2.50')) + True + >>> ExtendedContext.is_finite(Decimal('-0.3')) + True + >>> ExtendedContext.is_finite(Decimal('0')) + True + >>> ExtendedContext.is_finite(Decimal('Inf')) + False + >>> ExtendedContext.is_finite(Decimal('NaN')) + False + >>> ExtendedContext.is_finite(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_finite() + + def is_infinite(self, a): + """Return True if the operand is infinite; otherwise return False. + + >>> ExtendedContext.is_infinite(Decimal('2.50')) + False + >>> ExtendedContext.is_infinite(Decimal('-Inf')) + True + >>> ExtendedContext.is_infinite(Decimal('NaN')) + False + >>> ExtendedContext.is_infinite(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_infinite() + + def is_nan(self, a): + """Return True if the operand is a qNaN or sNaN; + otherwise return False. + + >>> ExtendedContext.is_nan(Decimal('2.50')) + False + >>> ExtendedContext.is_nan(Decimal('NaN')) + True + >>> ExtendedContext.is_nan(Decimal('-sNaN')) + True + >>> ExtendedContext.is_nan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_nan() + + def is_normal(self, a): + """Return True if the operand is a normal number; + otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_normal(Decimal('2.50')) + True + >>> c.is_normal(Decimal('0.1E-999')) + False + >>> c.is_normal(Decimal('0.00')) + False + >>> c.is_normal(Decimal('-Inf')) + False + >>> c.is_normal(Decimal('NaN')) + False + >>> c.is_normal(1) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_normal(context=self) + + def is_qnan(self, a): + """Return True if the operand is a quiet NaN; otherwise return False. + + >>> ExtendedContext.is_qnan(Decimal('2.50')) + False + >>> ExtendedContext.is_qnan(Decimal('NaN')) + True + >>> ExtendedContext.is_qnan(Decimal('sNaN')) + False + >>> ExtendedContext.is_qnan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_qnan() + + def is_signed(self, a): + """Return True if the operand is negative; otherwise return False. + + >>> ExtendedContext.is_signed(Decimal('2.50')) + False + >>> ExtendedContext.is_signed(Decimal('-12')) + True + >>> ExtendedContext.is_signed(Decimal('-0')) + True + >>> ExtendedContext.is_signed(8) + False + >>> ExtendedContext.is_signed(-8) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_signed() + + def is_snan(self, a): + """Return True if the operand is a signaling NaN; + otherwise return False. + + >>> ExtendedContext.is_snan(Decimal('2.50')) + False + >>> ExtendedContext.is_snan(Decimal('NaN')) + False + >>> ExtendedContext.is_snan(Decimal('sNaN')) + True + >>> ExtendedContext.is_snan(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_snan() + + def is_subnormal(self, a): + """Return True if the operand is subnormal; otherwise return False. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.is_subnormal(Decimal('2.50')) + False + >>> c.is_subnormal(Decimal('0.1E-999')) + True + >>> c.is_subnormal(Decimal('0.00')) + False + >>> c.is_subnormal(Decimal('-Inf')) + False + >>> c.is_subnormal(Decimal('NaN')) + False + >>> c.is_subnormal(1) + False + """ + a = _convert_other(a, raiseit=True) + return a.is_subnormal(context=self) + + def is_zero(self, a): + """Return True if the operand is a zero; otherwise return False. + + >>> ExtendedContext.is_zero(Decimal('0')) + True + >>> ExtendedContext.is_zero(Decimal('2.50')) + False + >>> ExtendedContext.is_zero(Decimal('-0E+2')) + True + >>> ExtendedContext.is_zero(1) + False + >>> ExtendedContext.is_zero(0) + True + """ + a = _convert_other(a, raiseit=True) + return a.is_zero() + + def ln(self, a): + """Returns the natural (base e) logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.ln(Decimal('0')) + Decimal('-Infinity') + >>> c.ln(Decimal('1.000')) + Decimal('0') + >>> c.ln(Decimal('2.71828183')) + Decimal('1.00000000') + >>> c.ln(Decimal('10')) + Decimal('2.30258509') + >>> c.ln(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.ln(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.ln(context=self) + + def log10(self, a): + """Returns the base 10 logarithm of the operand. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.log10(Decimal('0')) + Decimal('-Infinity') + >>> c.log10(Decimal('0.001')) + Decimal('-3') + >>> c.log10(Decimal('1.000')) + Decimal('0') + >>> c.log10(Decimal('2')) + Decimal('0.301029996') + >>> c.log10(Decimal('10')) + Decimal('1') + >>> c.log10(Decimal('70')) + Decimal('1.84509804') + >>> c.log10(Decimal('+Infinity')) + Decimal('Infinity') + >>> c.log10(0) + Decimal('-Infinity') + >>> c.log10(1) + Decimal('0') + """ + a = _convert_other(a, raiseit=True) + return a.log10(context=self) + + def logb(self, a): + """ Returns the exponent of the magnitude of the operand's MSD. + + The result is the integer which is the exponent of the magnitude + of the most significant digit of the operand (as though the + operand were truncated to a single digit while maintaining the + value of that digit and without limiting the resulting exponent). + + >>> ExtendedContext.logb(Decimal('250')) + Decimal('2') + >>> ExtendedContext.logb(Decimal('2.50')) + Decimal('0') + >>> ExtendedContext.logb(Decimal('0.03')) + Decimal('-2') + >>> ExtendedContext.logb(Decimal('0')) + Decimal('-Infinity') + >>> ExtendedContext.logb(1) + Decimal('0') + >>> ExtendedContext.logb(10) + Decimal('1') + >>> ExtendedContext.logb(100) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.logb(context=self) + + def logical_and(self, a, b): + """Applies the logical operation 'and' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010')) + Decimal('1000') + >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10')) + Decimal('10') + >>> ExtendedContext.logical_and(110, 1101) + Decimal('100') + >>> ExtendedContext.logical_and(Decimal(110), 1101) + Decimal('100') + >>> ExtendedContext.logical_and(110, Decimal(1101)) + Decimal('100') + """ + a = _convert_other(a, raiseit=True) + return a.logical_and(b, context=self) + + def logical_invert(self, a): + """Invert all the digits in the operand. + + The operand must be a logical number. + + >>> ExtendedContext.logical_invert(Decimal('0')) + Decimal('111111111') + >>> ExtendedContext.logical_invert(Decimal('1')) + Decimal('111111110') + >>> ExtendedContext.logical_invert(Decimal('111111111')) + Decimal('0') + >>> ExtendedContext.logical_invert(Decimal('101010101')) + Decimal('10101010') + >>> ExtendedContext.logical_invert(1101) + Decimal('111110010') + """ + a = _convert_other(a, raiseit=True) + return a.logical_invert(context=self) + + def logical_or(self, a, b): + """Applies the logical operation 'or' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010')) + Decimal('1110') + >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10')) + Decimal('1110') + >>> ExtendedContext.logical_or(110, 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(Decimal(110), 1101) + Decimal('1111') + >>> ExtendedContext.logical_or(110, Decimal(1101)) + Decimal('1111') + """ + a = _convert_other(a, raiseit=True) + return a.logical_or(b, context=self) + + def logical_xor(self, a, b): + """Applies the logical operation 'xor' between each operand's digits. + + The operands must be both logical numbers. + + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0')) + Decimal('1') + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1')) + Decimal('0') + >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010')) + Decimal('110') + >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10')) + Decimal('1101') + >>> ExtendedContext.logical_xor(110, 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(Decimal(110), 1101) + Decimal('1011') + >>> ExtendedContext.logical_xor(110, Decimal(1101)) + Decimal('1011') + """ + a = _convert_other(a, raiseit=True) + return a.logical_xor(b, context=self) + + def max(self, a, b): + """max compares two values numerically and returns the maximum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the maximum (closer to positive + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.max(Decimal('3'), Decimal('2')) + Decimal('3') + >>> ExtendedContext.max(Decimal('-10'), Decimal('3')) + Decimal('3') + >>> ExtendedContext.max(Decimal('1.0'), Decimal('1')) + Decimal('1') + >>> ExtendedContext.max(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max(1, 2) + Decimal('2') + >>> ExtendedContext.max(Decimal(1), 2) + Decimal('2') + >>> ExtendedContext.max(1, Decimal(2)) + Decimal('2') + """ + a = _convert_other(a, raiseit=True) + return a.max(b, context=self) + + def max_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.max_mag(Decimal('7'), Decimal('-10')) + Decimal('-10') + >>> ExtendedContext.max_mag(1, -2) + Decimal('-2') + >>> ExtendedContext.max_mag(Decimal(1), -2) + Decimal('-2') + >>> ExtendedContext.max_mag(1, Decimal(-2)) + Decimal('-2') + """ + a = _convert_other(a, raiseit=True) + return a.max_mag(b, context=self) + + def min(self, a, b): + """min compares two values numerically and returns the minimum. + + If either operand is a NaN then the general rules apply. + Otherwise, the operands are compared as though by the compare + operation. If they are numerically equal then the left-hand operand + is chosen as the result. Otherwise the minimum (closer to negative + infinity) of the two operands is chosen as the result. + + >>> ExtendedContext.min(Decimal('3'), Decimal('2')) + Decimal('2') + >>> ExtendedContext.min(Decimal('-10'), Decimal('3')) + Decimal('-10') + >>> ExtendedContext.min(Decimal('1.0'), Decimal('1')) + Decimal('1.0') + >>> ExtendedContext.min(Decimal('7'), Decimal('NaN')) + Decimal('7') + >>> ExtendedContext.min(1, 2) + Decimal('1') + >>> ExtendedContext.min(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.min(1, Decimal(29)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min(b, context=self) + + def min_mag(self, a, b): + """Compares the values numerically with their sign ignored. + + >>> ExtendedContext.min_mag(Decimal('3'), Decimal('-2')) + Decimal('-2') + >>> ExtendedContext.min_mag(Decimal('-3'), Decimal('NaN')) + Decimal('-3') + >>> ExtendedContext.min_mag(1, -2) + Decimal('1') + >>> ExtendedContext.min_mag(Decimal(1), -2) + Decimal('1') + >>> ExtendedContext.min_mag(1, Decimal(-2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.min_mag(b, context=self) + + def minus(self, a): + """Minus corresponds to unary prefix minus in Python. + + The operation is evaluated using the same rules as subtract; the + operation minus(a) is calculated as subtract('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.minus(Decimal('1.3')) + Decimal('-1.3') + >>> ExtendedContext.minus(Decimal('-1.3')) + Decimal('1.3') + >>> ExtendedContext.minus(1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__neg__(context=self) + + def multiply(self, a, b): + """multiply multiplies two operands. + + If either operand is a special value then the general rules apply. + Otherwise, the operands are multiplied together + ('long multiplication'), resulting in a number which may be as long as + the sum of the lengths of the two operands. + + >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) + Decimal('3.60') + >>> ExtendedContext.multiply(Decimal('7'), Decimal('3')) + Decimal('21') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('0.8')) + Decimal('0.72') + >>> ExtendedContext.multiply(Decimal('0.9'), Decimal('-0')) + Decimal('-0.0') + >>> ExtendedContext.multiply(Decimal('654321'), Decimal('654321')) + Decimal('4.28135971E+11') + >>> ExtendedContext.multiply(7, 7) + Decimal('49') + >>> ExtendedContext.multiply(Decimal(7), 7) + Decimal('49') + >>> ExtendedContext.multiply(7, Decimal(7)) + Decimal('49') + """ + a = _convert_other(a, raiseit=True) + r = a.__mul__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def next_minus(self, a): + """Returns the largest representable number smaller than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_minus(Decimal('1')) + Decimal('0.999999999') + >>> c.next_minus(Decimal('1E-1007')) + Decimal('0E-1007') + >>> ExtendedContext.next_minus(Decimal('-1.00000003')) + Decimal('-1.00000004') + >>> c.next_minus(Decimal('Infinity')) + Decimal('9.99999999E+999') + >>> c.next_minus(1) + Decimal('0.999999999') + """ + a = _convert_other(a, raiseit=True) + return a.next_minus(context=self) + + def next_plus(self, a): + """Returns the smallest representable number larger than a. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> ExtendedContext.next_plus(Decimal('1')) + Decimal('1.00000001') + >>> c.next_plus(Decimal('-1E-1007')) + Decimal('-0E-1007') + >>> ExtendedContext.next_plus(Decimal('-1.00000003')) + Decimal('-1.00000002') + >>> c.next_plus(Decimal('-Infinity')) + Decimal('-9.99999999E+999') + >>> c.next_plus(1) + Decimal('1.00000001') + """ + a = _convert_other(a, raiseit=True) + return a.next_plus(context=self) + + def next_toward(self, a, b): + """Returns the number closest to a, in direction towards b. + + The result is the closest representable number from the first + operand (but not the first operand) that is in the direction + towards the second operand, unless the operands have the same + value. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.next_toward(Decimal('1'), Decimal('2')) + Decimal('1.00000001') + >>> c.next_toward(Decimal('-1E-1007'), Decimal('1')) + Decimal('-0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('0')) + Decimal('-1.00000002') + >>> c.next_toward(Decimal('1'), Decimal('0')) + Decimal('0.999999999') + >>> c.next_toward(Decimal('1E-1007'), Decimal('-100')) + Decimal('0E-1007') + >>> c.next_toward(Decimal('-1.00000003'), Decimal('-10')) + Decimal('-1.00000004') + >>> c.next_toward(Decimal('0.00'), Decimal('-0.0000')) + Decimal('-0.00') + >>> c.next_toward(0, 1) + Decimal('1E-1007') + >>> c.next_toward(Decimal(0), 1) + Decimal('1E-1007') + >>> c.next_toward(0, Decimal(1)) + Decimal('1E-1007') + """ + a = _convert_other(a, raiseit=True) + return a.next_toward(b, context=self) + + def normalize(self, a): + """normalize reduces an operand to its simplest form. + + Essentially a plus operation with all trailing zeros removed from the + result. + + >>> ExtendedContext.normalize(Decimal('2.1')) + Decimal('2.1') + >>> ExtendedContext.normalize(Decimal('-2.0')) + Decimal('-2') + >>> ExtendedContext.normalize(Decimal('1.200')) + Decimal('1.2') + >>> ExtendedContext.normalize(Decimal('-120')) + Decimal('-1.2E+2') + >>> ExtendedContext.normalize(Decimal('120.00')) + Decimal('1.2E+2') + >>> ExtendedContext.normalize(Decimal('0.00')) + Decimal('0') + >>> ExtendedContext.normalize(6) + Decimal('6') + """ + a = _convert_other(a, raiseit=True) + return a.normalize(context=self) + + def number_class(self, a): + """Returns an indication of the class of the operand. + + The class is one of the following strings: + -sNaN + -NaN + -Infinity + -Normal + -Subnormal + -Zero + +Zero + +Subnormal + +Normal + +Infinity + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.number_class(Decimal('Infinity')) + '+Infinity' + >>> c.number_class(Decimal('1E-10')) + '+Normal' + >>> c.number_class(Decimal('2.50')) + '+Normal' + >>> c.number_class(Decimal('0.1E-999')) + '+Subnormal' + >>> c.number_class(Decimal('0')) + '+Zero' + >>> c.number_class(Decimal('-0')) + '-Zero' + >>> c.number_class(Decimal('-0.1E-999')) + '-Subnormal' + >>> c.number_class(Decimal('-1E-10')) + '-Normal' + >>> c.number_class(Decimal('-2.50')) + '-Normal' + >>> c.number_class(Decimal('-Infinity')) + '-Infinity' + >>> c.number_class(Decimal('NaN')) + 'NaN' + >>> c.number_class(Decimal('-NaN')) + 'NaN' + >>> c.number_class(Decimal('sNaN')) + 'sNaN' + >>> c.number_class(123) + '+Normal' + """ + a = _convert_other(a, raiseit=True) + return a.number_class(context=self) + + def plus(self, a): + """Plus corresponds to unary prefix plus in Python. + + The operation is evaluated using the same rules as add; the + operation plus(a) is calculated as add('0', a) where the '0' + has the same exponent as the operand. + + >>> ExtendedContext.plus(Decimal('1.3')) + Decimal('1.3') + >>> ExtendedContext.plus(Decimal('-1.3')) + Decimal('-1.3') + >>> ExtendedContext.plus(-1) + Decimal('-1') + """ + a = _convert_other(a, raiseit=True) + return a.__pos__(context=self) + + def power(self, a, b, modulo=None): + """Raises a to the power of b, to modulo if given. + + With two arguments, compute a**b. If a is negative then b + must be integral. The result will be inexact unless b is + integral and the result is finite and can be expressed exactly + in 'precision' digits. + + With three arguments, compute (a**b) % modulo. For the + three argument form, the following restrictions on the + arguments hold: + + - all three arguments must be integral + - b must be nonnegative + - at least one of a or b must be nonzero + - modulo must be nonzero and have at most 'precision' digits + + The result of pow(a, b, modulo) is identical to the result + that would be obtained by computing (a**b) % modulo with + unbounded precision, but is computed more efficiently. It is + always exact. + + >>> c = ExtendedContext.copy() + >>> c.Emin = -999 + >>> c.Emax = 999 + >>> c.power(Decimal('2'), Decimal('3')) + Decimal('8') + >>> c.power(Decimal('-2'), Decimal('3')) + Decimal('-8') + >>> c.power(Decimal('2'), Decimal('-3')) + Decimal('0.125') + >>> c.power(Decimal('1.7'), Decimal('8')) + Decimal('69.7575744') + >>> c.power(Decimal('10'), Decimal('0.301029996')) + Decimal('2.00000000') + >>> c.power(Decimal('Infinity'), Decimal('-1')) + Decimal('0') + >>> c.power(Decimal('Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('Infinity'), Decimal('1')) + Decimal('Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('-1')) + Decimal('-0') + >>> c.power(Decimal('-Infinity'), Decimal('0')) + Decimal('1') + >>> c.power(Decimal('-Infinity'), Decimal('1')) + Decimal('-Infinity') + >>> c.power(Decimal('-Infinity'), Decimal('2')) + Decimal('Infinity') + >>> c.power(Decimal('0'), Decimal('0')) + Decimal('NaN') + + >>> c.power(Decimal('3'), Decimal('7'), Decimal('16')) + Decimal('11') + >>> c.power(Decimal('-3'), Decimal('7'), Decimal('16')) + Decimal('-11') + >>> c.power(Decimal('-3'), Decimal('8'), Decimal('16')) + Decimal('1') + >>> c.power(Decimal('3'), Decimal('7'), Decimal('-16')) + Decimal('11') + >>> c.power(Decimal('23E12345'), Decimal('67E189'), Decimal('123456789')) + Decimal('11729830') + >>> c.power(Decimal('-0'), Decimal('17'), Decimal('1729')) + Decimal('-0') + >>> c.power(Decimal('-23'), Decimal('0'), Decimal('65537')) + Decimal('1') + >>> ExtendedContext.power(7, 7) + Decimal('823543') + >>> ExtendedContext.power(Decimal(7), 7) + Decimal('823543') + >>> ExtendedContext.power(7, Decimal(7), 2) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + r = a.__pow__(b, modulo, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def quantize(self, a, b): + """Returns a value equal to 'a' (rounded), having the exponent of 'b'. + + The coefficient of the result is derived from that of the left-hand + operand. It may be rounded using the current rounding setting (if the + exponent is being increased), multiplied by a positive power of ten (if + the exponent is being decreased), or is unchanged (if the exponent is + already equal to that of the right-hand operand). + + Unlike other operations, if the length of the coefficient after the + quantize operation would be greater than precision then an Invalid + operation condition is raised. This guarantees that, unless there is + an error condition, the exponent of the result of a quantize is always + equal to that of the right-hand operand. + + Also unlike other operations, quantize will never raise Underflow, even + if the result is subnormal and inexact. + + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) + Decimal('2.170') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.01')) + Decimal('2.17') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.1')) + Decimal('2.2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+0')) + Decimal('2') + >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('1e+1')) + Decimal('0E+1') + >>> ExtendedContext.quantize(Decimal('-Inf'), Decimal('Infinity')) + Decimal('-Infinity') + >>> ExtendedContext.quantize(Decimal('2'), Decimal('Infinity')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-0.1'), Decimal('1')) + Decimal('-0') + >>> ExtendedContext.quantize(Decimal('-0'), Decimal('1e+5')) + Decimal('-0E+5') + >>> ExtendedContext.quantize(Decimal('+35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('-35236450.6'), Decimal('1e-2')) + Decimal('NaN') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-1')) + Decimal('217.0') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e-0')) + Decimal('217') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+1')) + Decimal('2.2E+2') + >>> ExtendedContext.quantize(Decimal('217'), Decimal('1e+2')) + Decimal('2E+2') + >>> ExtendedContext.quantize(1, 2) + Decimal('1') + >>> ExtendedContext.quantize(Decimal(1), 2) + Decimal('1') + >>> ExtendedContext.quantize(1, Decimal(2)) + Decimal('1') + """ + a = _convert_other(a, raiseit=True) + return a.quantize(b, context=self) + + def radix(self): + """Just returns 10, as this is Decimal, :) + + >>> ExtendedContext.radix() + Decimal('10') + """ + return Decimal(10) + + def remainder(self, a, b): + """Returns the remainder from integer division. + + The result is the residue of the dividend after the operation of + calculating integer division as described for divide-integer, rounded + to precision digits if necessary. The sign of the result, if + non-zero, is the same as that of the original dividend. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) + Decimal('2.1') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder(Decimal('3.6'), Decimal('1.3')) + Decimal('1.0') + >>> ExtendedContext.remainder(22, 6) + Decimal('4') + >>> ExtendedContext.remainder(Decimal(22), 6) + Decimal('4') + >>> ExtendedContext.remainder(22, Decimal(6)) + Decimal('4') + """ + a = _convert_other(a, raiseit=True) + r = a.__mod__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def remainder_near(self, a, b): + """Returns to be "a - b * n", where n is the integer nearest the exact + value of "x / b" (if two integers are equally near then the even one + is chosen). If the result is equal to 0 then its sign will be the + sign of a. + + This operation will fail under the same conditions as integer division + (that is, if integer division on the same two operands would fail, the + remainder cannot be calculated). + + >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) + Decimal('-0.9') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('6')) + Decimal('-2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('3')) + Decimal('1') + >>> ExtendedContext.remainder_near(Decimal('-10'), Decimal('3')) + Decimal('-1') + >>> ExtendedContext.remainder_near(Decimal('10.2'), Decimal('1')) + Decimal('0.2') + >>> ExtendedContext.remainder_near(Decimal('10'), Decimal('0.3')) + Decimal('0.1') + >>> ExtendedContext.remainder_near(Decimal('3.6'), Decimal('1.3')) + Decimal('-0.3') + >>> ExtendedContext.remainder_near(3, 11) + Decimal('3') + >>> ExtendedContext.remainder_near(Decimal(3), 11) + Decimal('3') + >>> ExtendedContext.remainder_near(3, Decimal(11)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + return a.remainder_near(b, context=self) + + def rotate(self, a, b): + """Returns a rotated copy of a, b times. + + The coefficient of the result is a rotated copy of the digits in + the coefficient of the first operand. The number of places of + rotation is taken from the absolute value of the second operand, + with the rotation being to the left if the second operand is + positive or to the right otherwise. + + >>> ExtendedContext.rotate(Decimal('34'), Decimal('8')) + Decimal('400000003') + >>> ExtendedContext.rotate(Decimal('12'), Decimal('9')) + Decimal('12') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('-2')) + Decimal('891234567') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.rotate(Decimal('123456789'), Decimal('+2')) + Decimal('345678912') + >>> ExtendedContext.rotate(1333333, 1) + Decimal('13333330') + >>> ExtendedContext.rotate(Decimal(1333333), 1) + Decimal('13333330') + >>> ExtendedContext.rotate(1333333, Decimal(1)) + Decimal('13333330') + """ + a = _convert_other(a, raiseit=True) + return a.rotate(b, context=self) + + def same_quantum(self, a, b): + """Returns True if the two operands have the same exponent. + + The result is never affected by either the sign or the coefficient of + either operand. + + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) + False + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.01')) + True + >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('1')) + False + >>> ExtendedContext.same_quantum(Decimal('Inf'), Decimal('-Inf')) + True + >>> ExtendedContext.same_quantum(10000, -1) + True + >>> ExtendedContext.same_quantum(Decimal(10000), -1) + True + >>> ExtendedContext.same_quantum(10000, Decimal(-1)) + True + """ + a = _convert_other(a, raiseit=True) + return a.same_quantum(b) + + def scaleb (self, a, b): + """Returns the first operand after adding the second value its exp. + + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('-2')) + Decimal('0.0750') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('0')) + Decimal('7.50') + >>> ExtendedContext.scaleb(Decimal('7.50'), Decimal('3')) + Decimal('7.50E+3') + >>> ExtendedContext.scaleb(1, 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(Decimal(1), 4) + Decimal('1E+4') + >>> ExtendedContext.scaleb(1, Decimal(4)) + Decimal('1E+4') + """ + a = _convert_other(a, raiseit=True) + return a.scaleb(b, context=self) + + def shift(self, a, b): + """Returns a shifted copy of a, b times. + + The coefficient of the result is a shifted copy of the digits + in the coefficient of the first operand. The number of places + to shift is taken from the absolute value of the second operand, + with the shift being to the left if the second operand is + positive or to the right otherwise. Digits shifted into the + coefficient are zeros. + + >>> ExtendedContext.shift(Decimal('34'), Decimal('8')) + Decimal('400000000') + >>> ExtendedContext.shift(Decimal('12'), Decimal('9')) + Decimal('0') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('-2')) + Decimal('1234567') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('0')) + Decimal('123456789') + >>> ExtendedContext.shift(Decimal('123456789'), Decimal('+2')) + Decimal('345678900') + >>> ExtendedContext.shift(88888888, 2) + Decimal('888888800') + >>> ExtendedContext.shift(Decimal(88888888), 2) + Decimal('888888800') + >>> ExtendedContext.shift(88888888, Decimal(2)) + Decimal('888888800') + """ + a = _convert_other(a, raiseit=True) + return a.shift(b, context=self) + + def sqrt(self, a): + """Square root of a non-negative number to context precision. + + If the result must be inexact, it is rounded using the round-half-even + algorithm. + + >>> ExtendedContext.sqrt(Decimal('0')) + Decimal('0') + >>> ExtendedContext.sqrt(Decimal('-0')) + Decimal('-0') + >>> ExtendedContext.sqrt(Decimal('0.39')) + Decimal('0.624499800') + >>> ExtendedContext.sqrt(Decimal('100')) + Decimal('10') + >>> ExtendedContext.sqrt(Decimal('1')) + Decimal('1') + >>> ExtendedContext.sqrt(Decimal('1.0')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('1.00')) + Decimal('1.0') + >>> ExtendedContext.sqrt(Decimal('7')) + Decimal('2.64575131') + >>> ExtendedContext.sqrt(Decimal('10')) + Decimal('3.16227766') + >>> ExtendedContext.sqrt(2) + Decimal('1.41421356') + >>> ExtendedContext.prec + 9 + """ + a = _convert_other(a, raiseit=True) + return a.sqrt(context=self) + + def subtract(self, a, b): + """Return the difference between the two operands. + + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.07')) + Decimal('0.23') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('1.30')) + Decimal('0.00') + >>> ExtendedContext.subtract(Decimal('1.3'), Decimal('2.07')) + Decimal('-0.77') + >>> ExtendedContext.subtract(8, 5) + Decimal('3') + >>> ExtendedContext.subtract(Decimal(8), 5) + Decimal('3') + >>> ExtendedContext.subtract(8, Decimal(5)) + Decimal('3') + """ + a = _convert_other(a, raiseit=True) + r = a.__sub__(b, context=self) + if r is NotImplemented: + raise TypeError("Unable to convert %s to Decimal" % b) + else: + return r + + def to_eng_string(self, a): + """Converts a number to a string, using scientific notation. + + The operation is not affected by the context. + """ + a = _convert_other(a, raiseit=True) + return a.to_eng_string(context=self) + + def to_sci_string(self, a): + """Converts a number to a string, using scientific notation. + + The operation is not affected by the context. + """ + a = _convert_other(a, raiseit=True) + return a.__str__(context=self) + + def to_integral_exact(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting; Inexact and Rounded flags + are allowed in this operation. The rounding mode is taken from the + context. + + >>> ExtendedContext.to_integral_exact(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_exact(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_exact(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_exact(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_exact(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_exact(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_exact(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_exact(context=self) + + def to_integral_value(self, a): + """Rounds to an integer. + + When the operand has a negative exponent, the result is the same + as using the quantize() operation using the given operand as the + left-hand-operand, 1E+0 as the right-hand-operand, and the precision + of the operand as the precision setting, except that no flags will + be set. The rounding mode is taken from the context. + + >>> ExtendedContext.to_integral_value(Decimal('2.1')) + Decimal('2') + >>> ExtendedContext.to_integral_value(Decimal('100')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('100.0')) + Decimal('100') + >>> ExtendedContext.to_integral_value(Decimal('101.5')) + Decimal('102') + >>> ExtendedContext.to_integral_value(Decimal('-101.5')) + Decimal('-102') + >>> ExtendedContext.to_integral_value(Decimal('10E+5')) + Decimal('1.0E+6') + >>> ExtendedContext.to_integral_value(Decimal('7.89E+77')) + Decimal('7.89E+77') + >>> ExtendedContext.to_integral_value(Decimal('-Inf')) + Decimal('-Infinity') + """ + a = _convert_other(a, raiseit=True) + return a.to_integral_value(context=self) + + # the method name changed, but we provide also the old one, for compatibility + to_integral = to_integral_value + +class _WorkRep(object): + __slots__ = ('sign','int','exp') + # sign: 0 or 1 + # int: int + # exp: None, int, or string + + def __init__(self, value=None): + if value is None: + self.sign = None + self.int = 0 + self.exp = None + elif isinstance(value, Decimal): + self.sign = value._sign + self.int = int(value._int) + self.exp = value._exp + else: + # assert isinstance(value, tuple) + self.sign = value[0] + self.int = value[1] + self.exp = value[2] + + def __repr__(self): + return "(%r, %r, %r)" % (self.sign, self.int, self.exp) + + __str__ = __repr__ + + + +def _normalize(op1, op2, prec = 0): + """Normalizes op1, op2 to have the same exp and length of coefficient. + + Done during addition. + """ + if op1.exp < op2.exp: + tmp = op2 + other = op1 + else: + tmp = op1 + other = op2 + + # Let exp = min(tmp.exp - 1, tmp.adjusted() - precision - 1). + # Then adding 10**exp to tmp has the same effect (after rounding) + # as adding any positive quantity smaller than 10**exp; similarly + # for subtraction. So if other is smaller than 10**exp we replace + # it with 10**exp. This avoids tmp.exp - other.exp getting too large. + tmp_len = len(str(tmp.int)) + other_len = len(str(other.int)) + exp = tmp.exp + min(-1, tmp_len - prec - 2) + if other_len + other.exp - 1 < exp: + other.int = 1 + other.exp = exp + + tmp.int *= 10 ** (tmp.exp - other.exp) + tmp.exp = other.exp + return op1, op2 + +##### Integer arithmetic functions used by ln, log10, exp and __pow__ ##### + +_nbits = int.bit_length + +def _decimal_lshift_exact(n, e): + """ Given integers n and e, return n * 10**e if it's an integer, else None. + + The computation is designed to avoid computing large powers of 10 + unnecessarily. + + >>> _decimal_lshift_exact(3, 4) + 30000 + >>> _decimal_lshift_exact(300, -999999999) # returns None + + """ + if n == 0: + return 0 + elif e >= 0: + return n * 10**e + else: + # val_n = largest power of 10 dividing n. + str_n = str(abs(n)) + val_n = len(str_n) - len(str_n.rstrip('0')) + return None if val_n < -e else n // 10**-e + +def _sqrt_nearest(n, a): + """Closest integer to the square root of the positive integer n. a is + an initial approximation to the square root. Any positive integer + will do for a, but the closer a is to the square root of n the + faster convergence will be. + + """ + if n <= 0 or a <= 0: + raise ValueError("Both arguments to _sqrt_nearest should be positive.") + + b=0 + while a != b: + b, a = a, a--n//a>>1 + return a + +def _rshift_nearest(x, shift): + """Given an integer x and a nonnegative integer shift, return closest + integer to x / 2**shift; use round-to-even in case of a tie. + + """ + b, q = 1 << shift, x >> shift + return q + (2*(x & (b-1)) + (q&1) > b) + +def _div_nearest(a, b): + """Closest integer to a/b, a and b positive integers; rounds to even + in the case of a tie. + + """ + q, r = divmod(a, b) + return q + (2*r + (q&1) > b) + +def _ilog(x, M, L = 8): + """Integer approximation to M*log(x/M), with absolute error boundable + in terms only of x/M. + + Given positive integers x and M, return an integer approximation to + M * log(x/M). For L = 8 and 0.1 <= x/M <= 10 the difference + between the approximation and the exact result is at most 22. For + L = 8 and 1.0 <= x/M <= 10.0 the difference is at most 15. In + both cases these are upper bounds on the error; it will usually be + much smaller.""" + + # The basic algorithm is the following: let log1p be the function + # log1p(x) = log(1+x). Then log(x/M) = log1p((x-M)/M). We use + # the reduction + # + # log1p(y) = 2*log1p(y/(1+sqrt(1+y))) + # + # repeatedly until the argument to log1p is small (< 2**-L in + # absolute value). For small y we can use the Taylor series + # expansion + # + # log1p(y) ~ y - y**2/2 + y**3/3 - ... - (-y)**T/T + # + # truncating at T such that y**T is small enough. The whole + # computation is carried out in a form of fixed-point arithmetic, + # with a real number z being represented by an integer + # approximation to z*M. To avoid loss of precision, the y below + # is actually an integer approximation to 2**R*y*M, where R is the + # number of reductions performed so far. + + y = x-M + # argument reduction; R = number of reductions performed + R = 0 + while (R <= L and abs(y) << L-R >= M or + R > L and abs(y) >> R-L >= M): + y = _div_nearest((M*y) << 1, + M + _sqrt_nearest(M*(M+_rshift_nearest(y, R)), M)) + R += 1 + + # Taylor series with T terms + T = -int(-10*len(str(M))//(3*L)) + yshift = _rshift_nearest(y, R) + w = _div_nearest(M, T) + for k in range(T-1, 0, -1): + w = _div_nearest(M, k) - _div_nearest(yshift*w, M) + + return _div_nearest(w*y, M) + +def _dlog10(c, e, p): + """Given integers c, e and p with c > 0, p >= 0, compute an integer + approximation to 10**p * log10(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # increase precision by 2; compensate for this by dividing + # final result by 100 + p += 2 + + # write c*10**e as d*10**f with either: + # f >= 0 and 1 <= d <= 10, or + # f <= 0 and 0.1 <= d <= 1. + # Thus for c*10**e close to 1, f = 0 + l = len(str(c)) + f = e+l - (e+l >= 1) + + if p > 0: + M = 10**p + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) + + log_d = _ilog(c, M) # error < 5 + 22 = 27 + log_10 = _log10_digits(p) # error < 1 + log_d = _div_nearest(log_d*M, log_10) + log_tenpower = f*M # exact + else: + log_d = 0 # error < 2.31 + log_tenpower = _div_nearest(f, 10**-p) # error < 0.5 + + return _div_nearest(log_tenpower+log_d, 100) + +def _dlog(c, e, p): + """Given integers c, e and p with c > 0, compute an integer + approximation to 10**p * log(c*10**e), with an absolute error of + at most 1. Assumes that c*10**e is not exactly 1.""" + + # Increase precision by 2. The precision increase is compensated + # for at the end with a division by 100. + p += 2 + + # rewrite c*10**e as d*10**f with either f >= 0 and 1 <= d <= 10, + # or f <= 0 and 0.1 <= d <= 1. Then we can compute 10**p * log(c*10**e) + # as 10**p * log(d) + 10**p*f * log(10). + l = len(str(c)) + f = e+l - (e+l >= 1) + + # compute approximation to 10**p*log(d), with error < 27 + if p > 0: + k = e+p-f + if k >= 0: + c *= 10**k + else: + c = _div_nearest(c, 10**-k) # error of <= 0.5 in c + + # _ilog magnifies existing error in c by a factor of at most 10 + log_d = _ilog(c, 10**p) # error < 5 + 22 = 27 + else: + # p <= 0: just approximate the whole thing by 0; error < 2.31 + log_d = 0 + + # compute approximation to f*10**p*log(10), with error < 11. + if f: + extra = len(str(abs(f)))-1 + if p + extra >= 0: + # error in f * _log10_digits(p+extra) < |f| * 1 = |f| + # after division, error < |f|/10**extra + 0.5 < 10 + 0.5 < 11 + f_log_ten = _div_nearest(f*_log10_digits(p+extra), 10**extra) + else: + f_log_ten = 0 + else: + f_log_ten = 0 + + # error in sum < 11+27 = 38; error after division < 0.38 + 0.5 < 1 + return _div_nearest(f_log_ten + log_d, 100) + +class _Log10Memoize(object): + """Class to compute, store, and allow retrieval of, digits of the + constant log(10) = 2.302585.... This constant is needed by + Decimal.ln, Decimal.log10, Decimal.exp and Decimal.__pow__.""" + def __init__(self): + self.digits = "23025850929940456840179914546843642076011014886" + + def getdigits(self, p): + """Given an integer p >= 0, return floor(10**p)*log(10). + + For example, self.getdigits(3) returns 2302. + """ + # digits are stored as a string, for quick conversion to + # integer in the case that we've already computed enough + # digits; the stored digits should always be correct + # (truncated, not rounded to nearest). + if p < 0: + raise ValueError("p should be nonnegative") + + if p >= len(self.digits): + # compute p+3, p+6, p+9, ... digits; continue until at + # least one of the extra digits is nonzero + extra = 3 + while True: + # compute p+extra digits, correct to within 1ulp + M = 10**(p+extra+2) + digits = str(_div_nearest(_ilog(10*M, M), 100)) + if digits[-extra:] != '0'*extra: + break + extra += 3 + # keep all reliable digits so far; remove trailing zeros + # and next nonzero digit + self.digits = digits.rstrip('0')[:-1] + return int(self.digits[:p+1]) + +_log10_digits = _Log10Memoize().getdigits + +def _iexp(x, M, L=8): + """Given integers x and M, M > 0, such that x/M is small in absolute + value, compute an integer approximation to M*exp(x/M). For 0 <= + x/M <= 2.4, the absolute error in the result is bounded by 60 (and + is usually much smaller).""" + + # Algorithm: to compute exp(z) for a real number z, first divide z + # by a suitable power R of 2 so that |z/2**R| < 2**-L. Then + # compute expm1(z/2**R) = exp(z/2**R) - 1 using the usual Taylor + # series + # + # expm1(x) = x + x**2/2! + x**3/3! + ... + # + # Now use the identity + # + # expm1(2x) = expm1(x)*(expm1(x)+2) + # + # R times to compute the sequence expm1(z/2**R), + # expm1(z/2**(R-1)), ... , exp(z/2), exp(z). + + # Find R such that x/2**R/M <= 2**-L + R = _nbits((x< M + T = -int(-10*len(str(M))//(3*L)) + y = _div_nearest(x, T) + Mshift = M<= 0: + cshift = c*10**shift + else: + cshift = c//10**-shift + quot, rem = divmod(cshift, _log10_digits(q)) + + # reduce remainder back to original precision + rem = _div_nearest(rem, 10**extra) + + # error in result of _iexp < 120; error after division < 0.62 + return _div_nearest(_iexp(rem, 10**p), 1000), quot - p + 3 + +def _dpower(xc, xe, yc, ye, p): + """Given integers xc, xe, yc and ye representing Decimals x = xc*10**xe and + y = yc*10**ye, compute x**y. Returns a pair of integers (c, e) such that: + + 10**(p-1) <= c <= 10**p, and + (c-1)*10**e < x**y < (c+1)*10**e + + in other words, c*10**e is an approximation to x**y with p digits + of precision, and with an error in c of at most 1. (This is + almost, but not quite, the same as the error being < 1ulp: when c + == 10**(p-1) we can only guarantee error < 10ulp.) + + We assume that: x is positive and not equal to 1, and y is nonzero. + """ + + # Find b such that 10**(b-1) <= |y| <= 10**b + b = len(str(abs(yc))) + ye + + # log(x) = lxc*10**(-p-b-1), to p+b+1 places after the decimal point + lxc = _dlog(xc, xe, p+b+1) + + # compute product y*log(x) = yc*lxc*10**(-p-b-1+ye) = pc*10**(-p-1) + shift = ye-b + if shift >= 0: + pc = lxc*yc*10**shift + else: + pc = _div_nearest(lxc*yc, 10**-shift) + + if pc == 0: + # we prefer a result that isn't exactly 1; this makes it + # easier to compute a correctly rounded result in __pow__ + if ((len(str(xc)) + xe >= 1) == (yc > 0)): # if x**y > 1: + coeff, exp = 10**(p-1)+1, 1-p + else: + coeff, exp = 10**p-1, -p + else: + coeff, exp = _dexp(pc, -(p+1), p+1) + coeff = _div_nearest(coeff, 10) + exp += 1 + + return coeff, exp + +def _log10_lb(c, correction = { + '1': 100, '2': 70, '3': 53, '4': 40, '5': 31, + '6': 23, '7': 16, '8': 10, '9': 5}): + """Compute a lower bound for 100*log10(c) for a positive integer c.""" + if c <= 0: + raise ValueError("The argument to _log10_lb should be nonnegative.") + str_c = str(c) + return 100*len(str_c) - correction[str_c[0]] + +##### Helper Functions #################################################### + +def _convert_other(other, raiseit=False, allow_float=False): + """Convert other to Decimal. + + Verifies that it's ok to use in an implicit construction. + If allow_float is true, allow conversion from float; this + is used in the comparison methods (__eq__ and friends). + + """ + if isinstance(other, Decimal): + return other + if isinstance(other, int): + return Decimal(other) + if allow_float and isinstance(other, float): + return Decimal.from_float(other) + + if raiseit: + raise TypeError("Unable to convert %s to Decimal" % other) + return NotImplemented + +def _convert_for_comparison(self, other, equality_op=False): + """Given a Decimal instance self and a Python object other, return + a pair (s, o) of Decimal instances such that "s op o" is + equivalent to "self op other" for any of the 6 comparison + operators "op". + + """ + if isinstance(other, Decimal): + return self, other + + # Comparison with a Rational instance (also includes integers): + # self op n/d <=> self*d op n (for n and d integers, d positive). + # A NaN or infinity can be left unchanged without affecting the + # comparison result. + if isinstance(other, _numbers.Rational): + if not self._is_special: + self = _dec_from_triple(self._sign, + str(int(self._int) * other.denominator), + self._exp) + return self, Decimal(other.numerator) + + # Comparisons with float and complex types. == and != comparisons + # with complex numbers should succeed, returning either True or False + # as appropriate. Other comparisons return NotImplemented. + if equality_op and isinstance(other, _numbers.Complex) and other.imag == 0: + other = other.real + if isinstance(other, float): + context = getcontext() + if equality_op: + context.flags[FloatOperation] = 1 + else: + context._raise_error(FloatOperation, + "strict semantics for mixing floats and Decimals are enabled") + return self, Decimal.from_float(other) + return NotImplemented, NotImplemented + + +##### Setup Specific Contexts ############################################ + +# The default context prototype used by Context() +# Is mutable, so that new contexts can have different default values + +DefaultContext = Context( + prec=28, rounding=ROUND_HALF_EVEN, + traps=[DivisionByZero, Overflow, InvalidOperation], + flags=[], + Emax=999999, + Emin=-999999, + capitals=1, + clamp=0 +) + +# Pre-made alternate contexts offered by the specification +# Don't change these; the user should be able to select these +# contexts and be able to reproduce results from other implementations +# of the spec. + +BasicContext = Context( + prec=9, rounding=ROUND_HALF_UP, + traps=[DivisionByZero, Overflow, InvalidOperation, Clamped, Underflow], + flags=[], +) + +ExtendedContext = Context( + prec=9, rounding=ROUND_HALF_EVEN, + traps=[], + flags=[], +) + + +##### crud for parsing strings ############################################# +# +# Regular expression used for parsing numeric strings. Additional +# comments: +# +# 1. Uncomment the two '\s*' lines to allow leading and/or trailing +# whitespace. But note that the specification disallows whitespace in +# a numeric string. +# +# 2. For finite numbers (not infinities and NaNs) the body of the +# number between the optional sign and the optional exponent must have +# at least one decimal digit, possibly after the decimal point. The +# lookahead expression '(?=\d|\.\d)' checks this. + +import re +_parser = re.compile(r""" # A numeric string consists of: +# \s* + (?P[-+])? # an optional sign, followed by either... + ( + (?=\d|\.\d) # ...a number (with at least one digit) + (?P\d*) # having a (possibly empty) integer part + (\.(?P\d*))? # followed by an optional fractional part + (E(?P[-+]?\d+))? # followed by an optional exponent, or... + | + Inf(inity)? # ...an infinity, or... + | + (?Ps)? # ...an (optionally signaling) + NaN # NaN + (?P\d*) # with (possibly empty) diagnostic info. + ) +# \s* + \Z +""", re.VERBOSE | re.IGNORECASE).match + +_all_zeros = re.compile('0*$').match +_exact_half = re.compile('50*$').match + +##### PEP3101 support functions ############################################## +# The functions in this section have little to do with the Decimal +# class, and could potentially be reused or adapted for other pure +# Python numeric classes that want to implement __format__ +# +# A format specifier for Decimal looks like: +# +# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] + +_parse_format_specifier_regex = re.compile(r"""\A +(?: + (?P.)? + (?P[<>=^]) +)? +(?P[-+ ])? +(?P\#)? +(?P0)? +(?P(?!0)\d+)? +(?P,)? +(?:\.(?P0|(?!0)\d+))? +(?P[eEfFgGn%])? +\Z +""", re.VERBOSE) + +del re + +# The locale module is only needed for the 'n' format specifier. The +# rest of the PEP 3101 code functions quite happily without it, so we +# don't care too much if locale isn't present. +try: + import locale as _locale +except ImportError: + pass + +def _parse_format_specifier(format_spec, _localeconv=None): + """Parse and validate a format specifier. + + Turns a standard numeric format specifier into a dict, with the + following entries: + + fill: fill character to pad field to minimum width + align: alignment type, either '<', '>', '=' or '^' + sign: either '+', '-' or ' ' + minimumwidth: nonnegative integer giving minimum width + zeropad: boolean, indicating whether to pad with zeros + thousands_sep: string to use as thousands separator, or '' + grouping: grouping for thousands separators, in format + used by localeconv + decimal_point: string to use for decimal point + precision: nonnegative integer giving precision, or None + type: one of the characters 'eEfFgG%', or None + + """ + m = _parse_format_specifier_regex.match(format_spec) + if m is None: + raise ValueError("Invalid format specifier: " + format_spec) + + # get the dictionary + format_dict = m.groupdict() + + # zeropad; defaults for fill and alignment. If zero padding + # is requested, the fill and align fields should be absent. + fill = format_dict['fill'] + align = format_dict['align'] + format_dict['zeropad'] = (format_dict['zeropad'] is not None) + if format_dict['zeropad']: + if fill is not None: + raise ValueError("Fill character conflicts with '0'" + " in format specifier: " + format_spec) + if align is not None: + raise ValueError("Alignment conflicts with '0' in " + "format specifier: " + format_spec) + format_dict['fill'] = fill or ' ' + # PEP 3101 originally specified that the default alignment should + # be left; it was later agreed that right-aligned makes more sense + # for numeric types. See http://bugs.python.org/issue6857. + format_dict['align'] = align or '>' + + # default sign handling: '-' for negative, '' for positive + if format_dict['sign'] is None: + format_dict['sign'] = '-' + + # minimumwidth defaults to 0; precision remains None if not given + format_dict['minimumwidth'] = int(format_dict['minimumwidth'] or '0') + if format_dict['precision'] is not None: + format_dict['precision'] = int(format_dict['precision']) + + # if format type is 'g' or 'G' then a precision of 0 makes little + # sense; convert it to 1. Same if format type is unspecified. + if format_dict['precision'] == 0: + if format_dict['type'] is None or format_dict['type'] in 'gGn': + format_dict['precision'] = 1 + + # determine thousands separator, grouping, and decimal separator, and + # add appropriate entries to format_dict + if format_dict['type'] == 'n': + # apart from separators, 'n' behaves just like 'g' + format_dict['type'] = 'g' + if _localeconv is None: + _localeconv = _locale.localeconv() + if format_dict['thousands_sep'] is not None: + raise ValueError("Explicit thousands separator conflicts with " + "'n' type in format specifier: " + format_spec) + format_dict['thousands_sep'] = _localeconv['thousands_sep'] + format_dict['grouping'] = _localeconv['grouping'] + format_dict['decimal_point'] = _localeconv['decimal_point'] + else: + if format_dict['thousands_sep'] is None: + format_dict['thousands_sep'] = '' + format_dict['grouping'] = [3, 0] + format_dict['decimal_point'] = '.' + + return format_dict + +def _format_align(sign, body, spec): + """Given an unpadded, non-aligned numeric string 'body' and sign + string 'sign', add padding and alignment conforming to the given + format specifier dictionary 'spec' (as produced by + parse_format_specifier). + + """ + # how much extra space do we have to play with? + minimumwidth = spec['minimumwidth'] + fill = spec['fill'] + padding = fill*(minimumwidth - len(sign) - len(body)) + + align = spec['align'] + if align == '<': + result = sign + body + padding + elif align == '>': + result = padding + sign + body + elif align == '=': + result = sign + padding + body + elif align == '^': + half = len(padding)//2 + result = padding[:half] + sign + body + padding[half:] + else: + raise ValueError('Unrecognised alignment field') + + return result + +def _group_lengths(grouping): + """Convert a localeconv-style grouping into a (possibly infinite) + iterable of integers representing group lengths. + + """ + # The result from localeconv()['grouping'], and the input to this + # function, should be a list of integers in one of the + # following three forms: + # + # (1) an empty list, or + # (2) nonempty list of positive integers + [0] + # (3) list of positive integers + [locale.CHAR_MAX], or + + from itertools import chain, repeat + if not grouping: + return [] + elif grouping[-1] == 0 and len(grouping) >= 2: + return chain(grouping[:-1], repeat(grouping[-2])) + elif grouping[-1] == _locale.CHAR_MAX: + return grouping[:-1] + else: + raise ValueError('unrecognised format for grouping') + +def _insert_thousands_sep(digits, spec, min_width=1): + """Insert thousands separators into a digit string. + + spec is a dictionary whose keys should include 'thousands_sep' and + 'grouping'; typically it's the result of parsing the format + specifier using _parse_format_specifier. + + The min_width keyword argument gives the minimum length of the + result, which will be padded on the left with zeros if necessary. + + If necessary, the zero padding adds an extra '0' on the left to + avoid a leading thousands separator. For example, inserting + commas every three digits in '123456', with min_width=8, gives + '0,123,456', even though that has length 9. + + """ + + sep = spec['thousands_sep'] + grouping = spec['grouping'] + + groups = [] + for l in _group_lengths(grouping): + if l <= 0: + raise ValueError("group length should be positive") + # max(..., 1) forces at least 1 digit to the left of a separator + l = min(max(len(digits), min_width, 1), l) + groups.append('0'*(l - len(digits)) + digits[-l:]) + digits = digits[:-l] + min_width -= l + if not digits and min_width <= 0: + break + min_width -= len(sep) + else: + l = max(len(digits), min_width, 1) + groups.append('0'*(l - len(digits)) + digits[-l:]) + return sep.join(reversed(groups)) + +def _format_sign(is_negative, spec): + """Determine sign character.""" + + if is_negative: + return '-' + elif spec['sign'] in ' +': + return spec['sign'] + else: + return '' + +def _format_number(is_negative, intpart, fracpart, exp, spec): + """Format a number, given the following data: + + is_negative: true if the number is negative, else false + intpart: string of digits that must appear before the decimal point + fracpart: string of digits that must come after the point + exp: exponent, as an integer + spec: dictionary resulting from parsing the format specifier + + This function uses the information in spec to: + insert separators (decimal separator and thousands separators) + format the sign + format the exponent + add trailing '%' for the '%' type + zero-pad if necessary + fill and align if necessary + """ + + sign = _format_sign(is_negative, spec) + + if fracpart or spec['alt']: + fracpart = spec['decimal_point'] + fracpart + + if exp != 0 or spec['type'] in 'eE': + echar = {'E': 'E', 'e': 'e', 'G': 'E', 'g': 'e'}[spec['type']] + fracpart += "{0}{1:+}".format(echar, exp) + if spec['type'] == '%': + fracpart += '%' + + if spec['zeropad']: + min_width = spec['minimumwidth'] - len(fracpart) - len(sign) + else: + min_width = 0 + intpart = _insert_thousands_sep(intpart, spec, min_width) + + return _format_align(sign, intpart+fracpart, spec) + + +##### Useful Constants (internal use only) ################################ + +# Reusable defaults +_Infinity = Decimal('Inf') +_NegativeInfinity = Decimal('-Inf') +_NaN = Decimal('NaN') +_Zero = Decimal(0) +_One = Decimal(1) +_NegativeOne = Decimal(-1) + +# _SignedInfinity[sign] is infinity w/ that sign +_SignedInfinity = (_Infinity, _NegativeInfinity) + +# Constants related to the hash implementation; hash(x) is based +# on the reduction of x modulo _PyHASH_MODULUS +_PyHASH_MODULUS = sys.hash_info.modulus +# hash values to use for positive and negative infinities, and nans +_PyHASH_INF = sys.hash_info.inf +_PyHASH_NAN = sys.hash_info.nan + +# _PyHASH_10INV is the inverse of 10 modulo the prime _PyHASH_MODULUS +_PyHASH_10INV = pow(10, _PyHASH_MODULUS - 2, _PyHASH_MODULUS) +del sys + +try: + import _decimal +except ImportError: + pass +else: + s1 = set(dir()) + s2 = set(dir(_decimal)) + for name in s1 - s2: + del globals()[name] + del s1, s2, name from _decimal import * - from _decimal import __doc__ - from _decimal import __version__ - from _decimal import __libmpdec_version__ -except ImportError: - from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + +if __name__ == '__main__': + import doctest, decimal + doctest.testmod(decimal) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/difflib.py --- a/Lib/difflib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/difflib.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,5 @@ +#! /usr/bin/env python3 + """ Module difflib -- helpers for computing deltas between objects. @@ -28,9 +30,10 @@ __all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher', 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', - 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] + 'unified_diff', 'HtmlDiff', 'Match'] -from heapq import nlargest as _nlargest +import warnings +import heapq from collections import namedtuple as _namedtuple Match = _namedtuple('Match', 'a b size') @@ -333,6 +336,20 @@ for elt in popular: # ditto; as fast for 1% deletion del b2j[elt] + def isbjunk(self, item): + "Deprecated; use 'item in SequenceMatcher().bjunk'." + warnings.warn("'SequenceMatcher().isbjunk(item)' is deprecated;\n" + "use 'item in SMinstance.bjunk' instead.", + DeprecationWarning, 2) + return item in self.bjunk + + def isbpopular(self, item): + "Deprecated; use 'item in SequenceMatcher().bpopular'." + warnings.warn("'SequenceMatcher().isbpopular(item)' is deprecated;\n" + "use 'item in SMinstance.bpopular' instead.", + DeprecationWarning, 2) + return item in self.bpopular + def find_longest_match(self, alo, ahi, blo, bhi): """Find longest matching block in a[alo:ahi] and b[blo:bhi]. @@ -511,8 +528,8 @@ non_adjacent.append((i1, j1, k1)) non_adjacent.append( (la, lb, 0) ) - self.matching_blocks = list(map(Match._make, non_adjacent)) - return self.matching_blocks + self.matching_blocks = non_adjacent + return map(Match._make, self.matching_blocks) def get_opcodes(self): """Return list of 5-tuples describing how to turn a into b. @@ -572,7 +589,7 @@ def get_grouped_opcodes(self, n=3): """ Isolate change clusters by eliminating ranges with no changes. - Return a generator of groups with up to n lines of context. + Return a generator of groups with upto n lines of context. Each group is in the same format as returned by get_opcodes(). >>> from pprint import pprint @@ -729,7 +746,7 @@ result.append((s.ratio(), x)) # Move the best scorers to head of list - result = _nlargest(n, result) + result = heapq.nlargest(n, result) # Strip scores for the best n matches return [x for score, x in result] @@ -852,9 +869,10 @@ and return true iff the string is junk. The module-level function `IS_LINE_JUNK` may be used to filter out lines without visible characters, except for at most one splat ('#'). It is recommended - to leave linejunk None; the underlying SequenceMatcher class has - an adaptive notion of "noise" lines that's better than any static - definition the author has ever been able to craft. + to leave linejunk None; as of Python 2.3, the underlying + SequenceMatcher class has grown an adaptive notion of "noise" lines + that's better than any static definition the author has ever been + able to craft. - `charjunk`: A function that should accept a string of length 1. The module-level function `IS_CHARACTER_JUNK` may be used to filter out @@ -904,7 +922,8 @@ else: raise ValueError('unknown tag %r' % (tag,)) - yield from g + for line in g: + yield line def _dump(self, tag, x, lo, hi): """Generate comparison results for a same-tagged range.""" @@ -923,7 +942,8 @@ second = self._dump('+', b, blo, bhi) for g in first, second: - yield from g + for line in g: + yield line def _fancy_replace(self, a, alo, ahi, b, blo, bhi): r""" @@ -977,7 +997,8 @@ # no non-identical "pretty close" pair if eqi is None: # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) + for line in self._plain_replace(a, alo, ahi, b, blo, bhi): + yield line return # no close pair, but an identical pair -- synch up on that best_i, best_j, best_ratio = eqi, eqj, 1.0 @@ -989,7 +1010,8 @@ # identical # pump out diffs from before the synch point - yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) + for line in self._fancy_helper(a, alo, best_i, b, blo, best_j): + yield line # do intraline marking on the synch pair aelt, belt = a[best_i], b[best_j] @@ -1011,13 +1033,15 @@ btags += ' ' * lb else: raise ValueError('unknown tag %r' % (tag,)) - yield from self._qformat(aelt, belt, atags, btags) + for line in self._qformat(aelt, belt, atags, btags): + yield line else: # the synch pair is identical yield ' ' + aelt # pump out diffs from after the synch point - yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + for line in self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi): + yield line def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] @@ -1029,7 +1053,8 @@ elif blo < bhi: g = self._dump('+', b, blo, bhi) - yield from g + for line in g: + yield line def _qformat(self, aline, bline, atags, btags): r""" @@ -1174,7 +1199,6 @@ four """ - _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): if not started: @@ -1262,7 +1286,6 @@ four """ - _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): @@ -1294,70 +1317,22 @@ for line in b[j1:j2]: yield prefix[tag] + line -def _check_types(a, b, *args): - # Checking types is weird, but the alternative is garbled output when - # someone passes mixed bytes and str to {unified,context}_diff(). E.g. - # without this check, passing filenames as bytes results in output like - # --- b'oldfile.txt' - # +++ b'newfile.txt' - # because of how str.format() incorporates bytes objects. - if a and not isinstance(a[0], str): - raise TypeError('lines to compare must be str, not %s (%r)' % - (type(a[0]).__name__, a[0])) - if b and not isinstance(b[0], str): - raise TypeError('lines to compare must be str, not %s (%r)' % - (type(b[0]).__name__, b[0])) - for arg in args: - if not isinstance(arg, str): - raise TypeError('all arguments must be str, not: %r' % (arg,)) - -def diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', - fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n'): - r""" - Compare `a` and `b`, two sequences of lines represented as bytes rather - than str. This is a wrapper for `dfunc`, which is typically either - unified_diff() or context_diff(). Inputs are losslessly converted to - strings so that `dfunc` only has to worry about strings, and encoded - back to bytes on return. This is necessary to compare files with - unknown or inconsistent encoding. All other inputs (except `n`) must be - bytes rather than str. - """ - def decode(s): - try: - return s.decode('ascii', 'surrogateescape') - except AttributeError as err: - msg = ('all arguments must be bytes, not %s (%r)' % - (type(s).__name__, s)) - raise TypeError(msg) from err - a = list(map(decode, a)) - b = list(map(decode, b)) - fromfile = decode(fromfile) - tofile = decode(tofile) - fromfiledate = decode(fromfiledate) - tofiledate = decode(tofiledate) - lineterm = decode(lineterm) - - lines = dfunc(a, b, fromfile, tofile, fromfiledate, tofiledate, n, lineterm) - for line in lines: - yield line.encode('ascii', 'surrogateescape') - def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): r""" Compare `a` and `b` (lists of strings); return a `Differ`-style delta. Optional keyword parameters `linejunk` and `charjunk` are for filter - functions, or can be None: + functions (or None): - - linejunk: A function that should accept a single string argument and + - linejunk: A function that should accept a single string argument, and return true iff the string is junk. The default is None, and is - recommended; the underlying SequenceMatcher class has an adaptive - notion of "noise" lines. + recommended; as of Python 2.3, an adaptive notion of "noise" lines is + used that does a good job on its own. - - charjunk: A function that accepts a character (string of length - 1), and returns true iff the character is junk. The default is - the module-level function IS_CHARACTER_JUNK, which filters out - whitespace characters (a blank or tab; note: it's a bad idea to - include newline in this!). + - charjunk: A function that should accept a string of length 1. The + default is module-level function IS_CHARACTER_JUNK, which filters out + whitespace characters (a blank or tab; note: bad idea to include newline + in this!). Tools/scripts/ndiff.py is a command-line front-end to this function. @@ -1390,7 +1365,7 @@ linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) - This function returns an iterator which returns a tuple: + This function returns an interator which returns a tuple: (from line tuple, to line tuple, boolean flag) from/to line tuple -- (line num, line text) @@ -1459,7 +1434,7 @@ change_re.sub(record_sub_info,markers) # process each tuple inserting our special marks that won't be # noticed by an xml/html escaper. - for key,(begin,end) in reversed(sub_info): + for key,(begin,end) in sub_info[::-1]: text = text[0:begin]+'\0'+key+text[begin:end]+'\1'+text[end:] text = text[2:] # Handle case of add/delete entire line @@ -1497,7 +1472,10 @@ # are a concatenation of the first character of each of the 4 lines # so we can do some very readable comparisons. while len(lines) < 4: - lines.append(next(diff_lines_iterator, 'X')) + try: + lines.append(next(diff_lines_iterator)) + except StopIteration: + lines.append('X') s = ''.join([line[0] for line in lines]) if s.startswith('X'): # When no more lines, pump out any remaining blank lines so the @@ -1515,7 +1493,7 @@ yield _make_line(lines,'-',0), None, True continue elif s.startswith(('--?+', '--+', '- ')): - # in delete block and see an intraline change or unchanged line + # in delete block and see a intraline change or unchanged line # coming: yield the delete line and then blanks from_line,to_line = _make_line(lines,'-',0), None num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0 @@ -1560,7 +1538,7 @@ num_blanks_to_yield -= 1 yield ('','\n'),None,True if s.startswith('X'): - return + raise StopIteration else: yield from_line,to_line,True @@ -1582,10 +1560,7 @@ while True: # Collecting lines of text until we have a from/to pair while (len(fromlines)==0 or len(tolines)==0): - try: - from_line, to_line, found_diff = next(line_iterator) - except StopIteration: - return + from_line, to_line, found_diff = next(line_iterator) if from_line is not None: fromlines.append((from_line,found_diff)) if to_line is not None: @@ -1599,7 +1574,8 @@ # them up without doing anything else with them. line_pair_iterator = _line_pair_iterator() if context is None: - yield from line_pair_iterator + while True: + yield next(line_pair_iterator) # Handle case where user wants context differencing. We must do some # storage of lines until we know for sure that they are to be yielded. else: @@ -1612,10 +1588,7 @@ index, contextLines = 0, [None]*(context) found_diff = False while(found_diff is False): - try: - from_line, to_line, found_diff = next(line_pair_iterator) - except StopIteration: - return + from_line, to_line, found_diff = next(line_pair_iterator) i = index % context contextLines[i] = (from_line, to_line, found_diff) index += 1 @@ -1652,7 +1625,7 @@ + content="text/html; charset=ISO-8859-1" /> @@ -1730,7 +1703,7 @@ tabsize -- tab stop spacing, defaults to 8. wrapcolumn -- column number where lines are broken and wrapped, defaults to None where lines are not wrapped. - linejunk,charjunk -- keyword arguments passed into ndiff() (used by + linejunk,charjunk -- keyword arguments passed into ndiff() (used to by HtmlDiff() to generate the side by side HTML differences). See ndiff() documentation for argument default values and descriptions. """ @@ -1739,8 +1712,8 @@ self._linejunk = linejunk self._charjunk = charjunk - def make_file(self, fromlines, tolines, fromdesc='', todesc='', - context=False, numlines=5, *, charset='utf-8'): + def make_file(self,fromlines,tolines,fromdesc='',todesc='',context=False, + numlines=5): """Returns HTML file of side by side comparison with change highlights Arguments: @@ -1755,16 +1728,13 @@ When context is False, controls the number of lines to place the "next" link anchors before the next change (so click of "next" link jumps to just before the change). - charset -- charset of the HTML document """ - return (self._file_template % dict( - styles=self._styles, - legend=self._legend, - table=self.make_table(fromlines, tolines, fromdesc, todesc, - context=context, numlines=numlines), - charset=charset - )).encode(charset, 'xmlcharrefreplace').decode(charset) + return self._file_template % dict( + styles = self._styles, + legend = self._legend, + table = self.make_table(fromlines,tolines,fromdesc,todesc, + context=context,numlines=numlines)) def _tab_newline_replace(self,fromlines,tolines): """Returns from/to line lists with tabs expanded and newlines removed. @@ -1997,7 +1967,7 @@ self._make_prefix() # change tabs to spaces before it gets more difficult after we insert - # markup + # markkup fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff -r 6db40a9955dc -r 0d413f60cc23 Lib/dis.py --- a/Lib/dis.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/dis.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,15 +2,12 @@ import sys import types -import collections -import io from opcode import * from opcode import __all__ as _opcodes_all __all__ = ["code_info", "dis", "disassemble", "distb", "disco", - "findlinestarts", "findlabels", "show_code", - "get_instructions", "Instruction", "Bytecode"] + _opcodes_all + "findlinestarts", "findlabels", "show_code"] + _opcodes_all del _opcodes_all _have_code = (types.MethodType, types.FunctionType, types.CodeType, type) @@ -28,42 +25,40 @@ c = compile(source, name, 'exec') return c -def dis(x=None, *, file=None): - """Disassemble classes, methods, functions, generators, or code. +def dis(x=None): + """Disassemble classes, methods, functions, or code. With no argument, disassemble the last traceback. """ if x is None: - distb(file=file) + distb() return if hasattr(x, '__func__'): # Method x = x.__func__ if hasattr(x, '__code__'): # Function x = x.__code__ - if hasattr(x, 'gi_code'): # Generator - x = x.gi_code if hasattr(x, '__dict__'): # Class or module items = sorted(x.__dict__.items()) for name, x1 in items: if isinstance(x1, _have_code): - print("Disassembly of %s:" % name, file=file) + print("Disassembly of %s:" % name) try: - dis(x1, file=file) + dis(x1) except TypeError as msg: - print("Sorry:", msg, file=file) - print(file=file) + print("Sorry:", msg) + print() elif hasattr(x, 'co_code'): # Code object - disassemble(x, file=file) + disassemble(x) elif isinstance(x, (bytes, bytearray)): # Raw bytecode - _disassemble_bytes(x, file=file) + _disassemble_bytes(x) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file) + _disassemble_str(x) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None): +def distb(tb=None): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -71,7 +66,7 @@ except AttributeError: raise RuntimeError("no last traceback to disassemble") while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file) + disassemble(tb.tb_frame.f_code, tb.tb_lasti) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to @@ -84,8 +79,6 @@ 16: "NESTED", 32: "GENERATOR", 64: "NOFREE", - 128: "COROUTINE", - 256: "ITERABLE_COROUTINE", } def pretty_flags(flags): @@ -102,24 +95,19 @@ names.append(hex(flags)) return ", ".join(names) -def _get_code_object(x): - """Helper to handle methods, functions, generators, strings and raw code objects""" +def code_info(x): + """Formatted details of methods, functions, or code.""" if hasattr(x, '__func__'): # Method x = x.__func__ if hasattr(x, '__code__'): # Function x = x.__code__ - if hasattr(x, 'gi_code'): # Generator - x = x.gi_code if isinstance(x, str): # Source code - x = _try_compile(x, "") + x = _try_compile(x, "") if hasattr(x, 'co_code'): # Code object - return x - raise TypeError("don't know how to disassemble %s objects" % - type(x).__name__) - -def code_info(x): - """Formatted details of methods, functions, or code.""" - return _format_code_info(_get_code_object(x)) + return _format_code_info(x) + else: + raise TypeError("don't know how to disassemble %s objects" % + type(x).__name__) def _format_code_info(co): lines = [] @@ -152,215 +140,100 @@ lines.append("%4d: %s" % i_n) return "\n".join(lines) -def show_code(co, *, file=None): - """Print details of methods, functions, or code to *file*. +def show_code(co): + """Print details of methods, functions, or code to stdout.""" + print(code_info(co)) - If *file* is not provided, the output is printed on stdout. - """ - print(code_info(co), file=file) +def disassemble(co, lasti=-1): + """Disassemble a code object.""" + code = co.co_code + labels = findlabels(code) + linestarts = dict(findlinestarts(co)) + n = len(code) + i = 0 + extended_arg = 0 + free = None + while i < n: + op = code[i] + if i in linestarts: + if i > 0: + print() + print("%3d" % linestarts[i], end=' ') + else: + print(' ', end=' ') -_Instruction = collections.namedtuple("_Instruction", - "opname opcode arg argval argrepr offset starts_line is_jump_target") + if i == lasti: print('-->', end=' ') + else: print(' ', end=' ') + if i in labels: print('>>', end=' ') + else: print(' ', end=' ') + print(repr(i).rjust(4), end=' ') + print(opname[op].ljust(20), end=' ') + i = i+1 + if op >= HAVE_ARGUMENT: + oparg = code[i] + code[i+1]*256 + extended_arg + extended_arg = 0 + i = i+2 + if op == EXTENDED_ARG: + extended_arg = oparg*65536 + print(repr(oparg).rjust(5), end=' ') + if op in hasconst: + print('(' + repr(co.co_consts[oparg]) + ')', end=' ') + elif op in hasname: + print('(' + co.co_names[oparg] + ')', end=' ') + elif op in hasjrel: + print('(to ' + repr(i + oparg) + ')', end=' ') + elif op in haslocal: + print('(' + co.co_varnames[oparg] + ')', end=' ') + elif op in hascompare: + print('(' + cmp_op[oparg] + ')', end=' ') + elif op in hasfree: + if free is None: + free = co.co_cellvars + co.co_freevars + print('(' + free[oparg] + ')', end=' ') + print() -_Instruction.opname.__doc__ = "Human readable name for operation" -_Instruction.opcode.__doc__ = "Numeric code for operation" -_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" -_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" -_Instruction.argrepr.__doc__ = "Human readable description of operation argument" -_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" -_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" -_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" - -class Instruction(_Instruction): - """Details for a bytecode operation - - Defined fields: - opname - human readable name for operation - opcode - numeric code for operation - arg - numeric argument to operation (if any), otherwise None - argval - resolved arg value (if known), otherwise same as arg - argrepr - human readable description of operation argument - offset - start index of operation within bytecode sequence - starts_line - line started by this opcode (if any), otherwise None - is_jump_target - True if other code jumps to here, otherwise False - """ - - def _disassemble(self, lineno_width=3, mark_as_current=False): - """Format instruction details for inclusion in disassembly output - - *lineno_width* sets the width of the line number field (0 omits it) - *mark_as_current* inserts a '-->' marker arrow as part of the line - """ - fields = [] - # Column: Source code line number - if lineno_width: - if self.starts_line is not None: - lineno_fmt = "%%%dd" % lineno_width - fields.append(lineno_fmt % self.starts_line) - else: - fields.append(' ' * lineno_width) - # Column: Current instruction indicator - if mark_as_current: - fields.append('-->') - else: - fields.append(' ') - # Column: Jump target marker - if self.is_jump_target: - fields.append('>>') - else: - fields.append(' ') - # Column: Instruction offset from start of code sequence - fields.append(repr(self.offset).rjust(4)) - # Column: Opcode name - fields.append(self.opname.ljust(20)) - # Column: Opcode argument - if self.arg is not None: - fields.append(repr(self.arg).rjust(5)) - # Column: Opcode argument details - if self.argrepr: - fields.append('(' + self.argrepr + ')') - return ' '.join(fields).rstrip() - - -def get_instructions(x, *, first_line=None): - """Iterator for the opcodes in methods, functions or code - - Generates a series of Instruction named tuples giving the details of - each operations in the supplied code. - - If *first_line* is not None, it indicates the line number that should - be reported for the first source line in the disassembled code. - Otherwise, the source line information (if any) is taken directly from - the disassembled code object. - """ - co = _get_code_object(x) - cell_names = co.co_cellvars + co.co_freevars - linestarts = dict(findlinestarts(co)) - if first_line is not None: - line_offset = first_line - co.co_firstlineno - else: - line_offset = 0 - return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names, - co.co_consts, cell_names, linestarts, - line_offset) - -def _get_const_info(const_index, const_list): - """Helper to get optional details about const references - - Returns the dereferenced constant and its repr if the constant - list is defined. - Otherwise returns the constant index and its repr(). - """ - argval = const_index - if const_list is not None: - argval = const_list[const_index] - return argval, repr(argval) - -def _get_name_info(name_index, name_list): - """Helper to get optional details about named references - - Returns the dereferenced name as both value and repr if the name - list is defined. - Otherwise returns the name index and its repr(). - """ - argval = name_index - if name_list is not None: - argval = name_list[name_index] - argrepr = argval - else: - argrepr = repr(argval) - return argval, argrepr - - -def _get_instructions_bytes(code, varnames=None, names=None, constants=None, - cells=None, linestarts=None, line_offset=0): - """Iterate over the instructions in a bytecode string. - - Generates a sequence of Instruction namedtuples giving the details of each - opcode. Additional information about the code's runtime environment - (e.g. variable names, constants) can be specified using optional - arguments. - - """ +def _disassemble_bytes(code, lasti=-1, varnames=None, names=None, + constants=None): labels = findlabels(code) - extended_arg = 0 - starts_line = None - free = None - # enumerate() is not an option, since we sometimes process - # multiple elements on a single pass through the loop n = len(code) i = 0 while i < n: op = code[i] - offset = i - if linestarts is not None: - starts_line = linestarts.get(i, None) - if starts_line is not None: - starts_line += line_offset - is_jump_target = i in labels + if i == lasti: print('-->', end=' ') + else: print(' ', end=' ') + if i in labels: print('>>', end=' ') + else: print(' ', end=' ') + print(repr(i).rjust(4), end=' ') + print(opname[op].ljust(15), end=' ') i = i+1 - arg = None - argval = None - argrepr = '' if op >= HAVE_ARGUMENT: - arg = code[i] + code[i+1]*256 + extended_arg - extended_arg = 0 + oparg = code[i] + code[i+1]*256 i = i+2 - if op == EXTENDED_ARG: - extended_arg = arg*65536 - # Set argval to the dereferenced value of the argument when - # availabe, and argrepr to the string representation of argval. - # _disassemble_bytes needs the string repr of the - # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. - argval = arg + print(repr(oparg).rjust(5), end=' ') if op in hasconst: - argval, argrepr = _get_const_info(arg, constants) + if constants: + print('(' + repr(constants[oparg]) + ')', end=' ') + else: + print('(%d)'%oparg, end=' ') elif op in hasname: - argval, argrepr = _get_name_info(arg, names) + if names is not None: + print('(' + names[oparg] + ')', end=' ') + else: + print('(%d)'%oparg, end=' ') elif op in hasjrel: - argval = i + arg - argrepr = "to " + repr(argval) + print('(to ' + repr(i + oparg) + ')', end=' ') elif op in haslocal: - argval, argrepr = _get_name_info(arg, varnames) + if varnames: + print('(' + varnames[oparg] + ')', end=' ') + else: + print('(%d)' % oparg, end=' ') elif op in hascompare: - argval = cmp_op[arg] - argrepr = argval - elif op in hasfree: - argval, argrepr = _get_name_info(arg, cells) - elif op in hasnargs: - argrepr = "%d positional, %d keyword pair" % (code[i-2], code[i-1]) - yield Instruction(opname[op], op, - arg, argval, argrepr, - offset, starts_line, is_jump_target) + print('(' + cmp_op[oparg] + ')', end=' ') + print() -def disassemble(co, lasti=-1, *, file=None): - """Disassemble a code object.""" - cell_names = co.co_cellvars + co.co_freevars - linestarts = dict(findlinestarts(co)) - _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names, - co.co_consts, cell_names, linestarts, file=file) - -def _disassemble_bytes(code, lasti=-1, varnames=None, names=None, - constants=None, cells=None, linestarts=None, - *, file=None, line_offset=0): - # Omit the line number column entirely if we have no line number info - show_lineno = linestarts is not None - # TODO?: Adjust width upwards if max(linestarts.values()) >= 1000? - lineno_width = 3 if show_lineno else 0 - for instr in _get_instructions_bytes(code, varnames, names, - constants, cells, linestarts, - line_offset=line_offset): - new_source_line = (show_lineno and - instr.starts_line is not None and - instr.offset > 0) - if new_source_line: - print(file=file) - is_current_instr = instr.offset == lasti - print(instr._disassemble(lineno_width, is_current_instr), file=file) - -def _disassemble_str(source, *, file=None): +def _disassemble_str(source): """Compile the source string, then disassemble the code object.""" - disassemble(_try_compile(source, ''), file=file) + disassemble(_try_compile(source, '')) disco = disassemble # XXX For backwards compatibility @@ -371,21 +244,19 @@ """ labels = [] - # enumerate() is not an option, since we sometimes process - # multiple elements on a single pass through the loop n = len(code) i = 0 while i < n: op = code[i] i = i+1 if op >= HAVE_ARGUMENT: - arg = code[i] + code[i+1]*256 + oparg = code[i] + code[i+1]*256 i = i+2 label = -1 if op in hasjrel: - label = i+arg + label = i+oparg elif op in hasjabs: - label = arg + label = oparg if label >= 0: if label not in labels: labels.append(label) @@ -397,8 +268,8 @@ Generate pairs (offset, lineno) as described in Python/compile.c. """ - byte_increments = code.co_lnotab[0::2] - line_increments = code.co_lnotab[1::2] + byte_increments = list(code.co_lnotab[0::2]) + line_increments = list(code.co_lnotab[1::2]) lastlineno = None lineno = code.co_firstlineno @@ -409,84 +280,31 @@ yield (addr, lineno) lastlineno = lineno addr += byte_incr - if line_incr >= 0x80: - # line_increments is an array of 8-bit signed integers - line_incr -= 0x100 lineno += line_incr if lineno != lastlineno: yield (addr, lineno) -class Bytecode: - """The bytecode operations of a piece of code - - Instantiate this with a function, method, string of code, or a code object - (as returned by compile()). - - Iterating over this yields the bytecode operations as Instruction instances. - """ - def __init__(self, x, *, first_line=None, current_offset=None): - self.codeobj = co = _get_code_object(x) - if first_line is None: - self.first_line = co.co_firstlineno - self._line_offset = 0 - else: - self.first_line = first_line - self._line_offset = first_line - co.co_firstlineno - self._cell_names = co.co_cellvars + co.co_freevars - self._linestarts = dict(findlinestarts(co)) - self._original_object = x - self.current_offset = current_offset - - def __iter__(self): - co = self.codeobj - return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names, - co.co_consts, self._cell_names, - self._linestarts, - line_offset=self._line_offset) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, - self._original_object) - - @classmethod - def from_traceback(cls, tb): - """ Construct a Bytecode from the given traceback """ - while tb.tb_next: - tb = tb.tb_next - return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti) - - def info(self): - """Return formatted information about the code object.""" - return _format_code_info(self.codeobj) - - def dis(self): - """Return a formatted view of the bytecode operations.""" - co = self.codeobj - if self.current_offset is not None: - offset = self.current_offset - else: - offset = -1 - with io.StringIO() as output: - _disassemble_bytes(co.co_code, varnames=co.co_varnames, - names=co.co_names, constants=co.co_consts, - cells=self._cell_names, - linestarts=self._linestarts, - line_offset=self._line_offset, - file=output, - lasti=offset) - return output.getvalue() - - def _test(): """Simple test program to disassemble a file.""" - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-') - args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") + if sys.argv[1:]: + if sys.argv[2:]: + sys.stderr.write("usage: python dis.py [-|file]\n") + sys.exit(2) + fn = sys.argv[1] + if not fn or fn == "-": + fn = None + else: + fn = None + if fn is None: + f = sys.stdin + else: + f = open(fn) + source = f.read() + if fn is not None: + f.close() + else: + fn = "" + code = compile(source, fn, "exec") dis(code) if __name__ == "__main__": diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/__init__.py --- a/Lib/distutils/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,6 +8,10 @@ setup (...) """ -import sys - -__version__ = sys.version[:sys.version.index(' ')] +# Distutils version +# +# Updated automatically by the Python release process. +# +#--start constants-- +__version__ = "3.3.0a1" +#--end constants-- diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/_msvccompiler.py --- a/Lib/distutils/_msvccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,535 +0,0 @@ -"""distutils._msvccompiler - -Contains MSVCCompiler, an implementation of the abstract CCompiler class -for Microsoft Visual Studio 2015. - -The module is compatible with VS 2015 and later. You can find legacy support -for older versions in distutils.msvc9compiler and distutils.msvccompiler. -""" - -# Written by Perry Stoll -# hacked by Robin Becker and Thomas Heller to do a better job of -# finding DevStudio (through the registry) -# ported to VS 2005 and VS 2008 by Christian Heimes -# ported to VS 2015 by Steve Dower - -import os -import shutil -import stat -import subprocess - -from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ - CompileError, LibError, LinkError -from distutils.ccompiler import CCompiler, gen_lib_options -from distutils import log -from distutils.util import get_platform - -import winreg -from itertools import count - -def _find_vcvarsall(plat_spec): - try: - key = winreg.OpenKeyEx( - winreg.HKEY_LOCAL_MACHINE, - r"Software\Microsoft\VisualStudio\SxS\VC7", - access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY - ) - except OSError: - log.debug("Visual C++ is not registered") - return None, None - - with key: - best_version = 0 - best_dir = None - for i in count(): - try: - v, vc_dir, vt = winreg.EnumValue(key, i) - except OSError: - break - if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): - try: - version = int(float(v)) - except (ValueError, TypeError): - continue - if version >= 14 and version > best_version: - best_version, best_dir = version, vc_dir - if not best_version: - log.debug("No suitable Visual C++ version found") - return None, None - - vcvarsall = os.path.join(best_dir, "vcvarsall.bat") - if not os.path.isfile(vcvarsall): - log.debug("%s cannot be found", vcvarsall) - return None, None - - vcruntime = None - vcruntime_spec = _VCVARS_PLAT_TO_VCRUNTIME_REDIST.get(plat_spec) - if vcruntime_spec: - vcruntime = os.path.join(best_dir, - vcruntime_spec.format(best_version)) - if not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime - -def _get_vc_env(plat_spec): - if os.getenv("DISTUTILS_USE_SDK"): - return { - key.lower(): value - for key, value in os.environ.items() - } - - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) - if not vcvarsall: - raise DistutilsPlatformError("Unable to find vcvarsall.bat") - - try: - out = subprocess.check_output( - '"{}" {} && set'.format(vcvarsall, plat_spec), - shell=True, - stderr=subprocess.STDOUT, - universal_newlines=True, - ) - except subprocess.CalledProcessError as exc: - log.error(exc.output) - raise DistutilsPlatformError("Error executing {}" - .format(exc.cmd)) - - env = { - key.lower(): value - for key, _, value in - (line.partition('=') for line in out.splitlines()) - if key and value - } - - if vcruntime: - env['py_vcruntime_redist'] = vcruntime - return env - -def _find_exe(exe, paths=None): - """Return path to an MSVC executable program. - - Tries to find the program in several places: first, one of the - MSVC program search paths from the registry; next, the directories - in the PATH environment variable. If any of those work, return an - absolute path that is known to exist. If none of them work, just - return the original program name, 'exe'. - """ - if not paths: - paths = os.getenv('path').split(os.pathsep) - for p in paths: - fn = os.path.join(os.path.abspath(p), exe) - if os.path.isfile(fn): - return fn - return exe - -# A map keyed by get_platform() return values to values accepted by -# 'vcvarsall.bat'. Always cross-compile from x86 to work with the -# lighter-weight MSVC installs that do not include native 64-bit tools. -PLAT_TO_VCVARS = { - 'win32' : 'x86', - 'win-amd64' : 'x86_amd64', -} - -# A map keyed by get_platform() return values to the file under -# the VC install directory containing the vcruntime redistributable. -_VCVARS_PLAT_TO_VCRUNTIME_REDIST = { - 'x86' : 'redist\\x86\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll', - 'amd64' : 'redist\\x64\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll', - 'x86_amd64' : 'redist\\x64\\Microsoft.VC{0}0.CRT\\vcruntime{0}0.dll', -} - -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - -class MSVCCompiler(CCompiler) : - """Concrete class that implements an interface to Microsoft Visual C++, - as defined by the CCompiler abstract class.""" - - compiler_type = 'msvc' - - # Just set this so CCompiler's constructor doesn't barf. We currently - # don't use the 'set_executables()' bureaucracy provided by CCompiler, - # as it really isn't necessary for this sort of single-compiler class. - # Would be nice to have a consistent interface with UnixCCompiler, - # though, so it's worth thinking about. - executables = {} - - # Private class data (need to distinguish C from C++ source for compiler) - _c_extensions = ['.c'] - _cpp_extensions = ['.cc', '.cpp', '.cxx'] - _rc_extensions = ['.rc'] - _mc_extensions = ['.mc'] - - # Needed for the filename generation methods provided by the - # base class, CCompiler. - src_extensions = (_c_extensions + _cpp_extensions + - _rc_extensions + _mc_extensions) - res_extension = '.res' - obj_extension = '.obj' - static_lib_extension = '.lib' - shared_lib_extension = '.dll' - static_lib_format = shared_lib_format = '%s%s' - exe_extension = '.exe' - - - def __init__(self, verbose=0, dry_run=0, force=0): - CCompiler.__init__ (self, verbose, dry_run, force) - # target platform (.plat_name is consistent with 'bdist') - self.plat_name = None - self.initialized = False - - def initialize(self, plat_name=None): - # multi-init means we would need to check platform same each time... - assert not self.initialized, "don't init multiple times" - if plat_name is None: - plat_name = get_platform() - # sanity check for platforms to prevent obscure errors later. - if plat_name not in PLAT_TO_VCVARS: - raise DistutilsPlatformError("--plat-name must be one of {}" - .format(tuple(PLAT_TO_VCVARS))) - - # Get the vcvarsall.bat spec for the requested platform. - plat_spec = PLAT_TO_VCVARS[plat_name] - - vc_env = _get_vc_env(plat_spec) - if not vc_env: - raise DistutilsPlatformError("Unable to find a compatible " - "Visual Studio installation.") - - self._paths = vc_env.get('path', '') - paths = self._paths.split(os.pathsep) - self.cc = _find_exe("cl.exe", paths) - self.linker = _find_exe("link.exe", paths) - self.lib = _find_exe("lib.exe", paths) - self.rc = _find_exe("rc.exe", paths) # resource compiler - self.mc = _find_exe("mc.exe", paths) # message compiler - self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') - - for dir in vc_env.get('include', '').split(os.pathsep): - if dir: - self.add_include_dir(dir) - - for dir in vc_env.get('lib', '').split(os.pathsep): - if dir: - self.add_library_dir(dir) - - self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. - self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' - ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') - - self.compile_options_debug = [ - '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' - ] - - ldflags = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG' - ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) - - ldflags_debug = [ - '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' - ] - - self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] - self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] - self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] - self.ldflags_static = [*ldflags] - self.ldflags_static_debug = [*ldflags_debug] - - self._ldflags = { - (CCompiler.EXECUTABLE, None): self.ldflags_exe, - (CCompiler.EXECUTABLE, False): self.ldflags_exe, - (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug, - (CCompiler.SHARED_OBJECT, None): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, False): self.ldflags_shared, - (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug, - (CCompiler.SHARED_LIBRARY, None): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, False): self.ldflags_static, - (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug, - } - - self.initialized = True - - # -- Worker methods ------------------------------------------------ - - def object_filenames(self, - source_filenames, - strip_dir=0, - output_dir=''): - ext_map = { - **{ext: self.obj_extension for ext in self.src_extensions}, - **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions}, - } - - output_dir = output_dir or '' - - def make_out_path(p): - base, ext = os.path.splitext(p) - if strip_dir: - base = os.path.basename(base) - else: - _, base = os.path.splitdrive(base) - if base.startswith((os.path.sep, os.path.altsep)): - base = base[1:] - try: - # XXX: This may produce absurdly long paths. We should check - # the length of the result and trim base until we fit within - # 260 characters. - return os.path.join(output_dir, base + ext_map[ext]) - except LookupError: - # Better to raise an exception instead of silently continuing - # and later complain about sources and targets having - # different lengths - raise CompileError("Don't know how to compile {}".format(p)) - - return list(map(make_out_path, source_filenames)) - - - def compile(self, sources, - output_dir=None, macros=None, include_dirs=None, debug=0, - extra_preargs=None, extra_postargs=None, depends=None): - - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, - sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - compile_opts = extra_preargs or [] - compile_opts.append('/c') - if debug: - compile_opts.extend(self.compile_options_debug) - else: - compile_opts.extend(self.compile_options) - - - add_cpp_opts = False - - for obj in objects: - try: - src, ext = build[obj] - except KeyError: - continue - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - add_cpp_opts = True - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt, input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - continue - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * '-h dir' specifies the directory for the - # generated include file - # * '-r dir' specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src]) - base, _ = os.path.splitext(os.path.basename (src)) - rc_file = os.path.join(rc_dir, base + '.rc') - # then compile .RC to .RES file - self.spawn([self.rc, "/fo" + obj, rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - continue - else: - # how to handle this file? - raise CompileError("Don't know how to compile {} to {}" - .format(src, obj)) - - args = [self.cc] + compile_opts + pp_opts - if add_cpp_opts: - args.append('/EHsc') - args.append(input_opt) - args.append("/Fo" + obj) - args.extend(extra_postargs) - - try: - self.spawn(args) - except DistutilsExecError as msg: - raise CompileError(msg) - - return objects - - - def create_static_lib(self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - output_filename = self.library_filename(output_libname, - output_dir=output_dir) - - if self._need_link(objects, output_filename): - lib_args = objects + ['/OUT:' + output_filename] - if debug: - pass # XXX what goes here? - try: - log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args)) - self.spawn([self.lib] + lib_args) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - - def link(self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: - self.initialize() - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - if runtime_library_dirs: - self.warn("I don't know what to do with 'runtime_library_dirs': " - + str(runtime_library_dirs)) - - lib_opts = gen_lib_options(self, - library_dirs, runtime_library_dirs, - libraries) - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ldflags = self._ldflags[target_desc, debug] - - export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])] - - ld_args = (ldflags + lib_opts + export_opts + - objects + ['/OUT:' + output_filename]) - - # The MSVC linker generates .lib and .exp files, which cannot be - # suppressed by any linker switches. The .lib files may even be - # needed! Make sure they are generated in the temporary build - # directory. Since they have different names for debug and release - # builds, they can go into the same directory. - build_temp = os.path.dirname(objects[0]) - if export_symbols is not None: - (dll_name, dll_ext) = os.path.splitext( - os.path.basename(output_filename)) - implib_file = os.path.join( - build_temp, - self.library_filename(dll_name)) - ld_args.append ('/IMPLIB:' + implib_file) - - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - - output_dir = os.path.dirname(os.path.abspath(output_filename)) - self.mkpath(output_dir) - try: - log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) - self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - - def spawn(self, cmd): - old_path = os.getenv('path') - try: - os.environ['path'] = self._paths - return super().spawn(cmd) - finally: - os.environ['path'] = old_path - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "/LIBPATH:" + dir - - def runtime_library_dir_option(self, dir): - raise DistutilsPlatformError( - "don't know how to set runtime library search path for MSVC") - - def library_option(self, lib): - return self.library_filename(lib) - - def find_library_file(self, dirs, lib, debug=0): - # Prefer a debugging library if found (and requested), but deal - # with it if we don't have one. - if debug: - try_names = [lib + "_d", lib] - else: - try_names = [lib] - for dir in dirs: - for name in try_names: - libfile = os.path.join(dir, self.library_filename(name)) - if os.path.isfile(libfile): - return libfile - else: - # Oops, didn't find it in *any* of 'dirs' - return None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/archive_util.py --- a/Lib/distutils/archive_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/archive_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -18,67 +18,25 @@ from distutils.dir_util import mkpath from distutils import log -try: - from pwd import getpwnam -except ImportError: - getpwnam = None - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - -def _get_gid(name): - """Returns a gid, given a group name.""" - if getgrnam is None or name is None: - return None - try: - result = getgrnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def _get_uid(name): - """Returns an uid, given a user name.""" - if getpwnam is None or name is None: - return None - try: - result = getpwnam(name) - except KeyError: - result = None - if result is not None: - return result[2] - return None - -def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, - owner=None, group=None): +def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0): """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or - None. ("compress" will be deprecated in Python 3.2) - - 'owner' and 'group' can be used to define an owner and a group for the - archive that is being built. If not provided, the current owner and group - will be used. - + 'compress' must be "gzip" (the default), "compress", "bzip2", or None. + Both "tar" and the compression utility named by 'compress' must be on + the default program search path, so this is probably Unix-specific. The output tar file will be named 'base_dir' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z"). - + the appropriate compression extension (".gz", ".bz2" or ".Z"). Returns the output filename. """ - tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '', - 'compress': ''} - compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz', - 'compress': '.Z'} + tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', None: '', 'compress': ''} + compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'compress': '.Z'} # flags for compression program, each element of list will be an argument if compress is not None and compress not in compress_ext.keys(): raise ValueError( - "bad value for 'compress': must be None, 'gzip', 'bzip2', " - "'xz' or 'compress'") + "bad value for 'compress': must be None, 'gzip', 'bzip2' " + "or 'compress'") archive_name = base_name + '.tar' if compress != 'compress': @@ -90,23 +48,10 @@ import tarfile # late import so Python build itself doesn't break log.info('Creating tar archive') - - uid = _get_uid(owner) - gid = _get_gid(group) - - def _set_uid_gid(tarinfo): - if gid is not None: - tarinfo.gid = gid - tarinfo.gname = group - if uid is not None: - tarinfo.uid = uid - tarinfo.uname = owner - return tarinfo - if not dry_run: tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) try: - tar.add(base_dir, filter=_set_uid_gid) + tar.add(base_dir) finally: tar.close() @@ -179,7 +124,6 @@ ARCHIVE_FORMATS = { 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), - 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"), 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"), 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"), 'zip': (make_zipfile, [],"ZIP file") @@ -196,12 +140,12 @@ return None def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, - dry_run=0, owner=None, group=None): + dry_run=0): """Create an archive file (eg. zip or tar). 'base_name' is the name of the file to create, minus any format-specific - extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "xztar", or "ztar". + extension; 'format' is the archive format: one of "zip", "tar", "ztar", + or "gztar". 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -209,9 +153,6 @@ ie. 'base_dir' will be the common prefix of all files and directories in the archive. 'root_dir' and 'base_dir' both default to the current directory. Returns the name of the archive file. - - 'owner' and 'group' are used when creating a tar archive. By default, - uses the current owner and group. """ save_cwd = os.getcwd() if root_dir is not None: @@ -233,11 +174,6 @@ func = format_info[0] for arg, val in format_info[1]: kwargs[arg] = val - - if format != 'zip': - kwargs['owner'] = owner - kwargs['group'] = group - try: filename = func(base_name, base_dir, **kwargs) finally: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/ccompiler.py --- a/Lib/distutils/ccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/ccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -351,7 +351,7 @@ return macros, objects, extra, pp_opts, build def _get_cc_args(self, pp_opts, debug, before): - # works for unixccompiler, cygwinccompiler + # works for unixccompiler, emxccompiler, cygwinccompiler cc_args = pp_opts + ['-c'] if debug: cc_args[:0] = ['-g'] @@ -752,7 +752,7 @@ raise NotImplementedError def library_option(self, lib): - """Return the compiler option to add 'lib' to the list of libraries + """Return the compiler option to add 'dir' to the list of libraries linked into the shared library or executable. """ raise NotImplementedError @@ -926,6 +926,7 @@ # on a cygwin built python we can use gcc like an ordinary UNIXish # compiler ('cygwin.*', 'unix'), + ('os2emx', 'emx'), # OS name mappings ('posix', 'unix'), @@ -959,7 +960,7 @@ # is assumed to be in the 'distutils' package.) compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', "standard UNIX-style compiler"), - 'msvc': ('_msvccompiler', 'MSVCCompiler', + 'msvc': ('msvccompiler', 'MSVCCompiler', "Microsoft Visual C++"), 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', "Cygwin port of GNU C Compiler for Win32"), @@ -967,6 +968,8 @@ "Mingw32 port of GNU C Compiler for Win32"), 'bcpp': ('bcppcompiler', 'BCPPCompiler', "Borland C++ Compiler"), + 'emx': ('emxccompiler', 'EMXCCompiler', + "EMX port of GNU C Compiler for OS/2"), } def show_compilers(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/cmd.py --- a/Lib/distutils/cmd.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/cmd.py Mon Jan 25 17:05:13 2016 +0100 @@ -365,11 +365,9 @@ from distutils.spawn import spawn spawn(cmd, search_path, dry_run=self.dry_run) - def make_archive(self, base_name, format, root_dir=None, base_dir=None, - owner=None, group=None): + def make_archive(self, base_name, format, root_dir=None, base_dir=None): return archive_util.make_archive(base_name, format, root_dir, base_dir, - dry_run=self.dry_run, - owner=owner, group=group) + dry_run=self.dry_run) def make_file(self, infiles, outfile, func, args, exec_msg=None, skip_msg=None, level=1): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/bdist.py --- a/Lib/distutils/command/bdist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/bdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -37,12 +37,6 @@ "[default: dist]"), ('skip-build', None, "skip rebuilding everything (for testing/debugging)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), ] boolean_options = ['skip-build'] @@ -58,17 +52,17 @@ # This won't do in reality: will need to distinguish RPM-ish Linux, # Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS. default_format = {'posix': 'gztar', - 'nt': 'zip'} + 'nt': 'zip', + 'os2': 'zip'} # Establish the preferred order (for the --help-formats option). - format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', + format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar', 'wininst', 'zip', 'msi'] # And the real information. format_command = {'rpm': ('bdist_rpm', "RPM distribution"), 'gztar': ('bdist_dumb', "gzip'ed tar file"), 'bztar': ('bdist_dumb', "bzip2'ed tar file"), - 'xztar': ('bdist_dumb', "xz'ed tar file"), 'ztar': ('bdist_dumb', "compressed tar file"), 'tar': ('bdist_dumb', "tar file"), 'wininst': ('bdist_wininst', @@ -84,8 +78,6 @@ self.formats = None self.dist_dir = None self.skip_build = 0 - self.group = None - self.owner = None def finalize_options(self): # have to finalize 'plat_name' before 'bdist_base' @@ -131,11 +123,6 @@ if cmd_name not in self.no_format_option: sub_cmd.format = self.formats[i] - # passing the owner and group names for tar archiving - if cmd_name == 'bdist_dumb': - sub_cmd.owner = self.owner - sub_cmd.group = self.group - # If we're going to need to run this command again, tell it to # keep its temporary files around so subsequent runs go faster. if cmd_name in commands[i+1:]: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/bdist_dumb.py --- a/Lib/distutils/command/bdist_dumb.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/bdist_dumb.py Mon Jan 25 17:05:13 2016 +0100 @@ -22,8 +22,7 @@ "platform name to embed in generated filenames " "(default: %s)" % get_platform()), ('format=', 'f', - "archive format to create (tar, gztar, bztar, xztar, " - "ztar, zip)"), + "archive format to create (tar, ztar, gztar, zip)"), ('keep-temp', 'k', "keep the pseudo-installation tree around after " + "creating the distribution archive"), @@ -34,18 +33,13 @@ ('relative', None, "build the archive using relative paths" "(default: false)"), - ('owner=', 'u', - "Owner name used when creating a tar file" - " [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file" - " [default: current group]"), ] boolean_options = ['keep-temp', 'skip-build', 'relative'] default_format = { 'posix': 'gztar', - 'nt': 'zip' } + 'nt': 'zip', + 'os2': 'zip' } def initialize_options(self): self.bdist_dir = None @@ -55,8 +49,6 @@ self.dist_dir = None self.skip_build = None self.relative = 0 - self.owner = None - self.group = None def finalize_options(self): if self.bdist_dir is None: @@ -93,6 +85,11 @@ archive_basename = "%s.%s" % (self.distribution.get_fullname(), self.plat_name) + # OS/2 objects to any ":" characters in a filename (such as when + # a timestamp is used in a version) so change them to hyphens. + if os.name == "os2": + archive_basename = archive_basename.replace(":", "-") + pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) if not self.relative: archive_root = self.bdist_dir @@ -110,8 +107,7 @@ # Make the archive filename = self.make_archive(pseudoinstall_root, - self.format, root_dir=archive_root, - owner=self.owner, group=self.group) + self.format, root_dir=archive_root) if self.distribution.has_ext_modules(): pyversion = get_python_version() else: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/bdist_rpm.py --- a/Lib/distutils/command/bdist_rpm.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/bdist_rpm.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,7 @@ Implements the Distutils 'bdist_rpm' command (create RPM source and binary distributions).""" -import subprocess, sys, os +import sys, os from distutils.core import Command from distutils.debug import DEBUG from distutils.util import get_platform @@ -190,7 +190,7 @@ if self.fix_python: self.python = sys.executable else: - self.python = "python3" + self.python = "python" elif self.fix_python: raise DistutilsOptionError( "--python and --fix-python are mutually exclusive options") @@ -320,7 +320,6 @@ rpm_cmd.append('-bb') else: rpm_cmd.append('-ba') - rpm_cmd.extend(['--define', '__python %s' % self.python]) if self.rpm3_mode: rpm_cmd.extend(['--define', '_topdir %s' % os.path.abspath(self.rpm_base)]) @@ -406,21 +405,6 @@ 'Summary: ' + self.distribution.get_description(), ] - # Workaround for #14443 which affects some RPM based systems such as - # RHEL6 (and probably derivatives) - vendor_hook = subprocess.getoutput('rpm --eval %{__os_install_post}') - # Generate a potential replacement value for __os_install_post (whilst - # normalizing the whitespace to simplify the test for whether the - # invocation of brp-python-bytecompile passes in __python): - vendor_hook = '\n'.join([' %s \\' % line.strip() - for line in vendor_hook.splitlines()]) - problem = "brp-python-bytecompile \\\n" - fixed = "brp-python-bytecompile %{__python} \\\n" - fixed_hook = vendor_hook.replace(problem, fixed) - if fixed_hook != vendor_hook: - spec_file.append('# Workaround for http://bugs.python.org/issue14443') - spec_file.append('%define __os_install_post ' + fixed_hook + '\n') - # put locale summaries into spec file # XXX not supported for now (hard to put a dictionary # in a config file -- arg!) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/bdist_wininst.py --- a/Lib/distutils/command/bdist_wininst.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/bdist_wininst.py Mon Jan 25 17:05:13 2016 +0100 @@ -303,6 +303,7 @@ return installer_name def get_exe_bytes(self): + from distutils.msvccompiler import get_build_version # If a target-version other than the current version has been # specified, then using the MSVC version from *this* build is no good. # Without actually finding and executing the target version and parsing @@ -312,33 +313,20 @@ # We can then execute this program to obtain any info we need, such # as the real sys.version string for the build. cur_version = get_python_version() - - # If the target version is *later* than us, then we assume they - # use what we use - # string compares seem wrong, but are what sysconfig.py itself uses - if self.target_version and self.target_version < cur_version: - if self.target_version < "2.4": - bv = 6.0 - elif self.target_version == "2.4": - bv = 7.1 - elif self.target_version == "2.5": - bv = 8.0 - elif self.target_version <= "3.2": - bv = 9.0 - elif self.target_version <= "3.4": - bv = 10.0 + if self.target_version and self.target_version != cur_version: + # If the target version is *later* than us, then we assume they + # use what we use + # string compares seem wrong, but are what sysconfig.py itself uses + if self.target_version > cur_version: + bv = get_build_version() else: - bv = 14.0 + if self.target_version < "2.4": + bv = 6.0 + else: + bv = 7.1 else: # for current version - use authoritative check. - try: - from msvcrt import CRT_ASSEMBLY_VERSION - except ImportError: - # cross-building, so assume the latest version - bv = 14.0 - else: - bv = float('.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2])) - + bv = get_build_version() # wininst-x.y.exe is in the same directory as this file directory = os.path.dirname(__file__) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/build.py --- a/Lib/distutils/command/build.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/build.py Mon Jan 25 17:05:13 2016 +0100 @@ -36,8 +36,6 @@ "(default: %s)" % get_platform()), ('compiler=', 'c', "specify the compiler type"), - ('parallel=', 'j', - "number of parallel build jobs"), ('debug', 'g', "compile extensions and libraries with debugging information"), ('force', 'f', @@ -67,7 +65,6 @@ self.debug = None self.force = 0 self.executable = None - self.parallel = None def finalize_options(self): if self.plat_name is None: @@ -119,12 +116,6 @@ if self.executable is None: self.executable = os.path.normpath(sys.executable) - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - def run(self): # Run all relevant sub-commands. This will be some subset of: # - build_py - pure Python modules diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/build_ext.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,20 +4,26 @@ modules (currently limited to C extensions, should accommodate C++ extensions ASAP).""" -import contextlib -import os -import re -import sys +import sys, os, re from distutils.core import Command from distutils.errors import * from distutils.sysconfig import customize_compiler, get_python_version -from distutils.sysconfig import get_config_h_filename from distutils.dep_util import newer_group from distutils.extension import Extension from distutils.util import get_platform from distutils import log -from site import USER_BASE +# this keeps compatibility from 2.3 to 2.5 +if sys.version < "2.6": + USER_BASE = None + HAS_USER_SITE = False +else: + from site import USER_BASE + HAS_USER_SITE = True + +if os.name == 'nt': + from distutils.msvccompiler import get_build_version + MSVC_VERSION = int(get_build_version()) # An extension name is just a dot-separated list of Python NAMEs (ie. # the same as a fully-qualified module name). @@ -84,19 +90,20 @@ "forcibly build everything (ignore file timestamps)"), ('compiler=', 'c', "specify the compiler type"), - ('parallel=', 'j', - "number of parallel build jobs"), ('swig-cpp', None, "make SWIG create C++ files (default is C)"), ('swig-opts=', None, "list of SWIG command line options"), ('swig=', None, "path to the SWIG executable"), - ('user', None, - "add user include, library and rpath") ] - boolean_options = ['inplace', 'debug', 'force', 'swig-cpp', 'user'] + boolean_options = ['inplace', 'debug', 'force', 'swig-cpp'] + + if HAS_USER_SITE: + user_options.append(('user', None, + "add user include, library and rpath")) + boolean_options.append('user') help_options = [ ('help-compiler', None, @@ -125,7 +132,6 @@ self.swig_cpp = None self.swig_opts = None self.user = None - self.parallel = None def finalize_options(self): from distutils import sysconfig @@ -136,7 +142,6 @@ ('compiler', 'compiler'), ('debug', 'debug'), ('force', 'force'), - ('parallel', 'parallel'), ('plat_name', 'plat_name'), ) @@ -154,11 +159,6 @@ if isinstance(self.include_dirs, str): self.include_dirs = self.include_dirs.split(os.pathsep) - # If in a virtualenv, add its include directory - # Issue 16116 - if sys.exec_prefix != sys.base_exec_prefix: - self.include_dirs.append(os.path.join(sys.exec_prefix, 'include')) - # Put the Python "system" include dir at the end, so that # any local include dirs take precedence. self.include_dirs.append(py_include) @@ -189,8 +189,6 @@ # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) - if sys.base_exec_prefix != sys.prefix: # Issue 16116 - self.library_dirs.append(os.path.join(sys.base_exec_prefix, 'libs')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: @@ -198,21 +196,33 @@ # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.dirname(get_config_h_filename())) - _sys_home = getattr(sys, '_home', None) - if _sys_home: - self.library_dirs.append(_sys_home) + self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + if MSVC_VERSION == 9: + # Use the .lib files for the correct architecture + if self.plat_name == 'win32': + suffix = '' + else: + # win-amd64 or win-ia64 + suffix = self.plat_name[4:] + new_lib = os.path.join(sys.exec_prefix, 'PCbuild') + if suffix: + new_lib = os.path.join(new_lib, suffix) + self.library_dirs.append(new_lib) - # Use the .lib files for the correct architecture - if self.plat_name == 'win32': - suffix = 'win32' + elif MSVC_VERSION == 8: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS8.0')) + elif MSVC_VERSION == 7: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS7.1')) else: - # win-amd64 or win-ia64 - suffix = self.plat_name[4:] - new_lib = os.path.join(sys.exec_prefix, 'PCbuild') - if suffix: - new_lib = os.path.join(new_lib, suffix) - self.library_dirs.append(new_lib) + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VC6')) + + # OS/2 (EMX) doesn't support Debug vs Release builds, but has the + # import libraries in its "Config" subdirectory + if os.name == 'os2': + self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) # for extensions under Cygwin and AtheOS Python's library directory must be # appended to library_dirs @@ -226,11 +236,12 @@ # building python standard extensions self.library_dirs.append('.') - # For building extensions with a shared Python library, + # for extensions under Linux or Solaris with a shared Python library, # Python's library directory must be appended to library_dirs - # See Issues: #1600860, #4366 - if (sysconfig.get_config_var('Py_ENABLE_SHARED')): - if not sysconfig.python_build: + sysconfig.get_config_var('Py_ENABLE_SHARED') + if (sys.platform.startswith(('linux', 'gnu', 'sunos')) + and sysconfig.get_config_var('Py_ENABLE_SHARED')): + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: @@ -267,12 +278,6 @@ self.library_dirs.append(user_lib) self.rpath.append(user_lib) - if isinstance(self.parallel, str): - try: - self.parallel = int(self.parallel) - except ValueError: - raise DistutilsOptionError("parallel should be an integer") - def run(self): from distutils.ccompiler import new_compiler @@ -441,45 +446,15 @@ def build_extensions(self): # First, sanity-check the 'extensions' list self.check_extensions_list(self.extensions) - if self.parallel: - self._build_extensions_parallel() - else: - self._build_extensions_serial() - def _build_extensions_parallel(self): - workers = self.parallel - if self.parallel is True: - workers = os.cpu_count() # may return None - try: - from concurrent.futures import ThreadPoolExecutor - except ImportError: - workers = None - - if workers is None: - self._build_extensions_serial() - return - - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [executor.submit(self.build_extension, ext) - for ext in self.extensions] - for ext, fut in zip(self.extensions, futures): - with self._filter_build_errors(ext): - fut.result() - - def _build_extensions_serial(self): for ext in self.extensions: - with self._filter_build_errors(ext): + try: self.build_extension(ext) - - @contextlib.contextmanager - def _filter_build_errors(self, ext): - try: - yield - except (CCompilerError, DistutilsError, CompileError) as e: - if not ext.optional: - raise - self.warn('building extension "%s" failed: %s' % - (ext.name, e)) + except (CCompilerError, DistutilsError, CompileError) as e: + if not ext.optional: + raise + self.warn('building extension "%s" failed: %s' % + (ext.name, e)) def build_extension(self, ext): sources = ext.sources @@ -531,8 +506,15 @@ extra_postargs=extra_args, depends=ext.depends) - # XXX outdated variable, kept here in case third-part code - # needs it. + # XXX -- this is a Vile HACK! + # + # The setup.py script for Python on Unix needs to be able to + # get this list so it can perform all the clean up needed to + # avoid keeping object files around when cleaning out a failed + # build of an extension module. Since Distutils does not + # track dependencies, we have to get rid of intermediates to + # ensure all the intermediates will be properly re-built. + # self._built_objects = objects[:] # Now link the object files together into a "shared object" -- @@ -627,6 +609,9 @@ return fn else: return "swig.exe" + elif os.name == "os2": + # assume swig available in the PATH. + return "swig.exe" else: raise DistutilsPlatformError( "I don't know how to find (much less run) SWIG " @@ -677,8 +662,14 @@ """ from distutils.sysconfig import get_config_var ext_path = ext_name.split('.') - ext_suffix = get_config_var('EXT_SUFFIX') - return os.path.join(*ext_path) + ext_suffix + # OS/2 has an 8 character module (extension) limit :-( + if os.name == "os2": + ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] + # extensions in debug_mode are named 'module_d.pyd' under windows + so_ext = get_config_var('SO') + if os.name == 'nt' and self.debug: + return os.path.join(*ext_path) + '_d' + so_ext + return os.path.join(*ext_path) + so_ext def get_export_symbols(self, ext): """Return the list of symbols that a shared extension has to @@ -694,7 +685,7 @@ def get_libraries(self, ext): """Return the list of libraries to link against when building a shared extension. On most platforms, this is just 'ext.libraries'; - on Windows, we add the Python library (eg. python20.dll). + on Windows and OS/2, we add the Python library (eg. python20.dll). """ # The python library is always needed on Windows. For MSVC, this # is redundant, since the library is mentioned in a pragma in @@ -702,7 +693,7 @@ # to need it mentioned explicitly, though, so that's what we do. # Append '_d' to the python import library on debug builds. if sys.platform == "win32": - from distutils._msvccompiler import MSVCCompiler + from distutils.msvccompiler import MSVCCompiler if not isinstance(self.compiler, MSVCCompiler): template = "python%d%d" if self.debug: @@ -714,6 +705,19 @@ return ext.libraries + [pythonlib] else: return ext.libraries + elif sys.platform == "os2emx": + # EMX/GCC requires the python library explicitly, and I + # believe VACPP does as well (though not confirmed) - AIM Apr01 + template = "python%d%d" + # debug versions of the main DLL aren't supported, at least + # not at this time - AIM Apr01 + #if self.debug: + # template = template + '_d' + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] elif sys.platform[:6] == "cygwin": template = "python%d.%d" pythonlib = (template % diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/build_py.py --- a/Lib/distutils/command/build_py.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/build_py.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,7 +3,7 @@ Implements the Distutils 'build_py' command.""" import os -import importlib.util +import imp import sys from glob import glob @@ -127,8 +127,7 @@ # Each pattern has to be converted to a platform-specific path filelist = glob(os.path.join(src_dir, convert_path(pattern))) # Files that match more than one pattern are only added once - files.extend([fn for fn in filelist if fn not in files - and os.path.isfile(fn)]) + files.extend([fn for fn in filelist if fn not in files]) return files def build_package_data(self): @@ -313,11 +312,11 @@ outputs.append(filename) if include_bytecode: if self.compile: - outputs.append(importlib.util.cache_from_source( - filename, optimization='')) + outputs.append(imp.cache_from_source(filename, + debug_override=True)) if self.optimize > 0: - outputs.append(importlib.util.cache_from_source( - filename, optimization=self.optimize)) + outputs.append(imp.cache_from_source(filename, + debug_override=False)) outputs += [ os.path.join(build_dir, filename) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/build_scripts.py --- a/Lib/distutils/command/build_scripts.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/build_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -74,7 +74,7 @@ # script. try: f = open(script, "rb") - except OSError: + except IOError: if not self.dry_run: raise f = None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/check.py --- a/Lib/distutils/command/check.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/check.py Mon Jan 25 17:05:13 2016 +0100 @@ -23,9 +23,6 @@ def system_message(self, level, message, *children, **kwargs): self.messages.append((level, message, children, kwargs)) - return nodes.system_message(message, level=level, - type=self.levels[level], - *children, **kwargs) HAS_DOCUTILS = True except Exception: @@ -122,7 +119,7 @@ """Returns warnings when the provided data doesn't compile.""" source_path = StringIO() parser = Parser() - settings = frontend.OptionParser(components=(Parser,)).get_default_values() + settings = frontend.OptionParser().get_default_values() settings.tab_width = 4 settings.pep_references = None settings.rfc_references = None @@ -138,8 +135,8 @@ document.note_source(source_path, -1) try: parser.parse(data, document) - except AttributeError as e: - reporter.messages.append( - (-1, 'Could not finish the parsing: %s.' % e, '', {})) + except AttributeError: + reporter.messages.append((-1, 'Could not finish the parsing.', + '', {})) return reporter.messages diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/install.py --- a/Lib/distutils/command/install.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/install.py Mon Jan 25 17:05:13 2016 +0100 @@ -15,17 +15,32 @@ from distutils.util import get_platform from distutils.errors import DistutilsOptionError -from site import USER_BASE -from site import USER_SITE -HAS_USER_SITE = True +# this keeps compatibility from 2.3 to 2.5 +if sys.version < "2.6": + USER_BASE = None + USER_SITE = None + HAS_USER_SITE = False +else: + from site import USER_BASE + from site import USER_SITE + HAS_USER_SITE = True -WINDOWS_SCHEME = { - 'purelib': '$base/Lib/site-packages', - 'platlib': '$base/Lib/site-packages', - 'headers': '$base/Include/$dist_name', - 'scripts': '$base/Scripts', - 'data' : '$base', -} +if sys.version < "2.2": + WINDOWS_SCHEME = { + 'purelib': '$base', + 'platlib': '$base', + 'headers': '$base/Include/$dist_name', + 'scripts': '$base/Scripts', + 'data' : '$base', + } +else: + WINDOWS_SCHEME = { + 'purelib': '$base/Lib/site-packages', + 'platlib': '$base/Lib/site-packages', + 'headers': '$base/Include/$dist_name', + 'scripts': '$base/Scripts', + 'data' : '$base', + } INSTALL_SCHEMES = { 'unix_prefix': { @@ -43,6 +58,13 @@ 'data' : '$base', }, 'nt': WINDOWS_SCHEME, + 'os2': { + 'purelib': '$base/Lib/site-packages', + 'platlib': '$base/Lib/site-packages', + 'headers': '$base/Include/$dist_name', + 'scripts': '$base/Scripts', + 'data' : '$base', + }, } # user site schemes @@ -51,7 +73,7 @@ 'purelib': '$usersite', 'platlib': '$usersite', 'headers': '$userbase/Python$py_version_nodot/Include/$dist_name', - 'scripts': '$userbase/Python$py_version_nodot/Scripts', + 'scripts': '$userbase/Scripts', 'data' : '$userbase', } @@ -64,6 +86,14 @@ 'data' : '$userbase', } + INSTALL_SCHEMES['os2_home'] = { + 'purelib': '$usersite', + 'platlib': '$usersite', + 'headers': '$userbase/include/python$py_version_short/$dist_name', + 'scripts': '$userbase/bin', + 'data' : '$userbase', + } + # The keys to an installation scheme; if any new types of files are to be # installed, be sure to add an entry to every installation scheme above, # and to SCHEME_KEYS here. @@ -248,8 +278,8 @@ if self.user and (self.prefix or self.exec_prefix or self.home or self.install_base or self.install_platbase): - raise DistutilsOptionError("can't combine user with prefix, " - "exec_prefix/home, or install_(plat)base") + raise DistutilsOptionError("can't combine user with with prefix/" + "exec_prefix/home or install_(plat)base") # Next, stuff that's wrong (or dubious) only on certain platforms. if os.name != "posix": @@ -515,7 +545,7 @@ self.extra_dirs = extra_dirs def change_roots(self, *names): - """Change the install directories pointed by name using root.""" + """Change the install direcories pointed by name using root.""" for name in names: attr = "install_" + name setattr(self, attr, change_root(self.root, getattr(self, attr))) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/install_lib.py --- a/Lib/distutils/command/install_lib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/install_lib.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,7 @@ (install all Python modules).""" import os -import importlib.util +import imp import sys from distutils.core import Command @@ -22,15 +22,15 @@ # possible scenarios: # 1) no compilation at all (--no-compile --no-optimize) # 2) compile .pyc only (--compile --no-optimize; default) - # 3) compile .pyc and "opt-1" .pyc (--compile --optimize) - # 4) compile "opt-1" .pyc only (--no-compile --optimize) - # 5) compile .pyc and "opt-2" .pyc (--compile --optimize-more) - # 6) compile "opt-2" .pyc only (--no-compile --optimize-more) + # 3) compile .pyc and "level 1" .pyo (--compile --optimize) + # 4) compile "level 1" .pyo only (--no-compile --optimize) + # 5) compile .pyc and "level 2" .pyo (--compile --optimize-more) + # 6) compile "level 2" .pyo only (--no-compile --optimize-more) # - # The UI for this is two options, 'compile' and 'optimize'. + # The UI for this is two option, 'compile' and 'optimize'. # 'compile' is strictly boolean, and only decides whether to # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and - # decides both whether to generate .pyc files and what level of + # decides both whether to generate .pyo files and what level of # optimization to use. user_options = [ @@ -165,11 +165,11 @@ if ext != PYTHON_SOURCE_EXTENSION: continue if self.compile: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization='')) + bytecode_files.append(imp.cache_from_source( + py_file, debug_override=True)) if self.optimize > 0: - bytecode_files.append(importlib.util.cache_from_source( - py_file, optimization=self.optimize)) + bytecode_files.append(imp.cache_from_source( + py_file, debug_override=False)) return bytecode_files diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/register.py --- a/Lib/distutils/command/register.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/register.py Mon Jan 25 17:05:13 2016 +0100 @@ -87,7 +87,7 @@ ''' url = self.repository+'?:action=list_classifiers' response = urllib.request.urlopen(url) - log.info(self._read_pypi_response(response)) + log.info(response.read()) def verify_metadata(self): ''' Send the metadata to the package index server to be checked. @@ -300,5 +300,5 @@ result = 200, 'OK' if self.show_response: dashes = '-' * 75 - self.announce('%s%r%s' % (dashes, data, dashes)) + self.announce('%s%s%s' % (dashes, data, dashes)) return result diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/sdist.py --- a/Lib/distutils/command/sdist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/sdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -74,10 +74,6 @@ ('metadata-check', None, "Ensure that all required elements of meta-data " "are supplied. Warn if any missing. [default]"), - ('owner=', 'u', - "Owner name used when creating a tar file [default: current user]"), - ('group=', 'g', - "Group name used when creating a tar file [default: current group]"), ] boolean_options = ['use-defaults', 'prune', @@ -117,8 +113,6 @@ self.archive_files = None self.metadata_check = 1 - self.owner = None - self.group = None def finalize_options(self): if self.manifest is None: @@ -181,7 +175,7 @@ depends on the user's options. """ # new behavior when using a template: - # the file list is recalculated every time because + # the file list is recalculated everytime because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. @@ -450,8 +444,7 @@ self.formats.append(self.formats.pop(self.formats.index('tar'))) for fmt in self.formats: - file = self.make_archive(base_name, fmt, base_dir=base_dir, - owner=self.owner, group=self.group) + file = self.make_archive(base_name, fmt, base_dir=base_dir) archive_files.append(file) self.distribution.dist_files.append(('sdist', '', file)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/upload.py --- a/Lib/distutils/command/upload.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/command/upload.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,21 +1,25 @@ -""" -distutils.command.upload +"""distutils.command.upload -Implements the Distutils 'upload' subcommand (upload package to a package -index). -""" +Implements the Distutils 'upload' subcommand (upload package to PyPI).""" -import os -import io -import platform -import hashlib -from base64 import standard_b64encode -from urllib.request import urlopen, Request, HTTPError -from urllib.parse import urlparse -from distutils.errors import DistutilsError, DistutilsOptionError +from distutils.errors import * from distutils.core import PyPIRCCommand from distutils.spawn import spawn from distutils import log +import sys +import os, io +import socket +import platform +import configparser +import http.client as httpclient +from base64 import standard_b64encode +import urllib.parse + +# this keeps compatibility for 2.3 and 2.4 +if sys.version < "2.5": + from md5 import md5 +else: + from hashlib import md5 class upload(PyPIRCCommand): @@ -57,21 +61,11 @@ def run(self): if not self.distribution.dist_files: - msg = "No dist file created in earlier command" - raise DistutilsOptionError(msg) + raise DistutilsOptionError("No dist file created in earlier command") for command, pyversion, filename in self.distribution.dist_files: self.upload_file(command, pyversion, filename) def upload_file(self, command, pyversion, filename): - # Makes sure the repository URL is compliant - schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) - if params or query or fragments: - raise AssertionError("Incompatible url %s" % self.repository) - - if schema not in ('http', 'https'): - raise AssertionError("unsupported schema " + schema) - # Sign if requested if self.sign: gpg_args = ["gpg", "--detach-sign", "-a", filename] @@ -101,10 +95,10 @@ 'content': (os.path.basename(filename),content), 'filetype': command, 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), + 'md5_digest': md5(content).hexdigest(), # additional meta-data - 'metadata_version': '1.0', + 'metadata_version' : '1.0', 'summary': meta.get_description(), 'home_page': meta.get_url(), 'author': meta.get_contact(), @@ -131,7 +125,7 @@ if self.sign: data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc", "rb").read()) + open(filename+".asc").read()) # set up the authentication user_pass = (self.username + ":" + self.password).encode('ascii') @@ -141,13 +135,13 @@ # Build up the MIME payload for the POST data boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = b'\r\n--' + boundary.encode('ascii') - end_boundary = sep_boundary + b'--\r\n' + sep_boundary = b'\n--' + boundary.encode('ascii') + end_boundary = sep_boundary + b'--' body = io.BytesIO() for key, value in data.items(): - title = '\r\nContent-Disposition: form-data; name="%s"' % key + title = '\nContent-Disposition: form-data; name="%s"' % key # handle multiple entries for the same name - if not isinstance(value, list): + if type(value) != type([]): value = [value] for value in value: if type(value) is tuple: @@ -157,45 +151,52 @@ value = str(value).encode('utf-8') body.write(sep_boundary) body.write(title.encode('utf-8')) - body.write(b"\r\n\r\n") + body.write(b"\n\n") body.write(value) if value and value[-1:] == b'\r': body.write(b'\n') # write an extra newline (lurve Macs) body.write(end_boundary) + body.write(b"\n") body = body.getvalue() - msg = "Submitting %s to %s" % (filename, self.repository) - self.announce(msg, log.INFO) + self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO) # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s' % boundary, - 'Content-length': str(len(body)), - 'Authorization': auth, - } + # We can't use urllib since we need to send the Basic + # auth right with the first request + # TODO(jhylton): Can we fix urllib? + schema, netloc, url, params, query, fragments = \ + urllib.parse.urlparse(self.repository) + assert not params and not query and not fragments + if schema == 'http': + http = httpclient.HTTPConnection(netloc) + elif schema == 'https': + http = httpclient.HTTPSConnection(netloc) + else: + raise AssertionError("unsupported schema "+schema) - request = Request(self.repository, data=body, - headers=headers) - # send the data + data = '' + loglevel = log.INFO try: - result = urlopen(request) - status = result.getcode() - reason = result.msg - except OSError as e: + http.connect() + http.putrequest("POST", url) + http.putheader('Content-type', + 'multipart/form-data; boundary=%s'%boundary) + http.putheader('Content-length', str(len(body))) + http.putheader('Authorization', auth) + http.endheaders() + http.send(body) + except socket.error as e: self.announce(str(e), log.ERROR) - raise - except HTTPError as e: - status = e.code - reason = e.msg + return - if status == 200: - self.announce('Server response (%s): %s' % (status, reason), + r = http.getresponse() + if r.status == 200: + self.announce('Server response (%s): %s' % (r.status, r.reason), log.INFO) else: - msg = 'Upload failed (%s): %s' % (status, reason) - self.announce(msg, log.ERROR) - raise DistutilsError(msg) + self.announce('Upload failed (%s): %s' % (r.status, r.reason), + log.ERROR) if self.show_response: - text = self._read_pypi_response(result) - msg = '\n'.join(('-' * 75, text, '-' * 75)) + msg = '\n'.join(('-' * 75, r.read(), '-' * 75)) self.announce(msg, log.INFO) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/wininst-10.0-amd64.exe Binary file Lib/distutils/command/wininst-10.0-amd64.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/wininst-10.0.exe Binary file Lib/distutils/command/wininst-10.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/wininst-14.0-amd64.exe Binary file Lib/distutils/command/wininst-14.0-amd64.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/command/wininst-14.0.exe Binary file Lib/distutils/command/wininst-14.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/config.py --- a/Lib/distutils/config.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/config.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,7 @@ that uses .pypirc in the distutils.command package. """ import os +import sys from configparser import ConfigParser from distutils.cmd import Command @@ -21,7 +22,7 @@ class PyPIRCCommand(Command): """Base command that knows how to handle the .pypirc file """ - DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi' + DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi' DEFAULT_REALM = 'pypi' repository = None realm = None @@ -42,8 +43,16 @@ def _store_pypirc(self, username, password): """Creates a default .pypirc file.""" rc = self._get_rc_file() - with os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + f = open(rc, 'w') + try: f.write(DEFAULT_PYPIRC % (username, password)) + finally: + f.close() + try: + os.chmod(rc, 0o600) + except OSError: + # should do something better here + pass def _read_pypirc(self): """Reads the .pypirc file.""" @@ -83,15 +92,6 @@ current[key] = config.get(server, key) else: current[key] = default - - # work around people having "repository" for the "pypi" - # section of their config set to the HTTP (rather than - # HTTPS) URL - if (server == 'pypi' and - repository in (self.DEFAULT_REPOSITORY, 'pypi')): - current['repository'] = self.DEFAULT_REPOSITORY - return current - if (current['server'] == repository or current['repository'] == repository): return current @@ -110,13 +110,6 @@ return {} - def _read_pypi_response(self, response): - """Read and decode a PyPI HTTP response.""" - import cgi - content_type = response.getheader('content-type', 'text/plain') - encoding = cgi.parse_header(content_type)[1].get('charset', 'ascii') - return response.read().decode(encoding) - def initialize_options(self): """Initialize options.""" self.repository = None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/core.py --- a/Lib/distutils/core.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/core.py Mon Jan 25 17:05:13 2016 +0100 @@ -11,6 +11,7 @@ from distutils.debug import DEBUG from distutils.errors import * +from distutils.util import grok_environment_error # Mainly import these so setup scripts can "from distutils.core import" them. from distutils.dist import Distribution @@ -127,9 +128,8 @@ if _setup_stop_after == "config": return dist - # Parse the command line and override config files; any - # command-line errors are the end user's fault, so turn them into - # SystemExit to suppress tracebacks. + # Parse the command line; any command-line errors are the end user's + # fault, so turn them into SystemExit to suppress tracebacks. try: ok = dist.parse_command_line() except DistutilsArgError as msg: @@ -148,12 +148,14 @@ dist.run_commands() except KeyboardInterrupt: raise SystemExit("interrupted") - except OSError as exc: + except (IOError, os.error) as exc: + error = grok_environment_error(exc) + if DEBUG: - sys.stderr.write("error: %s\n" % (exc,)) + sys.stderr.write(error + "\n") raise else: - raise SystemExit("error: %s" % (exc,)) + raise SystemExit(error) except (DistutilsError, CCompilerError) as msg: @@ -204,15 +206,16 @@ global _setup_stop_after, _setup_distribution _setup_stop_after = stop_after - save_argv = sys.argv.copy() + save_argv = sys.argv g = {'__file__': script_name} + l = {} try: try: sys.argv[0] = script_name if script_args is not None: sys.argv[1:] = script_args with open(script_name, 'rb') as f: - exec(f.read(), g) + exec(f.read(), g, l) finally: sys.argv = save_argv _setup_stop_after = None @@ -220,6 +223,8 @@ # Hmm, should we do something if exiting with a non-zero code # (ie. error)? pass + except: + raise if _setup_distribution is None: raise RuntimeError(("'distutils.core.setup()' was never called -- " diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/cygwinccompiler.py --- a/Lib/distutils/cygwinccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/cygwinccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,9 +10,9 @@ # # * if you use a msvc compiled python version (1.5.2) # 1. you have to insert a __GNUC__ section in its config.h -# 2. you have to generate an import library for its dll +# 2. you have to generate a import library for its dll # - create a def-file for python??.dll -# - create an import library using +# - create a import library using # dlltool --dllname python15.dll --def python15.def \ # --output-lib libpython15.a # @@ -48,14 +48,13 @@ import os import sys import copy -from subprocess import Popen, PIPE, check_output +from subprocess import Popen, PIPE import re from distutils.ccompiler import gen_preprocess_options, gen_lib_options from distutils.unixccompiler import UnixCCompiler from distutils.file_util import write_file -from distutils.errors import (DistutilsExecError, CCompilerError, - CompileError, UnknownFileError) +from distutils.errors import DistutilsExecError, CompileError, UnknownFileError from distutils import log from distutils.version import LooseVersion from distutils.spawn import find_executable @@ -79,9 +78,6 @@ elif msc_ver == '1500': # VS2008 / MSVC 9.0 return ['msvcr90'] - elif msc_ver == '1600': - # VS2010 / MSVC 10.0 - return ['msvcr100'] else: raise ValueError("Unknown MS Compiler version %s " % msc_ver) @@ -295,15 +291,11 @@ else: entry_point = '' - if is_cygwingcc(): - raise CCompilerError( - 'Cygwin gcc cannot be used with --compiler=mingw32') - - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', - linker_so='%s %s %s' + self.set_executables(compiler='gcc -mno-cygwin -O -Wall', + compiler_so='gcc -mno-cygwin -mdll -O -Wall', + compiler_cxx='g++ -mno-cygwin -O -Wall', + linker_exe='gcc -mno-cygwin', + linker_so='%s -mno-cygwin %s %s' % (self.linker_dll, shared_option, entry_point)) # Maybe we should also append -mthreads, but then the finished @@ -318,7 +310,7 @@ self.dll_libraries = get_msvcr() # Because these compilers aren't configured in Python's pyconfig.h file by -# default, we should at least warn the user if he is using an unmodified +# default, we should at least warn the user if he is using a unmodified # version. CONFIG_H_OK = "ok" @@ -364,7 +356,7 @@ return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn finally: config_h.close() - except OSError as exc: + except IOError as exc: return (CONFIG_H_UNCERTAIN, "couldn't read '%s': %s" % (fn, exc.strerror)) @@ -398,8 +390,3 @@ """ commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] return tuple([_find_exe_version(cmd) for cmd in commands]) - -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) - return out_string.strip().endswith(b'cygwin') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/dir_util.py --- a/Lib/distutils/dir_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/dir_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,7 +2,7 @@ Utility functions for manipulating directories and directory trees.""" -import os +import os, sys import errno from distutils.errors import DistutilsFileError, DistutilsInternalError from distutils import log @@ -81,7 +81,7 @@ """Create all the empty directories under 'base_dir' needed to put 'files' there. - 'base_dir' is just the name of a directory which doesn't necessarily + 'base_dir' is just the a name of a directory which doesn't necessarily exist yet; 'files' is a list of filenames to be interpreted relative to 'base_dir'. 'base_dir' + the directory portion of every file in 'files' will be created if it doesn't already exist. 'mode', 'verbose' and @@ -124,12 +124,13 @@ "cannot copy tree '%s': not a directory" % src) try: names = os.listdir(src) - except OSError as e: + except os.error as e: + (errno, errstr) = e if dry_run: names = [] else: raise DistutilsFileError( - "error listing files in '%s': %s" % (src, e.strerror)) + "error listing files in '%s': %s" % (src, errstr)) if not dry_run: mkpath(dst, verbose=verbose) @@ -140,10 +141,6 @@ src_name = os.path.join(src, n) dst_name = os.path.join(dst, n) - if n.startswith('.nfs'): - # skip NFS rename files - continue - if preserve_symlinks and os.path.islink(src_name): link_dest = os.readlink(src_name) if verbose >= 1: @@ -181,6 +178,7 @@ Any errors are ignored (apart from being reported to stdout if 'verbose' is true). """ + from distutils.util import grok_environment_error global _path_created if verbose >= 1: @@ -196,8 +194,9 @@ abspath = os.path.abspath(cmd[1]) if abspath in _path_created: del _path_created[abspath] - except OSError as exc: - log.warn("error removing %s: %s", directory, exc) + except (IOError, OSError) as exc: + log.warn(grok_environment_error( + exc, "error removing %s: " % directory)) def ensure_relative(path): """Take the full path 'path', and make it a relative path. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/dist.py --- a/Lib/distutils/dist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,7 @@ being built/installed/distributed. """ -import sys -import os -import re -from email import message_from_file +import sys, os, re try: import warnings @@ -24,7 +21,7 @@ # the same as a Python NAME -- I don't allow leading underscores. The fact # that they're very similar is no coincidence; the default naming scheme is # to look for a Python module named after the command. -command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') +command_re = re.compile (r'^[a-zA-Z]([a-zA-Z0-9_]*)$') class Distribution: @@ -41,6 +38,7 @@ See the code for 'setup()', in core.py, for details. """ + # 'global_options' describes the command-line options that may be # supplied to the setup script prior to any actual commands. # Eg. "./setup.py -n" or "./setup.py --quiet" both take advantage of @@ -49,14 +47,11 @@ # don't want to pollute the commands with too many options that they # have minimal control over. # The fourth entry for verbose means that it can be repeated. - global_options = [ - ('verbose', 'v', "run verbosely (default)", 1), - ('quiet', 'q', "run quietly (turns verbosity off)"), - ('dry-run', 'n', "don't actually do anything"), - ('help', 'h', "show detailed help message"), - ('no-user-cfg', None, - 'ignore pydistutils.cfg in your home directory'), - ] + global_options = [('verbose', 'v', "run verbosely (default)", 1), + ('quiet', 'q', "run quietly (turns verbosity off)"), + ('dry-run', 'n', "don't actually do anything"), + ('help', 'h', "show detailed help message"), + ] # 'common_usage' is a short (2-3 line) string describing the common # usage of the setup script. @@ -117,9 +112,10 @@ # negative options are options that exclude other options negative_opt = {'quiet': 'verbose'} + # -- Creation/initialization methods ------------------------------- - def __init__(self, attrs=None): + def __init__ (self, attrs=None): """Construct a new Distribution instance: initialize all the attributes of a Distribution, and then use 'attrs' (a dictionary mapping attribute names to values) to assign some of those @@ -262,22 +258,6 @@ else: sys.stderr.write(msg + "\n") - # no-user-cfg is handled before other command line args - # because other args override the config files, and this - # one is needed before we can load the config files. - # If attrs['script_args'] wasn't passed, assume false. - # - # This also make sure we just look at the global options - self.want_user_cfg = True - - if self.script_args is not None: - for arg in self.script_args: - if not arg.startswith('-'): - break - if arg == '--no-user-cfg': - self.want_user_cfg = False - break - self.finalize_options() def get_option_dict(self, command): @@ -329,10 +309,7 @@ Distutils installation directory (ie. where the top-level Distutils __inst__.py file lives), a file in the user's home directory named .pydistutils.cfg on Unix and pydistutils.cfg - on Windows/Mac; and setup.cfg in the current directory. - - The file in the user's home directory can be disabled with the - --no-user-cfg option. + on Windows/Mac, and setup.cfg in the current directory. """ files = [] check_environ() @@ -352,36 +329,20 @@ user_filename = "pydistutils.cfg" # And look for the user config file - if self.want_user_cfg: - user_file = os.path.join(os.path.expanduser('~'), user_filename) - if os.path.isfile(user_file): - files.append(user_file) + user_file = os.path.join(os.path.expanduser('~'), user_filename) + if os.path.isfile(user_file): + files.append(user_file) # All platforms support local setup.cfg local_file = "setup.cfg" if os.path.isfile(local_file): files.append(local_file) - if DEBUG: - self.announce("using config files: %s" % ', '.join(files)) - return files def parse_config_files(self, filenames=None): from configparser import ConfigParser - # Ignore install directory options if we have a venv - if sys.prefix != sys.base_prefix: - ignore_options = [ - 'install-base', 'install-platbase', 'install-lib', - 'install-platlib', 'install-purelib', 'install-headers', - 'install-scripts', 'install-data', 'prefix', 'exec-prefix', - 'home', 'user', 'root'] - else: - ignore_options = [] - - ignore_options = frozenset(ignore_options) - if filenames is None: filenames = self.find_config_files() @@ -398,7 +359,7 @@ opt_dict = self.get_option_dict(section) for opt in options: - if opt != '__name__' and opt not in ignore_options: + if opt != '__name__': val = parser.get(section,opt) opt = opt.replace('-', '_') opt_dict[opt] = (filename, val) @@ -533,15 +494,15 @@ # to be sure that the basic "command" interface is implemented. if not issubclass(cmd_class, Command): raise DistutilsClassError( - "command class %s must subclass Command" % cmd_class) + "command class %s must subclass Command" % cmd_class) # Also make sure that the command object provides a list of its # known options. if not (hasattr(cmd_class, 'user_options') and isinstance(cmd_class.user_options, list)): - msg = ("command class %s must provide " - "'user_options' attribute (a list of tuples)") - raise DistutilsClassError(msg % cmd_class) + raise DistutilsClassError(("command class %s must provide " + + "'user_options' attribute (a list of tuples)") % \ + cmd_class) # If the command class has a list of negative alias options, # merge it in with the global negative aliases. @@ -553,11 +514,12 @@ # Check for help_options in command class. They have a different # format (tuple of four) so we need to preprocess them here. if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): + isinstance(cmd_class.help_options, list)): help_options = fix_help_options(cmd_class.help_options) else: help_options = [] + # All commands support the global options too, just by adding # in 'global_options'. parser.set_option_table(self.global_options + @@ -570,7 +532,7 @@ return if (hasattr(cmd_class, 'help_options') and - isinstance(cmd_class.help_options, list)): + isinstance(cmd_class.help_options, list)): help_option_found=0 for (help_option, short, desc, func) in cmd_class.help_options: if hasattr(opts, parser.get_attr_name(help_option)): @@ -647,7 +609,7 @@ else: klass = self.get_command_class(command) if (hasattr(klass, 'help_options') and - isinstance(klass.help_options, list)): + isinstance(klass.help_options, list)): parser.set_option_table(klass.user_options + fix_help_options(klass.help_options)) else: @@ -814,7 +776,7 @@ klass_name = command try: - __import__(module_name) + __import__ (module_name) module = sys.modules[module_name] except ImportError: continue @@ -823,8 +785,8 @@ klass = getattr(module, klass_name) except AttributeError: raise DistutilsModuleError( - "invalid command '%s' (no class '%s' in module '%s')" - % (command, klass_name, module_name)) + "invalid command '%s' (no class '%s' in module '%s')" + % (command, klass_name, module_name)) self.cmdclass[command] = klass return klass @@ -840,7 +802,7 @@ cmd_obj = self.command_obj.get(command) if not cmd_obj and create: if DEBUG: - self.announce("Distribution.get_command_obj(): " + self.announce("Distribution.get_command_obj(): " \ "creating '%s' command object" % command) klass = self.get_command_class(command) @@ -897,8 +859,8 @@ setattr(command_obj, option, value) else: raise DistutilsOptionError( - "error in %s: command '%s' has no such option '%s'" - % (source, command_name, option)) + "error in %s: command '%s' has no such option '%s'" + % (source, command_name, option)) except ValueError as msg: raise DistutilsOptionError(msg) @@ -974,6 +936,7 @@ cmd_obj.run() self.have_run[command] = 1 + # -- Distribution query methods ------------------------------------ def has_pure_modules(self): @@ -1024,80 +987,25 @@ "provides", "requires", "obsoletes", ) - def __init__(self, path=None): - if path is not None: - self.read_pkg_file(open(path)) - else: - self.name = None - self.version = None - self.author = None - self.author_email = None - self.maintainer = None - self.maintainer_email = None - self.url = None - self.license = None - self.description = None - self.long_description = None - self.keywords = None - self.platforms = None - self.classifiers = None - self.download_url = None - # PEP 314 - self.provides = None - self.requires = None - self.obsoletes = None - - def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) - - def _read_field(name): - value = msg[name] - if value == 'UNKNOWN': - return None - return value - - def _read_list(name): - values = msg.get_all(name, None) - if values == []: - return None - return values - - metadata_version = msg['metadata-version'] - self.name = _read_field('name') - self.version = _read_field('version') - self.description = _read_field('summary') - # we are filling author only. - self.author = _read_field('author') + def __init__ (self): + self.name = None + self.version = None + self.author = None + self.author_email = None self.maintainer = None - self.author_email = _read_field('author-email') self.maintainer_email = None - self.url = _read_field('home-page') - self.license = _read_field('license') - - if 'download-url' in msg: - self.download_url = _read_field('download-url') - else: - self.download_url = None - - self.long_description = _read_field('description') - self.description = _read_field('summary') - - if 'keywords' in msg: - self.keywords = _read_field('keywords').split(',') - - self.platforms = _read_list('platform') - self.classifiers = _read_list('classifier') - - # PEP 314 - these fields only exist in 1.1 - if metadata_version == '1.1': - self.requires = _read_list('requires') - self.provides = _read_list('provides') - self.obsoletes = _read_list('obsoletes') - else: - self.requires = None - self.provides = None - self.obsoletes = None + self.url = None + self.license = None + self.description = None + self.long_description = None + self.keywords = None + self.platforms = None + self.classifiers = None + self.download_url = None + # PEP 314 + self.provides = None + self.requires = None + self.obsoletes = None def write_pkg_info(self, base_dir): """Write the PKG-INFO file into the release tree. @@ -1111,17 +1019,17 @@ """ version = '1.0' if (self.provides or self.requires or self.obsoletes or - self.classifiers or self.download_url): + self.classifiers or self.download_url): version = '1.1' file.write('Metadata-Version: %s\n' % version) - file.write('Name: %s\n' % self.get_name()) - file.write('Version: %s\n' % self.get_version()) - file.write('Summary: %s\n' % self.get_description()) - file.write('Home-page: %s\n' % self.get_url()) - file.write('Author: %s\n' % self.get_contact()) - file.write('Author-email: %s\n' % self.get_contact_email()) - file.write('License: %s\n' % self.get_license()) + file.write('Name: %s\n' % self.get_name() ) + file.write('Version: %s\n' % self.get_version() ) + file.write('Summary: %s\n' % self.get_description() ) + file.write('Home-page: %s\n' % self.get_url() ) + file.write('Author: %s\n' % self.get_contact() ) + file.write('Author-email: %s\n' % self.get_contact_email() ) + file.write('License: %s\n' % self.get_license() ) if self.download_url: file.write('Download-URL: %s\n' % self.download_url) @@ -1130,7 +1038,7 @@ keywords = ','.join(self.get_keywords()) if keywords: - file.write('Keywords: %s\n' % keywords) + file.write('Keywords: %s\n' % keywords ) self._write_list(file, 'Platform', self.get_platforms()) self._write_list(file, 'Classifier', self.get_classifiers()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/emxccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/distutils/emxccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,315 @@ +"""distutils.emxccompiler + +Provides the EMXCCompiler class, a subclass of UnixCCompiler that +handles the EMX port of the GNU C compiler to OS/2. +""" + +# issues: +# +# * OS/2 insists that DLLs can have names no longer than 8 characters +# We put export_symbols in a def-file, as though the DLL can have +# an arbitrary length name, but truncate the output filename. +# +# * only use OMF objects and use LINK386 as the linker (-Zomf) +# +# * always build for multithreading (-Zmt) as the accompanying OS/2 port +# of Python is only distributed with threads enabled. +# +# tested configurations: +# +# * EMX gcc 2.81/EMX 0.9d fix03 + +import os,sys,copy +from distutils.ccompiler import gen_preprocess_options, gen_lib_options +from distutils.unixccompiler import UnixCCompiler +from distutils.file_util import write_file +from distutils.errors import DistutilsExecError, CompileError, UnknownFileError +from distutils import log + +class EMXCCompiler (UnixCCompiler): + + compiler_type = 'emx' + obj_extension = ".obj" + static_lib_extension = ".lib" + shared_lib_extension = ".dll" + static_lib_format = "%s%s" + shared_lib_format = "%s%s" + res_extension = ".res" # compiled resource file + exe_extension = ".exe" + + def __init__ (self, + verbose=0, + dry_run=0, + force=0): + + UnixCCompiler.__init__ (self, verbose, dry_run, force) + + (status, details) = check_config_h() + self.debug_print("Python's GCC status: %s (details: %s)" % + (status, details)) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + + ("Reason: %s." % details) + + "Compiling may fail because of undefined preprocessor macros.") + + (self.gcc_version, self.ld_version) = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s\n" % + (self.gcc_version, + self.ld_version) ) + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', + compiler_so='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall', + linker_exe='gcc -Zomf -Zmt -Zcrtdll', + linker_so='gcc -Zomf -Zmt -Zcrtdll -Zdll') + + # want the gcc library statically linked (so that we don't have + # to distribute a version dependent on the compiler we have) + self.dll_libraries=["gcc"] + + # __init__ () + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + if ext == '.rc': + # gcc requires '.rc' compiled to binary ('.res') files !!! + try: + self.spawn(["rc", "-r", src]) + except DistutilsExecError as msg: + raise CompileError(msg) + else: # for other files use the C-compiler + try: + self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) + + def link (self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + # use separate copies, so we can modify the lists + extra_preargs = copy.copy(extra_preargs or []) + libraries = copy.copy(libraries or []) + objects = copy.copy(objects or []) + + # Additional libraries + libraries.extend(self.dll_libraries) + + # handle export symbols by creating a def-file + # with executables this only works with gcc/ld as linker + if ((export_symbols is not None) and + (target_desc != self.EXECUTABLE)): + # (The linker doesn't do anything if output is up-to-date. + # So it would probably better to check if we really need this, + # but for this we had to insert some unchanged parts of + # UnixCCompiler, and this is not what we want.) + + # we want to put some files in the same directory as the + # object files are, build_temp doesn't help much + # where are the object files + temp_dir = os.path.dirname(objects[0]) + # name of dll to give the helper files the same base name + (dll_name, dll_extension) = os.path.splitext( + os.path.basename(output_filename)) + + # generate the filenames for these files + def_file = os.path.join(temp_dir, dll_name + ".def") + + # Generate .def file + contents = [ + "LIBRARY %s INITINSTANCE TERMINSTANCE" % \ + os.path.splitext(os.path.basename(output_filename))[0], + "DATA MULTIPLE NONSHARED", + "EXPORTS"] + for sym in export_symbols: + contents.append(' "%s"' % sym) + self.execute(write_file, (def_file, contents), + "writing %s" % def_file) + + # next add options for def-file and to creating import libraries + # for gcc/ld the def-file is specified as any other object files + objects.append(def_file) + + #end: if ((export_symbols is not None) and + # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): + + # who wants symbols and a many times larger output file + # should explicitly switch the debug mode on + # otherwise we let dllwrap/ld strip the output file + # (On my machine: 10KB < stripped_file < ??100KB + # unstripped_file = stripped_file + XXX KB + # ( XXX=254 for a typical python extension)) + if not debug: + extra_preargs.append("-s") + + UnixCCompiler.link(self, + target_desc, + objects, + output_filename, + output_dir, + libraries, + library_dirs, + runtime_library_dirs, + None, # export_symbols, we do this in our def-file + debug, + extra_preargs, + extra_postargs, + build_temp, + target_lang) + + # link () + + # -- Miscellaneous methods ----------------------------------------- + + # override the object_filenames method from CCompiler to + # support rc and res-files + def object_filenames (self, + source_filenames, + strip_dir=0, + output_dir=''): + if output_dir is None: output_dir = '' + obj_names = [] + for src_name in source_filenames: + # use normcase to make sure '.rc' is really '.rc' and not '.RC' + (base, ext) = os.path.splitext (os.path.normcase(src_name)) + if ext not in (self.src_extensions + ['.rc']): + raise UnknownFileError("unknown file type '%s' (from '%s')" % \ + (ext, src_name)) + if strip_dir: + base = os.path.basename (base) + if ext == '.rc': + # these need to be compiled to object files + obj_names.append (os.path.join (output_dir, + base + self.res_extension)) + else: + obj_names.append (os.path.join (output_dir, + base + self.obj_extension)) + return obj_names + + # object_filenames () + + # override the find_library_file method from UnixCCompiler + # to deal with file naming/searching differences + def find_library_file(self, dirs, lib, debug=0): + shortlib = '%s.lib' % lib + longlib = 'lib%s.lib' % lib # this form very rare + + # get EMX's default library directory search path + try: + emx_dirs = os.environ['LIBRARY_PATH'].split(';') + except KeyError: + emx_dirs = [] + + for dir in dirs + emx_dirs: + shortlibp = os.path.join(dir, shortlib) + longlibp = os.path.join(dir, longlib) + if os.path.exists(shortlibp): + return shortlibp + elif os.path.exists(longlibp): + return longlibp + + # Oops, didn't find it in *any* of 'dirs' + return None + +# class EMXCCompiler + + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using a unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + + """Check if the current Python installation (specifically, pyconfig.h) + appears amenable to building extensions with GCC. Returns a tuple + (status, details), where 'status' is one of the following constants: + CONFIG_H_OK + all is well, go ahead and compile + CONFIG_H_NOTOK + doesn't look good + CONFIG_H_UNCERTAIN + not sure -- unable to read pyconfig.h + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + + from distutils import sysconfig + # if sys.version contains GCC then python was compiled with + # GCC, and the pyconfig.h file should be OK + if sys.version.find("GCC") >= 0: + return (CONFIG_H_OK, "sys.version mentions 'GCC'") + + fn = sysconfig.get_config_h_filename() + try: + # It would probably better to read single lines to search. + # But we do this only once, and it is fast enough + f = open(fn) + try: + s = f.read() + finally: + f.close() + + except IOError as exc: + # if we can't read this file, we cannot say it is wrong + # the compiler will complain later about this file as missing + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) + + else: + # "pyconfig.h" contains an "#ifdef __GNUC__" or something similar + if s.find("__GNUC__") >= 0: + return (CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn) + else: + return (CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn) + + +def get_versions(): + """ Try to find out the versions of gcc and ld. + If not possible it returns None for it. + """ + from distutils.version import StrictVersion + from distutils.spawn import find_executable + import re + + gcc_exe = find_executable('gcc') + if gcc_exe: + out = os.popen(gcc_exe + ' -dumpversion','r') + try: + out_string = out.read() + finally: + out.close() + result = re.search('(\d+\.\d+\.\d+)', out_string, re.ASCII) + if result: + gcc_version = StrictVersion(result.group(1)) + else: + gcc_version = None + else: + gcc_version = None + # EMX ld has no way of reporting version number, and we use GCC + # anyway - so we can link OMF DLLs + ld_version = None + return (gcc_version, ld_version) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/errors.py --- a/Lib/distutils/errors.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/errors.py Mon Jan 25 17:05:13 2016 +0100 @@ -35,8 +35,8 @@ class DistutilsFileError (DistutilsError): """Any problems in the filesystem: expected file not found, etc. - Typically this is for problems that we detect before OSError - could be raised.""" + Typically this is for problems that we detect before IOError or + OSError could be raised.""" pass class DistutilsOptionError (DistutilsError): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/extension.py --- a/Lib/distutils/extension.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/extension.py Mon Jan 25 17:05:13 2016 +0100 @@ -131,14 +131,6 @@ msg = "Unknown Extension options: %s" % options warnings.warn(msg) - def __repr__(self): - return '<%s.%s(%r) at %#x>' % ( - self.__class__.__module__, - self.__class__.__qualname__, - self.name, - id(self)) - - def read_setup_file(filename): """Reads a Setup file and returns Extension instances.""" from distutils.sysconfig import (parse_makefile, expand_makefile_vars, diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/file_util.py --- a/Lib/distutils/file_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/file_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -27,26 +27,26 @@ try: try: fsrc = open(src, 'rb') - except OSError as e: + except os.error as e: raise DistutilsFileError("could not open '%s': %s" % (src, e.strerror)) if os.path.exists(dst): try: os.unlink(dst) - except OSError as e: + except os.error as e: raise DistutilsFileError( "could not delete '%s': %s" % (dst, e.strerror)) try: fdst = open(dst, 'wb') - except OSError as e: + except os.error as e: raise DistutilsFileError( "could not create '%s': %s" % (dst, e.strerror)) while True: try: buf = fsrc.read(buffer_size) - except OSError as e: + except os.error as e: raise DistutilsFileError( "could not read from '%s': %s" % (src, e.strerror)) @@ -55,7 +55,7 @@ try: fdst.write(buf) - except OSError as e: + except os.error as e: raise DistutilsFileError( "could not write to '%s': %s" % (dst, e.strerror)) finally: @@ -80,8 +80,7 @@ (os.symlink) instead of copying: set it to "hard" or "sym"; if it is None (the default), files are copied. Don't set 'link' on systems that don't support it: 'copy_file()' doesn't check if hard or symbolic - linking is available. If hardlink fails, falls back to - _copy_file_contents(). + linking is available. Under Mac OS, uses the native file copy function in macostools; on other systems, uses '_copy_file_contents()' to copy file contents. @@ -133,31 +132,24 @@ # (Unix only, of course, but that's the caller's responsibility) elif link == 'hard': if not (os.path.exists(dst) and os.path.samefile(src, dst)): - try: - os.link(src, dst) - return (dst, 1) - except OSError: - # If hard linking fails, fall back on copying file - # (some special filesystems don't support hard linking - # even under Unix, see issue #8876). - pass + os.link(src, dst) elif link == 'sym': if not (os.path.exists(dst) and os.path.samefile(src, dst)): os.symlink(src, dst) - return (dst, 1) # Otherwise (non-Mac, not linking), copy the file contents and # (optionally) copy the times and mode. - _copy_file_contents(src, dst) - if preserve_mode or preserve_times: - st = os.stat(src) + else: + _copy_file_contents(src, dst) + if preserve_mode or preserve_times: + st = os.stat(src) - # According to David Ascher , utime() should be done - # before chmod() (at least under NT). - if preserve_times: - os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) - if preserve_mode: - os.chmod(dst, S_IMODE(st[ST_MODE])) + # According to David Ascher , utime() should be done + # before chmod() (at least under NT). + if preserve_times: + os.utime(dst, (st[ST_ATIME], st[ST_MTIME])) + if preserve_mode: + os.chmod(dst, S_IMODE(st[ST_MODE])) return (dst, 1) @@ -201,8 +193,8 @@ copy_it = False try: os.rename(src, dst) - except OSError as e: - (num, msg) = e.args + except os.error as e: + (num, msg) = e if num == errno.EXDEV: copy_it = True else: @@ -213,11 +205,11 @@ copy_file(src, dst, verbose=verbose) try: os.unlink(src) - except OSError as e: - (num, msg) = e.args + except os.error as e: + (num, msg) = e try: os.unlink(dst) - except OSError: + except os.error: pass raise DistutilsFileError( "couldn't move '%s' to '%s' by copy/delete: " diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/msvc9compiler.py --- a/Lib/distutils/msvc9compiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/msvc9compiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -179,9 +179,6 @@ i = i + len(prefix) s, rest = sys.version[i:].split(" ", 1) majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 minorVersion = int(s[2:3]) / 10.0 # I don't think paths are affected by minor version in version 6 if majorVersion == 6: @@ -416,7 +413,7 @@ self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] if self.__version >= 7: self.ldflags_shared_debug = [ - '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' + '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG', '/pdb:None' ] self.ldflags_static = [ '/nologo'] @@ -732,7 +729,7 @@ return manifest_file finally: manifest_f.close() - except OSError: + except IOError: pass # -- Miscellaneous methods ----------------------------------------- diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/msvccompiler.py --- a/Lib/distutils/msvccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/msvccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -157,9 +157,6 @@ i = i + len(prefix) s, rest = sys.version[i:].split(" ", 1) majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - # v13 was skipped and should be v14 - majorVersion += 1 minorVersion = int(s[2:3]) / 10.0 # I don't think paths are affected by minor version in version 6 if majorVersion == 6: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/spawn.py --- a/Lib/distutils/spawn.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/spawn.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,7 +10,6 @@ import os from distutils.errors import DistutilsPlatformError, DistutilsExecError -from distutils.debug import DEBUG from distutils import log def spawn(cmd, search_path=1, verbose=0, dry_run=0): @@ -29,13 +28,12 @@ Raise DistutilsExecError if running the program fails in any way; just return on success. """ - # cmd is documented as a list, but just in case some code passes a tuple - # in, protect our %-formatting code against horrible death - cmd = list(cmd) if os.name == 'posix': _spawn_posix(cmd, search_path, dry_run=dry_run) elif os.name == 'nt': _spawn_nt(cmd, search_path, dry_run=dry_run) + elif os.name == 'os2': + _spawn_os2(cmd, search_path, dry_run=dry_run) else: raise DistutilsPlatformError( "don't know how to spawn programs on platform '%s'" % os.name) @@ -69,16 +67,32 @@ rc = os.spawnv(os.P_WAIT, executable, cmd) except OSError as exc: # this seems to happen when the command isn't found - if not DEBUG: - cmd = executable raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) + "command '%s' failed: %s" % (cmd[0], exc.args[-1])) if rc != 0: # and this reflects the command running but failing - if not DEBUG: - cmd = executable raise DistutilsExecError( - "command %r failed with exit status %d" % (cmd, rc)) + "command '%s' failed with exit status %d" % (cmd[0], rc)) + +def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0): + executable = cmd[0] + if search_path: + # either we find one or it stays the same + executable = find_executable(executable) or executable + log.info(' '.join([executable] + cmd[1:])) + if not dry_run: + # spawnv for OS/2 EMX requires a full path to the .exe + try: + rc = os.spawnv(os.P_WAIT, executable, cmd) + except OSError as exc: + # this seems to happen when the command isn't found + raise DistutilsExecError( + "command '%s' failed: %s" % (cmd[0], exc.args[-1])) + if rc != 0: + # and this reflects the command running but failing + log.debug("command '%s' failed with exit status %d" % (cmd[0], rc)) + raise DistutilsExecError( + "command '%s' failed with exit status %d" % (cmd[0], rc)) if sys.platform == 'darwin': from distutils import sysconfig @@ -89,9 +103,8 @@ log.info(' '.join(cmd)) if dry_run: return - executable = cmd[0] exec_fn = search_path and os.execvp or os.execv - env = None + exec_args = [cmd[0], cmd] if sys.platform == 'darwin': global _cfg_target, _cfg_target_split if _cfg_target is None: @@ -112,23 +125,17 @@ env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target) exec_fn = search_path and os.execvpe or os.execve + exec_args.append(env) pid = os.fork() if pid == 0: # in the child try: - if env is None: - exec_fn(executable, cmd) - else: - exec_fn(executable, cmd, env) + exec_fn(*exec_args) except OSError as e: - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r: %s\n" - % (cmd, e.strerror)) + sys.stderr.write("unable to execute %s: %s\n" + % (cmd[0], e.strerror)) os._exit(1) - if not DEBUG: - cmd = executable - sys.stderr.write("unable to execute %r for unknown reasons" % cmd) + sys.stderr.write("unable to execute %s for unknown reasons" % cmd[0]) os._exit(1) else: # in the parent # Loop until the child either exits or is terminated by a signal @@ -137,34 +144,29 @@ try: pid, status = os.waitpid(pid, 0) except OSError as exc: - if not DEBUG: - cmd = executable + import errno + if exc.errno == errno.EINTR: + continue raise DistutilsExecError( - "command %r failed: %s" % (cmd, exc.args[-1])) + "command '%s' failed: %s" % (cmd[0], exc.args[-1])) if os.WIFSIGNALED(status): - if not DEBUG: - cmd = executable raise DistutilsExecError( - "command %r terminated by signal %d" - % (cmd, os.WTERMSIG(status))) + "command '%s' terminated by signal %d" + % (cmd[0], os.WTERMSIG(status))) elif os.WIFEXITED(status): exit_status = os.WEXITSTATUS(status) if exit_status == 0: return # hey, it succeeded! else: - if not DEBUG: - cmd = executable raise DistutilsExecError( - "command %r failed with exit status %d" - % (cmd, exit_status)) + "command '%s' failed with exit status %d" + % (cmd[0], exit_status)) elif os.WIFSTOPPED(status): continue else: - if not DEBUG: - cmd = executable raise DistutilsExecError( - "unknown error executing %r: termination status %d" - % (cmd, status)) + "unknown error executing '%s': termination status %d" + % (cmd[0], status)) def find_executable(executable, path=None): """Tries to find 'executable' in the directories listed in 'path'. @@ -178,7 +180,7 @@ paths = path.split(os.pathsep) base, ext = os.path.splitext(executable) - if (sys.platform == 'win32') and (ext != '.exe'): + if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): executable = executable + '.exe' if not os.path.isfile(executable): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/sysconfig.py --- a/Lib/distutils/sysconfig.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/sysconfig.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,7 +9,6 @@ Email: """ -import _imp import os import re import sys @@ -19,38 +18,32 @@ # These are needed in a couple of spots, so just compute them once. PREFIX = os.path.normpath(sys.prefix) EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Path to the base directory of the project. On Windows the binary may -# live in project/PCBuild/win32 or project/PCBuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - project_base = os.path.dirname(os.path.abspath(sys.executable)) -if (os.name == 'nt' and - project_base.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - project_base = os.path.dirname(os.path.dirname(project_base)) +# live in project/PCBuild9. If we're dealing with an x64 Windows build, +# it'll live in project/PCbuild/amd64. +project_base = os.path.dirname(os.path.abspath(sys.executable)) +if os.name == "nt" and "pcbuild" in project_base[-8:].lower(): + project_base = os.path.abspath(os.path.join(project_base, os.path.pardir)) +# PC/VS7.1 +if os.name == "nt" and "\\pc\\v" in project_base[-10:].lower(): + project_base = os.path.abspath(os.path.join(project_base, os.path.pardir, + os.path.pardir)) +# PC/AMD64 +if os.name == "nt" and "\\pcbuild\\amd64" in project_base[-14:].lower(): + project_base = os.path.abspath(os.path.join(project_base, os.path.pardir, + os.path.pardir)) # python_build: (Boolean) if true, we're either building Python or # building an extension with an un-installed Python, so we use # different (hard-wired) directories. # Setup.local is available for Makefile builds including VPATH builds, # Setup.dist is available on Windows -def _is_python_source_dir(d): +def _python_build(): for fn in ("Setup.dist", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): + if os.path.isfile(os.path.join(project_base, "Modules", fn)): return True return False -_sys_home = getattr(sys, '_home', None) -if (_sys_home and os.name == 'nt' and - _sys_home.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - _sys_home = os.path.dirname(os.path.dirname(_sys_home)) -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) python_build = _python_build() # Calculate the build qualifier flags if they are defined. Adding the flags @@ -81,11 +74,11 @@ otherwise, this is the path to platform-specific header files (namely pyconfig.h). - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. + If 'prefix' is supplied, use it instead of sys.prefix or + sys.exec_prefix -- i.e., ignore 'plat_specific'. """ if prefix is None: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX + prefix = plat_specific and EXEC_PREFIX or PREFIX if os.name == "posix": if python_build: # Assume the executable is in the build directory. The @@ -93,18 +86,18 @@ # the build directory may not be the source directory, we # must use "srcdir" from the makefile to find the "Include" # directory. - base = _sys_home or project_base + base = os.path.dirname(os.path.abspath(sys.executable)) if plat_specific: return base - if _sys_home: - incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR')) else: incdir = os.path.join(get_config_var('srcdir'), 'Include') - return os.path.normpath(incdir) + return os.path.normpath(incdir) python_dir = 'python' + get_python_version() + build_flags return os.path.join(prefix, "include", python_dir) elif os.name == "nt": return os.path.join(prefix, "include") + elif os.name == "os2": + return os.path.join(prefix, "Include") else: raise DistutilsPlatformError( "I don't know where Python installs its C header files " @@ -122,14 +115,11 @@ containing standard Python library modules; otherwise, return the directory for site-specific modules. - If 'prefix' is supplied, use it instead of sys.base_prefix or - sys.base_exec_prefix -- i.e., ignore 'plat_specific'. + If 'prefix' is supplied, use it instead of sys.prefix or + sys.exec_prefix -- i.e., ignore 'plat_specific'. """ if prefix is None: - if standard_lib: - prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - else: - prefix = plat_specific and EXEC_PREFIX or PREFIX + prefix = plat_specific and EXEC_PREFIX or PREFIX if os.name == "posix": libpython = os.path.join(prefix, @@ -142,13 +132,21 @@ if standard_lib: return os.path.join(prefix, "Lib") else: + if get_python_version() < "2.2": + return prefix + else: + return os.path.join(prefix, "Lib", "site-packages") + elif os.name == "os2": + if standard_lib: + return os.path.join(prefix, "Lib") + else: return os.path.join(prefix, "Lib", "site-packages") else: raise DistutilsPlatformError( "I don't know where Python installs its library " "on platform '%s'" % os.name) - +_USE_CLANG = None def customize_compiler(compiler): """Do any platform-specific customization of a CCompiler instance. @@ -157,33 +155,40 @@ varies across Unices and is stored in Python's Makefile. """ if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' + (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \ + get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', + 'CCSHARED', 'LDSHARED', 'SO', 'AR', 'ARFLAGS') - (cc, cxx, opt, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - + newcc = None if 'CC' in os.environ: newcc = os.environ['CC'] + elif sys.platform == 'darwin' and cc == 'gcc-4.2': + # Issue #13590: + # Since Apple removed gcc-4.2 in Xcode 4.2, we can no + # longer assume it is available for extension module builds. + # If Python was built with gcc-4.2, check first to see if + # it is available on this system; if not, try to use clang + # instead unless the caller explicitly set CC. + global _USE_CLANG + if _USE_CLANG is None: + from distutils import log + from subprocess import Popen, PIPE + p = Popen("! type gcc-4.2 && type clang && exit 2", + shell=True, stdout=PIPE, stderr=PIPE) + p.wait() + if p.returncode == 2: + _USE_CLANG = True + log.warn("gcc-4.2 not found, using clang instead") + else: + _USE_CLANG = False + if _USE_CLANG: + newcc = 'clang' + if newcc: + # On OS X, if CC is overridden, use that as the default + # command for LDSHARED as well if (sys.platform == 'darwin' and 'LDSHARED' not in os.environ and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well ldshared = newcc + ldshared[len(cc):] cc = newcc if 'CXX' in os.environ: @@ -220,26 +225,30 @@ linker_exe=cc, archiver=archiver) - compiler.shared_lib_extension = shlib_suffix + compiler.shared_lib_extension = so_ext def get_config_h_filename(): """Return full pathname of installed pyconfig.h file.""" if python_build: if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") + inc_dir = os.path.join(project_base, "PC") else: - inc_dir = _sys_home or project_base + inc_dir = project_base else: inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') + if get_python_version() < '2.2': + config_h = 'config.h' + else: + # The name of the config.h file changed in 2.2 + config_h = 'pyconfig.h' + return os.path.join(inc_dir, config_h) def get_makefile_filename(): """Return full pathname of installed Makefile from the Python build.""" if python_build: - return os.path.join(_sys_home or project_base, "Makefile") + return os.path.join(os.path.dirname(sys.executable), "Makefile") lib_dir = get_python_lib(plat_specific=0, standard_lib=1) config_file = 'config-{}{}'.format(get_python_version(), build_flags) return os.path.join(lib_dir, config_file, 'Makefile') @@ -420,7 +429,7 @@ try: filename = get_makefile_filename() parse_makefile(filename, g) - except OSError as msg: + except IOError as msg: my_msg = "invalid Python installation: unable to open %s" % filename if hasattr(msg, "strerror"): my_msg = my_msg + " (%s)" % msg.strerror @@ -432,7 +441,7 @@ filename = get_config_h_filename() with open(filename) as file: parse_config_h(file, g) - except OSError as msg: + except IOError as msg: my_msg = "invalid Python installation: unable to open %s" % filename if hasattr(msg, "strerror"): my_msg = my_msg + " (%s)" % msg.strerror @@ -445,6 +454,17 @@ if python_build: g['LDSHARED'] = g['BLDSHARED'] + elif get_python_version() < '2.1': + # The following two branches are for 1.5.2 compatibility. + if sys.platform == 'aix4': # what about AIX 3.x ? + # Linker script is in the config directory, not in Modules as the + # Makefile says. + python_lib = get_python_lib(standard_lib=1) + ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') + python_exp = os.path.join(python_lib, 'config', 'python.exp') + + g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) + global _config_vars _config_vars = g @@ -459,7 +479,7 @@ # XXX hmmm.. a normal install puts include files here g['INCLUDEPY'] = get_python_inc(plat_specific=0) - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] + g['SO'] = '.pyd' g['EXE'] = ".exe" g['VERSION'] = get_python_version().replace(".", "") g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) @@ -468,12 +488,29 @@ _config_vars = g +def _init_os2(): + """Initialize the module as appropriate for OS/2""" + g = {} + # set basic install directories + g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) + g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) + + # XXX hmmm.. a normal install puts include files here + g['INCLUDEPY'] = get_python_inc(plat_specific=0) + + g['SO'] = '.pyd' + g['EXE'] = ".exe" + + global _config_vars + _config_vars = g + + def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. Generally this includes everything needed to build extensions and install both pure modules and extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. + installed Makefile; on Windows and Mac OS it's a much smaller set. With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. @@ -492,34 +529,12 @@ _config_vars['prefix'] = PREFIX _config_vars['exec_prefix'] = EXEC_PREFIX - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - # Convert srcdir into an absolute path if it appears necessary. # Normally it is relative to the build directory. However, during # testing, for example, we might be running a non-installed python # from a different directory. if python_build and os.name == "posix": - base = project_base + base = os.path.dirname(os.path.abspath(sys.executable)) if (not os.path.isabs(_config_vars['srcdir']) and base != os.getcwd()): # srcdir is relative and we are not in the same directory @@ -528,11 +543,43 @@ srcdir = os.path.join(base, _config_vars['srcdir']) _config_vars['srcdir'] = os.path.normpath(srcdir) - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) + kernel_version = os.uname()[2] # Kernel version (8.4.3) + major_version = int(kernel_version.split('.')[0]) + + if major_version < 8: + # On Mac OS X before 10.4, check if -arch and -isysroot + # are in CFLAGS or LDFLAGS and remove them if they are. + # This is needed when building extensions on a 10.3 system + # using a universal build of python. + for key in ('LDFLAGS', 'BASECFLAGS', + # a number of derived variables. These need to be + # patched up as well. + 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): + flags = _config_vars[key] + flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub('-isysroot [^ \t]*', ' ', flags) + _config_vars[key] = flags + + else: + + # Allow the user to override the architecture flags using + # an environment variable. + # NOTE: This name was introduced by Apple in OSX 10.5 and + # is used by several scripting languages distributed with + # that OS release. + + if 'ARCHFLAGS' in os.environ: + arch = os.environ['ARCHFLAGS'] + for key in ('LDFLAGS', 'BASECFLAGS', + # a number of derived variables. These need to be + # patched up as well. + 'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'): + + flags = _config_vars[key] + flags = re.sub('-arch\s+\w+\s', ' ', flags) + flags = flags + ' ' + arch + _config_vars[key] = flags if args: vals = [] @@ -547,7 +594,4 @@ returned by 'get_config_vars()'. Equivalent to get_config_vars().get(name) """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) return get_config_vars().get(name) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/support.py --- a/Lib/distutils/tests/support.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/support.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,15 +32,14 @@ def _log(self, level, msg, args): if level not in (DEBUG, INFO, WARN, ERROR, FATAL): raise ValueError('%s wrong log level' % str(level)) - if not isinstance(msg, str): - raise TypeError("msg should be str, not '%.200s'" - % (type(msg).__name__)) self.logs.append((level, msg, args)) def get_logs(self, *levels): def _format(msg, args): + if len(args) == 0: + return msg return msg % args - return [msg % args for level, msg, args + return [_format(msg, args) for level, msg, args in self.logs if level in levels] def clear_logs(self): @@ -207,4 +206,4 @@ cmd.library_dirs = [] else: name, equals, value = runshared.partition('=') - cmd.library_dirs = [d for d in value.split(os.pathsep) if d] + cmd.library_dirs = value.split(os.pathsep) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_archive_util.py --- a/Lib/distutils/tests/test_archive_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_archive_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -13,14 +13,7 @@ ARCHIVE_FORMATS) from distutils.spawn import find_executable, spawn from distutils.tests import support -from test.support import check_warnings, run_unittest, patch, change_cwd - -try: - import grp - import pwd - UID_GID_SUPPORT = True -except ImportError: - UID_GID_SUPPORT = False +from test.support import check_warnings, run_unittest, patch try: import zipfile @@ -34,16 +27,6 @@ except ImportError: ZLIB_SUPPORT = False -try: - import bz2 -except ImportError: - bz2 = None - -try: - import lzma -except ImportError: - lzma = None - def can_fs_encode(filename): """ Return True if the filename can be saved in the file system. @@ -62,36 +45,19 @@ unittest.TestCase): @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') - def test_make_tarball(self, name='archive'): - # creating something to tar - tmpdir = self._create_files() - self._make_tarball(tmpdir, name, '.tar.gz') - # trying an uncompressed one - self._make_tarball(tmpdir, name, '.tar', compress=None) + def test_make_tarball(self): + self._make_tarball('archive') @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') - def test_make_tarball_gzip(self): - tmpdir = self._create_files() - self._make_tarball(tmpdir, 'archive', '.tar.gz', compress='gzip') - - @unittest.skipUnless(bz2, 'Need bz2 support to run') - def test_make_tarball_bzip2(self): - tmpdir = self._create_files() - self._make_tarball(tmpdir, 'archive', '.tar.bz2', compress='bzip2') - - @unittest.skipUnless(lzma, 'Need lzma support to run') - def test_make_tarball_xz(self): - tmpdir = self._create_files() - self._make_tarball(tmpdir, 'archive', '.tar.xz', compress='xz') - @unittest.skipUnless(can_fs_encode('årchiv'), 'File system cannot handle this filename') def test_make_tarball_latin1(self): """ Mirror test_make_tarball, except filename contains latin characters. """ - self.test_make_tarball('årchiv') # note this isn't a real word + self._make_tarball('årchiv') # note this isn't a real word + @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') @unittest.skipUnless(can_fs_encode('のアーカイブ'), 'File system cannot handle this filename') def test_make_tarball_extended(self): @@ -99,23 +65,44 @@ Mirror test_make_tarball, except filename contains extended characters outside the latin charset. """ - self.test_make_tarball('のアーカイブ') # japanese for archive + self._make_tarball('のアーカイブ') # japanese for archive - def _make_tarball(self, tmpdir, target_name, suffix, **kwargs): + def _make_tarball(self, target_name): + # creating something to tar + tmpdir = self.mkdtemp() + self.write_file([tmpdir, 'file1'], 'xxx') + self.write_file([tmpdir, 'file2'], 'xxx') + os.mkdir(os.path.join(tmpdir, 'sub')) + self.write_file([tmpdir, 'sub', 'file3'], 'xxx') + tmpdir2 = self.mkdtemp() unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0], - "source and target should be on same drive") + "Source and target should be on same drive") base_name = os.path.join(tmpdir2, target_name) # working with relative paths to avoid tar warnings - with change_cwd(tmpdir): - make_tarball(splitdrive(base_name)[1], 'dist', **kwargs) + old_dir = os.getcwd() + os.chdir(tmpdir) + try: + make_tarball(splitdrive(base_name)[1], '.') + finally: + os.chdir(old_dir) # check if the compressed tarball was created - tarball = base_name + suffix + tarball = base_name + '.tar.gz' self.assertTrue(os.path.exists(tarball)) - self.assertEqual(self._tarinfo(tarball), self._created_files) + + # trying an uncompressed one + base_name = os.path.join(tmpdir2, target_name) + old_dir = os.getcwd() + os.chdir(tmpdir) + try: + make_tarball(splitdrive(base_name)[1], '.', compress=None) + finally: + os.chdir(old_dir) + tarball = base_name + '.tar' + self.assertTrue(os.path.exists(tarball)) def _tarinfo(self, path): tar = tarfile.open(path) @@ -126,9 +113,6 @@ finally: tar.close() - _created_files = ('dist', 'dist/file1', 'dist/file2', - 'dist/sub', 'dist/sub/file3', 'dist/sub2') - def _create_files(self): # creating something to tar tmpdir = self.mkdtemp() @@ -139,15 +123,15 @@ os.mkdir(os.path.join(dist, 'sub')) self.write_file([dist, 'sub', 'file3'], 'xxx') os.mkdir(os.path.join(dist, 'sub2')) - return tmpdir + tmpdir2 = self.mkdtemp() + base_name = os.path.join(tmpdir2, 'archive') + return tmpdir, tmpdir2, base_name @unittest.skipUnless(find_executable('tar') and find_executable('gzip') and ZLIB_SUPPORT, 'Need the tar, gzip and zlib command to run') def test_tarfile_vs_tar(self): - tmpdir = self._create_files() - tmpdir2 = self.mkdtemp() - base_name = os.path.join(tmpdir2, 'archive') + tmpdir, tmpdir2, base_name = self._create_files() old_dir = os.getcwd() os.chdir(tmpdir) try: @@ -173,8 +157,7 @@ self.assertTrue(os.path.exists(tarball2)) # let's compare both tarballs - self.assertEqual(self._tarinfo(tarball), self._created_files) - self.assertEqual(self._tarinfo(tarball2), self._created_files) + self.assertEqual(self._tarinfo(tarball), self._tarinfo(tarball2)) # trying an uncompressed one base_name = os.path.join(tmpdir2, 'archive') @@ -201,8 +184,7 @@ @unittest.skipUnless(find_executable('compress'), 'The compress program is required') def test_compress_deprecated(self): - tmpdir = self._create_files() - base_name = os.path.join(self.mkdtemp(), 'archive') + tmpdir, tmpdir2, base_name = self._create_files() # using compress and testing the PendingDeprecationWarning old_dir = os.getcwd() @@ -228,24 +210,24 @@ dry_run=True) finally: os.chdir(old_dir) - self.assertFalse(os.path.exists(tarball)) + self.assertTrue(not os.path.exists(tarball)) self.assertEqual(len(w.warnings), 1) @unittest.skipUnless(ZIP_SUPPORT and ZLIB_SUPPORT, 'Need zip and zlib support to run') def test_make_zipfile(self): # creating something to tar - tmpdir = self._create_files() - base_name = os.path.join(self.mkdtemp(), 'archive') - with change_cwd(tmpdir): - make_zipfile(base_name, 'dist') + tmpdir = self.mkdtemp() + self.write_file([tmpdir, 'file1'], 'xxx') + self.write_file([tmpdir, 'file2'], 'xxx') + + tmpdir2 = self.mkdtemp() + base_name = os.path.join(tmpdir2, 'archive') + make_zipfile(base_name, tmpdir) # check if the compressed tarball was created tarball = base_name + '.zip' self.assertTrue(os.path.exists(tarball)) - with zipfile.ZipFile(tarball) as zf: - self.assertEqual(sorted(zf.namelist()), - ['dist/file1', 'dist/file2', 'dist/sub/file3']) @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run') def test_make_zipfile_no_zlib(self): @@ -261,24 +243,18 @@ patch(self, archive_util.zipfile, 'ZipFile', fake_zipfile) # create something to tar and compress - tmpdir = self._create_files() - base_name = os.path.join(self.mkdtemp(), 'archive') - with change_cwd(tmpdir): - make_zipfile(base_name, 'dist') + tmpdir, tmpdir2, base_name = self._create_files() + make_zipfile(base_name, tmpdir) tarball = base_name + '.zip' self.assertEqual(called, [((tarball, "w"), {'compression': zipfile.ZIP_STORED})]) self.assertTrue(os.path.exists(tarball)) - with zipfile.ZipFile(tarball) as zf: - self.assertEqual(sorted(zf.namelist()), - ['dist/file1', 'dist/file2', 'dist/sub/file3']) def test_check_archive_formats(self): self.assertEqual(check_archive_formats(['gztar', 'xxx', 'zip']), 'xxx') - self.assertIsNone(check_archive_formats(['gztar', 'bztar', 'xztar', - 'ztar', 'tar', 'zip'])) + self.assertEqual(check_archive_formats(['gztar', 'zip']), None) def test_make_archive(self): tmpdir = self.mkdtemp() @@ -299,95 +275,6 @@ finally: del ARCHIVE_FORMATS['xxx'] - def test_make_archive_tar(self): - base_dir = self._create_files() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'tar', base_dir, 'dist') - self.assertTrue(os.path.exists(res)) - self.assertEqual(os.path.basename(res), 'archive.tar') - self.assertEqual(self._tarinfo(res), self._created_files) - - @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') - def test_make_archive_gztar(self): - base_dir = self._create_files() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'gztar', base_dir, 'dist') - self.assertTrue(os.path.exists(res)) - self.assertEqual(os.path.basename(res), 'archive.tar.gz') - self.assertEqual(self._tarinfo(res), self._created_files) - - @unittest.skipUnless(bz2, 'Need bz2 support to run') - def test_make_archive_bztar(self): - base_dir = self._create_files() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'bztar', base_dir, 'dist') - self.assertTrue(os.path.exists(res)) - self.assertEqual(os.path.basename(res), 'archive.tar.bz2') - self.assertEqual(self._tarinfo(res), self._created_files) - - @unittest.skipUnless(lzma, 'Need xz support to run') - def test_make_archive_xztar(self): - base_dir = self._create_files() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'xztar', base_dir, 'dist') - self.assertTrue(os.path.exists(res)) - self.assertEqual(os.path.basename(res), 'archive.tar.xz') - self.assertEqual(self._tarinfo(res), self._created_files) - - def test_make_archive_owner_group(self): - # testing make_archive with owner and group, with various combinations - # this works even if there's not gid/uid support - if UID_GID_SUPPORT: - group = grp.getgrgid(0)[0] - owner = pwd.getpwuid(0)[0] - else: - group = owner = 'root' - - base_dir = self._create_files() - root_dir = self.mkdtemp() - base_name = os.path.join(self.mkdtemp() , 'archive') - res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner, - group=group) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'zip', root_dir, base_dir) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'tar', root_dir, base_dir, - owner=owner, group=group) - self.assertTrue(os.path.exists(res)) - - res = make_archive(base_name, 'tar', root_dir, base_dir, - owner='kjhkjhkjg', group='oihohoh') - self.assertTrue(os.path.exists(res)) - - @unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") - def test_tarfile_root_owner(self): - tmpdir = self._create_files() - base_name = os.path.join(self.mkdtemp(), 'archive') - old_dir = os.getcwd() - os.chdir(tmpdir) - group = grp.getgrgid(0)[0] - owner = pwd.getpwuid(0)[0] - try: - archive_name = make_tarball(base_name, 'dist', compress=None, - owner=owner, group=group) - finally: - os.chdir(old_dir) - - # check if the compressed tarball was created - self.assertTrue(os.path.exists(archive_name)) - - # now checks the rights - archive = tarfile.open(archive_name) - try: - for member in archive.getmembers(): - self.assertEqual(member.uid, 0) - self.assertEqual(member.gid, 0) - finally: - archive.close() - def test_suite(): return unittest.makeSuite(ArchiveUtilTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_bdist.py --- a/Lib/distutils/tests/test_bdist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_bdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -21,7 +21,7 @@ # what formats does bdist offer? formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar', - 'wininst', 'xztar', 'zip', 'ztar'] + 'wininst', 'zip', 'ztar'] found = sorted(cmd.format_command) self.assertEqual(found, formats) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_bdist_dumb.py --- a/Lib/distutils/tests/test_bdist_dumb.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_bdist_dumb.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Tests for distutils.command.bdist_dumb.""" import os +import imp import sys import zipfile import unittest @@ -74,6 +75,8 @@ # see what we have dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name) + if os.name == 'os2': + base = base.replace(':', '-') self.assertEqual(dist_created, [base]) @@ -85,9 +88,9 @@ fp.close() contents = sorted(os.path.basename(fn) for fn in contents) - wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py'] - if not sys.dont_write_bytecode: - wanted.append('foo.%s.pyc' % sys.implementation.cache_tag) + wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], + 'foo.%s.pyc' % imp.get_tag(), + 'foo.py'] self.assertEqual(contents, sorted(wanted)) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_bdist_rpm.py --- a/Lib/distutils/tests/test_bdist_rpm.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_bdist_rpm.py Mon Jan 25 17:05:13 2016 +0100 @@ -24,7 +24,6 @@ """ class BuildRpmTestCase(support.TempdirManager, - support.EnvironGuard, support.LoggingSilencer, unittest.TestCase): @@ -44,18 +43,20 @@ sys.argv[:] = self.old_sys_argv[1] super(BuildRpmTestCase, self).tearDown() - # XXX I am unable yet to make this test work without - # spurious sdtout/stderr output under Mac OS X - @unittest.skipUnless(sys.platform.startswith('linux'), - 'spurious sdtout/stderr output under Mac OS X') - @unittest.skipIf(find_executable('rpm') is None, - 'the rpm command is not found') - @unittest.skipIf(find_executable('rpmbuild') is None, - 'the rpmbuild command is not found') def test_quiet(self): + + # XXX I am unable yet to make this test work without + # spurious sdtout/stderr output under Mac OS X + if not sys.platform.startswith('linux'): + return + + # this test will run only if the rpm commands are found + if (find_executable('rpm') is None or + find_executable('rpmbuild') is None): + return + # let's create a package tmp_dir = self.mkdtemp() - os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -80,25 +81,27 @@ cmd.run() dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) - self.assertIn('foo-0.1-1.noarch.rpm', dist_created) + self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) # bug #2945: upload ignores bdist_rpm files self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files) - # XXX I am unable yet to make this test work without - # spurious sdtout/stderr output under Mac OS X - @unittest.skipUnless(sys.platform.startswith('linux'), - 'spurious sdtout/stderr output under Mac OS X') - # http://bugs.python.org/issue1533164 - @unittest.skipIf(find_executable('rpm') is None, - 'the rpm command is not found') - @unittest.skipIf(find_executable('rpmbuild') is None, - 'the rpmbuild command is not found') def test_no_optimize_flag(self): + + # XXX I am unable yet to make this test work without + # spurious sdtout/stderr output under Mac OS X + if not sys.platform.startswith('linux'): + return + + # http://bugs.python.org/issue1533164 + # this test will run only if the rpm command is found + if (find_executable('rpm') is None or + find_executable('rpmbuild') is None): + return + # let's create a package that brakes bdist_rpm tmp_dir = self.mkdtemp() - os.environ['HOME'] = tmp_dir # to confine dir '.rpmdb' creation pkg_dir = os.path.join(tmp_dir, 'foo') os.mkdir(pkg_dir) self.write_file((pkg_dir, 'setup.py'), SETUP_PY) @@ -122,7 +125,7 @@ cmd.run() dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) - self.assertIn('foo-0.1-1.noarch.rpm', dist_created) + self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) # bug #2945: upload ignores bdist_rpm files self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_bdist_wininst.py --- a/Lib/distutils/tests/test_bdist_wininst.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_bdist_wininst.py Mon Jan 25 17:05:13 2016 +0100 @@ -22,7 +22,7 @@ # and make sure it finds it and returns its content # no matter what platform we have exe_file = cmd.get_exe_bytes() - self.assertGreater(len(exe_file), 10) + self.assertTrue(len(exe_file) > 10) def test_suite(): return unittest.makeSuite(BuildWinInstTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_build_clib.py --- a/Lib/distutils/tests/test_build_clib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_build_clib.py Mon Jan 25 17:05:13 2016 +0100 @@ -77,7 +77,7 @@ cmd.compiler = FakeCompiler() - # build_libraries is also doing a bit of typo checking + # build_libraries is also doing a bit of typoe checking lib = [('name', {'sources': 'notvalid'})] self.assertRaises(DistutilsSetupError, cmd.build_libraries, lib) @@ -102,8 +102,11 @@ cmd.distribution.libraries = 'WONTWORK' self.assertRaises(DistutilsSetupError, cmd.finalize_options) - @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_run(self): + # can't test on windows + if sys.platform == 'win32': + return + pkg_dir, dist = self.create_dist() cmd = build_clib(dist) @@ -128,13 +131,13 @@ if ccmd is None: continue if find_executable(ccmd[0]) is None: - self.skipTest('The %r command is not found' % ccmd[0]) + return # can't test # this should work cmd.run() # let's check the result - self.assertIn('libfoo.a', os.listdir(build_temp)) + self.assertTrue('libfoo.a' in os.listdir(build_temp)) def test_suite(): return unittest.makeSuite(BuildCLibTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_build_ext.py --- a/Lib/distutils/tests/test_build_ext.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_build_ext.py Mon Jan 25 17:05:13 2016 +0100 @@ -31,14 +31,12 @@ self.tmp_dir = self.mkdtemp() self.sys_path = sys.path, sys.path[:] sys.path.append(self.tmp_dir) - import site - self.old_user_base = site.USER_BASE - site.USER_BASE = self.mkdtemp() - from distutils.command import build_ext - build_ext.USER_BASE = site.USER_BASE - - def build_ext(self, *args, **kwargs): - return build_ext(*args, **kwargs) + if sys.version > "2.6": + import site + self.old_user_base = site.USER_BASE + site.USER_BASE = self.mkdtemp() + from distutils.command import build_ext + build_ext.USER_BASE = site.USER_BASE def test_build_ext(self): global ALREADY_TESTED @@ -47,7 +45,7 @@ xx_ext = Extension('xx', [xx_c]) dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) dist.package_dir = self.tmp_dir - cmd = self.build_ext(dist) + cmd = build_ext(dist) fixup_build_ext(cmd) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -63,9 +61,9 @@ sys.stdout = old_stdout if ALREADY_TESTED: - self.skipTest('Already tested in %s' % ALREADY_TESTED) + return else: - ALREADY_TESTED = type(self).__name__ + ALREADY_TESTED = True import xx @@ -75,26 +73,26 @@ self.assertEqual(xx.foo(2, 5), 7) self.assertEqual(xx.foo(13,15), 28) self.assertEqual(xx.new().demo(), None) - if support.HAVE_DOCSTRINGS: - doc = 'This is a template module just for instruction.' - self.assertEqual(xx.__doc__, doc) - self.assertIsInstance(xx.Null(), xx.Null) - self.assertIsInstance(xx.Str(), xx.Str) + doc = 'This is a template module just for instruction.' + self.assertEqual(xx.__doc__, doc) + self.assertTrue(isinstance(xx.Null(), xx.Null)) + self.assertTrue(isinstance(xx.Str(), xx.Str)) def tearDown(self): # Get everything back to normal support.unload('xx') sys.path = self.sys_path[0] sys.path[:] = self.sys_path[1] - import site - site.USER_BASE = self.old_user_base - from distutils.command import build_ext - build_ext.USER_BASE = self.old_user_base + if sys.version > "2.6": + import site + site.USER_BASE = self.old_user_base + from distutils.command import build_ext + build_ext.USER_BASE = self.old_user_base super(BuildExtTestCase, self).tearDown() def test_solaris_enable_shared(self): dist = Distribution({'name': 'xx'}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) old = sys.platform sys.platform = 'sunos' # fooling finalize_options @@ -111,17 +109,21 @@ _config_vars['Py_ENABLE_SHARED'] = old_var # make sure we get some library dirs under solaris - self.assertGreater(len(cmd.library_dirs), 0) + self.assertTrue(len(cmd.library_dirs) > 0) def test_user_site(self): + # site.USER_SITE was introduced in 2.6 + if sys.version < '2.6': + return + import site dist = Distribution({'name': 'xx'}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) # making sure the user option is there options = [name for name, short, lable in cmd.user_options] - self.assertIn('user', options) + self.assertTrue('user' in options) # setting a value cmd.user = 1 @@ -147,14 +149,14 @@ # with the optional argument. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.ensure_finalized() self.assertRaises((UnknownFileError, CompileError), cmd.run) # should raise an error modules = [Extension('foo', ['xxx'], optional=True)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.ensure_finalized() cmd.run() # should pass @@ -163,26 +165,26 @@ # etc.) are in the include search path. modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.finalize_options() from distutils import sysconfig py_include = sysconfig.get_python_inc() - self.assertIn(py_include, cmd.include_dirs) + self.assertTrue(py_include in cmd.include_dirs) plat_py_include = sysconfig.get_python_inc(plat_specific=1) - self.assertIn(plat_py_include, cmd.include_dirs) + self.assertTrue(plat_py_include in cmd.include_dirs) # make sure cmd.libraries is turned into a list # if it's a string - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.libraries = 'my_lib, other_lib lastlib' cmd.finalize_options() self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib']) # make sure cmd.library_dirs is turned into a list # if it's a string - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep cmd.finalize_options() self.assertIn('my_lib_dir', cmd.library_dirs) @@ -190,7 +192,7 @@ # make sure rpath is turned into a list # if it's a string - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.rpath = 'one%stwo' % os.pathsep cmd.finalize_options() self.assertEqual(cmd.rpath, ['one', 'two']) @@ -199,32 +201,32 @@ # make sure define is turned into 2-tuples # strings if they are ','-separated strings - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.define = 'one,two' cmd.finalize_options() self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) # make sure undef is turned into a list of # strings if they are ','-separated strings - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.undef = 'one,two' cmd.finalize_options() self.assertEqual(cmd.undef, ['one', 'two']) # make sure swig_opts is turned into a list - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.swig_opts = None cmd.finalize_options() self.assertEqual(cmd.swig_opts, []) - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.swig_opts = '1 2' cmd.finalize_options() self.assertEqual(cmd.swig_opts, ['1', '2']) def test_check_extensions_list(self): dist = Distribution() - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.finalize_options() #'extensions' option must be a list of Extension instances @@ -252,13 +254,13 @@ 'some': 'bar'})] cmd.check_extensions_list(exts) ext = exts[0] - self.assertIsInstance(ext, Extension) + self.assertTrue(isinstance(ext, Extension)) # check_extensions_list adds in ext the values passed # when they are in ('include_dirs', 'library_dirs', 'libraries' # 'extra_objects', 'extra_compile_args', 'extra_link_args') self.assertEqual(ext.libraries, 'foo') - self.assertFalse(hasattr(ext, 'some')) + self.assertTrue(not hasattr(ext, 'some')) # 'macros' element of build info dict must be 1- or 2-tuple exts = [('foo.bar', {'sources': [''], 'libraries': 'foo', @@ -273,7 +275,7 @@ def test_get_source_files(self): modules = [Extension('foo', ['xxx'], optional=False)] dist = Distribution({'name': 'xx', 'ext_modules': modules}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.ensure_finalized() self.assertEqual(cmd.get_source_files(), ['xxx']) @@ -282,7 +284,7 @@ # should not be overriden by a compiler instance # when the command is run dist = Distribution() - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.compiler = 'unix' cmd.ensure_finalized() cmd.run() @@ -295,7 +297,7 @@ ext = Extension('foo', [c_file], optional=False) dist = Distribution({'name': 'xx', 'ext_modules': [ext]}) - cmd = self.build_ext(dist) + cmd = build_ext(dist) fixup_build_ext(cmd) cmd.ensure_finalized() self.assertEqual(len(cmd.get_outputs()), 1) @@ -315,8 +317,8 @@ finally: os.chdir(old_wd) self.assertTrue(os.path.exists(so_file)) - ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(so_file.endswith(ext_suffix)) + so_ext = sysconfig.get_config_var('SO') + self.assertTrue(so_file.endswith(so_ext)) so_dir = os.path.dirname(so_file) self.assertEqual(so_dir, other_tmp_dir) @@ -325,7 +327,7 @@ cmd.run() so_file = cmd.get_outputs()[0] self.assertTrue(os.path.exists(so_file)) - self.assertTrue(so_file.endswith(ext_suffix)) + self.assertTrue(so_file.endswith(so_ext)) so_dir = os.path.dirname(so_file) self.assertEqual(so_dir, cmd.build_lib) @@ -352,13 +354,13 @@ self.assertEqual(lastdir, 'bar') def test_ext_fullpath(self): - ext = sysconfig.get_config_var('EXT_SUFFIX') + ext = sysconfig.get_config_vars()['SO'] # building lxml.etree inplace #etree_c = os.path.join(self.tmp_dir, 'lxml.etree.c') #etree_ext = Extension('lxml.etree', [etree_c]) #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) dist = Distribution() - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.inplace = 1 cmd.distribution.package_dir = {'': 'src'} cmd.distribution.packages = ['lxml', 'lxml.html'] @@ -445,16 +447,8 @@ # get the deployment target that the interpreter was built with target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') - target = tuple(map(int, target.split('.')[0:2])) - # format the target value as defined in the Apple - # Availability Macros. We can't use the macro names since - # at least one value we test with will not exist yet. - if target[1] < 10: - # for 10.1 through 10.9.x -> "10n0" - target = '%02d%01d0' % target - else: - # for 10.10 and beyond -> "10nn00" - target = '%02d%02d00' % target + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target deptarget_ext = Extension( 'deptarget', [deptarget_c], @@ -465,7 +459,7 @@ 'ext_modules': [deptarget_ext] }) dist.package_dir = self.tmp_dir - cmd = self.build_ext(dist) + cmd = build_ext(dist) cmd.build_lib = self.tmp_dir cmd.build_temp = self.tmp_dir @@ -484,19 +478,8 @@ self.fail("Wrong deployment target during compilation") -class ParallelBuildExtTestCase(BuildExtTestCase): - - def build_ext(self, *args, **kwargs): - build_ext = super().build_ext(*args, **kwargs) - build_ext.parallel = True - return build_ext - - def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(BuildExtTestCase)) - suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase)) - return suite + return unittest.makeSuite(BuildExtTestCase) if __name__ == '__main__': - support.run_unittest(__name__) + support.run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_build_py.py --- a/Lib/distutils/tests/test_build_py.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_build_py.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,6 +2,7 @@ import os import sys +import imp import unittest from distutils.command.build_py import build_py @@ -62,8 +63,7 @@ self.assertFalse(os.path.exists(pycache_dir)) else: pyc_files = os.listdir(pycache_dir) - self.assertIn("__init__.%s.pyc" % sys.implementation.cache_tag, - pyc_files) + self.assertIn("__init__.%s.pyc" % imp.get_tag(), pyc_files) def test_empty_package_dir(self): # See bugs #1668596/#1720897 @@ -102,8 +102,7 @@ found = os.listdir(cmd.build_lib) self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) - self.assertEqual(found, - ['boiledeggs.%s.pyc' % sys.implementation.cache_tag]) + self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()]) @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled') def test_byte_compile_optimized(self): @@ -120,39 +119,7 @@ found = os.listdir(cmd.build_lib) self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) - expect = 'boiledeggs.{}.opt-1.pyc'.format(sys.implementation.cache_tag) - self.assertEqual(sorted(found), [expect]) - - def test_dir_in_package_data(self): - """ - A directory in package_data should not be added to the filelist. - """ - # See bug 19286 - sources = self.mkdtemp() - pkg_dir = os.path.join(sources, "pkg") - - os.mkdir(pkg_dir) - open(os.path.join(pkg_dir, "__init__.py"), "w").close() - - docdir = os.path.join(pkg_dir, "doc") - os.mkdir(docdir) - open(os.path.join(docdir, "testfile"), "w").close() - - # create the directory that could be incorrectly detected as a file - os.mkdir(os.path.join(docdir, 'otherdir')) - - os.chdir(sources) - dist = Distribution({"packages": ["pkg"], - "package_data": {"pkg": ["doc/*"]}}) - # script_name need not exist, it just need to be initialized - dist.script_name = os.path.join(sources, "setup.py") - dist.script_args = ["build"] - dist.parse_command_line() - - try: - dist.run_commands() - except DistutilsFileError: - self.fail("failed package_data when data dir includes a dir") + self.assertEqual(sorted(found), ['boiledeggs.%s.pyo' % imp.get_tag()]) def test_dont_write_bytecode(self): # makes sure byte_compile is not used diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_build_scripts.py --- a/Lib/distutils/tests/test_build_scripts.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_build_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -17,8 +17,8 @@ def test_default_settings(self): cmd = self.get_build_scripts_cmd("/foo/bar", []) - self.assertFalse(cmd.force) - self.assertIsNone(cmd.build_dir) + self.assertTrue(not cmd.force) + self.assertTrue(cmd.build_dir is None) cmd.finalize_options() @@ -38,7 +38,7 @@ built = os.listdir(target) for name in expected: - self.assertIn(name, built) + self.assertTrue(name in built) def get_build_scripts_cmd(self, target, scripts): import sys @@ -103,7 +103,7 @@ built = os.listdir(target) for name in expected: - self.assertIn(name, built) + self.assertTrue(name in built) def test_suite(): return unittest.makeSuite(BuildScriptsTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_check.py --- a/Lib/distutils/tests/test_check.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_check.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ """Tests for distutils.command.check.""" -import textwrap import unittest from test.support import run_unittest @@ -56,8 +55,9 @@ cmd = self._run(metadata) self.assertEqual(cmd._warnings, 0) - @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils") def test_check_document(self): + if not HAS_DOCUTILS: # won't test without docutils + return pkg_info, dist = self.create_dist() cmd = check(dist) @@ -71,8 +71,9 @@ msgs = cmd._check_rst_data(rest) self.assertEqual(len(msgs), 0) - @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils") def test_check_restructuredtext(self): + if not HAS_DOCUTILS: # won't test without docutils + return # let's see if it detects broken rest in long_description broken_rest = 'title\n===\n\ntest' pkg_info, dist = self.create_dist(long_description=broken_rest) @@ -93,36 +94,6 @@ cmd = self._run(metadata, strict=1, restructuredtext=1) self.assertEqual(cmd._warnings, 0) - @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils") - def test_check_restructuredtext_with_syntax_highlight(self): - # Don't fail if there is a `code` or `code-block` directive - - example_rst_docs = [] - example_rst_docs.append(textwrap.dedent("""\ - Here's some code: - - .. code:: python - - def foo(): - pass - """)) - example_rst_docs.append(textwrap.dedent("""\ - Here's some code: - - .. code-block:: python - - def foo(): - pass - """)) - - for rest_with_code in example_rst_docs: - pkg_info, dist = self.create_dist(long_description=rest_with_code) - cmd = check(dist) - cmd.check_restructuredtext() - self.assertEqual(cmd._warnings, 0) - msgs = cmd._check_rst_data(rest_with_code) - self.assertEqual(len(msgs), 0) - def test_check_all(self): metadata = {'url': 'xxx', 'author': 'xxx'} diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_clean.py --- a/Lib/distutils/tests/test_clean.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_clean.py Mon Jan 25 17:05:13 2016 +0100 @@ -36,7 +36,7 @@ # make sure the files where removed for name, path in dirs: - self.assertFalse(os.path.exists(path), + self.assertTrue(not os.path.exists(path), '%s was not removed' % path) # let's run the command again (should spit warnings but succeed) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_cmd.py --- a/Lib/distutils/tests/test_cmd.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_cmd.py Mon Jan 25 17:05:13 2016 +0100 @@ -34,18 +34,6 @@ self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, 'not_string_list2') - cmd.option1 = 'ok,dok' - cmd.ensure_string_list('option1') - self.assertEqual(cmd.option1, ['ok', 'dok']) - - cmd.option2 = ['xxx', 'www'] - cmd.ensure_string_list('option2') - - cmd.option3 = ['ok', 2] - self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, - 'option3') - - def test_make_file(self): cmd = self.cmd @@ -89,6 +77,19 @@ cmd.option3 = 1 self.assertRaises(DistutilsOptionError, cmd.ensure_string, 'option3') + def test_ensure_string_list(self): + cmd = self.cmd + cmd.option1 = 'ok,dok' + cmd.ensure_string_list('option1') + self.assertEqual(cmd.option1, ['ok', 'dok']) + + cmd.option2 = ['xxx', 'www'] + cmd.ensure_string_list('option2') + + cmd.option3 = ['ok', 2] + self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, + 'option3') + def test_ensure_filename(self): cmd = self.cmd cmd.option1 = __file__ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_config.py --- a/Lib/distutils/tests/test_config.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_config.py Mon Jan 25 17:05:13 2016 +0100 @@ -87,7 +87,7 @@ config = list(sorted(config.items())) waited = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'https://pypi.python.org/pypi'), + ('repository', 'http://pypi.python.org/pypi'), ('server', 'server1'), ('username', 'me')] self.assertEqual(config, waited) @@ -96,14 +96,14 @@ config = cmd._read_pypirc() config = list(sorted(config.items())) waited = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'https://pypi.python.org/pypi'), + ('repository', 'http://pypi.python.org/pypi'), ('server', 'server-login'), ('username', 'tarek')] self.assertEqual(config, waited) def test_server_empty_registration(self): cmd = self._cmd(self.dist) rc = cmd._get_rc_file() - self.assertFalse(os.path.exists(rc)) + self.assertTrue(not os.path.exists(rc)) cmd._store_pypirc('tarek', 'xxx') self.assertTrue(os.path.exists(rc)) f = open(rc) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_config_cmd.py --- a/Lib/distutils/tests/test_config_cmd.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_config_cmd.py Mon Jan 25 17:05:13 2016 +0100 @@ -37,8 +37,9 @@ dump_file(this_file, 'I am the header') self.assertEqual(len(self._logs), numlines+1) - @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_search_cpp(self): + if sys.platform == 'win32': + return pkg_dir, dist = self.create_dist() cmd = config(dist) @@ -80,7 +81,7 @@ cmd._clean(f1, f2) for f in (f1, f2): - self.assertFalse(os.path.exists(f)) + self.assertTrue(not os.path.exists(f)) def test_suite(): return unittest.makeSuite(ConfigTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_core.py --- a/Lib/distutils/tests/test_core.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_core.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,7 +9,6 @@ from test.support import captured_stdout, run_unittest import unittest from distutils.tests import support -from distutils import log # setup script that uses __file__ setup_using___file__ = """\ @@ -29,21 +28,6 @@ setup() """ -setup_does_nothing = """\ -from distutils.core import setup -setup() -""" - - -setup_defines_subclass = """\ -from distutils.core import setup -from distutils.command.install import install as _install - -class install(_install): - sub_commands = _install.sub_commands + ['cmd'] - -setup(cmdclass={'install': install}) -""" class CoreTestCase(support.EnvironGuard, unittest.TestCase): @@ -52,7 +36,6 @@ self.old_stdout = sys.stdout self.cleanup_testfn() self.old_argv = sys.argv, sys.argv[:] - self.addCleanup(log.set_threshold, log._global_log.threshold) def tearDown(self): sys.stdout = self.old_stdout @@ -82,21 +65,6 @@ distutils.core.run_setup( self.write_setup(setup_using___file__)) - def test_run_setup_preserves_sys_argv(self): - # Make sure run_setup does not clobber sys.argv - argv_copy = sys.argv.copy() - distutils.core.run_setup( - self.write_setup(setup_does_nothing)) - self.assertEqual(sys.argv, argv_copy) - - def test_run_setup_defines_subclass(self): - # Make sure the script can use __file__; if that's missing, the test - # setup.py script will raise NameError. - dist = distutils.core.run_setup( - self.write_setup(setup_defines_subclass)) - install = dist.get_command_obj('install') - self.assertIn('cmd', install.sub_commands) - def test_run_setup_uses_current_dir(self): # This tests that the setup script is run with the current directory # as its own current directory; this was temporarily broken by a diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_dir_util.py --- a/Lib/distutils/tests/test_dir_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_dir_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,10 +2,9 @@ import unittest import os import stat +import shutil import sys -from unittest.mock import patch -from distutils import dir_util, errors from distutils.dir_util import (mkpath, remove_tree, create_tree, copy_tree, ensure_relative) @@ -13,7 +12,6 @@ from distutils.tests import support from test.support import run_unittest - class DirUtilTestCase(support.TempdirManager, unittest.TestCase): def _log(self, msg, *args): @@ -54,7 +52,7 @@ self.assertEqual(self._logs, wanted) @unittest.skipIf(sys.platform.startswith('win'), - "This test is only appropriate for POSIX-like systems.") + "This test is only appropriate for POSIX-like systems.") def test_mkpath_with_custom_mode(self): # Get and set the current umask value for testing mode bits. umask = os.umask(0o002) @@ -78,6 +76,7 @@ remove_tree(self.root_target, verbose=0) + def test_copy_tree_verbosity(self): mkpath(self.target, verbose=0) @@ -89,8 +88,11 @@ mkpath(self.target, verbose=0) a_file = os.path.join(self.target, 'ok.txt') - with open(a_file, 'w') as f: + f = open(a_file, 'w') + try: f.write('some content') + finally: + f.close() wanted = ['copying %s -> %s' % (a_file, self.target2)] copy_tree(self.target, self.target2, verbose=1) @@ -99,21 +101,6 @@ remove_tree(self.root_target, verbose=0) remove_tree(self.target2, verbose=0) - def test_copy_tree_skips_nfs_temp_files(self): - mkpath(self.target, verbose=0) - - a_file = os.path.join(self.target, 'ok.txt') - nfs_file = os.path.join(self.target, '.nfs123abc') - for f in a_file, nfs_file: - with open(f, 'w') as fh: - fh.write('some content') - - copy_tree(self.target, self.target2) - self.assertEqual(os.listdir(self.target2), ['ok.txt']) - - remove_tree(self.root_target, verbose=0) - remove_tree(self.target2, verbose=0) - def test_ensure_relative(self): if os.sep == '/': self.assertEqual(ensure_relative('/home/foo'), 'home/foo') @@ -122,16 +109,6 @@ self.assertEqual(ensure_relative('c:\\home\\foo'), 'c:home\\foo') self.assertEqual(ensure_relative('home\\foo'), 'home\\foo') - def test_copy_tree_exception_in_listdir(self): - """ - An exception in listdir should raise a DistutilsFileError - """ - with patch("os.listdir", side_effect=OSError()), \ - self.assertRaises(errors.DistutilsFileError): - src = self.tempdirs[-1] - dir_util.copy_tree(src, None) - - def test_suite(): return unittest.makeSuite(DirUtilTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_dist.py --- a/Lib/distutils/tests/test_dist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,14 +6,11 @@ import warnings import textwrap -from unittest import mock - -from distutils.dist import Distribution, fix_help_options, DistributionMetadata +from distutils.dist import Distribution, fix_help_options from distutils.cmd import Command from test.support import TESTFN, captured_stdout, run_unittest from distutils.tests import support -from distutils import log class test_dist(Command): @@ -21,7 +18,7 @@ user_options = [ ("sample-option=", "S", "help text"), - ] + ] def initialize_options(self): self.sample_option = None @@ -40,7 +37,6 @@ class DistributionTestCase(support.LoggingSilencer, - support.TempdirManager, support.EnvironGuard, unittest.TestCase): @@ -81,64 +77,6 @@ self.assertIsInstance(cmd, test_dist) self.assertEqual(cmd.sample_option, "sometext") - def test_venv_install_options(self): - sys.argv.append("install") - self.addCleanup(os.unlink, TESTFN) - - fakepath = '/somedir' - - with open(TESTFN, "w") as f: - print(("[install]\n" - "install-base = {0}\n" - "install-platbase = {0}\n" - "install-lib = {0}\n" - "install-platlib = {0}\n" - "install-purelib = {0}\n" - "install-headers = {0}\n" - "install-scripts = {0}\n" - "install-data = {0}\n" - "prefix = {0}\n" - "exec-prefix = {0}\n" - "home = {0}\n" - "user = {0}\n" - "root = {0}").format(fakepath), file=f) - - # Base case: Not in a Virtual Environment - with mock.patch.multiple(sys, prefix='/a', base_prefix='/a') as values: - d = self.create_distribution([TESTFN]) - - option_tuple = (TESTFN, fakepath) - - result_dict = { - 'install_base': option_tuple, - 'install_platbase': option_tuple, - 'install_lib': option_tuple, - 'install_platlib': option_tuple, - 'install_purelib': option_tuple, - 'install_headers': option_tuple, - 'install_scripts': option_tuple, - 'install_data': option_tuple, - 'prefix': option_tuple, - 'exec_prefix': option_tuple, - 'home': option_tuple, - 'user': option_tuple, - 'root': option_tuple, - } - - self.assertEqual( - sorted(d.command_options.get('install').keys()), - sorted(result_dict.keys())) - - for (key, value) in d.command_options.get('install').items(): - self.assertEqual(value, result_dict[key]) - - # Test case: In a Virtual Environment - with mock.patch.multiple(sys, prefix='/a', base_prefix='/b') as values: - d = self.create_distribution([TESTFN]) - - for key in result_dict.keys(): - self.assertNotIn(key, d.command_options.get('install', {})) - def test_command_packages_configfile(self): sys.argv.append("build") self.addCleanup(os.unlink, TESTFN) @@ -215,34 +153,6 @@ self.assertRaises(ValueError, dist.announce, args, kwargs) - def test_find_config_files_disable(self): - # Ticket #1180: Allow user to disable their home config file. - temp_home = self.mkdtemp() - if os.name == 'posix': - user_filename = os.path.join(temp_home, ".pydistutils.cfg") - else: - user_filename = os.path.join(temp_home, "pydistutils.cfg") - - with open(user_filename, 'w') as f: - f.write('[distutils]\n') - - def _expander(path): - return temp_home - - old_expander = os.path.expanduser - os.path.expanduser = _expander - try: - d = Distribution() - all_files = d.find_config_files() - - d = Distribution(attrs={'script_args': ['--no-user-cfg']}) - files = d.find_config_files() - finally: - os.path.expanduser = old_expander - - # make sure --no-user-cfg disables the user cfg file - self.assertEqual(len(all_files)-1, len(files)) - class MetadataTestCase(support.TempdirManager, support.EnvironGuard, unittest.TestCase): @@ -394,7 +304,7 @@ os.environ['HOME'] = temp_dir files = dist.find_config_files() self.assertIn(user_filename, files, - '%r not found in %r' % (user_filename, files)) + '%r not found in %r' % (user_filename, files)) finally: os.remove(user_filename) @@ -406,7 +316,6 @@ def test_show_help(self): # smoke test, just makes sure some help is displayed - self.addCleanup(log.set_threshold, log._global_log.threshold) dist = Distribution() sys.argv = [] dist.help = 1 @@ -419,33 +328,6 @@ self.assertTrue(output) - def test_read_metadata(self): - attrs = {"name": "package", - "version": "1.0", - "long_description": "desc", - "description": "xxx", - "download_url": "http://example.com", - "keywords": ['one', 'two'], - "requires": ['foo']} - - dist = Distribution(attrs) - metadata = dist.metadata - - # write it then reloads it - PKG_INFO = io.StringIO() - metadata.write_pkg_file(PKG_INFO) - PKG_INFO.seek(0) - metadata.read_pkg_file(PKG_INFO) - - self.assertEqual(metadata.name, "package") - self.assertEqual(metadata.version, "1.0") - self.assertEqual(metadata.description, "xxx") - self.assertEqual(metadata.download_url, 'http://example.com') - self.assertEqual(metadata.keywords, ['one', 'two']) - self.assertEqual(metadata.platforms, ['UNKNOWN']) - self.assertEqual(metadata.obsoletes, None) - self.assertEqual(metadata.requires, ['foo']) - def test_suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(DistributionTestCase)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_file_util.py --- a/Lib/distutils/tests/test_file_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_file_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,13 +2,10 @@ import unittest import os import shutil -import errno -from unittest.mock import patch -from distutils.file_util import move_file, copy_file +from distutils.file_util import move_file from distutils import log from distutils.tests import support -from distutils.errors import DistutilsFileError from test.support import run_unittest class FileUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -61,52 +58,6 @@ wanted = ['moving %s -> %s' % (self.source, self.target_dir)] self.assertEqual(self._logs, wanted) - def test_move_file_exception_unpacking_rename(self): - # see issue 22182 - with patch("os.rename", side_effect=OSError("wrong", 1)), \ - self.assertRaises(DistutilsFileError): - with open(self.source, 'w') as fobj: - fobj.write('spam eggs') - move_file(self.source, self.target, verbose=0) - - def test_move_file_exception_unpacking_unlink(self): - # see issue 22182 - with patch("os.rename", side_effect=OSError(errno.EXDEV, "wrong")), \ - patch("os.unlink", side_effect=OSError("wrong", 1)), \ - self.assertRaises(DistutilsFileError): - with open(self.source, 'w') as fobj: - fobj.write('spam eggs') - move_file(self.source, self.target, verbose=0) - - def test_copy_file_hard_link(self): - with open(self.source, 'w') as f: - f.write('some content') - st = os.stat(self.source) - copy_file(self.source, self.target, link='hard') - st2 = os.stat(self.source) - st3 = os.stat(self.target) - self.assertTrue(os.path.samestat(st, st2), (st, st2)) - self.assertTrue(os.path.samestat(st2, st3), (st2, st3)) - with open(self.source, 'r') as f: - self.assertEqual(f.read(), 'some content') - - def test_copy_file_hard_link_failure(self): - # If hard linking fails, copy_file() falls back on copying file - # (some special filesystems don't support hard linking even under - # Unix, see issue #8876). - with open(self.source, 'w') as f: - f.write('some content') - st = os.stat(self.source) - with patch("os.link", side_effect=OSError(0, "linking unsupported")): - copy_file(self.source, self.target, link='hard') - st2 = os.stat(self.source) - st3 = os.stat(self.target) - self.assertTrue(os.path.samestat(st, st2), (st, st2)) - self.assertFalse(os.path.samestat(st2, st3), (st2, st3)) - for fn in (self.source, self.target): - with open(fn, 'r') as f: - self.assertEqual(f.read(), 'some content') - def test_suite(): return unittest.makeSuite(FileUtilTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_install.py --- a/Lib/distutils/tests/test_install.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_install.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Tests for distutils.command.install.""" import os +import imp import sys import unittest import site @@ -20,7 +21,9 @@ def _make_ext_name(modname): - return modname + sysconfig.get_config_var('EXT_SUFFIX') + if os.name == 'nt' and sys.executable.endswith('_d.exe'): + modname += '_d' + return modname + sysconfig.get_config_var('SO') class InstallTestCase(support.TempdirManager, @@ -91,7 +94,7 @@ self.addCleanup(cleanup) - for key in ('nt_user', 'unix_user'): + for key in ('nt_user', 'unix_user', 'os2_home'): self.assertIn(key, INSTALL_SCHEMES) dist = Distribution({'name': 'xx'}) @@ -162,7 +165,7 @@ cmd.home = 'home' self.assertRaises(DistutilsOptionError, cmd.finalize_options) - # can't combine user with prefix/exec_prefix/home or + # can't combine user with with prefix/exec_prefix/home or # install_(plat)base cmd.prefix = None cmd.user = 'user' @@ -190,8 +193,7 @@ f.close() found = [os.path.basename(line) for line in content.splitlines()] - expected = ['hello.py', 'hello.%s.pyc' % sys.implementation.cache_tag, - 'sayhi', + expected = ['hello.py', 'hello.%s.pyc' % imp.get_tag(), 'sayhi', 'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]] self.assertEqual(found, expected) @@ -234,7 +236,7 @@ self.test_record() finally: install_module.DEBUG = False - self.assertGreater(len(self.logs), old_logs_len) + self.assertTrue(len(self.logs) > old_logs_len) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_install_lib.py --- a/Lib/distutils/tests/test_install_lib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_install_lib.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,7 @@ """Tests for distutils.command.install_data.""" import sys import os -import importlib.util +import imp import unittest from distutils.command.install_lib import install_lib @@ -44,11 +44,10 @@ f = os.path.join(project_dir, 'foo.py') self.write_file(f, '# python file') cmd.byte_compile([f]) - pyc_file = importlib.util.cache_from_source('foo.py', optimization='') - pyc_opt_file = importlib.util.cache_from_source('foo.py', - optimization=cmd.optimize) + pyc_file = imp.cache_from_source('foo.py', debug_override=True) + pyo_file = imp.cache_from_source('foo.py', debug_override=False) self.assertTrue(os.path.exists(pyc_file)) - self.assertTrue(os.path.exists(pyc_opt_file)) + self.assertTrue(os.path.exists(pyo_file)) def test_get_outputs(self): project_dir, dist = self.create_dist() @@ -65,8 +64,8 @@ cmd.distribution.packages = ['spam'] cmd.distribution.script_name = 'setup.py' - # get_outputs should return 4 elements: spam/__init__.py and .pyc, - # foo.import-tag-abiflags.so / foo.pyd + # get_outputs should return 4 elements: spam/__init__.py, .pyc and + # .pyo, foo.import-tag-abiflags.so / foo.pyd outputs = cmd.get_outputs() self.assertEqual(len(outputs), 4, outputs) @@ -104,7 +103,7 @@ finally: sys.dont_write_bytecode = old_dont_write_bytecode - self.assertIn('byte-compiling is disabled', self.logs[0][1]) + self.assertTrue('byte-compiling is disabled' in self.logs[0][1]) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_install_scripts.py --- a/Lib/distutils/tests/test_install_scripts.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_install_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -24,10 +24,10 @@ skip_build=1, ) cmd = install_scripts(dist) - self.assertFalse(cmd.force) - self.assertFalse(cmd.skip_build) - self.assertIsNone(cmd.build_dir) - self.assertIsNone(cmd.install_dir) + self.assertTrue(not cmd.force) + self.assertTrue(not cmd.skip_build) + self.assertTrue(cmd.build_dir is None) + self.assertTrue(cmd.install_dir is None) cmd.finalize_options() @@ -72,7 +72,7 @@ installed = os.listdir(target) for name in expected: - self.assertIn(name, installed) + self.assertTrue(name in installed) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_log.py --- a/Lib/distutils/tests/test_log.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_log.py Mon Jan 25 17:05:13 2016 +0100 @@ -14,8 +14,8 @@ # error handler) old_stdout = sys.stdout old_stderr = sys.stderr - old_threshold = log.set_threshold(log.DEBUG) try: + log.set_threshold(log.DEBUG) with NamedTemporaryFile(mode="w+", encoding='ascii') as stdout, \ NamedTemporaryFile(mode="w+", encoding='ascii') as stderr: sys.stdout = stdout @@ -27,7 +27,6 @@ stderr.seek(0) self.assertEqual(stderr.read().rstrip(), "fatal:\\xe9") finally: - log.set_threshold(old_threshold) sys.stdout = old_stdout sys.stderr = old_stderr diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_msvc9compiler.py --- a/Lib/distutils/tests/test_msvc9compiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_msvc9compiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -104,7 +104,7 @@ unittest.TestCase): def test_no_compiler(self): - # makes sure query_vcvarsall raises + # makes sure query_vcvarsall throws # a DistutilsPlatformError if the compiler # is not found from distutils.msvc9compiler import query_vcvarsall @@ -128,7 +128,7 @@ # windows registeries versions. path = r'Control Panel\Desktop' v = Reg.get_value(path, 'dragfullwindows') - self.assertIn(v, ('0', '1', '2')) + self.assertTrue(v in ('0', '1', '2')) import winreg HKCU = winreg.HKEY_CURRENT_USER @@ -136,7 +136,7 @@ self.assertEqual(keys, None) keys = Reg.read_keys(HKCU, r'Control Panel') - self.assertIn('Desktop', keys) + self.assertTrue('Desktop' in keys) def test_remove_visual_c_ref(self): from distutils.msvc9compiler import MSVCCompiler @@ -174,7 +174,7 @@ compiler = MSVCCompiler() got = compiler._remove_visual_c_ref(manifest) - self.assertIsNone(got) + self.assertIs(got, None) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_msvccompiler.py --- a/Lib/distutils/tests/test_msvccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,90 +0,0 @@ -"""Tests for distutils._msvccompiler.""" -import sys -import unittest -import os - -from distutils.errors import DistutilsPlatformError -from distutils.tests import support -from test.support import run_unittest - - -SKIP_MESSAGE = (None if sys.platform == "win32" else - "These tests are only for win32") - -@unittest.skipUnless(SKIP_MESSAGE is None, SKIP_MESSAGE) -class msvccompilerTestCase(support.TempdirManager, - unittest.TestCase): - - def test_no_compiler(self): - import distutils._msvccompiler as _msvccompiler - # makes sure query_vcvarsall raises - # a DistutilsPlatformError if the compiler - # is not found - def _find_vcvarsall(plat_spec): - return None, None - - old_find_vcvarsall = _msvccompiler._find_vcvarsall - _msvccompiler._find_vcvarsall = _find_vcvarsall - try: - self.assertRaises(DistutilsPlatformError, - _msvccompiler._get_vc_env, - 'wont find this version') - finally: - _msvccompiler._find_vcvarsall = old_find_vcvarsall - - def test_compiler_options(self): - import distutils._msvccompiler as _msvccompiler - # suppress path to vcruntime from _find_vcvarsall to - # check that /MT is added to compile options - old_find_vcvarsall = _msvccompiler._find_vcvarsall - def _find_vcvarsall(plat_spec): - return old_find_vcvarsall(plat_spec)[0], None - _msvccompiler._find_vcvarsall = _find_vcvarsall - try: - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - - self.assertIn('/MT', compiler.compile_options) - self.assertNotIn('/MD', compiler.compile_options) - finally: - _msvccompiler._find_vcvarsall = old_find_vcvarsall - - def test_vcruntime_copy(self): - import distutils._msvccompiler as _msvccompiler - # force path to a known file - it doesn't matter - # what we copy as long as its name is not in - # _msvccompiler._BUNDLED_DLLS - old_find_vcvarsall = _msvccompiler._find_vcvarsall - def _find_vcvarsall(plat_spec): - return old_find_vcvarsall(plat_spec)[0], __file__ - _msvccompiler._find_vcvarsall = _find_vcvarsall - try: - tempdir = self.mkdtemp() - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - compiler._copy_vcruntime(tempdir) - - self.assertTrue(os.path.isfile(os.path.join( - tempdir, os.path.basename(__file__)))) - finally: - _msvccompiler._find_vcvarsall = old_find_vcvarsall - - def test_vcruntime_skip_copy(self): - import distutils._msvccompiler as _msvccompiler - - tempdir = self.mkdtemp() - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - dll = compiler._vcruntime_redist - self.assertTrue(os.path.isfile(dll)) - - compiler._copy_vcruntime(tempdir) - - self.assertFalse(os.path.isfile(os.path.join( - tempdir, os.path.basename(dll)))) - -def test_suite(): - return unittest.makeSuite(msvccompilerTestCase) - -if __name__ == "__main__": - run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_register.py --- a/Lib/distutils/tests/test_register.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_register.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,5 @@ """Tests for distutils.command.register.""" +import sys import os import unittest import getpass @@ -9,15 +10,11 @@ from distutils.command import register as register_module from distutils.command.register import register +from distutils.core import Distribution from distutils.errors import DistutilsSetupError -from distutils.log import INFO -from distutils.tests.test_config import PyPIRCCommandTestCase - -try: - import docutils -except ImportError: - docutils = None +from distutils.tests import support +from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase PYPIRC_NOPASSWORD = """\ [distutils] @@ -59,18 +56,12 @@ def __call__(self, *args): return self - def open(self, req, data=None, timeout=None): + def open(self, req): self.reqs.append(req) return self def read(self): - return b'xxx' - - def getheader(self, name, default=None): - return { - 'content-type': 'text/plain; charset=utf-8', - }.get(name.lower(), default) - + return 'xxx' class RegisterTestCase(PyPIRCCommandTestCase): @@ -81,13 +72,11 @@ def _getpass(prompt): return 'password' getpass.getpass = _getpass - urllib.request._opener = None self.old_opener = urllib.request.build_opener self.conn = urllib.request.build_opener = FakeOpener() def tearDown(self): getpass.getpass = self._old_getpass - urllib.request._opener = None urllib.request.build_opener = self.old_opener super(RegisterTestCase, self).tearDown() @@ -107,7 +96,7 @@ cmd = self._get_cmd() # we shouldn't have a .pypirc file yet - self.assertFalse(os.path.exists(self.rc)) + self.assertTrue(not os.path.exists(self.rc)) # patching input and getpass.getpass # so register gets happy @@ -154,7 +143,7 @@ self.assertEqual(req1['Content-length'], '1374') self.assertEqual(req2['Content-length'], '1374') - self.assertIn(b'xxx', self.conn.reqs[1].data) + self.assertTrue((b'xxx') in self.conn.reqs[1].data) def test_password_not_in_file(self): @@ -184,7 +173,7 @@ req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '608') - self.assertIn(b'tarek', req.data) + self.assertTrue((b'tarek') in req.data) def test_password_reset(self): # this test runs choice 3 @@ -202,9 +191,8 @@ req = self.conn.reqs[0] headers = dict(req.headers) self.assertEqual(headers['Content-length'], '290') - self.assertIn(b'tarek', req.data) + self.assertTrue((b'tarek') in req.data) - @unittest.skipUnless(docutils is not None, 'needs docutils') def test_strict(self): # testing the script option # when on, the register command stops if @@ -217,6 +205,13 @@ cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) + # we don't test the reSt feature if docutils + # is not installed + try: + import docutils + except ImportError: + return + # metadata are OK but long_description is broken metadata = {'url': 'xxx', 'author': 'xxx', 'author_email': 'éxéxé', @@ -270,22 +265,6 @@ finally: del register_module.input - @unittest.skipUnless(docutils is not None, 'needs docutils') - def test_register_invalid_long_description(self): - description = ':funkie:`str`' # mimic Sphinx-specific markup - metadata = {'url': 'xxx', 'author': 'xxx', - 'author_email': 'xxx', - 'name': 'xxx', 'version': 'xxx', - 'long_description': description} - cmd = self._get_cmd(metadata) - cmd.ensure_finalized() - cmd.strict = True - inputs = Inputs('2', 'tarek', 'tarek@ziade.org') - register_module.input = inputs - self.addCleanup(delattr, register_module, 'input') - - self.assertRaises(DistutilsSetupError, cmd.run) - def test_check_metadata_deprecated(self): # makes sure make_metadata is deprecated cmd = self._get_cmd() @@ -294,14 +273,6 @@ cmd.check_metadata() self.assertEqual(len(w.warnings), 1) - def test_list_classifiers(self): - cmd = self._get_cmd() - cmd.list_classifiers = 1 - cmd.run() - results = self.get_logs(INFO) - self.assertEqual(results, ['running check', 'xxx']) - - def test_suite(): return unittest.makeSuite(RegisterTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_sdist.py --- a/Lib/distutils/tests/test_sdist.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_sdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -14,12 +14,6 @@ except ImportError: ZLIB_SUPPORT = False -try: - import grp - import pwd - UID_GID_SUPPORT = True -except ImportError: - UID_GID_SUPPORT = False from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution @@ -89,8 +83,9 @@ @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') def test_prune_file_list(self): - # this test creates a project with some VCS dirs and an NFS rename - # file, then launches sdist to check they get pruned on all systems + # this test creates a package with some vcs dirs in it + # and launch sdist to make sure they get pruned + # on all systems # creating VCS directories with some files in them os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) @@ -104,8 +99,6 @@ self.write_file((self.tmp_dir, 'somecode', '.git', 'ok'), 'xxx') - self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx') - # now building a sdist dist, cmd = self.get_cmd() @@ -131,11 +124,13 @@ self.assertEqual(len(content), 4) @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run') - @unittest.skipIf(find_executable('tar') is None, - "The tar command is not found") - @unittest.skipIf(find_executable('gzip') is None, - "The gzip command is not found") def test_make_distribution(self): + + # check if tar and gzip are installed + if (find_executable('tar') is None or + find_executable('gzip') is None): + return + # now building a sdist dist, cmd = self.get_cmd() @@ -429,54 +424,6 @@ self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO', 'fake-1.0/README.manual']) - @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") - @unittest.skipIf(find_executable('tar') is None, - "The tar command is not found") - @unittest.skipIf(find_executable('gzip') is None, - "The gzip command is not found") - def test_make_distribution_owner_group(self): - # now building a sdist - dist, cmd = self.get_cmd() - - # creating a gztar and specifying the owner+group - cmd.formats = ['gztar'] - cmd.owner = pwd.getpwuid(0)[0] - cmd.group = grp.getgrgid(0)[0] - cmd.ensure_finalized() - cmd.run() - - # making sure we have the good rights - archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') - archive = tarfile.open(archive_name) - try: - for member in archive.getmembers(): - self.assertEqual(member.uid, 0) - self.assertEqual(member.gid, 0) - finally: - archive.close() - - # building a sdist again - dist, cmd = self.get_cmd() - - # creating a gztar - cmd.formats = ['gztar'] - cmd.ensure_finalized() - cmd.run() - - # making sure we have the good rights - archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') - archive = tarfile.open(archive_name) - - # note that we are not testing the group ownership here - # because, depending on the platforms and the container - # rights (see #7408) - try: - for member in archive.getmembers(): - self.assertEqual(member.uid, os.getuid()) - finally: - archive.close() - def test_suite(): return unittest.makeSuite(SDistTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_sysconfig.py --- a/Lib/distutils/tests/test_sysconfig.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_sysconfig.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,17 +1,16 @@ """Tests for distutils.sysconfig.""" import os import shutil -import subprocess -import sys -import textwrap +import test import unittest from distutils import sysconfig from distutils.ccompiler import get_default_compiler from distutils.tests import support -from test.support import TESTFN, run_unittest, check_warnings +from test.support import TESTFN, run_unittest -class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): +class SysconfigTestCase(support.EnvironGuard, + unittest.TestCase): def setUp(self): super(SysconfigTestCase, self).setUp() self.makefile = None @@ -33,6 +32,7 @@ self.assertTrue(os.path.isfile(config_h), config_h) def test_get_python_lib(self): + lib_dir = sysconfig.get_python_lib() # XXX doesn't work on Linux when Python was never installed before #self.assertTrue(os.path.isdir(lib_dir), lib_dir) # test for pythonxx.lib? @@ -50,42 +50,15 @@ def test_get_config_vars(self): cvars = sysconfig.get_config_vars() - self.assertIsInstance(cvars, dict) + self.assertTrue(isinstance(cvars, dict)) self.assertTrue(cvars) - def test_srcdir(self): - # See Issues #15322, #15364. - srcdir = sysconfig.get_config_var('srcdir') + def test_customize_compiler(self): - self.assertTrue(os.path.isabs(srcdir), srcdir) - self.assertTrue(os.path.isdir(srcdir), srcdir) + # not testing if default compiler is not unix + if get_default_compiler() != 'unix': + return - if sysconfig.python_build: - # The python executable has not been installed so srcdir - # should be a full source checkout. - Python_h = os.path.join(srcdir, 'Include', 'Python.h') - self.assertTrue(os.path.exists(Python_h), Python_h) - self.assertTrue(sysconfig._is_python_source_dir(srcdir)) - elif os.name == 'posix': - self.assertEqual( - os.path.dirname(sysconfig.get_makefile_filename()), - srcdir) - - def test_srcdir_independent_of_cwd(self): - # srcdir should be independent of the current working directory - # See Issues #15322, #15364. - srcdir = sysconfig.get_config_var('srcdir') - cwd = os.getcwd() - try: - os.chdir('..') - srcdir2 = sysconfig.get_config_var('srcdir') - finally: - os.chdir(cwd) - self.assertEqual(srcdir, srcdir2) - - @unittest.skipUnless(get_default_compiler() == 'unix', - 'not testing if default compiler is not unix') - def test_customize_compiler(self): os.environ['AR'] = 'my_ar' os.environ['ARFLAGS'] = '-arflags' @@ -127,74 +100,11 @@ def test_sysconfig_module(self): import sysconfig as global_sysconfig - self.assertEqual(global_sysconfig.get_config_var('CFLAGS'), - sysconfig.get_config_var('CFLAGS')) - self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'), - sysconfig.get_config_var('LDFLAGS')) + self.assertEqual(global_sysconfig.get_config_var('CFLAGS'), sysconfig.get_config_var('CFLAGS')) + self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'), sysconfig.get_config_var('LDFLAGS')) + self.assertEqual(global_sysconfig.get_config_var('LDSHARED'),sysconfig.get_config_var('LDSHARED')) + self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC')) - @unittest.skipIf(sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'), - 'compiler flags customized') - def test_sysconfig_compiler_vars(self): - # On OS X, binary installers support extension module building on - # various levels of the operating system with differing Xcode - # configurations. This requires customization of some of the - # compiler configuration directives to suit the environment on - # the installed machine. Some of these customizations may require - # running external programs and, so, are deferred until needed by - # the first extension module build. With Python 3.3, only - # the Distutils version of sysconfig is used for extension module - # builds, which happens earlier in the Distutils tests. This may - # cause the following tests to fail since no tests have caused - # the global version of sysconfig to call the customization yet. - # The solution for now is to simply skip this test in this case. - # The longer-term solution is to only have one version of sysconfig. - - import sysconfig as global_sysconfig - if sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'): - self.skipTest('compiler flags customized') - self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), - sysconfig.get_config_var('LDSHARED')) - self.assertEqual(global_sysconfig.get_config_var('CC'), - sysconfig.get_config_var('CC')) - - @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, - 'EXT_SUFFIX required for this test') - def test_SO_deprecation(self): - self.assertWarns(DeprecationWarning, - sysconfig.get_config_var, 'SO') - - @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, - 'EXT_SUFFIX required for this test') - def test_SO_value(self): - with check_warnings(('', DeprecationWarning)): - self.assertEqual(sysconfig.get_config_var('SO'), - sysconfig.get_config_var('EXT_SUFFIX')) - - @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, - 'EXT_SUFFIX required for this test') - def test_SO_in_vars(self): - vars = sysconfig.get_config_vars() - self.assertIsNotNone(vars['SO']) - self.assertEqual(vars['SO'], vars['EXT_SUFFIX']) - - def test_customize_compiler_before_get_config_vars(self): - # Issue #21923: test that a Distribution compiler - # instance can be called without an explicit call to - # get_config_vars(). - with open(TESTFN, 'w') as f: - f.writelines(textwrap.dedent('''\ - from distutils.core import Distribution - config = Distribution().get_command_obj('config') - # try_compile may pass or it may fail if no compiler - # is found but it should not raise an exception. - rc = config.try_compile('int x;') - ''')) - p = subprocess.Popen([str(sys.executable), TESTFN], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - outs, errs = p.communicate() - self.assertEqual(0, p.returncode, "Subprocess failed: " + outs) def test_suite(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_unixccompiler.py --- a/Lib/distutils/tests/test_unixccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_unixccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,8 +1,7 @@ """Tests for distutils.unixccompiler.""" -import os import sys import unittest -from test.support import EnvironmentVarGuard, run_unittest +from test.support import run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -21,8 +20,12 @@ sys.platform = self._backup_platform sysconfig.get_config_var = self._backup_get_config_var - @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_runtime_libdir_option(self): + + # not tested under windows + if sys.platform == 'win32': + return + # Issue#5900 # # Ensure RUNPATH is added to extension modules with RPATH if @@ -91,6 +94,7 @@ sysconfig.get_config_var = gcv self.assertEqual(self.cc.rpath_foo(), '-Wl,--enable-new-dtags,-R/foo') + # non-GCC GNULD sys.platform = 'bar' def gcv(v): @@ -111,38 +115,6 @@ sysconfig.get_config_var = gcv self.assertEqual(self.cc.rpath_foo(), '-R/foo') - @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') - def test_osx_cc_overrides_ldshared(self): - # Issue #18080: - # ensure that setting CC env variable also changes default linker - def gcv(v): - if v == 'LDSHARED': - return 'gcc-4.2 -bundle -undefined dynamic_lookup ' - return 'gcc-4.2' - sysconfig.get_config_var = gcv - with EnvironmentVarGuard() as env: - env['CC'] = 'my_cc' - del env['LDSHARED'] - sysconfig.customize_compiler(self.cc) - self.assertEqual(self.cc.linker_so[0], 'my_cc') - - @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') - def test_osx_explict_ldshared(self): - # Issue #18080: - # ensure that setting CC env variable does not change - # explicit LDSHARED setting for linker - def gcv(v): - if v == 'LDSHARED': - return 'gcc-4.2 -bundle -undefined dynamic_lookup ' - return 'gcc-4.2' - sysconfig.get_config_var = gcv - with EnvironmentVarGuard() as env: - env['CC'] = 'my_cc' - env['LDSHARED'] = 'my_ld -bundle -dynamic' - sysconfig.customize_compiler(self.cc) - self.assertEqual(self.cc.linker_so[0], 'my_ld') - - def test_suite(): return unittest.makeSuite(UnixCCompilerTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_upload.py --- a/Lib/distutils/tests/test_upload.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_upload.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,13 +1,11 @@ """Tests for distutils.command.upload.""" import os import unittest +import http.client as httpclient from test.support import run_unittest -from distutils.command import upload as upload_mod from distutils.command.upload import upload from distutils.core import Distribution -from distutils.errors import DistutilsError -from distutils.log import INFO from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase @@ -39,48 +37,48 @@ [server1] username:me """ +class Response(object): + def __init__(self, status=200, reason='OK'): + self.status = status + self.reason = reason -class FakeOpen(object): +class FakeConnection(object): - def __init__(self, url, msg=None, code=None): - self.url = url - if not isinstance(url, str): - self.req = url - else: - self.req = None - self.msg = msg or 'OK' - self.code = code or 200 + def __init__(self): + self.requests = [] + self.headers = [] + self.body = '' - def getheader(self, name, default=None): - return { - 'content-type': 'text/plain; charset=utf-8', - }.get(name.lower(), default) + def __call__(self, netloc): + return self - def read(self): - return b'xyzzy' + def connect(self): + pass + endheaders = connect - def getcode(self): - return self.code + def putrequest(self, method, url): + self.requests.append((method, url)) + def putheader(self, name, value): + self.headers.append((name, value)) + + def send(self, body): + self.body = body + + def getresponse(self): + return Response() class uploadTestCase(PyPIRCCommandTestCase): def setUp(self): super(uploadTestCase, self).setUp() - self.old_open = upload_mod.urlopen - upload_mod.urlopen = self._urlopen - self.last_open = None - self.next_msg = None - self.next_code = None + self.old_class = httpclient.HTTPConnection + self.conn = httpclient.HTTPConnection = FakeConnection() def tearDown(self): - upload_mod.urlopen = self.old_open + httpclient.HTTPConnection = self.old_class super(uploadTestCase, self).tearDown() - def _urlopen(self, url): - self.last_open = FakeOpen(url, msg=self.next_msg, code=self.next_code) - return self.last_open - def test_finalize_options(self): # new format @@ -90,7 +88,7 @@ cmd.finalize_options() for attr, waited in (('username', 'me'), ('password', 'secret'), ('realm', 'pypi'), - ('repository', 'https://pypi.python.org/pypi')): + ('repository', 'http://pypi.python.org/pypi')): self.assertEqual(getattr(cmd, attr), waited) def test_saved_password(self): @@ -121,28 +119,17 @@ # lets run it pkg_dir, dist = self.create_dist(dist_files=dist_files) cmd = upload(dist) - cmd.show_response = 1 cmd.ensure_finalized() cmd.run() # what did we send ? - headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2161') - content_type = headers['Content-type'] - self.assertTrue(content_type.startswith('multipart/form-data')) - self.assertEqual(self.last_open.req.get_method(), 'POST') - expected_url = 'https://pypi.python.org/pypi' - self.assertEqual(self.last_open.req.get_full_url(), expected_url) - self.assertTrue(b'xxx' in self.last_open.req.data) + headers = dict(self.conn.headers) + self.assertEqual(headers['Content-length'], '2087') + self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) + self.assertFalse('\n' in headers['Authorization']) - # The PyPI response body was echoed - results = self.get_logs(INFO) - self.assertIn('xyzzy\n', results[-1]) - - def test_upload_fails(self): - self.next_msg = "Not Found" - self.next_code = 404 - self.assertRaises(DistutilsError, self.test_upload) + self.assertEqual(self.conn.requests, [('POST', '/pypi')]) + self.assertTrue((b'xxx') in self.conn.body) def test_suite(): return unittest.makeSuite(uploadTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/tests/test_util.py --- a/Lib/distutils/tests/test_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/tests/test_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,13 +8,11 @@ from distutils.errors import DistutilsPlatformError, DistutilsByteCompileError from distutils.util import (get_platform, convert_path, change_root, check_environ, split_quoted, strtobool, - rfc822_escape, byte_compile, - grok_environment_error) + rfc822_escape, byte_compile) from distutils import util # used to patch _environ_checked from distutils.sysconfig import get_config_vars from distutils import sysconfig from distutils.tests import support -import _osx_support class UtilTestCase(support.EnvironGuard, unittest.TestCase): @@ -94,7 +92,6 @@ ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' 'root:xnu-792.25.20~1/RELEASE_I386'), 'i386')) - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3' get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g ' @@ -108,7 +105,6 @@ sys.maxsize = cursize # macbook with fat binaries (fat, universal or fat64) - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.4' get_config_vars()['CFLAGS'] = ('-arch ppc -arch i386 -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' @@ -117,12 +113,10 @@ self.assertEqual(get_platform(), 'macosx-10.4-fat') - _osx_support._remove_original_values(get_config_vars()) os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.1' self.assertEqual(get_platform(), 'macosx-10.4-fat') - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch i386 -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' @@ -130,21 +124,18 @@ self.assertEqual(get_platform(), 'macosx-10.4-intel') - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch ppc -arch i386 -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' '-dynamic -DNDEBUG -g -O3') self.assertEqual(get_platform(), 'macosx-10.4-fat3') - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['CFLAGS'] = ('-arch ppc64 -arch x86_64 -arch ppc -arch i386 -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' '-dynamic -DNDEBUG -g -O3') self.assertEqual(get_platform(), 'macosx-10.4-universal') - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['CFLAGS'] = ('-arch x86_64 -arch ppc64 -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' @@ -153,7 +144,6 @@ self.assertEqual(get_platform(), 'macosx-10.4-fat64') for arch in ('ppc', 'i386', 'x86_64', 'ppc64'): - _osx_support._remove_original_values(get_config_vars()) get_config_vars()['CFLAGS'] = ('-arch %s -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' @@ -237,7 +227,7 @@ self.assertRaises(DistutilsPlatformError, change_root, 'c:\\root', 'its\\here') - # XXX platforms to be covered: mac + # XXX platforms to be covered: os2, mac def test_check_environ(self): util._environ_checked = 0 @@ -267,7 +257,7 @@ self.assertTrue(strtobool(y)) for n in no: - self.assertFalse(strtobool(n)) + self.assertTrue(not strtobool(n)) def test_rfc822_escape(self): header = 'I am a\npoor\nlonesome\nheader\n' @@ -286,13 +276,6 @@ finally: sys.dont_write_bytecode = old_dont_write_bytecode - def test_grok_environment_error(self): - # test obsolete function to ensure backward compat (#4931) - exc = IOError("Unable to find batch file") - msg = grok_environment_error(exc) - self.assertEqual(msg, "error: Unable to find batch file") - - def test_suite(): return unittest.makeSuite(UtilTestCase) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/text_file.py --- a/Lib/distutils/text_file.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/text_file.py Mon Jan 25 17:05:13 2016 +0100 @@ -118,11 +118,10 @@ def close(self): """Close the current file and forget everything we know about it (filename, current line number).""" - file = self.file + self.file.close() self.file = None self.filename = None self.current_line = None - file.close() def gen_error(self, msg, line=None): outmsg = [] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/unixccompiler.py --- a/Lib/distutils/unixccompiler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/unixccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -23,9 +23,6 @@ DistutilsExecError, CompileError, LibError, LinkError from distutils import log -if sys.platform == 'darwin': - import _osx_support - # XXX Things not currently handled: # * optimization/debug/warning flags; we just use whatever's in Python's # Makefile and live with it. Is this adequate? If not, we might @@ -41,6 +38,68 @@ # should just happily stuff them into the preprocessor/compiler/linker # options and carry on. +def _darwin_compiler_fixup(compiler_so, cc_args): + """ + This function will strip '-isysroot PATH' and '-arch ARCH' from the + compile flags if the user has specified one them in extra_compile_flags. + + This is needed because '-arch ARCH' adds another architecture to the + build, without a way to remove an architecture. Furthermore GCC will + barf if multiple '-isysroot' arguments are present. + """ + stripArch = stripSysroot = False + + compiler_so = list(compiler_so) + kernel_version = os.uname()[2] # 8.4.3 + major_version = int(kernel_version.split('.')[0]) + + if major_version < 8: + # OSX before 10.4.0, these don't support -arch and -isysroot at + # all. + stripArch = stripSysroot = True + else: + stripArch = '-arch' in cc_args + stripSysroot = '-isysroot' in cc_args + + if stripArch or 'ARCHFLAGS' in os.environ: + while True: + try: + index = compiler_so.index('-arch') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + if 'ARCHFLAGS' in os.environ and not stripArch: + # User specified different -arch flags in the environ, + # see also distutils.sysconfig + compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() + + if stripSysroot: + try: + index = compiler_so.index('-isysroot') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + pass + + # Check if the SDK that is used during compilation actually exists, + # the universal build requires the usage of a universal SDK and not all + # users have that installed by default. + sysroot = None + if '-isysroot' in cc_args: + idx = cc_args.index('-isysroot') + sysroot = cc_args[idx+1] + elif '-isysroot' in compiler_so: + idx = compiler_so.index('-isysroot') + sysroot = compiler_so[idx+1] + + if sysroot and not os.path.isdir(sysroot): + log.warn("Compiling with an SDK that doesn't seem to exist: %s", + sysroot) + log.warn("Please check your Xcode installation") + + return compiler_so class UnixCCompiler(CCompiler): @@ -109,8 +168,7 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): compiler_so = self.compiler_so if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) + compiler_so = _darwin_compiler_fixup(compiler_so, cc_args + extra_postargs) try: self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs) @@ -189,7 +247,7 @@ linker[i] = self.compiler_cxx[i] if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) + linker = _darwin_compiler_fixup(linker, ld_args) self.spawn(linker + ld_args) except DistutilsExecError as msg: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/util.py --- a/Lib/distutils/util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,9 +6,9 @@ import os import re -import importlib.util +import imp +import sys import string -import sys from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -53,10 +53,6 @@ return 'win-ia64' return sys.platform - # Set for cross builds explicitly - if "_PYTHON_HOST_PLATFORM" in os.environ: - return os.environ["_PYTHON_HOST_PLATFORM"] - if os.name != "posix" or not hasattr(os, 'uname'): # XXX what about the architecture? NT is Intel or Alpha, # Mac OS is M68k or PPC, etc. @@ -98,10 +94,94 @@ if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support, distutils.sysconfig - osname, release, machine = _osx_support.get_platform_osx( - distutils.sysconfig.get_config_vars(), - osname, release, machine) + # + # For our purposes, we'll assume that the system version from + # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set + # to. This makes the compatibility story a bit more sane because the + # machine is going to compile and link as if it were + # MACOSX_DEPLOYMENT_TARGET. + from distutils.sysconfig import get_config_vars + cfgvars = get_config_vars() + + macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') + + if 1: + # Always calculate the release of the running machine, + # needed to determine if we can build fat binaries or not. + + macrelease = macver + # Get the system version. Reading this plist is a documented + # way to get the system version (see the documentation for + # the Gestalt Manager) + try: + f = open('/System/Library/CoreServices/SystemVersion.plist') + except IOError: + # We're on a plain darwin box, fall back to the default + # behaviour. + pass + else: + try: + m = re.search( + r'ProductUserVisibleVersion\s*' + + r'(.*?)', f.read()) + if m is not None: + macrelease = '.'.join(m.group(1).split('.')[:2]) + # else: fall back to the default behaviour + finally: + f.close() + + if not macver: + macver = macrelease + + if macver: + from distutils.sysconfig import get_config_vars + release = macver + osname = "macosx" + + if (macrelease + '.') >= '10.4.' and \ + '-arch' in get_config_vars().get('CFLAGS', '').strip(): + # The universal build will build fat binaries, but not on + # systems before 10.4 + # + # Try to detect 4-way universal builds, those have machine-type + # 'universal' instead of 'fat'. + + machine = 'fat' + cflags = get_config_vars().get('CFLAGS') + + archs = re.findall('-arch\s+(\S+)', cflags) + archs = tuple(sorted(set(archs))) + + if len(archs) == 1: + machine = archs[0] + elif archs == ('i386', 'ppc'): + machine = 'fat' + elif archs == ('i386', 'x86_64'): + machine = 'intel' + elif archs == ('i386', 'ppc', 'x86_64'): + machine = 'fat3' + elif archs == ('ppc64', 'x86_64'): + machine = 'fat64' + elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): + machine = 'universal' + else: + raise ValueError( + "Don't know machine value for archs=%r"%(archs,)) + + elif machine == 'i386': + # On OSX the machine type returned by uname is always the + # 32-bit variant, even if the executable architecture is + # the 64-bit variant + if sys.maxsize >= 2**32: + machine = 'x86_64' + + elif machine in ('PowerPC', 'Power_Macintosh'): + # Pick a sane name for the PPC architecture. + machine = 'ppc' + + # See 'i386' case + if sys.maxsize >= 2**32: + machine = 'ppc64' return "%s-%s-%s" % (osname, release, machine) @@ -154,6 +234,12 @@ path = path[1:] return os.path.join(new_root, path) + elif os.name == 'os2': + (drive, path) = os.path.splitdrive(pathname) + if path[0] == os.sep: + path = path[1:] + return os.path.join(new_root, path) + else: raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) @@ -207,10 +293,25 @@ def grok_environment_error (exc, prefix="error: "): - # Function kept for backward compatibility. - # Used to try clever things with EnvironmentErrors, - # but nowadays str(exception) produces good messages. - return prefix + str(exc) + """Generate a useful error message from an EnvironmentError (IOError or + OSError) exception object. Handles Python 1.5.1 and 1.5.2 styles, and + does what it can to deal with exception objects that don't have a + filename (which happens when the error is due to a two-file operation, + such as 'rename()' or 'link()'. Returns the error message as a string + prefixed with 'prefix'. + """ + # check for Python 1.5.2-style {IO,OS}Error exception objects + if hasattr(exc, 'filename') and hasattr(exc, 'strerror'): + if exc.filename: + error = prefix + "%s: %s" % (exc.filename, exc.strerror) + else: + # two-argument functions in posix module don't + # include the filename in the exception object! + error = prefix + "%s" % exc.strerror + else: + error = prefix + str(exc.args[-1]) + + return error # Needed by 'split_quoted()' @@ -322,11 +423,11 @@ prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None): - """Byte-compile a collection of Python source files to .pyc - files in a __pycache__ subdirectory. 'py_files' is a list + """Byte-compile a collection of Python source files to either .pyc + or .pyo files in a __pycache__ subdirectory. 'py_files' is a list of files to compile; any files that don't end in ".py" are silently skipped. 'optimize' must be one of the following: - 0 - don't optimize + 0 - don't optimize (generate .pyc) 1 - normal optimization (like "python -O") 2 - extra optimization (like "python -OO") If 'force' is true, all files are recompiled regardless of @@ -350,11 +451,6 @@ generated in indirect mode; unless you know what you're doing, leave it set to None. """ - - # Late import to fix a bootstrap issue: _posixsubprocess is built by - # setup.py, but setup.py uses distutils. - import subprocess - # nothing is done if sys.dont_write_bytecode is True if sys.dont_write_bytecode: raise DistutilsByteCompileError('byte-compiling is disabled.') @@ -417,9 +513,11 @@ script.close() - cmd = [sys.executable] - cmd.extend(subprocess._optim_args_from_interpreter_flags()) - cmd.append(script_name) + cmd = [sys.executable, script_name] + if optimize == 1: + cmd.insert(1, "-O") + elif optimize == 2: + cmd.insert(1, "-OO") spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run) @@ -441,11 +539,9 @@ # cfile - byte-compiled file # dfile - purported source filename (same as 'file' by default) if optimize >= 0: - opt = '' if optimize == 0 else optimize - cfile = importlib.util.cache_from_source( - file, optimization=opt) + cfile = imp.cache_from_source(file, debug_override=not optimize) else: - cfile = importlib.util.cache_from_source(file) + cfile = imp.cache_from_source(file) dfile = file if prefix: if file[:len(prefix)] != prefix: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/distutils/version.py --- a/Lib/distutils/version.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/distutils/version.py Mon Jan 25 17:05:13 2016 +0100 @@ -48,6 +48,12 @@ return c return c == 0 + def __ne__(self, other): + c = self._cmp(other) + if c is NotImplemented: + return c + return c != 0 + def __lt__(self, other): c = self._cmp(other) if c is NotImplemented: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/doctest.py --- a/Lib/doctest.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/doctest.py Mon Jan 25 17:05:13 2016 +0100 @@ -62,7 +62,6 @@ 'REPORT_NDIFF', 'REPORT_ONLY_FIRST_FAILURE', 'REPORTING_FLAGS', - 'FAIL_FAST', # 1. Utility Functions # 2. Example & DocTest 'Example', @@ -93,7 +92,6 @@ ] import __future__ -import argparse import difflib import inspect import linecache @@ -152,13 +150,11 @@ REPORT_CDIFF = register_optionflag('REPORT_CDIFF') REPORT_NDIFF = register_optionflag('REPORT_NDIFF') REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') -FAIL_FAST = register_optionflag('FAIL_FAST') REPORTING_FLAGS = (REPORT_UDIFF | REPORT_CDIFF | REPORT_NDIFF | - REPORT_ONLY_FIRST_FAILURE | - FAIL_FAST) + REPORT_ONLY_FIRST_FAILURE) # Special string markers for use in `want` strings: BLANKLINE_MARKER = '' @@ -216,7 +212,7 @@ if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) - if getattr(package, '__loader__', None) is not None: + if hasattr(package, '__loader__'): if hasattr(package.__loader__, 'get_data'): file_contents = package.__loader__.get_data(filename) file_contents = file_contents.decode(encoding) @@ -318,32 +314,6 @@ else: return '#' -def _strip_exception_details(msg): - # Support for IGNORE_EXCEPTION_DETAIL. - # Get rid of everything except the exception name; in particular, drop - # the possibly dotted module path (if any) and the exception message (if - # any). We assume that a colon is never part of a dotted name, or of an - # exception name. - # E.g., given - # "foo.bar.MyError: la di da" - # return "MyError" - # Or for "abc.def" or "abc.def:\n" return "def". - - start, end = 0, len(msg) - # The exception name must appear on the first line. - i = msg.find("\n") - if i >= 0: - end = i - # retain up to the first colon (if any) - i = msg.find(':', 0, end) - if i >= 0: - end = i - # retain just the exception name - i = msg.rfind('.', 0, end) - if i >= 0: - start = i+1 - return msg[start: end] - class _OutputRedirectingPdb(pdb.Pdb): """ A specialized version of the python debugger that redirects stdout @@ -443,7 +413,7 @@ zero-based, with respect to the beginning of the DocTest. - indent: The example's indentation in the DocTest string. - I.e., the number of space characters that precede the + I.e., the number of space characters that preceed the example's first prompt. - options: A dictionary mapping from option flags to True or @@ -481,6 +451,9 @@ self.options == other.options and \ self.exc_msg == other.exc_msg + def __ne__(self, other): + return not self == other + def __hash__(self): return hash((self.source, self.want, self.lineno, self.indent, self.exc_msg)) @@ -530,9 +503,8 @@ examples = '1 example' else: examples = '%d examples' % len(self.examples) - return ('<%s %s from %s:%s (%s)>' % - (self.__class__.__name__, - self.name, self.filename, self.lineno, examples)) + return ('' % + (self.name, self.filename, self.lineno, examples)) def __eq__(self, other): if type(self) is not type(other): @@ -545,6 +517,9 @@ self.filename == other.filename and \ self.lineno == other.lineno + def __ne__(self, other): + return not self == other + def __hash__(self): return hash((self.docstring, self.name, self.filename, self.lineno)) @@ -578,7 +553,7 @@ # Want consists of any non-blank lines that do not start with PS1. (?P (?:(?![ ]*$) # Not a blank line (?![ ]*>>>) # Not a line starting with PS1 - .+$\n? # But any other line + .*$\n? # But any other line )*) ''', re.MULTILINE | re.VERBOSE) @@ -918,7 +893,7 @@ if '__name__' not in globs: globs['__name__'] = '__main__' # provide a default module name - # Recursively explore `obj`, extracting DocTests. + # Recursively expore `obj`, extracting DocTests. tests = [] self._find(tests, obj, name, module, source_lines, globs, {}) # Sort the tests by alpha order of names, for consistency in @@ -939,14 +914,6 @@ return module is inspect.getmodule(object) elif inspect.isfunction(object): return module.__dict__ is object.__globals__ - elif inspect.ismethoddescriptor(object): - if hasattr(object, '__objclass__'): - obj_mod = object.__objclass__.__module__ - elif hasattr(object, '__module__'): - obj_mod = object.__module__ - else: - return True # [XX] no easy way to tell otherwise - return module.__name__ == obj_mod elif inspect.isclass(object): return module.__name__ == object.__module__ elif hasattr(object, '__module__'): @@ -979,8 +946,7 @@ for valname, val in obj.__dict__.items(): valname = '%s.%s' % (name, valname) # Recurse to functions & classes. - if ((inspect.isroutine(inspect.unwrap(val)) - or inspect.isclass(val)) and + if ((inspect.isfunction(val) or inspect.isclass(val)) and self._from_module(module, val)): self._find(tests, val, valname, module, source_lines, globs, seen) @@ -992,8 +958,9 @@ raise ValueError("DocTestFinder.find: __test__ keys " "must be strings: %r" % (type(valname),)) - if not (inspect.isroutine(val) or inspect.isclass(val) or - inspect.ismodule(val) or isinstance(val, str)): + if not (inspect.isfunction(val) or inspect.isclass(val) or + inspect.ismethod(val) or inspect.ismodule(val) or + isinstance(val, str)): raise ValueError("DocTestFinder.find: __test__ values " "must be strings, functions, methods, " "classes, or modules: %r" % @@ -1012,7 +979,7 @@ val = getattr(obj, valname).__func__ # Recurse to methods, properties, and nested classes. - if ((inspect.isroutine(val) or inspect.isclass(val) or + if ((inspect.isfunction(val) or inspect.isclass(val) or isinstance(val, property)) and self._from_module(module, val)): valname = '%s.%s' % (name, valname) @@ -1051,7 +1018,7 @@ filename = None else: filename = getattr(module, '__file__', module.__name__) - if filename[-4:] == ".pyc": + if filename[-4:] in (".pyc", ".pyo"): filename = filename[:-1] return self._parser.get_doctest(docstring, globs, name, filename, lineno) @@ -1353,9 +1320,10 @@ # Another chance if they didn't care about the detail. elif self.optionflags & IGNORE_EXCEPTION_DETAIL: - if check(_strip_exception_details(example.exc_msg), - _strip_exception_details(exc_msg), - self.optionflags): + m1 = re.match(r'(?:[^:]*\.)?([^:]*:)', example.exc_msg) + m2 = re.match(r'(?:[^:]*\.)?([^:]*:)', exc_msg) + if m1 and m2 and check(m1.group(1), m2.group(1), + self.optionflags): outcome = SUCCESS # Report the outcome. @@ -1374,9 +1342,6 @@ else: assert False, ("unknown outcome", outcome) - if failures and self.optionflags & FAIL_FAST: - break - # Restore the option flags (in case they were modified) self.optionflags = original_optionflags @@ -1575,7 +1540,7 @@ # If `want` contains hex-escaped character such as "\u1234", # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]). - # On the other hand, `got` could be another sequence of + # On the other hand, `got` could be an another sequence of # characters such as [\u1234], so `want` and `got` should # be folded to hex-escaped ASCII string to compare. got = self._toAscii(got) @@ -2285,6 +2250,9 @@ self._dt_tearDown == other._dt_tearDown and \ self._dt_checker == other._dt_checker + def __ne__(self, other): + return not self == other + def __hash__(self): return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, self._dt_checker)) @@ -2315,12 +2283,6 @@ __str__ = shortDescription -class _DocTestSuite(unittest.TestSuite): - - def _removeTestAtIndex(self, index): - pass - - def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, **options): """ @@ -2366,19 +2328,23 @@ if not tests and sys.flags.optimize >=2: # Skip doctests when running with -O2 - suite = _DocTestSuite() + suite = unittest.TestSuite() suite.addTest(SkipDocTestCase(module)) return suite + elif not tests: + # Why do we want to do this? Because it reveals a bug that might + # otherwise be hidden. + raise ValueError(module, "has no tests") tests.sort() - suite = _DocTestSuite() + suite = unittest.TestSuite() for test in tests: if len(test.examples) == 0: continue if not test.filename: filename = module.__file__ - if filename[-4:] == ".pyc": + if filename[-4:] in (".pyc", ".pyo"): filename = filename[:-1] test.filename = filename suite.addTest(DocTestCase(test, **options)) @@ -2481,7 +2447,7 @@ encoding An encoding that will be used to convert the files to unicode. """ - suite = _DocTestSuite() + suite = unittest.TestSuite() # We do this here so that _normalize_module is called at the right # level. If it were called in DocFileTest, then this function @@ -2731,30 +2697,13 @@ def _test(): - parser = argparse.ArgumentParser(description="doctest runner") - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='print very verbose output for all tests') - parser.add_argument('-o', '--option', action='append', - choices=OPTIONFLAGS_BY_NAME.keys(), default=[], - help=('specify a doctest option flag to apply' - ' to the test run; may be specified more' - ' than once to apply multiple options')) - parser.add_argument('-f', '--fail-fast', action='store_true', - help=('stop running tests after first failure (this' - ' is a shorthand for -o FAIL_FAST, and is' - ' in addition to any other -o options)')) - parser.add_argument('file', nargs='+', - help='file containing the tests to run') - args = parser.parse_args() - testfiles = args.file - # Verbose used to be handled by the "inspect argv" magic in DocTestRunner, - # but since we are using argparse we are passing it manually now. - verbose = args.verbose - options = 0 - for option in args.option: - options |= OPTIONFLAGS_BY_NAME[option] - if args.fail_fast: - options |= FAIL_FAST + testfiles = [arg for arg in sys.argv[1:] if arg and arg[0] != '-'] + if not testfiles: + name = os.path.basename(sys.argv[0]) + if '__loader__' in globals(): # python -m + name, _ = os.path.splitext(name) + print("usage: {0} [-v] file ...".format(name)) + return 2 for filename in testfiles: if filename.endswith(".py"): # It is a module -- insert its dir into sys.path and try to @@ -2764,10 +2713,9 @@ sys.path.insert(0, dirname) m = __import__(filename[:-3]) del sys.path[0] - failures, _ = testmod(m, verbose=verbose, optionflags=options) + failures, _ = testmod(m) else: - failures, _ = testfile(filename, module_relative=False, - verbose=verbose, optionflags=options) + failures, _ = testfile(filename, module_relative=False) if failures: return 1 return 0 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/__init__.py --- a/Lib/email/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,6 +4,8 @@ """A package for parsing, handling, and generating email messages.""" +__version__ = '5.1.0' + __all__ = [ 'base64mime', 'charset', diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/_encoded_words.py --- a/Lib/email/_encoded_words.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,221 +0,0 @@ -""" Routines for manipulating RFC2047 encoded words. - -This is currently a package-private API, but will be considered for promotion -to a public API if there is demand. - -""" - -# An ecoded word looks like this: -# -# =?charset[*lang]?cte?encoded_string?= -# -# for more information about charset see the charset module. Here it is one -# of the preferred MIME charset names (hopefully; you never know when parsing). -# cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In -# theory other letters could be used for other encodings, but in practice this -# (almost?) never happens. There could be a public API for adding entries -# to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is -# Base64. The meaning of encoded_string should be obvious. 'lang' is optional -# as indicated by the brackets (they are not part of the syntax) but is almost -# never encountered in practice. -# -# The general interface for a CTE decoder is that it takes the encoded_string -# as its argument, and returns a tuple (cte_decoded_string, defects). The -# cte_decoded_string is the original binary that was encoded using the -# specified cte. 'defects' is a list of MessageDefect instances indicating any -# problems encountered during conversion. 'charset' and 'lang' are the -# corresponding strings extracted from the EW, case preserved. -# -# The general interface for a CTE encoder is that it takes a binary sequence -# as input and returns the cte_encoded_string, which is an ascii-only string. -# -# Each decoder must also supply a length function that takes the binary -# sequence as its argument and returns the length of the resulting encoded -# string. -# -# The main API functions for the module are decode, which calls the decoder -# referenced by the cte specifier, and encode, which adds the appropriate -# RFC 2047 "chrome" to the encoded string, and can optionally automatically -# select the shortest possible encoding. See their docstrings below for -# details. - -import re -import base64 -import binascii -import functools -from string import ascii_letters, digits -from email import errors - -__all__ = ['decode_q', - 'encode_q', - 'decode_b', - 'encode_b', - 'len_q', - 'len_b', - 'decode', - 'encode', - ] - -# -# Quoted Printable -# - -# regex based decoder. -_q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, - lambda m: bytes([int(m.group(1), 16)])) - -def decode_q(encoded): - encoded = encoded.replace(b'_', b' ') - return _q_byte_subber(encoded), [] - - -# dict mapping bytes to their encoded form -class _QByteMap(dict): - - safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii') - - def __missing__(self, key): - if key in self.safe: - self[key] = chr(key) - else: - self[key] = "={:02X}".format(key) - return self[key] - -_q_byte_map = _QByteMap() - -# In headers spaces are mapped to '_'. -_q_byte_map[ord(' ')] = '_' - -def encode_q(bstring): - return ''.join(_q_byte_map[x] for x in bstring) - -def len_q(bstring): - return sum(len(_q_byte_map[x]) for x in bstring) - - -# -# Base64 -# - -def decode_b(encoded): - defects = [] - pad_err = len(encoded) % 4 - if pad_err: - defects.append(errors.InvalidBase64PaddingDefect()) - padded_encoded = encoded + b'==='[:4-pad_err] - else: - padded_encoded = encoded - try: - return base64.b64decode(padded_encoded, validate=True), defects - except binascii.Error: - # Since we had correct padding, this must an invalid char error. - defects = [errors.InvalidBase64CharactersDefect()] - # The non-alphabet characters are ignored as far as padding - # goes, but we don't know how many there are. So we'll just - # try various padding lengths until something works. - for i in 0, 1, 2, 3: - try: - return base64.b64decode(encoded+b'='*i, validate=False), defects - except binascii.Error: - if i==0: - defects.append(errors.InvalidBase64PaddingDefect()) - else: - # This should never happen. - raise AssertionError("unexpected binascii.Error") - -def encode_b(bstring): - return base64.b64encode(bstring).decode('ascii') - -def len_b(bstring): - groups_of_3, leftover = divmod(len(bstring), 3) - # 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. - return groups_of_3 * 4 + (4 if leftover else 0) - - -_cte_decoders = { - 'q': decode_q, - 'b': decode_b, - } - -def decode(ew): - """Decode encoded word and return (string, charset, lang, defects) tuple. - - An RFC 2047/2243 encoded word has the form: - - =?charset*lang?cte?encoded_string?= - - where '*lang' may be omitted but the other parts may not be. - - This function expects exactly such a string (that is, it does not check the - syntax and may raise errors if the string is not well formed), and returns - the encoded_string decoded first from its Content Transfer Encoding and - then from the resulting bytes into unicode using the specified charset. If - the cte-decoded string does not successfully decode using the specified - character set, a defect is added to the defects list and the unknown octets - are replaced by the unicode 'unknown' character \\uFDFF. - - The specified charset and language are returned. The default for language, - which is rarely if ever encountered, is the empty string. - - """ - _, charset, cte, cte_string, _ = ew.split('?') - charset, _, lang = charset.partition('*') - cte = cte.lower() - # Recover the original bytes and do CTE decoding. - bstring = cte_string.encode('ascii', 'surrogateescape') - bstring, defects = _cte_decoders[cte](bstring) - # Turn the CTE decoded bytes into unicode. - try: - string = bstring.decode(charset) - except UnicodeError: - defects.append(errors.UndecodableBytesDefect("Encoded word " - "contains bytes not decodable using {} charset".format(charset))) - string = bstring.decode(charset, 'surrogateescape') - except LookupError: - string = bstring.decode('ascii', 'surrogateescape') - if charset.lower() != 'unknown-8bit': - defects.append(errors.CharsetError("Unknown charset {} " - "in encoded word; decoded as unknown bytes".format(charset))) - return string, charset, lang, defects - - -_cte_encoders = { - 'q': encode_q, - 'b': encode_b, - } - -_cte_encode_length = { - 'q': len_q, - 'b': len_b, - } - -def encode(string, charset='utf-8', encoding=None, lang=''): - """Encode string using the CTE encoding that produces the shorter result. - - Produces an RFC 2047/2243 encoded word of the form: - - =?charset*lang?cte?encoded_string?= - - where '*lang' is omitted unless the 'lang' parameter is given a value. - Optional argument charset (defaults to utf-8) specifies the charset to use - to encode the string to binary before CTE encoding it. Optional argument - 'encoding' is the cte specifier for the encoding that should be used ('q' - or 'b'); if it is None (the default) the encoding which produces the - shortest encoded sequence is used, except that 'q' is preferred if it is up - to five characters longer. Optional argument 'lang' (default '') gives the - RFC 2243 language string to specify in the encoded word. - - """ - if charset == 'unknown-8bit': - bstring = string.encode('ascii', 'surrogateescape') - else: - bstring = string.encode(charset) - if encoding is None: - qlen = _cte_encode_length['q'](bstring) - blen = _cte_encode_length['b'](bstring) - # Bias toward q. 5 is arbitrary. - encoding = 'q' if qlen - blen < 5 else 'b' - encoded = _cte_encoders[encoding](bstring) - if lang: - lang = '*' + lang - return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/_header_value_parser.py --- a/Lib/email/_header_value_parser.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2970 +0,0 @@ -"""Header value parser implementing various email-related RFC parsing rules. - -The parsing methods defined in this module implement various email related -parsing rules. Principal among them is RFC 5322, which is the followon -to RFC 2822 and primarily a clarification of the former. It also implements -RFC 2047 encoded word decoding. - -RFC 5322 goes to considerable trouble to maintain backward compatibility with -RFC 822 in the parse phase, while cleaning up the structure on the generation -phase. This parser supports correct RFC 5322 generation by tagging white space -as folding white space only when folding is allowed in the non-obsolete rule -sets. Actually, the parser is even more generous when accepting input than RFC -5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages. -Where possible deviations from the standard are annotated on the 'defects' -attribute of tokens that deviate. - -The general structure of the parser follows RFC 5322, and uses its terminology -where there is a direct correspondence. Where the implementation requires a -somewhat different structure than that used by the formal grammar, new terms -that mimic the closest existing terms are used. Thus, it really helps to have -a copy of RFC 5322 handy when studying this code. - -Input to the parser is a string that has already been unfolded according to -RFC 5322 rules. According to the RFC this unfolding is the very first step, and -this parser leaves the unfolding step to a higher level message parser, which -will have already detected the line breaks that need unfolding while -determining the beginning and end of each header. - -The output of the parser is a TokenList object, which is a list subclass. A -TokenList is a recursive data structure. The terminal nodes of the structure -are Terminal objects, which are subclasses of str. These do not correspond -directly to terminal objects in the formal grammar, but are instead more -practical higher level combinations of true terminals. - -All TokenList and Terminal objects have a 'value' attribute, which produces the -semantically meaningful value of that part of the parse subtree. The value of -all whitespace tokens (no matter how many sub-tokens they may contain) is a -single space, as per the RFC rules. This includes 'CFWS', which is herein -included in the general class of whitespace tokens. There is one exception to -the rule that whitespace tokens are collapsed into single spaces in values: in -the value of a 'bare-quoted-string' (a quoted-string with no leading or -trailing whitespace), any whitespace that appeared between the quotation marks -is preserved in the returned value. Note that in all Terminal strings quoted -pairs are turned into their unquoted values. - -All TokenList and Terminal objects also have a string value, which attempts to -be a "canonical" representation of the RFC-compliant form of the substring that -produced the parsed subtree, including minimal use of quoted pair quoting. -Whitespace runs are not collapsed. - -Comment tokens also have a 'content' attribute providing the string found -between the parens (including any nested comments) with whitespace preserved. - -All TokenList and Terminal objects have a 'defects' attribute which is a -possibly empty list all of the defects found while creating the token. Defects -may appear on any token in the tree, and a composite list of all defects in the -subtree is available through the 'all_defects' attribute of any node. (For -Terminal notes x.defects == x.all_defects.) - -Each object in a parse tree is called a 'token', and each has a 'token_type' -attribute that gives the name from the RFC 5322 grammar that it represents. -Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that -may be produced: 'ptext'. A 'ptext' is a string of printable ascii characters. -It is returned in place of lists of (ctext/quoted-pair) and -(qtext/quoted-pair). - -XXX: provide complete list of token types. -""" - -import re -import urllib # For urllib.parse.unquote -from string import hexdigits -from collections import OrderedDict -from operator import itemgetter -from email import _encoded_words as _ew -from email import errors -from email import utils - -# -# Useful constants and functions -# - -WSP = set(' \t') -CFWS_LEADER = WSP | set('(') -SPECIALS = set(r'()<>@,:;.\"[]') -ATOM_ENDS = SPECIALS | WSP -DOT_ATOM_ENDS = ATOM_ENDS - set('.') -# '.', '"', and '(' do not end phrases in order to support obs-phrase -PHRASE_ENDS = SPECIALS - set('."(') -TSPECIALS = (SPECIALS | set('/?=')) - set('.') -TOKEN_ENDS = TSPECIALS | WSP -ASPECIALS = TSPECIALS | set("*'%") -ATTRIBUTE_ENDS = ASPECIALS | WSP -EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') - -def quote_string(value): - return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' - -# -# Accumulator for header folding -# - -class _Folded: - - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] - - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - - def finalize(self): - if self.current: - self.newline() - - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False - -# -# TokenList and its subclasses -# - -class TokenList(list): - - token_type = None - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self.defects = [] - - def __str__(self): - return ''.join(str(x) for x in self) - - def __repr__(self): - return '{}({})'.format(self.__class__.__name__, - super().__repr__()) - - @property - def value(self): - return ''.join(x.value for x in self if x.value) - - @property - def all_defects(self): - return sum((x.all_defects for x in self), self.defects) - - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - - def startswith_fws(self): - return self[0].startswith_fws() - - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() - - @property - def comments(self): - comments = [] - for token in self: - comments.extend(token.comments) - return comments - - def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - # Peel off the leading whitespace and make it sticky, to - # avoid infinite recursion. - folded.stickyspace = str(part.pop(0)) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - - def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) - - def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) - - def _pp(self, indent=''): - yield '{}{}/{}('.format( - indent, - self.__class__.__name__, - self.token_type) - for token in self: - if not hasattr(token, '_pp'): - yield (indent + ' !! invalid element in token ' - 'list: {!r}'.format(token)) - else: - yield from token._pp(indent+' ') - if self.defects: - extra = ' Defects: {}'.format(self.defects) - else: - extra = '' - yield '{}){}'.format(indent, extra) - - -class WhiteSpaceTokenList(TokenList): - - @property - def value(self): - return ' ' - - @property - def comments(self): - return [x.content for x in self if x.token_type=='comment'] - - -class UnstructuredTokenList(TokenList): - - token_type = 'unstructured' - - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part.fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word()) - return ''.join(res) - - -class Phrase(TokenList): - - token_type = 'phrase' - - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - -class Word(TokenList): - - token_type = 'word' - - -class CFWSList(WhiteSpaceTokenList): - - token_type = 'cfws' - - def has_leading_comment(self): - return bool(self.comments) - - -class Atom(TokenList): - - token_type = 'atom' - - -class Token(TokenList): - - token_type = 'token' - - -class EncodedWord(TokenList): - - token_type = 'encoded-word' - cte = None - charset = None - lang = None - - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - - -class QuotedString(TokenList): - - token_type = 'quoted-string' - - @property - def content(self): - for x in self: - if x.token_type == 'bare-quoted-string': - return x.value - - @property - def quoted_value(self): - res = [] - for x in self: - if x.token_type == 'bare-quoted-string': - res.append(str(x)) - else: - res.append(x.value) - return ''.join(res) - - @property - def stripped_value(self): - for token in self: - if token.token_type == 'bare-quoted-string': - return token.value - - -class BareQuotedString(QuotedString): - - token_type = 'bare-quoted-string' - - def __str__(self): - return quote_string(''.join(str(x) for x in self)) - - @property - def value(self): - return ''.join(str(x) for x in self) - - -class Comment(WhiteSpaceTokenList): - - token_type = 'comment' - - def __str__(self): - return ''.join(sum([ - ["("], - [self.quote(x) for x in self], - [")"], - ], [])) - - def quote(self, value): - if value.token_type == 'comment': - return str(value) - return str(value).replace('\\', '\\\\').replace( - '(', '\(').replace( - ')', '\)') - - @property - def content(self): - return ''.join(str(x) for x in self) - - @property - def comments(self): - return [self.content] - -class AddressList(TokenList): - - token_type = 'address-list' - - @property - def addresses(self): - return [x for x in self if x.token_type=='address'] - - @property - def mailboxes(self): - return sum((x.mailboxes - for x in self if x.token_type=='address'), []) - - @property - def all_mailboxes(self): - return sum((x.all_mailboxes - for x in self if x.token_type=='address'), []) - - -class Address(TokenList): - - token_type = 'address' - - @property - def display_name(self): - if self[0].token_type == 'group': - return self[0].display_name - - @property - def mailboxes(self): - if self[0].token_type == 'mailbox': - return [self[0]] - elif self[0].token_type == 'invalid-mailbox': - return [] - return self[0].mailboxes - - @property - def all_mailboxes(self): - if self[0].token_type == 'mailbox': - return [self[0]] - elif self[0].token_type == 'invalid-mailbox': - return [self[0]] - return self[0].all_mailboxes - -class MailboxList(TokenList): - - token_type = 'mailbox-list' - - @property - def mailboxes(self): - return [x for x in self if x.token_type=='mailbox'] - - @property - def all_mailboxes(self): - return [x for x in self - if x.token_type in ('mailbox', 'invalid-mailbox')] - - -class GroupList(TokenList): - - token_type = 'group-list' - - @property - def mailboxes(self): - if not self or self[0].token_type != 'mailbox-list': - return [] - return self[0].mailboxes - - @property - def all_mailboxes(self): - if not self or self[0].token_type != 'mailbox-list': - return [] - return self[0].all_mailboxes - - -class Group(TokenList): - - token_type = "group" - - @property - def mailboxes(self): - if self[2].token_type != 'group-list': - return [] - return self[2].mailboxes - - @property - def all_mailboxes(self): - if self[2].token_type != 'group-list': - return [] - return self[2].all_mailboxes - - @property - def display_name(self): - return self[0].display_name - - -class NameAddr(TokenList): - - token_type = 'name-addr' - - @property - def display_name(self): - if len(self) == 1: - return None - return self[0].display_name - - @property - def local_part(self): - return self[-1].local_part - - @property - def domain(self): - return self[-1].domain - - @property - def route(self): - return self[-1].route - - @property - def addr_spec(self): - return self[-1].addr_spec - - -class AngleAddr(TokenList): - - token_type = 'angle-addr' - - @property - def local_part(self): - for x in self: - if x.token_type == 'addr-spec': - return x.local_part - - @property - def domain(self): - for x in self: - if x.token_type == 'addr-spec': - return x.domain - - @property - def route(self): - for x in self: - if x.token_type == 'obs-route': - return x.domains - - @property - def addr_spec(self): - for x in self: - if x.token_type == 'addr-spec': - return x.addr_spec - else: - return '<>' - - -class ObsRoute(TokenList): - - token_type = 'obs-route' - - @property - def domains(self): - return [x.domain for x in self if x.token_type == 'domain'] - - -class Mailbox(TokenList): - - token_type = 'mailbox' - - @property - def display_name(self): - if self[0].token_type == 'name-addr': - return self[0].display_name - - @property - def local_part(self): - return self[0].local_part - - @property - def domain(self): - return self[0].domain - - @property - def route(self): - if self[0].token_type == 'name-addr': - return self[0].route - - @property - def addr_spec(self): - return self[0].addr_spec - - -class InvalidMailbox(TokenList): - - token_type = 'invalid-mailbox' - - @property - def display_name(self): - return None - - local_part = domain = route = addr_spec = display_name - - -class Domain(TokenList): - - token_type = 'domain' - - @property - def domain(self): - return ''.join(super().value.split()) - - -class DotAtom(TokenList): - - token_type = 'dot-atom' - - -class DotAtomText(TokenList): - - token_type = 'dot-atom-text' - - -class AddrSpec(TokenList): - - token_type = 'addr-spec' - - @property - def local_part(self): - return self[0].local_part - - @property - def domain(self): - if len(self) < 3: - return None - return self[-1].domain - - @property - def value(self): - if len(self) < 3: - return self[0].value - return self[0].value.rstrip()+self[1].value+self[2].value.lstrip() - - @property - def addr_spec(self): - nameset = set(self.local_part) - if len(nameset) > len(nameset-DOT_ATOM_ENDS): - lp = quote_string(self.local_part) - else: - lp = self.local_part - if self.domain is not None: - return lp + '@' + self.domain - return lp - - -class ObsLocalPart(TokenList): - - token_type = 'obs-local-part' - - -class DisplayName(Phrase): - - token_type = 'display-name' - - @property - def display_name(self): - res = TokenList(self) - if res[0].token_type == 'cfws': - res.pop(0) - else: - if res[0][0].token_type == 'cfws': - res[0] = TokenList(res[0][1:]) - if res[-1].token_type == 'cfws': - res.pop() - else: - if res[-1][-1].token_type == 'cfws': - res[-1] = TokenList(res[-1][:-1]) - return res.value - - @property - def value(self): - quote = False - if self.defects: - quote = True - else: - for x in self: - if x.token_type == 'quoted-string': - quote = True - if quote: - pre = post = '' - if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': - pre = ' ' - if self[-1].token_type=='cfws' or self[-1][-1].token_type=='cfws': - post = ' ' - return pre+quote_string(self.display_name)+post - else: - return super().value - - -class LocalPart(TokenList): - - token_type = 'local-part' - - @property - def value(self): - if self[0].token_type == "quoted-string": - return self[0].quoted_value - else: - return self[0].value - - @property - def local_part(self): - # Strip whitespace from front, back, and around dots. - res = [DOT] - last = DOT - last_is_tl = False - for tok in self[0] + [DOT]: - if tok.token_type == 'cfws': - continue - if (last_is_tl and tok.token_type == 'dot' and - last[-1].token_type == 'cfws'): - res[-1] = TokenList(last[:-1]) - is_tl = isinstance(tok, TokenList) - if (is_tl and last.token_type == 'dot' and - tok[0].token_type == 'cfws'): - res.append(TokenList(tok[1:])) - else: - res.append(tok) - last = res[-1] - last_is_tl = is_tl - res = TokenList(res[1:-1]) - return res.value - - -class DomainLiteral(TokenList): - - token_type = 'domain-literal' - - @property - def domain(self): - return ''.join(super().value.split()) - - @property - def ip(self): - for x in self: - if x.token_type == 'ptext': - return x.value - - -class MIMEVersion(TokenList): - - token_type = 'mime-version' - major = None - minor = None - - -class Parameter(TokenList): - - token_type = 'parameter' - sectioned = False - extended = False - charset = 'us-ascii' - - @property - def section_number(self): - # Because the first token, the attribute (name) eats CFWS, the second - # token is always the section if there is one. - return self[1].number if self.sectioned else 0 - - @property - def param_value(self): - # This is part of the "handle quoted extended parameters" hack. - for token in self: - if token.token_type == 'value': - return token.stripped_value - if token.token_type == 'quoted-string': - for token in token: - if token.token_type == 'bare-quoted-string': - for token in token: - if token.token_type == 'value': - return token.stripped_value - return '' - - -class InvalidParameter(Parameter): - - token_type = 'invalid-parameter' - - -class Attribute(TokenList): - - token_type = 'attribute' - - @property - def stripped_value(self): - for token in self: - if token.token_type.endswith('attrtext'): - return token.value - -class Section(TokenList): - - token_type = 'section' - number = None - - -class Value(TokenList): - - token_type = 'value' - - @property - def stripped_value(self): - token = self[0] - if token.token_type == 'cfws': - token = self[1] - if token.token_type.endswith( - ('quoted-string', 'attribute', 'extended-attribute')): - return token.stripped_value - return self.value - - -class MimeParameters(TokenList): - - token_type = 'mime-parameters' - - @property - def params(self): - # The RFC specifically states that the ordering of parameters is not - # guaranteed and may be reordered by the transport layer. So we have - # to assume the RFC 2231 pieces can come in any order. However, we - # output them in the order that we first see a given name, which gives - # us a stable __str__. - params = OrderedDict() - for token in self: - if not token.token_type.endswith('parameter'): - continue - if token[0].token_type != 'attribute': - continue - name = token[0].value.strip() - if name not in params: - params[name] = [] - params[name].append((token.section_number, token)) - for name, parts in params.items(): - parts = sorted(parts, key=itemgetter(0)) - first_param = parts[0][1] - charset = first_param.charset - # Our arbitrary error recovery is to ignore duplicate parameters, - # to use appearance order if there are duplicate rfc 2231 parts, - # and to ignore gaps. This mimics the error recovery of get_param. - if not first_param.extended and len(parts) > 1: - if parts[1][0] == 0: - parts[1][1].defects.append(errors.InvalidHeaderDefect( - 'duplicate parameter name; duplicate(s) ignored')) - parts = parts[:1] - # Else assume the *0* was missing...note that this is different - # from get_param, but we registered a defect for this earlier. - value_parts = [] - i = 0 - for section_number, param in parts: - if section_number != i: - # We could get fancier here and look for a complete - # duplicate extended parameter and ignore the second one - # seen. But we're not doing that. The old code didn't. - if not param.extended: - param.defects.append(errors.InvalidHeaderDefect( - 'duplicate parameter name; duplicate ignored')) - continue - else: - param.defects.append(errors.InvalidHeaderDefect( - "inconsistent RFC2231 parameter numbering")) - i += 1 - value = param.param_value - if param.extended: - try: - value = urllib.parse.unquote_to_bytes(value) - except UnicodeEncodeError: - # source had surrogate escaped bytes. What we do now - # is a bit of an open question. I'm not sure this is - # the best choice, but it is what the old algorithm did - value = urllib.parse.unquote(value, encoding='latin-1') - else: - try: - value = value.decode(charset, 'surrogateescape') - except LookupError: - # XXX: there should really be a custom defect for - # unknown character set to make it easy to find, - # because otherwise unknown charset is a silent - # failure. - value = value.decode('us-ascii', 'surrogateescape') - if utils._has_surrogates(value): - param.defects.append(errors.UndecodableBytesDefect()) - value_parts.append(value) - value = ''.join(value_parts) - yield name, value - - def __str__(self): - params = [] - for name, value in self.params: - if value: - params.append('{}={}'.format(name, quote_string(value))) - else: - params.append(name) - params = '; '.join(params) - return ' ' + params if params else '' - - -class ParameterizedHeaderValue(TokenList): - - @property - def params(self): - for token in reversed(self): - if token.token_type == 'mime-parameters': - return token.params - return {} - - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - - -class ContentType(ParameterizedHeaderValue): - - token_type = 'content-type' - maintype = 'text' - subtype = 'plain' - - -class ContentDisposition(ParameterizedHeaderValue): - - token_type = 'content-disposition' - content_disposition = None - - -class ContentTransferEncoding(TokenList): - - token_type = 'content-transfer-encoding' - cte = '7bit' - - -class HeaderLabel(TokenList): - - token_type = 'header-label' - - -class Header(TokenList): - - token_type = 'header' - - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) - - -# -# Terminal classes and instances -# - -class Terminal(str): - - def __new__(cls, value, token_type): - self = super().__new__(cls, value) - self.token_type = token_type - self.defects = [] - return self - - def __repr__(self): - return "{}({})".format(self.__class__.__name__, super().__repr__()) - - @property - def all_defects(self): - return list(self.defects) - - def _pp(self, indent=''): - return ["{}{}/{}({}){}".format( - indent, - self.__class__.__name__, - self.token_type, - super().__repr__(), - '' if not self.defects else ' {}'.format(self.defects), - )] - - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - - def pop_trailing_ws(self): - # This terminates the recursion. - return None - - def pop_leading_fws(self): - # This terminates the recursion. - return None - - @property - def comments(self): - return [] - - def has_leading_comment(self): - return False - - def __getnewargs__(self): - return(str(self), self.token_type) - - -class WhiteSpaceTerminal(Terminal): - - @property - def value(self): - return ' ' - - def startswith_fws(self): - return True - - has_fws = True - - -class ValueTerminal(Terminal): - - @property - def value(self): - return self - - def startswith_fws(self): - return False - - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - - -class EWWhiteSpaceTerminal(WhiteSpaceTerminal): - - @property - def value(self): - return '' - - @property - def encoded(self): - return self[:] - - def __str__(self): - return '' - - has_fws = True - - -# XXX these need to become classes and used as instances so -# that a program can't change them in a parse tree and screw -# up other parse trees. Maybe should have tests for that, too. -DOT = ValueTerminal('.', 'dot') -ListSeparator = ValueTerminal(',', 'list-separator') -RouteComponentMarker = ValueTerminal('@', 'route-component-marker') - -# -# Parser -# - -# Parse strings according to RFC822/2047/2822/5322 rules. -# -# This is a stateless parser. Each get_XXX function accepts a string and -# returns either a Terminal or a TokenList representing the RFC object named -# by the method and a string containing the remaining unparsed characters -# from the input. Thus a parser method consumes the next syntactic construct -# of a given type and returns a token representing the construct plus the -# unparsed remainder of the input string. -# -# For example, if the first element of a structured header is a 'phrase', -# then: -# -# phrase, value = get_phrase(value) -# -# returns the complete phrase from the start of the string value, plus any -# characters left in the string after the phrase is removed. - -_wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split -_non_atom_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']','\]'))).match -_non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall -_non_token_end_matcher = re.compile(r"[^{}]+".format( - ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']','\]'))).match -_non_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']','\]'))).match -_non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(EXTENDED_ATTRIBUTE_ENDS).replace( - '\\','\\\\').replace(']','\]'))).match - -def _validate_xtext(xtext): - """If input token contains ASCII non-printables, register a defect.""" - - non_printables = _non_printable_finder(xtext) - if non_printables: - xtext.defects.append(errors.NonPrintableDefect(non_printables)) - if utils._has_surrogates(xtext): - xtext.defects.append(errors.UndecodableBytesDefect( - "Non-ASCII characters found in header token")) - -def _get_ptext_to_endchars(value, endchars): - """Scan printables/quoted-pairs until endchars and return unquoted ptext. - - This function turns a run of qcontent, ccontent-without-comments, or - dtext-with-quoted-printables into a single string by unquoting any - quoted printables. It returns the string, the remaining value, and - a flag that is True iff there were any quoted printables decoded. - - """ - fragment, *remainder = _wsp_splitter(value, 1) - vchars = [] - escape = False - had_qp = False - for pos in range(len(fragment)): - if fragment[pos] == '\\': - if escape: - escape = False - had_qp = True - else: - escape = True - continue - if escape: - escape = False - elif fragment[pos] in endchars: - break - vchars.append(fragment[pos]) - else: - pos = pos + 1 - return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp - -def get_fws(value): - """FWS = 1*WSP - - This isn't the RFC definition. We're using fws to represent tokens where - folding can be done, but when we are parsing the *un*folding has already - been done so we don't need to watch out for CRLF. - - """ - newvalue = value.lstrip() - fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') - return fws, newvalue - -def get_encoded_word(value): - """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" - - """ - ew = EncodedWord() - if not value.startswith('=?'): - raise errors.HeaderParseError( - "expected encoded word but found {}".format(value)) - tok, *remainder = value[2:].split('?=', 1) - if tok == value[2:]: - raise errors.HeaderParseError( - "expected encoded word but found {}".format(value)) - remstr = ''.join(remainder) - if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits: - # The ? after the CTE was followed by an encoded word escape (=XX). - rest, *remainder = remstr.split('?=', 1) - tok = tok + '?=' + rest - if len(tok.split()) > 1: - ew.defects.append(errors.InvalidHeaderDefect( - "whitespace inside encoded word")) - ew.cte = value - value = ''.join(remainder) - try: - text, charset, lang, defects = _ew.decode('=?' + tok + '?=') - except ValueError: - raise errors.HeaderParseError( - "encoded word format invalid: '{}'".format(ew.cte)) - ew.charset = charset - ew.lang = lang - ew.defects.extend(defects) - while text: - if text[0] in WSP: - token, text = get_fws(text) - ew.append(token) - continue - chars, *remainder = _wsp_splitter(text, 1) - vtext = ValueTerminal(chars, 'vtext') - _validate_xtext(vtext) - ew.append(vtext) - text = ''.join(remainder) - return ew, value - -def get_unstructured(value): - """unstructured = (*([FWS] vchar) *WSP) / obs-unstruct - obs-unstruct = *((*LF *CR *(obs-utext) *LF *CR)) / FWS) - obs-utext = %d0 / obs-NO-WS-CTL / LF / CR - - obs-NO-WS-CTL is control characters except WSP/CR/LF. - - So, basically, we have printable runs, plus control characters or nulls in - the obsolete syntax, separated by whitespace. Since RFC 2047 uses the - obsolete syntax in its specification, but requires whitespace on either - side of the encoded words, I can see no reason to need to separate the - non-printable-non-whitespace from the printable runs if they occur, so we - parse this into xtext tokens separated by WSP tokens. - - Because an 'unstructured' value must by definition constitute the entire - value, this 'get' routine does not return a remaining value, only the - parsed TokenList. - - """ - # XXX: but what about bare CR and LF? They might signal the start or - # end of an encoded word. YAGNI for now, since our current parsers - # will never send us strings with bare CR or LF. - - unstructured = UnstructuredTokenList() - while value: - if value[0] in WSP: - token, value = get_fws(value) - unstructured.append(token) - continue - if value.startswith('=?'): - try: - token, value = get_encoded_word(value) - except errors.HeaderParseError: - # XXX: Need to figure out how to register defects when - # appropriate here. - pass - else: - have_ws = True - if len(unstructured) > 0: - if unstructured[-1].token_type != 'fws': - unstructured.defects.append(errors.InvalidHeaderDefect( - "missing whitespace before encoded word")) - have_ws = False - if have_ws and len(unstructured) > 1: - if unstructured[-2].token_type == 'encoded-word': - unstructured[-1] = EWWhiteSpaceTerminal( - unstructured[-1], 'fws') - unstructured.append(token) - continue - tok, *remainder = _wsp_splitter(value, 1) - vtext = ValueTerminal(tok, 'vtext') - _validate_xtext(vtext) - unstructured.append(vtext) - value = ''.join(remainder) - return unstructured - -def get_qp_ctext(value): - """ctext = - - This is not the RFC ctext, since we are handling nested comments in comment - and unquoting quoted-pairs here. We allow anything except the '()' - characters, but if we find any ASCII other than the RFC defined printable - ASCII an NonPrintableDefect is added to the token's defects list. Since - quoted pairs are converted to their unquoted values, what is returned is - a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value - is ' '. - - """ - ptext, value, _ = _get_ptext_to_endchars(value, '()') - ptext = WhiteSpaceTerminal(ptext, 'ptext') - _validate_xtext(ptext) - return ptext, value - -def get_qcontent(value): - """qcontent = qtext / quoted-pair - - We allow anything except the DQUOTE character, but if we find any ASCII - other than the RFC defined printable ASCII an NonPrintableDefect is - added to the token's defects list. Any quoted pairs are converted to their - unquoted values, so what is returned is a 'ptext' token. In this case it - is a ValueTerminal. - - """ - ptext, value, _ = _get_ptext_to_endchars(value, '"') - ptext = ValueTerminal(ptext, 'ptext') - _validate_xtext(ptext) - return ptext, value - -def get_atext(value): - """atext = - - We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to - the token's defects list if we find non-atext characters. - """ - m = _non_atom_end_matcher(value) - if not m: - raise errors.HeaderParseError( - "expected atext but found '{}'".format(value)) - atext = m.group() - value = value[len(atext):] - atext = ValueTerminal(atext, 'atext') - _validate_xtext(atext) - return atext, value - -def get_bare_quoted_string(value): - """bare-quoted-string = DQUOTE *([FWS] qcontent) [FWS] DQUOTE - - A quoted-string without the leading or trailing white space. Its - value is the text between the quote marks, with whitespace - preserved and quoted pairs decoded. - """ - if value[0] != '"': - raise errors.HeaderParseError( - "expected '\"' but found '{}'".format(value)) - bare_quoted_string = BareQuotedString() - value = value[1:] - while value and value[0] != '"': - if value[0] in WSP: - token, value = get_fws(value) - elif value[:2] == '=?': - try: - token, value = get_encoded_word(value) - bare_quoted_string.defects.append(errors.InvalidHeaderDefect( - "encoded word inside quoted string")) - except errors.HeaderParseError: - token, value = get_qcontent(value) - else: - token, value = get_qcontent(value) - bare_quoted_string.append(token) - if not value: - bare_quoted_string.defects.append(errors.InvalidHeaderDefect( - "end of header inside quoted string")) - return bare_quoted_string, value - return bare_quoted_string, value[1:] - -def get_comment(value): - """comment = "(" *([FWS] ccontent) [FWS] ")" - ccontent = ctext / quoted-pair / comment - - We handle nested comments here, and quoted-pair in our qp-ctext routine. - """ - if value and value[0] != '(': - raise errors.HeaderParseError( - "expected '(' but found '{}'".format(value)) - comment = Comment() - value = value[1:] - while value and value[0] != ")": - if value[0] in WSP: - token, value = get_fws(value) - elif value[0] == '(': - token, value = get_comment(value) - else: - token, value = get_qp_ctext(value) - comment.append(token) - if not value: - comment.defects.append(errors.InvalidHeaderDefect( - "end of header inside comment")) - return comment, value - return comment, value[1:] - -def get_cfws(value): - """CFWS = (1*([FWS] comment) [FWS]) / FWS - - """ - cfws = CFWSList() - while value and value[0] in CFWS_LEADER: - if value[0] in WSP: - token, value = get_fws(value) - else: - token, value = get_comment(value) - cfws.append(token) - return cfws, value - -def get_quoted_string(value): - """quoted-string = [CFWS] [CFWS] - - 'bare-quoted-string' is an intermediate class defined by this - parser and not by the RFC grammar. It is the quoted string - without any attached CFWS. - """ - quoted_string = QuotedString() - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - quoted_string.append(token) - token, value = get_bare_quoted_string(value) - quoted_string.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - quoted_string.append(token) - return quoted_string, value - -def get_atom(value): - """atom = [CFWS] 1*atext [CFWS] - - An atom could be an rfc2047 encoded word. - """ - atom = Atom() - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - atom.append(token) - if value and value[0] in ATOM_ENDS: - raise errors.HeaderParseError( - "expected atom but found '{}'".format(value)) - if value.startswith('=?'): - try: - token, value = get_encoded_word(value) - except errors.HeaderParseError: - # XXX: need to figure out how to register defects when - # appropriate here. - token, value = get_atext(value) - else: - token, value = get_atext(value) - atom.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - atom.append(token) - return atom, value - -def get_dot_atom_text(value): - """ dot-text = 1*atext *("." 1*atext) - - """ - dot_atom_text = DotAtomText() - if not value or value[0] in ATOM_ENDS: - raise errors.HeaderParseError("expected atom at a start of " - "dot-atom-text but found '{}'".format(value)) - while value and value[0] not in ATOM_ENDS: - token, value = get_atext(value) - dot_atom_text.append(token) - if value and value[0] == '.': - dot_atom_text.append(DOT) - value = value[1:] - if dot_atom_text[-1] is DOT: - raise errors.HeaderParseError("expected atom at end of dot-atom-text " - "but found '{}'".format('.'+value)) - return dot_atom_text, value - -def get_dot_atom(value): - """ dot-atom = [CFWS] dot-atom-text [CFWS] - - Any place we can have a dot atom, we could instead have an rfc2047 encoded - word. - """ - dot_atom = DotAtom() - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - dot_atom.append(token) - if value.startswith('=?'): - try: - token, value = get_encoded_word(value) - except errors.HeaderParseError: - # XXX: need to figure out how to register defects when - # appropriate here. - token, value = get_dot_atom_text(value) - else: - token, value = get_dot_atom_text(value) - dot_atom.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - dot_atom.append(token) - return dot_atom, value - -def get_word(value): - """word = atom / quoted-string - - Either atom or quoted-string may start with CFWS. We have to peel off this - CFWS first to determine which type of word to parse. Afterward we splice - the leading CFWS, if any, into the parsed sub-token. - - If neither an atom or a quoted-string is found before the next special, a - HeaderParseError is raised. - - The token returned is either an Atom or a QuotedString, as appropriate. - This means the 'word' level of the formal grammar is not represented in the - parse tree; this is because having that extra layer when manipulating the - parse tree is more confusing than it is helpful. - - """ - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - else: - leader = None - if value[0]=='"': - token, value = get_quoted_string(value) - elif value[0] in SPECIALS: - raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " - "but found '{}'".format(value)) - else: - token, value = get_atom(value) - if leader is not None: - token[:0] = [leader] - return token, value - -def get_phrase(value): - """ phrase = 1*word / obs-phrase - obs-phrase = word *(word / "." / CFWS) - - This means a phrase can be a sequence of words, periods, and CFWS in any - order as long as it starts with at least one word. If anything other than - words is detected, an ObsoleteHeaderDefect is added to the token's defect - list. We also accept a phrase that starts with CFWS followed by a dot; - this is registered as an InvalidHeaderDefect, since it is not supported by - even the obsolete grammar. - - """ - phrase = Phrase() - try: - token, value = get_word(value) - phrase.append(token) - except errors.HeaderParseError: - phrase.defects.append(errors.InvalidHeaderDefect( - "phrase does not start with word")) - while value and value[0] not in PHRASE_ENDS: - if value[0]=='.': - phrase.append(DOT) - phrase.defects.append(errors.ObsoleteHeaderDefect( - "period in 'phrase'")) - value = value[1:] - else: - try: - token, value = get_word(value) - except errors.HeaderParseError: - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - phrase.defects.append(errors.ObsoleteHeaderDefect( - "comment found without atom")) - else: - raise - phrase.append(token) - return phrase, value - -def get_local_part(value): - """ local-part = dot-atom / quoted-string / obs-local-part - - """ - local_part = LocalPart() - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - raise errors.HeaderParseError( - "expected local-part but found '{}'".format(value)) - try: - token, value = get_dot_atom(value) - except errors.HeaderParseError: - try: - token, value = get_word(value) - except errors.HeaderParseError: - if value[0] != '\\' and value[0] in PHRASE_ENDS: - raise - token = TokenList() - if leader is not None: - token[:0] = [leader] - local_part.append(token) - if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): - obs_local_part, value = get_obs_local_part(str(local_part) + value) - if obs_local_part.token_type == 'invalid-obs-local-part': - local_part.defects.append(errors.InvalidHeaderDefect( - "local-part is not dot-atom, quoted-string, or obs-local-part")) - else: - local_part.defects.append(errors.ObsoleteHeaderDefect( - "local-part is not a dot-atom (contains CFWS)")) - local_part[0] = obs_local_part - try: - local_part.value.encode('ascii') - except UnicodeEncodeError: - local_part.defects.append(errors.NonASCIILocalPartDefect( - "local-part contains non-ASCII characters)")) - return local_part, value - -def get_obs_local_part(value): - """ obs-local-part = word *("." word) - """ - obs_local_part = ObsLocalPart() - last_non_ws_was_dot = False - while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): - if value[0] == '.': - if last_non_ws_was_dot: - obs_local_part.defects.append(errors.InvalidHeaderDefect( - "invalid repeated '.'")) - obs_local_part.append(DOT) - last_non_ws_was_dot = True - value = value[1:] - continue - elif value[0]=='\\': - obs_local_part.append(ValueTerminal(value[0], - 'misplaced-special')) - value = value[1:] - obs_local_part.defects.append(errors.InvalidHeaderDefect( - "'\\' character outside of quoted-string/ccontent")) - last_non_ws_was_dot = False - continue - if obs_local_part and obs_local_part[-1].token_type != 'dot': - obs_local_part.defects.append(errors.InvalidHeaderDefect( - "missing '.' between words")) - try: - token, value = get_word(value) - last_non_ws_was_dot = False - except errors.HeaderParseError: - if value[0] not in CFWS_LEADER: - raise - token, value = get_cfws(value) - obs_local_part.append(token) - if (obs_local_part[0].token_type == 'dot' or - obs_local_part[0].token_type=='cfws' and - obs_local_part[1].token_type=='dot'): - obs_local_part.defects.append(errors.InvalidHeaderDefect( - "Invalid leading '.' in local part")) - if (obs_local_part[-1].token_type == 'dot' or - obs_local_part[-1].token_type=='cfws' and - obs_local_part[-2].token_type=='dot'): - obs_local_part.defects.append(errors.InvalidHeaderDefect( - "Invalid trailing '.' in local part")) - if obs_local_part.defects: - obs_local_part.token_type = 'invalid-obs-local-part' - return obs_local_part, value - -def get_dtext(value): - """ dtext = / obs-dtext - obs-dtext = obs-NO-WS-CTL / quoted-pair - - We allow anything except the excluded characters, but if we find any - ASCII other than the RFC defined printable ASCII an NonPrintableDefect is - added to the token's defects list. Quoted pairs are converted to their - unquoted values, so what is returned is a ptext token, in this case a - ValueTerminal. If there were quoted-printables, an ObsoleteHeaderDefect is - added to the returned token's defect list. - - """ - ptext, value, had_qp = _get_ptext_to_endchars(value, '[]') - ptext = ValueTerminal(ptext, 'ptext') - if had_qp: - ptext.defects.append(errors.ObsoleteHeaderDefect( - "quoted printable found in domain-literal")) - _validate_xtext(ptext) - return ptext, value - -def _check_for_early_dl_end(value, domain_literal): - if value: - return False - domain_literal.append(errors.InvalidHeaderDefect( - "end of input inside domain-literal")) - domain_literal.append(ValueTerminal(']', 'domain-literal-end')) - return True - -def get_domain_literal(value): - """ domain-literal = [CFWS] "[" *([FWS] dtext) [FWS] "]" [CFWS] - - """ - domain_literal = DomainLiteral() - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - domain_literal.append(token) - if not value: - raise errors.HeaderParseError("expected domain-literal") - if value[0] != '[': - raise errors.HeaderParseError("expected '[' at start of domain-literal " - "but found '{}'".format(value)) - value = value[1:] - if _check_for_early_dl_end(value, domain_literal): - return domain_literal, value - domain_literal.append(ValueTerminal('[', 'domain-literal-start')) - if value[0] in WSP: - token, value = get_fws(value) - domain_literal.append(token) - token, value = get_dtext(value) - domain_literal.append(token) - if _check_for_early_dl_end(value, domain_literal): - return domain_literal, value - if value[0] in WSP: - token, value = get_fws(value) - domain_literal.append(token) - if _check_for_early_dl_end(value, domain_literal): - return domain_literal, value - if value[0] != ']': - raise errors.HeaderParseError("expected ']' at end of domain-literal " - "but found '{}'".format(value)) - domain_literal.append(ValueTerminal(']', 'domain-literal-end')) - value = value[1:] - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - domain_literal.append(token) - return domain_literal, value - -def get_domain(value): - """ domain = dot-atom / domain-literal / obs-domain - obs-domain = atom *("." atom)) - - """ - domain = Domain() - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - raise errors.HeaderParseError( - "expected domain but found '{}'".format(value)) - if value[0] == '[': - token, value = get_domain_literal(value) - if leader is not None: - token[:0] = [leader] - domain.append(token) - return domain, value - try: - token, value = get_dot_atom(value) - except errors.HeaderParseError: - token, value = get_atom(value) - if leader is not None: - token[:0] = [leader] - domain.append(token) - if value and value[0] == '.': - domain.defects.append(errors.ObsoleteHeaderDefect( - "domain is not a dot-atom (contains CFWS)")) - if domain[0].token_type == 'dot-atom': - domain[:] = domain[0] - while value and value[0] == '.': - domain.append(DOT) - token, value = get_atom(value[1:]) - domain.append(token) - return domain, value - -def get_addr_spec(value): - """ addr-spec = local-part "@" domain - - """ - addr_spec = AddrSpec() - token, value = get_local_part(value) - addr_spec.append(token) - if not value or value[0] != '@': - addr_spec.defects.append(errors.InvalidHeaderDefect( - "add-spec local part with no domain")) - return addr_spec, value - addr_spec.append(ValueTerminal('@', 'address-at-symbol')) - token, value = get_domain(value[1:]) - addr_spec.append(token) - return addr_spec, value - -def get_obs_route(value): - """ obs-route = obs-domain-list ":" - obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain]) - - Returns an obs-route token with the appropriate sub-tokens (that is, - there is no obs-domain-list in the parse tree). - """ - obs_route = ObsRoute() - while value and (value[0]==',' or value[0] in CFWS_LEADER): - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - obs_route.append(token) - elif value[0] == ',': - obs_route.append(ListSeparator) - value = value[1:] - if not value or value[0] != '@': - raise errors.HeaderParseError( - "expected obs-route domain but found '{}'".format(value)) - obs_route.append(RouteComponentMarker) - token, value = get_domain(value[1:]) - obs_route.append(token) - while value and value[0]==',': - obs_route.append(ListSeparator) - value = value[1:] - if not value: - break - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - obs_route.append(token) - if value[0] == '@': - obs_route.append(RouteComponentMarker) - token, value = get_domain(value[1:]) - obs_route.append(token) - if not value: - raise errors.HeaderParseError("end of header while parsing obs-route") - if value[0] != ':': - raise errors.HeaderParseError( "expected ':' marking end of " - "obs-route but found '{}'".format(value)) - obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker')) - return obs_route, value[1:] - -def get_angle_addr(value): - """ angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr - obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS] - - """ - angle_addr = AngleAddr() - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - angle_addr.append(token) - if not value or value[0] != '<': - raise errors.HeaderParseError( - "expected angle-addr but found '{}'".format(value)) - angle_addr.append(ValueTerminal('<', 'angle-addr-start')) - value = value[1:] - # Although it is not legal per RFC5322, SMTP uses '<>' in certain - # circumstances. - if value[0] == '>': - angle_addr.append(ValueTerminal('>', 'angle-addr-end')) - angle_addr.defects.append(errors.InvalidHeaderDefect( - "null addr-spec in angle-addr")) - value = value[1:] - return angle_addr, value - try: - token, value = get_addr_spec(value) - except errors.HeaderParseError: - try: - token, value = get_obs_route(value) - angle_addr.defects.append(errors.ObsoleteHeaderDefect( - "obsolete route specification in angle-addr")) - except errors.HeaderParseError: - raise errors.HeaderParseError( - "expected addr-spec or obs-route but found '{}'".format(value)) - angle_addr.append(token) - token, value = get_addr_spec(value) - angle_addr.append(token) - if value and value[0] == '>': - value = value[1:] - else: - angle_addr.defects.append(errors.InvalidHeaderDefect( - "missing trailing '>' on angle-addr")) - angle_addr.append(ValueTerminal('>', 'angle-addr-end')) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - angle_addr.append(token) - return angle_addr, value - -def get_display_name(value): - """ display-name = phrase - - Because this is simply a name-rule, we don't return a display-name - token containing a phrase, but rather a display-name token with - the content of the phrase. - - """ - display_name = DisplayName() - token, value = get_phrase(value) - display_name.extend(token[:]) - display_name.defects = token.defects[:] - return display_name, value - - -def get_name_addr(value): - """ name-addr = [display-name] angle-addr - - """ - name_addr = NameAddr() - # Both the optional display name and the angle-addr can start with cfws. - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(leader)) - if value[0] != '<': - if value[0] in PHRASE_ENDS: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(value)) - token, value = get_display_name(value) - if not value: - raise errors.HeaderParseError( - "expected name-addr but found '{}'".format(token)) - if leader is not None: - token[0][:0] = [leader] - leader = None - name_addr.append(token) - token, value = get_angle_addr(value) - if leader is not None: - token[:0] = [leader] - name_addr.append(token) - return name_addr, value - -def get_mailbox(value): - """ mailbox = name-addr / addr-spec - - """ - # The only way to figure out if we are dealing with a name-addr or an - # addr-spec is to try parsing each one. - mailbox = Mailbox() - try: - token, value = get_name_addr(value) - except errors.HeaderParseError: - try: - token, value = get_addr_spec(value) - except errors.HeaderParseError: - raise errors.HeaderParseError( - "expected mailbox but found '{}'".format(value)) - if any(isinstance(x, errors.InvalidHeaderDefect) - for x in token.all_defects): - mailbox.token_type = 'invalid-mailbox' - mailbox.append(token) - return mailbox, value - -def get_invalid_mailbox(value, endchars): - """ Read everything up to one of the chars in endchars. - - This is outside the formal grammar. The InvalidMailbox TokenList that is - returned acts like a Mailbox, but the data attributes are None. - - """ - invalid_mailbox = InvalidMailbox() - while value and value[0] not in endchars: - if value[0] in PHRASE_ENDS: - invalid_mailbox.append(ValueTerminal(value[0], - 'misplaced-special')) - value = value[1:] - else: - token, value = get_phrase(value) - invalid_mailbox.append(token) - return invalid_mailbox, value - -def get_mailbox_list(value): - """ mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list - obs-mbox-list = *([CFWS] ",") mailbox *("," [mailbox / CFWS]) - - For this routine we go outside the formal grammar in order to improve error - handling. We recognize the end of the mailbox list only at the end of the - value or at a ';' (the group terminator). This is so that we can turn - invalid mailboxes into InvalidMailbox tokens and continue parsing any - remaining valid mailboxes. We also allow all mailbox entries to be null, - and this condition is handled appropriately at a higher level. - - """ - mailbox_list = MailboxList() - while value and value[0] != ';': - try: - token, value = get_mailbox(value) - mailbox_list.append(token) - except errors.HeaderParseError: - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value or value[0] in ',;': - mailbox_list.append(leader) - mailbox_list.defects.append(errors.ObsoleteHeaderDefect( - "empty element in mailbox-list")) - else: - token, value = get_invalid_mailbox(value, ',;') - if leader is not None: - token[:0] = [leader] - mailbox_list.append(token) - mailbox_list.defects.append(errors.InvalidHeaderDefect( - "invalid mailbox in mailbox-list")) - elif value[0] == ',': - mailbox_list.defects.append(errors.ObsoleteHeaderDefect( - "empty element in mailbox-list")) - else: - token, value = get_invalid_mailbox(value, ',;') - if leader is not None: - token[:0] = [leader] - mailbox_list.append(token) - mailbox_list.defects.append(errors.InvalidHeaderDefect( - "invalid mailbox in mailbox-list")) - if value and value[0] not in ',;': - # Crap after mailbox; treat it as an invalid mailbox. - # The mailbox info will still be available. - mailbox = mailbox_list[-1] - mailbox.token_type = 'invalid-mailbox' - token, value = get_invalid_mailbox(value, ',;') - mailbox.extend(token) - mailbox_list.defects.append(errors.InvalidHeaderDefect( - "invalid mailbox in mailbox-list")) - if value and value[0] == ',': - mailbox_list.append(ListSeparator) - value = value[1:] - return mailbox_list, value - - -def get_group_list(value): - """ group-list = mailbox-list / CFWS / obs-group-list - obs-group-list = 1*([CFWS] ",") [CFWS] - - """ - group_list = GroupList() - if not value: - group_list.defects.append(errors.InvalidHeaderDefect( - "end of header before group-list")) - return group_list, value - leader = None - if value and value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - # This should never happen in email parsing, since CFWS-only is a - # legal alternative to group-list in a group, which is the only - # place group-list appears. - group_list.defects.append(errors.InvalidHeaderDefect( - "end of header in group-list")) - group_list.append(leader) - return group_list, value - if value[0] == ';': - group_list.append(leader) - return group_list, value - token, value = get_mailbox_list(value) - if len(token.all_mailboxes)==0: - if leader is not None: - group_list.append(leader) - group_list.extend(token) - group_list.defects.append(errors.ObsoleteHeaderDefect( - "group-list with empty entries")) - return group_list, value - if leader is not None: - token[:0] = [leader] - group_list.append(token) - return group_list, value - -def get_group(value): - """ group = display-name ":" [group-list] ";" [CFWS] - - """ - group = Group() - token, value = get_display_name(value) - if not value or value[0] != ':': - raise errors.HeaderParseError("expected ':' at end of group " - "display name but found '{}'".format(value)) - group.append(token) - group.append(ValueTerminal(':', 'group-display-name-terminator')) - value = value[1:] - if value and value[0] == ';': - group.append(ValueTerminal(';', 'group-terminator')) - return group, value[1:] - token, value = get_group_list(value) - group.append(token) - if not value: - group.defects.append(errors.InvalidHeaderDefect( - "end of header in group")) - if value[0] != ';': - raise errors.HeaderParseError( - "expected ';' at end of group but found {}".format(value)) - group.append(ValueTerminal(';', 'group-terminator')) - value = value[1:] - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - group.append(token) - return group, value - -def get_address(value): - """ address = mailbox / group - - Note that counter-intuitively, an address can be either a single address or - a list of addresses (a group). This is why the returned Address object has - a 'mailboxes' attribute which treats a single address as a list of length - one. When you need to differentiate between to two cases, extract the single - element, which is either a mailbox or a group token. - - """ - # The formal grammar isn't very helpful when parsing an address. mailbox - # and group, especially when allowing for obsolete forms, start off very - # similarly. It is only when you reach one of @, <, or : that you know - # what you've got. So, we try each one in turn, starting with the more - # likely of the two. We could perhaps make this more efficient by looking - # for a phrase and then branching based on the next character, but that - # would be a premature optimization. - address = Address() - try: - token, value = get_group(value) - except errors.HeaderParseError: - try: - token, value = get_mailbox(value) - except errors.HeaderParseError: - raise errors.HeaderParseError( - "expected address but found '{}'".format(value)) - address.append(token) - return address, value - -def get_address_list(value): - """ address_list = (address *("," address)) / obs-addr-list - obs-addr-list = *([CFWS] ",") address *("," [address / CFWS]) - - We depart from the formal grammar here by continuing to parse until the end - of the input, assuming the input to be entirely composed of an - address-list. This is always true in email parsing, and allows us - to skip invalid addresses to parse additional valid ones. - - """ - address_list = AddressList() - while value: - try: - token, value = get_address(value) - address_list.append(token) - except errors.HeaderParseError as err: - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value or value[0] == ',': - address_list.append(leader) - address_list.defects.append(errors.ObsoleteHeaderDefect( - "address-list entry with no content")) - else: - token, value = get_invalid_mailbox(value, ',') - if leader is not None: - token[:0] = [leader] - address_list.append(Address([token])) - address_list.defects.append(errors.InvalidHeaderDefect( - "invalid address in address-list")) - elif value[0] == ',': - address_list.defects.append(errors.ObsoleteHeaderDefect( - "empty element in address-list")) - else: - token, value = get_invalid_mailbox(value, ',') - if leader is not None: - token[:0] = [leader] - address_list.append(Address([token])) - address_list.defects.append(errors.InvalidHeaderDefect( - "invalid address in address-list")) - if value and value[0] != ',': - # Crap after address; treat it as an invalid mailbox. - # The mailbox info will still be available. - mailbox = address_list[-1][0] - mailbox.token_type = 'invalid-mailbox' - token, value = get_invalid_mailbox(value, ',') - mailbox.extend(token) - address_list.defects.append(errors.InvalidHeaderDefect( - "invalid address in address-list")) - if value: # Must be a , at this point. - address_list.append(ValueTerminal(',', 'list-separator')) - value = value[1:] - return address_list, value - -# -# XXX: As I begin to add additional header parsers, I'm realizing we probably -# have two level of parser routines: the get_XXX methods that get a token in -# the grammar, and parse_XXX methods that parse an entire field value. So -# get_address_list above should really be a parse_ method, as probably should -# be get_unstructured. -# - -def parse_mime_version(value): - """ mime-version = [CFWS] 1*digit [CFWS] "." [CFWS] 1*digit [CFWS] - - """ - # The [CFWS] is implicit in the RFC 2045 BNF. - # XXX: This routine is a bit verbose, should factor out a get_int method. - mime_version = MIMEVersion() - if not value: - mime_version.defects.append(errors.HeaderMissingRequiredValue( - "Missing MIME version number (eg: 1.0)")) - return mime_version - if value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mime_version.append(token) - if not value: - mime_version.defects.append(errors.HeaderMissingRequiredValue( - "Expected MIME version number but found only CFWS")) - digits = '' - while value and value[0] != '.' and value[0] not in CFWS_LEADER: - digits += value[0] - value = value[1:] - if not digits.isdigit(): - mime_version.defects.append(errors.InvalidHeaderDefect( - "Expected MIME major version number but found {!r}".format(digits))) - mime_version.append(ValueTerminal(digits, 'xtext')) - else: - mime_version.major = int(digits) - mime_version.append(ValueTerminal(digits, 'digits')) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mime_version.append(token) - if not value or value[0] != '.': - if mime_version.major is not None: - mime_version.defects.append(errors.InvalidHeaderDefect( - "Incomplete MIME version; found only major number")) - if value: - mime_version.append(ValueTerminal(value, 'xtext')) - return mime_version - mime_version.append(ValueTerminal('.', 'version-separator')) - value = value[1:] - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mime_version.append(token) - if not value: - if mime_version.major is not None: - mime_version.defects.append(errors.InvalidHeaderDefect( - "Incomplete MIME version; found only major number")) - return mime_version - digits = '' - while value and value[0] not in CFWS_LEADER: - digits += value[0] - value = value[1:] - if not digits.isdigit(): - mime_version.defects.append(errors.InvalidHeaderDefect( - "Expected MIME minor version number but found {!r}".format(digits))) - mime_version.append(ValueTerminal(digits, 'xtext')) - else: - mime_version.minor = int(digits) - mime_version.append(ValueTerminal(digits, 'digits')) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mime_version.append(token) - if value: - mime_version.defects.append(errors.InvalidHeaderDefect( - "Excess non-CFWS text after MIME version")) - mime_version.append(ValueTerminal(value, 'xtext')) - return mime_version - -def get_invalid_parameter(value): - """ Read everything up to the next ';'. - - This is outside the formal grammar. The InvalidParameter TokenList that is - returned acts like a Parameter, but the data attributes are None. - - """ - invalid_parameter = InvalidParameter() - while value and value[0] != ';': - if value[0] in PHRASE_ENDS: - invalid_parameter.append(ValueTerminal(value[0], - 'misplaced-special')) - value = value[1:] - else: - token, value = get_phrase(value) - invalid_parameter.append(token) - return invalid_parameter, value - -def get_ttext(value): - """ttext = - - We allow any non-TOKEN_ENDS in ttext, but add defects to the token's - defects list if we find non-ttext characters. We also register defects for - *any* non-printables even though the RFC doesn't exclude all of them, - because we follow the spirit of RFC 5322. - - """ - m = _non_token_end_matcher(value) - if not m: - raise errors.HeaderParseError( - "expected ttext but found '{}'".format(value)) - ttext = m.group() - value = value[len(ttext):] - ttext = ValueTerminal(ttext, 'ttext') - _validate_xtext(ttext) - return ttext, value - -def get_token(value): - """token = [CFWS] 1*ttext [CFWS] - - The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or - tspecials. We also exclude tabs even though the RFC doesn't. - - The RFC implies the CFWS but is not explicit about it in the BNF. - - """ - mtoken = Token() - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mtoken.append(token) - if value and value[0] in TOKEN_ENDS: - raise errors.HeaderParseError( - "expected token but found '{}'".format(value)) - token, value = get_ttext(value) - mtoken.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - mtoken.append(token) - return mtoken, value - -def get_attrtext(value): - """attrtext = 1*(any non-ATTRIBUTE_ENDS character) - - We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the - token's defects list if we find non-attrtext characters. We also register - defects for *any* non-printables even though the RFC doesn't exclude all of - them, because we follow the spirit of RFC 5322. - - """ - m = _non_attribute_end_matcher(value) - if not m: - raise errors.HeaderParseError( - "expected attrtext but found {!r}".format(value)) - attrtext = m.group() - value = value[len(attrtext):] - attrtext = ValueTerminal(attrtext, 'attrtext') - _validate_xtext(attrtext) - return attrtext, value - -def get_attribute(value): - """ [CFWS] 1*attrtext [CFWS] - - This version of the BNF makes the CFWS explicit, and as usual we use a - value terminal for the actual run of characters. The RFC equivalent of - attrtext is the token characters, with the subtraction of '*', "'", and '%'. - We include tab in the excluded set just as we do for token. - - """ - attribute = Attribute() - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - attribute.append(token) - if value and value[0] in ATTRIBUTE_ENDS: - raise errors.HeaderParseError( - "expected token but found '{}'".format(value)) - token, value = get_attrtext(value) - attribute.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - attribute.append(token) - return attribute, value - -def get_extended_attrtext(value): - """attrtext = 1*(any non-ATTRIBUTE_ENDS character plus '%') - - This is a special parsing routine so that we get a value that - includes % escapes as a single string (which we decode as a single - string later). - - """ - m = _non_extended_attribute_end_matcher(value) - if not m: - raise errors.HeaderParseError( - "expected extended attrtext but found {!r}".format(value)) - attrtext = m.group() - value = value[len(attrtext):] - attrtext = ValueTerminal(attrtext, 'extended-attrtext') - _validate_xtext(attrtext) - return attrtext, value - -def get_extended_attribute(value): - """ [CFWS] 1*extended_attrtext [CFWS] - - This is like the non-extended version except we allow % characters, so that - we can pick up an encoded value as a single string. - - """ - # XXX: should we have an ExtendedAttribute TokenList? - attribute = Attribute() - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - attribute.append(token) - if value and value[0] in EXTENDED_ATTRIBUTE_ENDS: - raise errors.HeaderParseError( - "expected token but found '{}'".format(value)) - token, value = get_extended_attrtext(value) - attribute.append(token) - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - attribute.append(token) - return attribute, value - -def get_section(value): - """ '*' digits - - The formal BNF is more complicated because leading 0s are not allowed. We - check for that and add a defect. We also assume no CFWS is allowed between - the '*' and the digits, though the RFC is not crystal clear on that. - The caller should already have dealt with leading CFWS. - - """ - section = Section() - if not value or value[0] != '*': - raise errors.HeaderParseError("Expected section but found {}".format( - value)) - section.append(ValueTerminal('*', 'section-marker')) - value = value[1:] - if not value or not value[0].isdigit(): - raise errors.HeaderParseError("Expected section number but " - "found {}".format(value)) - digits = '' - while value and value[0].isdigit(): - digits += value[0] - value = value[1:] - if digits[0] == '0' and digits != '0': - section.defects.append(errors.InvalidHeaderError("section number" - "has an invalid leading 0")) - section.number = int(digits) - section.append(ValueTerminal(digits, 'digits')) - return section, value - - -def get_value(value): - """ quoted-string / attribute - - """ - v = Value() - if not value: - raise errors.HeaderParseError("Expected value but found end of string") - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - raise errors.HeaderParseError("Expected value but found " - "only {}".format(leader)) - if value[0] == '"': - token, value = get_quoted_string(value) - else: - token, value = get_extended_attribute(value) - if leader is not None: - token[:0] = [leader] - v.append(token) - return v, value - -def get_parameter(value): - """ attribute [section] ["*"] [CFWS] "=" value - - The CFWS is implied by the RFC but not made explicit in the BNF. This - simplified form of the BNF from the RFC is made to conform with the RFC BNF - through some extra checks. We do it this way because it makes both error - recovery and working with the resulting parse tree easier. - """ - # It is possible CFWS would also be implicitly allowed between the section - # and the 'extended-attribute' marker (the '*') , but we've never seen that - # in the wild and we will therefore ignore the possibility. - param = Parameter() - token, value = get_attribute(value) - param.append(token) - if not value or value[0] == ';': - param.defects.append(errors.InvalidHeaderDefect("Parameter contains " - "name ({}) but no value".format(token))) - return param, value - if value[0] == '*': - try: - token, value = get_section(value) - param.sectioned = True - param.append(token) - except errors.HeaderParseError: - pass - if not value: - raise errors.HeaderParseError("Incomplete parameter") - if value[0] == '*': - param.append(ValueTerminal('*', 'extended-parameter-marker')) - value = value[1:] - param.extended = True - if value[0] != '=': - raise errors.HeaderParseError("Parameter not followed by '='") - param.append(ValueTerminal('=', 'parameter-separator')) - value = value[1:] - leader = None - if value and value[0] in CFWS_LEADER: - token, value = get_cfws(value) - param.append(token) - remainder = None - appendto = param - if param.extended and value and value[0] == '"': - # Now for some serious hackery to handle the common invalid case of - # double quotes around an extended value. We also accept (with defect) - # a value marked as encoded that isn't really. - qstring, remainder = get_quoted_string(value) - inner_value = qstring.stripped_value - semi_valid = False - if param.section_number == 0: - if inner_value and inner_value[0] == "'": - semi_valid = True - else: - token, rest = get_attrtext(inner_value) - if rest and rest[0] == "'": - semi_valid = True - else: - try: - token, rest = get_extended_attrtext(inner_value) - except: - pass - else: - if not rest: - semi_valid = True - if semi_valid: - param.defects.append(errors.InvalidHeaderDefect( - "Quoted string value for extended parameter is invalid")) - param.append(qstring) - for t in qstring: - if t.token_type == 'bare-quoted-string': - t[:] = [] - appendto = t - break - value = inner_value - else: - remainder = None - param.defects.append(errors.InvalidHeaderDefect( - "Parameter marked as extended but appears to have a " - "quoted string value that is non-encoded")) - if value and value[0] == "'": - token = None - else: - token, value = get_value(value) - if not param.extended or param.section_number > 0: - if not value or value[0] != "'": - appendto.append(token) - if remainder is not None: - assert not value, value - value = remainder - return param, value - param.defects.append(errors.InvalidHeaderDefect( - "Apparent initial-extended-value but attribute " - "was not marked as extended or was not initial section")) - if not value: - # Assume the charset/lang is missing and the token is the value. - param.defects.append(errors.InvalidHeaderDefect( - "Missing required charset/lang delimiters")) - appendto.append(token) - if remainder is None: - return param, value - else: - if token is not None: - for t in token: - if t.token_type == 'extended-attrtext': - break - t.token_type == 'attrtext' - appendto.append(t) - param.charset = t.value - if value[0] != "'": - raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " - "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) - value = value[1:] - if value and value[0] != "'": - token, value = get_attrtext(value) - appendto.append(token) - param.lang = token.value - if not value or value[0] != "'": - raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " - "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) - value = value[1:] - if remainder is not None: - # Treat the rest of value as bare quoted string content. - v = Value() - while value: - if value[0] in WSP: - token, value = get_fws(value) - else: - token, value = get_qcontent(value) - v.append(token) - token = v - else: - token, value = get_value(value) - appendto.append(token) - if remainder is not None: - assert not value, value - value = remainder - return param, value - -def parse_mime_parameters(value): - """ parameter *( ";" parameter ) - - That BNF is meant to indicate this routine should only be called after - finding and handling the leading ';'. There is no corresponding rule in - the formal RFC grammar, but it is more convenient for us for the set of - parameters to be treated as its own TokenList. - - This is 'parse' routine because it consumes the reminaing value, but it - would never be called to parse a full header. Instead it is called to - parse everything after the non-parameter value of a specific MIME header. - - """ - mime_parameters = MimeParameters() - while value: - try: - token, value = get_parameter(value) - mime_parameters.append(token) - except errors.HeaderParseError as err: - leader = None - if value[0] in CFWS_LEADER: - leader, value = get_cfws(value) - if not value: - mime_parameters.append(leader) - return mime_parameters - if value[0] == ';': - if leader is not None: - mime_parameters.append(leader) - mime_parameters.defects.append(errors.InvalidHeaderDefect( - "parameter entry with no content")) - else: - token, value = get_invalid_parameter(value) - if leader: - token[:0] = [leader] - mime_parameters.append(token) - mime_parameters.defects.append(errors.InvalidHeaderDefect( - "invalid parameter {!r}".format(token))) - if value and value[0] != ';': - # Junk after the otherwise valid parameter. Mark it as - # invalid, but it will have a value. - param = mime_parameters[-1] - param.token_type = 'invalid-parameter' - token, value = get_invalid_parameter(value) - param.extend(token) - mime_parameters.defects.append(errors.InvalidHeaderDefect( - "parameter with invalid trailing text {!r}".format(token))) - if value: - # Must be a ';' at this point. - mime_parameters.append(ValueTerminal(';', 'parameter-separator')) - value = value[1:] - return mime_parameters - -def _find_mime_parameters(tokenlist, value): - """Do our best to find the parameters in an invalid MIME header - - """ - while value and value[0] != ';': - if value[0] in PHRASE_ENDS: - tokenlist.append(ValueTerminal(value[0], 'misplaced-special')) - value = value[1:] - else: - token, value = get_phrase(value) - tokenlist.append(token) - if not value: - return - tokenlist.append(ValueTerminal(';', 'parameter-separator')) - tokenlist.append(parse_mime_parameters(value[1:])) - -def parse_content_type_header(value): - """ maintype "/" subtype *( ";" parameter ) - - The maintype and substype are tokens. Theoretically they could - be checked against the official IANA list + x-token, but we - don't do that. - """ - ctype = ContentType() - recover = False - if not value: - ctype.defects.append(errors.HeaderMissingRequiredValue( - "Missing content type specification")) - return ctype - try: - token, value = get_token(value) - except errors.HeaderParseError: - ctype.defects.append(errors.InvalidHeaderDefect( - "Expected content maintype but found {!r}".format(value))) - _find_mime_parameters(ctype, value) - return ctype - ctype.append(token) - # XXX: If we really want to follow the formal grammer we should make - # mantype and subtype specialized TokenLists here. Probably not worth it. - if not value or value[0] != '/': - ctype.defects.append(errors.InvalidHeaderDefect( - "Invalid content type")) - if value: - _find_mime_parameters(ctype, value) - return ctype - ctype.maintype = token.value.strip().lower() - ctype.append(ValueTerminal('/', 'content-type-separator')) - value = value[1:] - try: - token, value = get_token(value) - except errors.HeaderParseError: - ctype.defects.append(errors.InvalidHeaderDefect( - "Expected content subtype but found {!r}".format(value))) - _find_mime_parameters(ctype, value) - return ctype - ctype.append(token) - ctype.subtype = token.value.strip().lower() - if not value: - return ctype - if value[0] != ';': - ctype.defects.append(errors.InvalidHeaderDefect( - "Only parameters are valid after content type, but " - "found {!r}".format(value))) - # The RFC requires that a syntactically invalid content-type be treated - # as text/plain. Perhaps we should postel this, but we should probably - # only do that if we were checking the subtype value against IANA. - del ctype.maintype, ctype.subtype - _find_mime_parameters(ctype, value) - return ctype - ctype.append(ValueTerminal(';', 'parameter-separator')) - ctype.append(parse_mime_parameters(value[1:])) - return ctype - -def parse_content_disposition_header(value): - """ disposition-type *( ";" parameter ) - - """ - disp_header = ContentDisposition() - if not value: - disp_header.defects.append(errors.HeaderMissingRequiredValue( - "Missing content disposition")) - return disp_header - try: - token, value = get_token(value) - except errors.HeaderParseError: - disp_header.defects.append(errors.InvalidHeaderDefect( - "Expected content disposition but found {!r}".format(value))) - _find_mime_parameters(disp_header, value) - return disp_header - disp_header.append(token) - disp_header.content_disposition = token.value.strip().lower() - if not value: - return disp_header - if value[0] != ';': - disp_header.defects.append(errors.InvalidHeaderDefect( - "Only parameters are valid after content disposition, but " - "found {!r}".format(value))) - _find_mime_parameters(disp_header, value) - return disp_header - disp_header.append(ValueTerminal(';', 'parameter-separator')) - disp_header.append(parse_mime_parameters(value[1:])) - return disp_header - -def parse_content_transfer_encoding_header(value): - """ mechanism - - """ - # We should probably validate the values, since the list is fixed. - cte_header = ContentTransferEncoding() - if not value: - cte_header.defects.append(errors.HeaderMissingRequiredValue( - "Missing content transfer encoding")) - return cte_header - try: - token, value = get_token(value) - except errors.HeaderParseError: - cte_header.defects.append(errors.InvalidHeaderDefect( - "Expected content transfer encoding but found {!r}".format(value))) - else: - cte_header.append(token) - cte_header.cte = token.value.strip().lower() - if not value: - return cte_header - while value: - cte_header.defects.append(errors.InvalidHeaderDefect( - "Extra text after content transfer encoding")) - if value[0] in PHRASE_ENDS: - cte_header.append(ValueTerminal(value[0], 'misplaced-special')) - value = value[1:] - else: - token, value = get_phrase(value) - cte_header.append(token) - return cte_header diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/_parseaddr.py --- a/Lib/email/_parseaddr.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/_parseaddr.py Mon Jan 25 17:05:13 2016 +0100 @@ -13,7 +13,7 @@ 'quote', ] -import time, calendar +import time SPACE = ' ' EMPTYSTRING = '' @@ -48,8 +48,6 @@ Accounts for military timezones. """ res = _parsedate_tz(data) - if not res: - return if res[9] is None: res[9] = 0 return tuple(res) @@ -64,8 +62,6 @@ source timezone really was UTC. """ - if not data: - return data = data.split() # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. @@ -181,13 +177,13 @@ def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a POSIX timestamp.""" + """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" if data[9] is None: # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: - t = calendar.timegm(data) - return t - data[9] + t = time.mktime(data[:8] + (0,)) + return t - data[9] - time.timezone def quote(str): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/_policybase.py --- a/Lib/email/_policybase.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,366 +0,0 @@ -"""Policy framework for the email package. - -Allows fine grained feature control of how the package parses and emits data. -""" - -import abc -from email import header -from email import charset as _charset -from email.utils import _has_surrogates - -__all__ = [ - 'Policy', - 'Compat32', - 'compat32', - ] - - -class _PolicyBase: - - """Policy Object basic framework. - - This class is useless unless subclassed. A subclass should define - class attributes with defaults for any values that are to be - managed by the Policy object. The constructor will then allow - non-default values to be set for these attributes at instance - creation time. The instance will be callable, taking these same - attributes keyword arguments, and returning a new instance - identical to the called instance except for those values changed - by the keyword arguments. Instances may be added, yielding new - instances with any non-default values from the right hand - operand overriding those in the left hand operand. That is, - - A + B == A() - - The repr of an instance can be used to reconstruct the object - if and only if the repr of the values can be used to reconstruct - those values. - - """ - - def __init__(self, **kw): - """Create new Policy, possibly overriding some defaults. - - See class docstring for a list of overridable attributes. - - """ - for name, value in kw.items(): - if hasattr(self, name): - super(_PolicyBase,self).__setattr__(name, value) - else: - raise TypeError( - "{!r} is an invalid keyword argument for {}".format( - name, self.__class__.__name__)) - - def __repr__(self): - args = [ "{}={!r}".format(name, value) - for name, value in self.__dict__.items() ] - return "{}({})".format(self.__class__.__name__, ', '.join(args)) - - def clone(self, **kw): - """Return a new instance with specified attributes changed. - - The new instance has the same attribute values as the current object, - except for the changes passed in as keyword arguments. - - """ - newpolicy = self.__class__.__new__(self.__class__) - for attr, value in self.__dict__.items(): - object.__setattr__(newpolicy, attr, value) - for attr, value in kw.items(): - if not hasattr(self, attr): - raise TypeError( - "{!r} is an invalid keyword argument for {}".format( - attr, self.__class__.__name__)) - object.__setattr__(newpolicy, attr, value) - return newpolicy - - def __setattr__(self, name, value): - if hasattr(self, name): - msg = "{!r} object attribute {!r} is read-only" - else: - msg = "{!r} object has no attribute {!r}" - raise AttributeError(msg.format(self.__class__.__name__, name)) - - def __add__(self, other): - """Non-default values from right operand override those from left. - - The object returned is a new instance of the subclass. - - """ - return self.clone(**other.__dict__) - - -def _append_doc(doc, added_doc): - doc = doc.rsplit('\n', 1)[0] - added_doc = added_doc.split('\n', 1)[1] - return doc + '\n' + added_doc - -def _extend_docstrings(cls): - if cls.__doc__ and cls.__doc__.startswith('+'): - cls.__doc__ = _append_doc(cls.__bases__[0].__doc__, cls.__doc__) - for name, attr in cls.__dict__.items(): - if attr.__doc__ and attr.__doc__.startswith('+'): - for c in (c for base in cls.__bases__ for c in base.mro()): - doc = getattr(getattr(c, name), '__doc__') - if doc: - attr.__doc__ = _append_doc(doc, attr.__doc__) - break - return cls - - -class Policy(_PolicyBase, metaclass=abc.ABCMeta): - - r"""Controls for how messages are interpreted and formatted. - - Most of the classes and many of the methods in the email package accept - Policy objects as parameters. A Policy object contains a set of values and - functions that control how input is interpreted and how output is rendered. - For example, the parameter 'raise_on_defect' controls whether or not an RFC - violation results in an error being raised or not, while 'max_line_length' - controls the maximum length of output lines when a Message is serialized. - - Any valid attribute may be overridden when a Policy is created by passing - it as a keyword argument to the constructor. Policy objects are immutable, - but a new Policy object can be created with only certain values changed by - calling the Policy instance with keyword arguments. Policy objects can - also be added, producing a new Policy object in which the non-default - attributes set in the right hand operand overwrite those specified in the - left operand. - - Settable attributes: - - raise_on_defect -- If true, then defects should be raised as errors. - Default: False. - - linesep -- string containing the value to use as separation - between output lines. Default '\n'. - - cte_type -- Type of allowed content transfer encodings - - 7bit -- ASCII only - 8bit -- Content-Transfer-Encoding: 8bit is allowed - - Default: 8bit. Also controls the disposition of - (RFC invalid) binary data in headers; see the - documentation of the binary_fold method. - - max_line_length -- maximum length of lines, excluding 'linesep', - during serialization. None or 0 means no line - wrapping is done. Default is 78. - - mangle_from_ -- a flag that, when True escapes From_ lines in the - body of the message by putting a `>' in front of - them. This is used when the message is being - serialized by a generator. Default: True. - - """ - - raise_on_defect = False - linesep = '\n' - cte_type = '8bit' - max_line_length = 78 - mangle_from_ = False - - def handle_defect(self, obj, defect): - """Based on policy, either raise defect or call register_defect. - - handle_defect(obj, defect) - - defect should be a Defect subclass, but in any case must be an - Exception subclass. obj is the object on which the defect should be - registered if it is not raised. If the raise_on_defect is True, the - defect is raised as an error, otherwise the object and the defect are - passed to register_defect. - - This method is intended to be called by parsers that discover defects. - The email package parsers always call it with Defect instances. - - """ - if self.raise_on_defect: - raise defect - self.register_defect(obj, defect) - - def register_defect(self, obj, defect): - """Record 'defect' on 'obj'. - - Called by handle_defect if raise_on_defect is False. This method is - part of the Policy API so that Policy subclasses can implement custom - defect handling. The default implementation calls the append method of - the defects attribute of obj. The objects used by the email package by - default that get passed to this method will always have a defects - attribute with an append method. - - """ - obj.defects.append(defect) - - def header_max_count(self, name): - """Return the maximum allowed number of headers named 'name'. - - Called when a header is added to a Message object. If the returned - value is not 0 or None, and there are already a number of headers with - the name 'name' equal to the value returned, a ValueError is raised. - - Because the default behavior of Message's __setitem__ is to append the - value to the list of headers, it is easy to create duplicate headers - without realizing it. This method allows certain headers to be limited - in the number of instances of that header that may be added to a - Message programmatically. (The limit is not observed by the parser, - which will faithfully produce as many headers as exist in the message - being parsed.) - - The default implementation returns None for all header names. - """ - return None - - @abc.abstractmethod - def header_source_parse(self, sourcelines): - """Given a list of linesep terminated strings constituting the lines of - a single header, return the (name, value) tuple that should be stored - in the model. The input lines should retain their terminating linesep - characters. The lines passed in by the email package may contain - surrogateescaped binary data. - """ - raise NotImplementedError - - @abc.abstractmethod - def header_store_parse(self, name, value): - """Given the header name and the value provided by the application - program, return the (name, value) that should be stored in the model. - """ - raise NotImplementedError - - @abc.abstractmethod - def header_fetch_parse(self, name, value): - """Given the header name and the value from the model, return the value - to be returned to the application program that is requesting that - header. The value passed in by the email package may contain - surrogateescaped binary data if the lines were parsed by a BytesParser. - The returned value should not contain any surrogateescaped data. - - """ - raise NotImplementedError - - @abc.abstractmethod - def fold(self, name, value): - """Given the header name and the value from the model, return a string - containing linesep characters that implement the folding of the header - according to the policy controls. The value passed in by the email - package may contain surrogateescaped binary data if the lines were - parsed by a BytesParser. The returned value should not contain any - surrogateescaped data. - - """ - raise NotImplementedError - - @abc.abstractmethod - def fold_binary(self, name, value): - """Given the header name and the value from the model, return binary - data containing linesep characters that implement the folding of the - header according to the policy controls. The value passed in by the - email package may contain surrogateescaped binary data. - - """ - raise NotImplementedError - - -@_extend_docstrings -class Compat32(Policy): - - """+ - This particular policy is the backward compatibility Policy. It - replicates the behavior of the email package version 5.1. - """ - - mangle_from_ = True - - def _sanitize_header(self, name, value): - # If the header value contains surrogates, return a Header using - # the unknown-8bit charset to encode the bytes as encoded words. - if not isinstance(value, str): - # Assume it is already a header object - return value - if _has_surrogates(value): - return header.Header(value, charset=_charset.UNKNOWN8BIT, - header_name=name) - else: - return value - - def header_source_parse(self, sourcelines): - """+ - The name is parsed as everything up to the ':' and returned unmodified. - The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and - stripping any trailing carriage return or linefeed characters. - - """ - name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) - return (name, value.rstrip('\r\n')) - - def header_store_parse(self, name, value): - """+ - The name and value are returned unmodified. - """ - return (name, value) - - def header_fetch_parse(self, name, value): - """+ - If the value contains binary data, it is converted into a Header object - using the unknown-8bit charset. Otherwise it is returned unmodified. - """ - return self._sanitize_header(name, value) - - def fold(self, name, value): - """+ - Headers are folded using the Header folding algorithm, which preserves - existing line breaks in the value, and wraps each resulting line to the - max_line_length. Non-ASCII binary data are CTE encoded using the - unknown-8bit charset. - - """ - return self._fold(name, value, sanitize=True) - - def fold_binary(self, name, value): - """+ - Headers are folded using the Header folding algorithm, which preserves - existing line breaks in the value, and wraps each resulting line to the - max_line_length. If cte_type is 7bit, non-ascii binary data is CTE - encoded using the unknown-8bit charset. Otherwise the original source - header is used, with its existing line breaks and/or binary data. - - """ - folded = self._fold(name, value, sanitize=self.cte_type=='7bit') - return folded.encode('ascii', 'surrogateescape') - - def _fold(self, name, value, sanitize): - parts = [] - parts.append('%s: ' % name) - if isinstance(value, str): - if _has_surrogates(value): - if sanitize: - h = header.Header(value, - charset=_charset.UNKNOWN8BIT, - header_name=name) - else: - # If we have raw 8bit data in a byte string, we have no idea - # what the encoding is. There is no safe way to split this - # string. If it's ascii-subset, then we could do a normal - # ascii split, but if it's multibyte then we could break the - # string. There's no way to know so the least harm seems to - # be to not split the string and risk it being too long. - parts.append(value) - h = None - else: - h = header.Header(value, header_name=name) - else: - # Assume it is a Header-like object. - h = value - if h is not None: - parts.append(h.encode(linesep=self.linesep, - maxlinelen=self.max_line_length)) - parts.append(self.linesep) - return ''.join(parts) - - -compat32 = Compat32() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/architecture.rst --- a/Lib/email/architecture.rst Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,216 +0,0 @@ -:mod:`email` Package Architecture -================================= - -Overview --------- - -The email package consists of three major components: - - Model - An object structure that represents an email message, and provides an - API for creating, querying, and modifying a message. - - Parser - Takes a sequence of characters or bytes and produces a model of the - email message represented by those characters or bytes. - - Generator - Takes a model and turns it into a sequence of characters or bytes. The - sequence can either be intended for human consumption (a printable - unicode string) or bytes suitable for transmission over the wire. In - the latter case all data is properly encoded using the content transfer - encodings specified by the relevant RFCs. - -Conceptually the package is organized around the model. The model provides both -"external" APIs intended for use by application programs using the library, -and "internal" APIs intended for use by the Parser and Generator components. -This division is intentionally a bit fuzzy; the API described by this -documentation is all a public, stable API. This allows for an application -with special needs to implement its own parser and/or generator. - -In addition to the three major functional components, there is a third key -component to the architecture: - - Policy - An object that specifies various behavioral settings and carries - implementations of various behavior-controlling methods. - -The Policy framework provides a simple and convenient way to control the -behavior of the library, making it possible for the library to be used in a -very flexible fashion while leveraging the common code required to parse, -represent, and generate message-like objects. For example, in addition to the -default :rfc:`5322` email message policy, we also have a policy that manages -HTTP headers in a fashion compliant with :rfc:`2616`. Individual policy -controls, such as the maximum line length produced by the generator, can also -be controlled individually to meet specialized application requirements. - - -The Model ---------- - -The message model is implemented by the :class:`~email.message.Message` class. -The model divides a message into the two fundamental parts discussed by the -RFC: the header section and the body. The `Message` object acts as a -pseudo-dictionary of named headers. Its dictionary interface provides -convenient access to individual headers by name. However, all headers are kept -internally in an ordered list, so that the information about the order of the -headers in the original message is preserved. - -The `Message` object also has a `payload` that holds the body. A `payload` can -be one of two things: data, or a list of `Message` objects. The latter is used -to represent a multipart MIME message. Lists can be nested arbitrarily deeply -in order to represent the message, with all terminal leaves having non-list -data payloads. - - -Message Lifecycle ------------------ - -The general lifecyle of a message is: - - Creation - A `Message` object can be created by a Parser, or it can be - instantiated as an empty message by an application. - - Manipulation - The application may examine one or more headers, and/or the - payload, and it may modify one or more headers and/or - the payload. This may be done on the top level `Message` - object, or on any sub-object. - - Finalization - The Model is converted into a unicode or binary stream, - or the model is discarded. - - - -Header Policy Control During Lifecycle --------------------------------------- - -One of the major controls exerted by the Policy is the management of headers -during the `Message` lifecycle. Most applications don't need to be aware of -this. - -A header enters the model in one of two ways: via a Parser, or by being set to -a specific value by an application program after the Model already exists. -Similarly, a header exits the model in one of two ways: by being serialized by -a Generator, or by being retrieved from a Model by an application program. The -Policy object provides hooks for all four of these pathways. - -The model storage for headers is a list of (name, value) tuples. - -The Parser identifies headers during parsing, and passes them to the -:meth:`~email.policy.Policy.header_source_parse` method of the Policy. The -result of that method is the (name, value) tuple to be stored in the model. - -When an application program supplies a header value (for example, through the -`Message` object `__setitem__` interface), the name and the value are passed to -the :meth:`~email.policy.Policy.header_store_parse` method of the Policy, which -returns the (name, value) tuple to be stored in the model. - -When an application program retrieves a header (through any of the dict or list -interfaces of `Message`), the name and value are passed to the -:meth:`~email.policy.Policy.header_fetch_parse` method of the Policy to -obtain the value returned to the application. - -When a Generator requests a header during serialization, the name and value are -passed to the :meth:`~email.policy.Policy.fold` method of the Policy, which -returns a string containing line breaks in the appropriate places. The -:meth:`~email.policy.Policy.cte_type` Policy control determines whether or -not Content Transfer Encoding is performed on the data in the header. There is -also a :meth:`~email.policy.Policy.binary_fold` method for use by generators -that produce binary output, which returns the folded header as binary data, -possibly folded at different places than the corresponding string would be. - - -Handling Binary Data --------------------- - -In an ideal world all message data would conform to the RFCs, meaning that the -parser could decode the message into the idealized unicode message that the -sender originally wrote. In the real world, the email package must also be -able to deal with badly formatted messages, including messages containing -non-ASCII characters that either have no indicated character set or are not -valid characters in the indicated character set. - -Since email messages are *primarily* text data, and operations on message data -are primarily text operations (except for binary payloads of course), the model -stores all text data as unicode strings. Un-decodable binary inside text -data is handled by using the `surrogateescape` error handler of the ASCII -codec. As with the binary filenames the error handler was introduced to -handle, this allows the email package to "carry" the binary data received -during parsing along until the output stage, at which time it is regenerated -in its original form. - -This carried binary data is almost entirely an implementation detail. The one -place where it is visible in the API is in the "internal" API. A Parser must -do the `surrogateescape` encoding of binary input data, and pass that data to -the appropriate Policy method. The "internal" interface used by the Generator -to access header values preserves the `surrogateescaped` bytes. All other -interfaces convert the binary data either back into bytes or into a safe form -(losing information in some cases). - - -Backward Compatibility ----------------------- - -The :class:`~email.policy.Policy.Compat32` Policy provides backward -compatibility with version 5.1 of the email package. It does this via the -following implementation of the four+1 Policy methods described above: - -header_source_parse - Splits the first line on the colon to obtain the name, discards any spaces - after the colon, and joins the remainder of the line with all of the - remaining lines, preserving the linesep characters to obtain the value. - Trailing carriage return and/or linefeed characters are stripped from the - resulting value string. - -header_store_parse - Returns the name and value exactly as received from the application. - -header_fetch_parse - If the value contains any `surrogateescaped` binary data, return the value - as a :class:`~email.header.Header` object, using the character set - `unknown-8bit`. Otherwise just returns the value. - -fold - Uses :class:`~email.header.Header`'s folding to fold headers in the - same way the email5.1 generator did. - -binary_fold - Same as fold, but encodes to 'ascii'. - - -New Algorithm -------------- - -header_source_parse - Same as legacy behavior. - -header_store_parse - Same as legacy behavior. - -header_fetch_parse - If the value is already a header object, returns it. Otherwise, parses the - value using the new parser, and returns the resulting object as the value. - `surrogateescaped` bytes get turned into unicode unknown character code - points. - -fold - Uses the new header folding algorithm, respecting the policy settings. - surrogateescaped bytes are encoded using the ``unknown-8bit`` charset for - ``cte_type=7bit`` or ``8bit``. Returns a string. - - At some point there will also be a ``cte_type=unicode``, and for that - policy fold will serialize the idealized unicode message with RFC-like - folding, converting any surrogateescaped bytes into the unicode - unknown character glyph. - -binary_fold - Uses the new header folding algorithm, respecting the policy settings. - surrogateescaped bytes are encoded using the `unknown-8bit` charset for - ``cte_type=7bit``, and get turned back into bytes for ``cte_type=8bit``. - Returns bytes. - - At some point there will also be a ``cte_type=unicode``, and for that - policy binary_fold will serialize the message according to :rfc:``5335``. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/charset.py --- a/Lib/email/charset.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/charset.py Mon Jan 25 17:05:13 2016 +0100 @@ -194,7 +194,7 @@ header encoding. Charset.SHORTEST is not allowed for body_encoding. - output_charset: Some character sets must be converted before they can be + output_charset: Some character sets must be converted before the can be used in email headers or bodies. If the input_charset is one of them, this attribute will contain the name of the charset output will be converted to. Otherwise, it will @@ -249,6 +249,9 @@ def __eq__(self, other): return str(self) == str(other).lower() + def __ne__(self, other): + return not self.__eq__(other) + def get_body_encoding(self): """Return the content-transfer-encoding used for body encoding. @@ -383,22 +386,12 @@ string using the ascii codec produces the correct string version of the content. """ - if not string: - return string + # 7bit/8bit encodings return the string unchanged (module conversions) if self.body_encoding is BASE64: if isinstance(string, str): string = string.encode(self.output_charset) return email.base64mime.body_encode(string) elif self.body_encoding is QP: - # quopromime.body_encode takes a string, but operates on it as if - # it were a list of byte codes. For a (minimal) history on why - # this is so, see changeset 0cf700464177. To correctly encode a - # character set, then, we must turn it into pseudo bytes via the - # latin1 charset, which will encode any byte as a single code point - # between 0 and 255, which is what body_encode is expecting. - if isinstance(string, str): - string = string.encode(self.output_charset) - string = string.decode('latin1') return email.quoprimime.body_encode(string) else: if isinstance(string, str): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/contentmanager.py --- a/Lib/email/contentmanager.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,249 +0,0 @@ -import binascii -import email.charset -import email.message -import email.errors -from email import quoprimime - -class ContentManager: - - def __init__(self): - self.get_handlers = {} - self.set_handlers = {} - - def add_get_handler(self, key, handler): - self.get_handlers[key] = handler - - def get_content(self, msg, *args, **kw): - content_type = msg.get_content_type() - if content_type in self.get_handlers: - return self.get_handlers[content_type](msg, *args, **kw) - maintype = msg.get_content_maintype() - if maintype in self.get_handlers: - return self.get_handlers[maintype](msg, *args, **kw) - if '' in self.get_handlers: - return self.get_handlers[''](msg, *args, **kw) - raise KeyError(content_type) - - def add_set_handler(self, typekey, handler): - self.set_handlers[typekey] = handler - - def set_content(self, msg, obj, *args, **kw): - if msg.get_content_maintype() == 'multipart': - # XXX: is this error a good idea or not? We can remove it later, - # but we can't add it later, so do it for now. - raise TypeError("set_content not valid on multipart") - handler = self._find_set_handler(msg, obj) - msg.clear_content() - handler(msg, obj, *args, **kw) - - def _find_set_handler(self, msg, obj): - full_path_for_error = None - for typ in type(obj).__mro__: - if typ in self.set_handlers: - return self.set_handlers[typ] - qname = typ.__qualname__ - modname = getattr(typ, '__module__', '') - full_path = '.'.join((modname, qname)) if modname else qname - if full_path_for_error is None: - full_path_for_error = full_path - if full_path in self.set_handlers: - return self.set_handlers[full_path] - if qname in self.set_handlers: - return self.set_handlers[qname] - name = typ.__name__ - if name in self.set_handlers: - return self.set_handlers[name] - if None in self.set_handlers: - return self.set_handlers[None] - raise KeyError(full_path_for_error) - - -raw_data_manager = ContentManager() - - -def get_text_content(msg, errors='replace'): - content = msg.get_payload(decode=True) - charset = msg.get_param('charset', 'ASCII') - return content.decode(charset, errors=errors) -raw_data_manager.add_get_handler('text', get_text_content) - - -def get_non_text_content(msg): - return msg.get_payload(decode=True) -for maintype in 'audio image video application'.split(): - raw_data_manager.add_get_handler(maintype, get_non_text_content) - - -def get_message_content(msg): - return msg.get_payload(0) -for subtype in 'rfc822 external-body'.split(): - raw_data_manager.add_get_handler('message/'+subtype, get_message_content) - - -def get_and_fixup_unknown_message_content(msg): - # If we don't understand a message subtype, we are supposed to treat it as - # if it were application/octet-stream, per - # tools.ietf.org/html/rfc2046#section-5.2.4. Feedparser doesn't do that, - # so do our best to fix things up. Note that it is *not* appropriate to - # model message/partial content as Message objects, so they are handled - # here as well. (How to reassemble them is out of scope for this comment :) - return bytes(msg.get_payload(0)) -raw_data_manager.add_get_handler('message', - get_and_fixup_unknown_message_content) - - -def _prepare_set(msg, maintype, subtype, headers): - msg['Content-Type'] = '/'.join((maintype, subtype)) - if headers: - if not hasattr(headers[0], 'name'): - mp = msg.policy - headers = [mp.header_factory(*mp.header_source_parse([header])) - for header in headers] - try: - for header in headers: - if header.defects: - raise header.defects[0] - msg[header.name] = header - except email.errors.HeaderDefect as exc: - raise ValueError("Invalid header: {}".format( - header.fold(policy=msg.policy))) from exc - - -def _finalize_set(msg, disposition, filename, cid, params): - if disposition is None and filename is not None: - disposition = 'attachment' - if disposition is not None: - msg['Content-Disposition'] = disposition - if filename is not None: - msg.set_param('filename', - filename, - header='Content-Disposition', - replace=True) - if cid is not None: - msg['Content-ID'] = cid - if params is not None: - for key, value in params.items(): - msg.set_param(key, value) - - -# XXX: This is a cleaned-up version of base64mime.body_encode. It would -# be nice to drop both this and quoprimime.body_encode in favor of -# enhanced binascii routines that accepted a max_line_length parameter. -def _encode_base64(data, max_line_length): - encoded_lines = [] - unencoded_bytes_per_line = max_line_length * 3 // 4 - for i in range(0, len(data), unencoded_bytes_per_line): - thisline = data[i:i+unencoded_bytes_per_line] - encoded_lines.append(binascii.b2a_base64(thisline).decode('ascii')) - return ''.join(encoded_lines) - - -def _encode_text(string, charset, cte, policy): - lines = string.encode(charset).splitlines() - linesep = policy.linesep.encode('ascii') - def embeded_body(lines): return linesep.join(lines) + linesep - def normal_body(lines): return b'\n'.join(lines) + b'\n' - if cte==None: - # Use heuristics to decide on the "best" encoding. - try: - return '7bit', normal_body(lines).decode('ascii') - except UnicodeDecodeError: - pass - if (policy.cte_type == '8bit' and - max(len(x) for x in lines) <= policy.max_line_length): - return '8bit', normal_body(lines).decode('ascii', 'surrogateescape') - sniff = embeded_body(lines[:10]) - sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'), - policy.max_line_length) - sniff_base64 = binascii.b2a_base64(sniff) - # This is a little unfair to qp; it includes lineseps, base64 doesn't. - if len(sniff_qp) > len(sniff_base64): - cte = 'base64' - else: - cte = 'quoted-printable' - if len(lines) <= 10: - return cte, sniff_qp - if cte == '7bit': - data = normal_body(lines).decode('ascii') - elif cte == '8bit': - data = normal_body(lines).decode('ascii', 'surrogateescape') - elif cte == 'quoted-printable': - data = quoprimime.body_encode(normal_body(lines).decode('latin-1'), - policy.max_line_length) - elif cte == 'base64': - data = _encode_base64(embeded_body(lines), policy.max_line_length) - else: - raise ValueError("Unknown content transfer encoding {}".format(cte)) - return cte, data - - -def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None, - disposition=None, filename=None, cid=None, - params=None, headers=None): - _prepare_set(msg, 'text', subtype, headers) - cte, payload = _encode_text(string, charset, cte, msg.policy) - msg.set_payload(payload) - msg.set_param('charset', - email.charset.ALIASES.get(charset, charset), - replace=True) - msg['Content-Transfer-Encoding'] = cte - _finalize_set(msg, disposition, filename, cid, params) -raw_data_manager.add_set_handler(str, set_text_content) - - -def set_message_content(msg, message, subtype="rfc822", cte=None, - disposition=None, filename=None, cid=None, - params=None, headers=None): - if subtype == 'partial': - raise ValueError("message/partial is not supported for Message objects") - if subtype == 'rfc822': - if cte not in (None, '7bit', '8bit', 'binary'): - # http://tools.ietf.org/html/rfc2046#section-5.2.1 mandate. - raise ValueError( - "message/rfc822 parts do not support cte={}".format(cte)) - # 8bit will get coerced on serialization if policy.cte_type='7bit'. We - # may end up claiming 8bit when it isn't needed, but the only negative - # result of that should be a gateway that needs to coerce to 7bit - # having to look through the whole embedded message to discover whether - # or not it actually has to do anything. - cte = '8bit' if cte is None else cte - elif subtype == 'external-body': - if cte not in (None, '7bit'): - # http://tools.ietf.org/html/rfc2046#section-5.2.3 mandate. - raise ValueError( - "message/external-body parts do not support cte={}".format(cte)) - cte = '7bit' - elif cte is None: - # http://tools.ietf.org/html/rfc2046#section-5.2.4 says all future - # subtypes should be restricted to 7bit, so assume that. - cte = '7bit' - _prepare_set(msg, 'message', subtype, headers) - msg.set_payload([message]) - msg['Content-Transfer-Encoding'] = cte - _finalize_set(msg, disposition, filename, cid, params) -raw_data_manager.add_set_handler(email.message.Message, set_message_content) - - -def set_bytes_content(msg, data, maintype, subtype, cte='base64', - disposition=None, filename=None, cid=None, - params=None, headers=None): - _prepare_set(msg, maintype, subtype, headers) - if cte == 'base64': - data = _encode_base64(data, max_line_length=msg.policy.max_line_length) - elif cte == 'quoted-printable': - # XXX: quoprimime.body_encode won't encode newline characters in data, - # so we can't use it. This means max_line_length is ignored. Another - # bug to fix later. (Note: encoders.quopri is broken on line ends.) - data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True) - data = data.decode('ascii') - elif cte == '7bit': - # Make sure it really is only ASCII. The early warning here seems - # worth the overhead...if you care write your own content manager :). - data.encode('ascii') - elif cte in ('8bit', 'binary'): - data = data.decode('ascii', 'surrogateescape') - msg.set_payload(data) - msg['Content-Transfer-Encoding'] = cte - _finalize_set(msg, disposition, filename, cid, params) -for typ in (bytes, bytearray, memoryview): - raw_data_manager.add_set_handler(typ, set_bytes_content) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/encoders.py --- a/Lib/email/encoders.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/encoders.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,7 +20,7 @@ def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do - return enc.replace(b' ', b'=20') + return enc.replace(' ', '=20') def encode_base64(msg): @@ -28,7 +28,7 @@ Also, add an appropriate Content-Transfer-Encoding header. """ - orig = msg.get_payload(decode=True) + orig = msg.get_payload() encdata = str(_bencode(orig), 'ascii') msg.set_payload(encdata) msg['Content-Transfer-Encoding'] = 'base64' @@ -40,7 +40,7 @@ Also, add an appropriate Content-Transfer-Encoding header. """ - orig = msg.get_payload(decode=True) + orig = msg.get_payload() encdata = _qencode(orig) msg.set_payload(encdata) msg['Content-Transfer-Encoding'] = 'quoted-printable' @@ -49,17 +49,26 @@ def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" - orig = msg.get_payload(decode=True) + orig = msg.get_payload() if orig is None: # There's no payload. For backwards compatibility we use 7bit msg['Content-Transfer-Encoding'] = '7bit' return - # We play a trick to make this go fast. If decoding from ASCII succeeds, - # we know the data must be 7bit, otherwise treat it as 8bit. + # We play a trick to make this go fast. If encoding/decode to ASCII + # succeeds, we know the data must be 7bit, otherwise treat it as 8bit. try: - orig.decode('ascii') + if isinstance(orig, str): + orig.encode('ascii') + else: + orig.decode('ascii') except UnicodeError: - msg['Content-Transfer-Encoding'] = '8bit' + # iso-2022-* is non-ASCII but still 7-bit + charset = msg.get_charset() + output_cset = charset and charset.output_charset + if output_cset and output_cset.lower().startswith('iso-2022-'): + msg['Content-Transfer-Encoding'] = '7bit' + else: + msg['Content-Transfer-Encoding'] = '8bit' else: msg['Content-Transfer-Encoding'] = '7bit' diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/errors.py --- a/Lib/email/errors.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/errors.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,6 +5,7 @@ """email package exception classes.""" + class MessageError(Exception): """Base class for errors in the email package.""" @@ -29,13 +30,12 @@ """An illegal charset was given.""" + # These are parsing defects which the parser was able to work around. -class MessageDefect(ValueError): +class MessageDefect(Exception): """Base class for a message defect.""" def __init__(self, line=None): - if line is not None: - super().__init__(line) self.line = line class NoBoundaryInMultipartDefect(MessageDefect): @@ -44,64 +44,17 @@ class StartBoundaryNotFoundDefect(MessageDefect): """The claimed start boundary was never found.""" -class CloseBoundaryNotFoundDefect(MessageDefect): - """A start boundary was found, but not the corresponding close boundary.""" - class FirstHeaderLineIsContinuationDefect(MessageDefect): """A message had a continuation line as its first header line.""" class MisplacedEnvelopeHeaderDefect(MessageDefect): """A 'Unix-from' header was found in the middle of a header block.""" -class MissingHeaderBodySeparatorDefect(MessageDefect): - """Found line with no leading whitespace and no colon before blank line.""" -# XXX: backward compatibility, just in case (it was never emitted). -MalformedHeaderDefect = MissingHeaderBodySeparatorDefect +class MalformedHeaderDefect(MessageDefect): + """Found a header that was missing a colon, or was otherwise malformed.""" class MultipartInvariantViolationDefect(MessageDefect): """A message claimed to be a multipart but no subparts were found.""" class InvalidMultipartContentTransferEncodingDefect(MessageDefect): """An invalid content transfer encoding was set on the multipart itself.""" - -class UndecodableBytesDefect(MessageDefect): - """Header contained bytes that could not be decoded""" - -class InvalidBase64PaddingDefect(MessageDefect): - """base64 encoded sequence had an incorrect length""" - -class InvalidBase64CharactersDefect(MessageDefect): - """base64 encoded sequence had characters not in base64 alphabet""" - -# These errors are specific to header parsing. - -class HeaderDefect(MessageDefect): - """Base class for a header defect.""" - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - -class InvalidHeaderDefect(HeaderDefect): - """Header is not valid, message gives details.""" - -class HeaderMissingRequiredValue(HeaderDefect): - """A header that must have a value had none""" - -class NonPrintableDefect(HeaderDefect): - """ASCII characters outside the ascii-printable range found""" - - def __init__(self, non_printables): - super().__init__(non_printables) - self.non_printables = non_printables - - def __str__(self): - return ("the following ASCII non-printables found in header: " - "{}".format(self.non_printables)) - -class ObsoleteHeaderDefect(HeaderDefect): - """Header uses syntax declared obsolete by RFC 5322""" - -class NonASCIILocalPartDefect(HeaderDefect): - """local_part contains non-ASCII characters""" - # This defect only occurs during unicode parsing, not when - # parsing messages decoded from binary. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/feedparser.py --- a/Lib/email/feedparser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/feedparser.py Mon Jan 25 17:05:13 2016 +0100 @@ -13,7 +13,7 @@ data. When you have no more data to push into the parser, call .close(). This completes the parsing and returns the root message object. -The other advantage of this parser is that it will never raise a parsing +The other advantage of this parser is that it will never throw a parsing exception. Instead, when it finds something unexpected, it adds a 'defect' to the current message. Defects are just instances that live on the message object's .defects attribute. @@ -25,8 +25,7 @@ from email import errors from email import message -from email._policybase import compat32 -from collections import deque +from email import policy NLCRE = re.compile('\r\n|\r|\n') NLCRE_bol = re.compile('(\r\n|\r|\n)') @@ -34,7 +33,7 @@ NLCRE_crack = re.compile('(\r\n|\r|\n)') # RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character # except controls, SP, and ":". -headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') +headerRE = re.compile(r'^(From |[\041-\071\073-\176]{1,}:|[\t ])') EMPTYSTRING = '' NL = '\n' @@ -51,10 +50,10 @@ simple abstraction -- it parses until EOF closes the current message. """ def __init__(self): - # Chunks of the last partial line pushed into this object. - self._partial = [] - # A deque of full, pushed lines - self._lines = deque() + # The last partial line pushed into this object. + self._partial = '' + # The list of full, pushed lines, in reverse order + self._lines = [] # The stack of false-EOF checking predicates. self._eofstack = [] # A flag indicating whether the file has been closed or not. @@ -68,8 +67,8 @@ def close(self): # Don't forget any trailing partial line. - self.pushlines(''.join(self._partial).splitlines(True)) - self._partial = [] + self._lines.append(self._partial) + self._partial = '' self._closed = True def readline(self): @@ -79,48 +78,48 @@ return NeedMoreData # Pop the line off the stack and see if it matches the current # false-EOF predicate. - line = self._lines.popleft() + line = self._lines.pop() # RFC 2046, section 5.1.2 requires us to recognize outer level # boundaries at any level of inner nesting. Do this, but be sure it's # in the order of most to least nested. - for ateof in reversed(self._eofstack): + for ateof in self._eofstack[::-1]: if ateof(line): # We're at the false EOF. But push the last line back first. - self._lines.appendleft(line) + self._lines.append(line) return '' return line def unreadline(self, line): # Let the consumer push a line back into the buffer. assert line is not NeedMoreData - self._lines.appendleft(line) + self._lines.append(line) def push(self, data): """Push some new data into this object.""" - # Crack into lines, but preserve the linesep characters on the end of each - parts = data.splitlines(True) - - if not parts or not parts[0].endswith(('\n', '\r')): - # No new complete lines, so just accumulate partials - self._partial += parts - return - - if self._partial: - # If there are previous leftovers, complete them now - self._partial.append(parts[0]) - parts[0:1] = ''.join(self._partial).splitlines(True) - del self._partial[:] - - # If the last element of the list does not end in a newline, then treat - # it as a partial line. We only check for '\n' here because a line - # ending with '\r' might be a line that was split in the middle of a - # '\r\n' sequence (see bugs 1555570 and 1721862). - if not parts[-1].endswith('\n'): - self._partial = [parts.pop()] - self.pushlines(parts) + # Handle any previous leftovers + data, self._partial = self._partial + data, '' + # Crack into lines, but preserve the newlines on the end of each + parts = NLCRE_crack.split(data) + # The *ahem* interesting behaviour of re.split when supplied grouping + # parentheses is that the last element of the resulting list is the + # data after the final RE. In the case of a NL/CR terminated string, + # this is the empty string. + self._partial = parts.pop() + #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r: + # is there a \n to follow later? + if not self._partial and parts and parts[-1].endswith('\r'): + self._partial = parts.pop(-2)+parts.pop() + # parts is a list of strings, alternating between the line contents + # and the eol character(s). Gather up a list of lines after + # re-attaching the newlines. + lines = [] + for i in range(len(parts) // 2): + lines.append(parts[i*2] + parts[i*2+1]) + self.pushlines(lines) def pushlines(self, lines): - self._lines.extend(lines) + # Reverse and insert at the front of the lines. + self._lines[:0] = lines[::-1] def __iter__(self): return self @@ -136,7 +135,7 @@ class FeedParser: """A feed-style parser of email.""" - def __init__(self, _factory=None, *, policy=compat32): + def __init__(self, _factory=message.Message, *, policy=policy.default): """_factory is called with no arguments to create a new message obj The policy keyword specifies a policy object that controls a number of @@ -144,23 +143,8 @@ backward compatibility. """ + self._factory = _factory self.policy = policy - self._factory_kwds = lambda: {'policy': self.policy} - if _factory is None: - # What this should be: - #self._factory = policy.default_message_factory - # but, because we are post 3.4 feature freeze, fix with temp hack: - if self.policy is compat32: - self._factory = message.Message - else: - self._factory = message.EmailMessage - else: - self._factory = _factory - try: - _factory(policy=self.policy) - except TypeError: - # Assume this is an old-style factory - self._factory_kwds = lambda: {} self._input = BufferedSubFile() self._msgstack = [] self._parse = self._parsegen().__next__ @@ -197,7 +181,7 @@ return root def _new_message(self): - msg = self._factory(**self._factory_kwds()) + msg = self._factory() if self._cur and self._cur.get_content_type() == 'multipart/digest': msg.set_default_type('message/rfc822') if self._msgstack: @@ -229,8 +213,6 @@ # (i.e. newline), just throw it away. Otherwise the line is # part of the body so push it back. if not NLCRE.match(line): - defect = errors.MissingHeaderBodySeparatorDefect() - self.policy.handle_defect(self._cur, defect) self._input.unreadline(line) break headers.append(line) @@ -238,7 +220,7 @@ # supposed to see in the body of the message. self._parse_headers(headers) # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could raise errors. All + # necessary in the older parser, which could throw errors. All # remaining lines in the input are thrown into the message body. if self._headersonly: lines = [] @@ -334,7 +316,6 @@ capturing_preamble = True preamble = [] linesep = False - close_boundary_seen = False while True: line = self._input.readline() if line is NeedMoreData: @@ -349,7 +330,6 @@ # the closing boundary, then we need to initialize the # epilogue with the empty string (see below). if mo.group('end'): - close_boundary_seen = True linesep = mo.group('linesep') break # We saw an inter-part boundary. Were we in the preamble? @@ -418,6 +398,7 @@ # We've seen either the EOF or the end boundary. If we're still # capturing the preamble, we never saw the start boundary. Note # that as a defect and store the captured text as the payload. + # Everything from here to the EOF is epilogue. if capturing_preamble: defect = errors.StartBoundaryNotFoundDefect() self.policy.handle_defect(self._cur, defect) @@ -429,15 +410,8 @@ continue self._cur.epilogue = EMPTYSTRING.join(epilogue) return - # If we're not processing the preamble, then we might have seen - # EOF without seeing that end boundary...that is also a defect. - if not close_boundary_seen: - defect = errors.CloseBoundaryNotFoundDefect() - self.policy.handle_defect(self._cur, defect) - return - # Everything from here to the EOF is epilogue. If the end boundary - # ended in a newline, we'll need to make sure the epilogue isn't - # None + # If the end boundary ended in a newline, we'll need to make sure + # the epilogue isn't None if linesep: epilogue = [''] else: @@ -484,7 +458,9 @@ lastvalue.append(line) continue if lastheader: - self._cur.set_raw(*self.policy.header_source_parse(lastvalue)) + # XXX reconsider the joining of folded lines + lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n') + self._cur[lastheader] = lhdr lastheader, lastvalue = '', [] # Check for envelope header, i.e. unix-from if line.startswith('From '): @@ -508,26 +484,19 @@ self._cur.defects.append(defect) continue # Split the line on the colon separating field name from value. - # There will always be a colon, because if there wasn't the part of - # the parser that calls us would have started parsing the body. i = line.find(':') - - # If the colon is on the start of the line the header is clearly - # malformed, but we might be able to salvage the rest of the - # message. Track the error but keep going. - if i == 0: - defect = errors.InvalidHeaderDefect("Missing header name.") + if i < 0: + defect = errors.MalformedHeaderDefect(line) self._cur.defects.append(defect) continue - - assert i>0, "_parse_headers fed line with no : and no leading WS" lastheader = line[:i] - lastvalue = [line] + lastvalue = [line[i+1:].lstrip()] # Done with all the lines, so handle the last header. if lastheader: - self._cur.set_raw(*self.policy.header_source_parse(lastvalue)) + # XXX reconsider the joining of folded lines + self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n') - + class BytesFeedParser(FeedParser): """Like FeedParser, but feed accepts bytes.""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/generator.py --- a/Lib/email/generator.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/generator.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,10 +10,13 @@ import sys import time import random +import warnings -from copy import deepcopy from io import StringIO, BytesIO -from email.utils import _has_surrogates +from email import policy +from email.header import Header +from email.message import _has_surrogates +import email.charset as _charset UNDERSCORE = '_' NL = '\n' # XXX: no longer used by the code below. @@ -32,16 +35,16 @@ # Public interface # - def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *, - policy=None): + def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *, + policy=policy.default): """Create the generator for message flattening. outfp is the output file-like object for writing the message to. It must have a write() method. - Optional mangle_from_ is a flag that, when True (the default if policy - is not set), escapes From_ lines in the body of the message by putting - a `>' in front of them. + Optional mangle_from_ is a flag that, when True (the default), escapes + From_ lines in the body of the message by putting a `>' in front of + them. Optional maxheaderlen specifies the longest length for a non-continued header. When a header line is longer (in characters, with tabs @@ -51,17 +54,14 @@ by RFC 2822. The policy keyword specifies a policy object that controls a number of - aspects of the generator's operation. If no policy is specified, - the policy associated with the Message object passed to the - flatten method is used. + aspects of the generator's operation. The default policy maintains + backward compatibility. """ - - if mangle_from_ is None: - mangle_from_ = True if policy is None else policy.mangle_from_ self._fp = outfp self._mangle_from_ = mangle_from_ - self.maxheaderlen = maxheaderlen + self._maxheaderlen = (maxheaderlen if maxheaderlen is not None else + policy.max_line_length) self.policy = policy def write(self, s): @@ -80,49 +80,27 @@ Note that for subobjects, no From_ line is printed. linesep specifies the characters used to indicate a new line in - the output. The default value is determined by the policy specified - when the Generator instance was created or, if none was specified, - from the policy associated with the msg. + the output. The default value is determined by the policy. """ # We use the _XXX constants for operating on data that comes directly # from the msg, and _encoded_XXX constants for operating on data that # has already been converted (to bytes in the BytesGenerator) and # inserted into a temporary buffer. - policy = msg.policy if self.policy is None else self.policy - if linesep is not None: - policy = policy.clone(linesep=linesep) - if self.maxheaderlen is not None: - policy = policy.clone(max_line_length=self.maxheaderlen) - self._NL = policy.linesep + self._NL = linesep if linesep is not None else self.policy.linesep self._encoded_NL = self._encode(self._NL) self._EMPTY = '' self._encoded_EMTPY = self._encode('') - # Because we use clone (below) when we recursively process message - # subparts, and because clone uses the computed policy (not None), - # submessages will automatically get set to the computed policy when - # they are processed by this code. - old_gen_policy = self.policy - old_msg_policy = msg.policy - try: - self.policy = policy - msg.policy = policy - if unixfrom: - ufrom = msg.get_unixfrom() - if not ufrom: - ufrom = 'From nobody ' + time.ctime(time.time()) - self.write(ufrom + self._NL) - self._write(msg) - finally: - self.policy = old_gen_policy - msg.policy = old_msg_policy + if unixfrom: + ufrom = msg.get_unixfrom() + if not ufrom: + ufrom = 'From nobody ' + time.ctime(time.time()) + self.write(ufrom + self._NL) + self._write(msg) def clone(self, fp): """Clone this generator with the exact same options.""" - return self.__class__(fp, - self._mangle_from_, - None, # Use policy setting, which we've adjusted - policy=self.policy) + return self.__class__(fp, self._mangle_from_, self._maxheaderlen) # # Protected interface - undocumented ;/ @@ -149,19 +127,6 @@ # BytesGenerator overrides this to encode strings to bytes. return s - def _write_lines(self, lines): - # We have to transform the line endings. - if not lines: - return - lines = lines.splitlines(True) - for line in lines[:-1]: - self.write(line.rstrip('\r\n')) - self.write(self._NL) - laststripped = lines[-1].rstrip('\r\n') - self.write(laststripped) - if len(lines[-1]) != len(laststripped): - self.write(self._NL) - def _write(self, msg): # We can't write the headers yet because of the following scenario: # say a multipart message includes the boundary string somewhere in @@ -176,18 +141,10 @@ # necessary. oldfp = self._fp try: - self._munge_cte = None self._fp = sfp = self._new_buffer() self._dispatch(msg) finally: self._fp = oldfp - munge_cte = self._munge_cte - del self._munge_cte - # If we munged the cte, copy the message again and re-fix the CTE. - if munge_cte: - msg = deepcopy(msg) - msg.replace_header('content-transfer-encoding', munge_cte[0]) - msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. meth = getattr(msg, '_write_headers', None) @@ -218,8 +175,16 @@ # def _write_headers(self, msg): - for h, v in msg.raw_items(): - self.write(self.policy.fold(h, v)) + for h, v in msg.items(): + self.write('%s: ' % h) + if isinstance(v, Header): + self.write(v.encode( + maxlinelen=self._maxheaderlen, linesep=self._NL)+self._NL) + else: + # Header's got lots of smarts, so use it. + header = Header(v, maxlinelen=self._maxheaderlen, + header_name=h) + self.write(header.encode(linesep=self._NL)+self._NL) # A blank line always separates headers from body self.write(self._NL) @@ -236,17 +201,12 @@ if _has_surrogates(msg._payload): charset = msg.get_param('charset') if charset is not None: - # XXX: This copy stuff is an ugly hack to avoid modifying the - # existing message. - msg = deepcopy(msg) del msg['content-transfer-encoding'] msg.set_payload(payload, charset) payload = msg.get_payload() - self._munge_cte = (msg['content-transfer-encoding'], - msg['content-type']) if self._mangle_from_: payload = fcre.sub('>From ', payload) - self._write_lines(payload) + self.write(payload) # Default body handler _writeBody = _handle_text @@ -281,12 +241,7 @@ msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: - if self._mangle_from_: - preamble = fcre.sub('>From ', msg.preamble) - else: - preamble = msg.preamble - self._write_lines(preamble) - self.write(self._NL) + self.write(msg.preamble + self._NL) # dash-boundary transport-padding CRLF self.write('--' + boundary + self._NL) # body-part @@ -301,24 +256,21 @@ # body-part self._fp.write(body_part) # close-delimiter transport-padding - self.write(self._NL + '--' + boundary + '--' + self._NL) + self.write(self._NL + '--' + boundary + '--') if msg.epilogue is not None: - if self._mangle_from_: - epilogue = fcre.sub('>From ', msg.epilogue) - else: - epilogue = msg.epilogue - self._write_lines(epilogue) + self.write(self._NL) + self.write(msg.epilogue) def _handle_multipart_signed(self, msg): # The contents of signed parts has to stay unmodified in order to keep # the signature intact per RFC1847 2.1, so we disable header wrapping. # RDM: This isn't enough to completely preserve the part, but it helps. - p = self.policy - self.policy = p.clone(max_line_length=0) + old_maxheaderlen = self._maxheaderlen try: + self._maxheaderlen = 0 self._handle_multipart(msg) finally: - self.policy = p + self._maxheaderlen = old_maxheaderlen def _handle_message_delivery_status(self, msg): # We can't just write the headers directly to self's file object @@ -364,7 +316,7 @@ # This used to be a module level function; we use a classmethod for this # and _compile_re so we can continue to provide the module level function # for backward compatibility by doing - # _make_boundary = Generator._make_boundary + # _make_boudary = Generator._make_boundary # at the end of the module. It *is* internal, so we could drop that... @classmethod def _make_boundary(cls, text=None): @@ -395,9 +347,9 @@ Functionally identical to the base Generator except that the output is bytes and not string. When surrogates were used in the input to encode bytes, these are decoded back to bytes for output. If the policy has - cte_type set to 7bit, then the message is transformed such that the - non-ASCII bytes are properly content transfer encoded, using the charset - unknown-8bit. + must_be_7bit set true, then the message is transformed such that the + non-ASCII bytes are properly content transfer encoded, using the + charset unknown-8bit. The outfp object must accept bytes in its write method. """ @@ -418,8 +370,27 @@ def _write_headers(self, msg): # This is almost the same as the string version, except for handling # strings with 8bit bytes. - for h, v in msg.raw_items(): - self._fp.write(self.policy.fold_binary(h, v)) + for h, v in msg._headers: + self.write('%s: ' % h) + if isinstance(v, str): + if _has_surrogates(v): + if not self.policy.must_be_7bit: + # If we have raw 8bit data in a byte string, we have no idea + # what the encoding is. There is no safe way to split this + # string. If it's ascii-subset, then we could do a normal + # ascii split, but if it's multibyte then we could break the + # string. There's no way to know so the least harm seems to + # be to not split the string and risk it being too long. + self.write(v+NL) + continue + h = Header(v, charset=_charset.UNKNOWN8BIT, header_name=h) + else: + h = Header(v, header_name=h) + else: + # Assume it is a Header-like object. + h = v + self.write(h.encode(linesep=self._NL, + maxlinelen=self._maxheaderlen)+self._NL) # A blank line always separates headers from body self.write(self._NL) @@ -428,16 +399,11 @@ # just write it back out. if msg._payload is None: return - if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit': - if self._mangle_from_: - msg._payload = fcre.sub(">From ", msg._payload) - self._write_lines(msg._payload) + if _has_surrogates(msg._payload) and not self.policy.must_be_7bit: + self.write(msg._payload) else: super(BytesGenerator,self)._handle_text(msg) - # Default body handler - _writeBody = _handle_text - @classmethod def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) @@ -452,7 +418,7 @@ Like the Generator base class, except that non-text parts are substituted with a format string representing the part. """ - def __init__(self, outfp, mangle_from_=None, maxheaderlen=78, fmt=None): + def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None): """Like Generator.__init__() except that an additional optional argument is allowed. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/header.py --- a/Lib/email/header.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/header.py Mon Jan 25 17:05:13 2016 +0100 @@ -40,6 +40,7 @@ \? # literal ? (?P.*?) # non-greedy up to the next ?= is the encoded string \?= # literal ?= + (?=[ \t]|$) # whitespace or the end of the string ''', re.VERBOSE | re.IGNORECASE | re.MULTILINE) # Field name regexp, including trailing colon, but not separating whitespace, @@ -85,12 +86,8 @@ words = [] for line in header.splitlines(): parts = ecre.split(line) - first = True while parts: - unencoded = parts.pop(0) - if first: - unencoded = unencoded.lstrip() - first = False + unencoded = parts.pop(0).strip() if unencoded: words.append((unencoded, None, None)) if parts: @@ -98,15 +95,6 @@ encoding = parts.pop(0).lower() encoded = parts.pop(0) words.append((encoded, encoding, charset)) - # Now loop over words and remove words that consist of whitespace - # between two encoded strings. - droplist = [] - for n, w in enumerate(words): - if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace(): - droplist.append(n-1) - for d in reversed(droplist): - del words[d] - # The next step is to decode each encoded word by applying the reverse # base64 or quopri transformation. decoded_words is now a list of the # form (decoded_word, charset). @@ -229,27 +217,22 @@ self._normalize() uchunks = [] lastcs = None - lastspace = None for string, charset in self._chunks: # We must preserve spaces between encoded and non-encoded word # boundaries, which means for us we need to add a space when we go # from a charset to None/us-ascii, or from None/us-ascii to a # charset. Only do this for the second and subsequent chunks. - # Don't add a space if the None/us-ascii string already has - # a space (trailing or leading depending on transition) nextcs = charset if nextcs == _charset.UNKNOWN8BIT: original_bytes = string.encode('ascii', 'surrogateescape') string = original_bytes.decode('ascii', 'replace') if uchunks: - hasspace = string and self._nonctext(string[0]) if lastcs not in (None, 'us-ascii'): - if nextcs in (None, 'us-ascii') and not hasspace: + if nextcs in (None, 'us-ascii'): uchunks.append(SPACE) nextcs = None - elif nextcs not in (None, 'us-ascii') and not lastspace: + elif nextcs not in (None, 'us-ascii'): uchunks.append(SPACE) - lastspace = string and self._nonctext(string[-1]) lastcs = nextcs uchunks.append(string) return EMPTYSTRING.join(uchunks) @@ -262,6 +245,9 @@ # args and do another comparison. return other == str(self) + def __ne__(self, other): + return not self == other + def append(self, s, charset=None, errors='strict'): """Append a string to the MIME header. @@ -294,7 +280,7 @@ else: s = s.decode(input_charset, errors) # Ensure that the bytes we're storing can be decoded to the output - # character set, otherwise an early error is raised. + # character set, otherwise an early error is thrown. output_charset = charset.output_codec or 'us-ascii' if output_charset != _charset.UNKNOWN8BIT: try: @@ -305,11 +291,6 @@ charset = UTF8 self._chunks.append((s, charset)) - def _nonctext(self, s): - """True if string s is not a ctext character of RFC822. - """ - return s.isspace() or s in ('(', ')', '\\') - def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): r"""Encode a message header into an RFC-compliant format. @@ -353,19 +334,7 @@ maxlinelen = 1000000 formatter = _ValueFormatter(self._headerlen, maxlinelen, self._continuation_ws, splitchars) - lastcs = None - hasspace = lastspace = None for string, charset in self._chunks: - if hasspace is not None: - hasspace = string and self._nonctext(string[0]) - if lastcs not in (None, 'us-ascii'): - if not hasspace or charset not in (None, 'us-ascii'): - formatter.add_transition() - elif charset not in (None, 'us-ascii') and not lastspace: - formatter.add_transition() - lastspace = string and self._nonctext(string[-1]) - lastcs = charset - hasspace = False lines = string.splitlines() if lines: formatter.feed('', lines[0], charset) @@ -382,7 +351,6 @@ formatter.feed(fws, sline, charset) if len(lines) > 1: formatter.newline() - if self._chunks: formatter.add_transition() value = formatter._str(linesep) if _embeded_header.search(value): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/headerregistry.py --- a/Lib/email/headerregistry.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,586 +0,0 @@ -"""Representing and manipulating email headers via custom objects. - -This module provides an implementation of the HeaderRegistry API. -The implementation is designed to flexibly follow RFC5322 rules. - -Eventually HeaderRegistry will be a public API, but it isn't yet, -and will probably change some before that happens. - -""" -from types import MappingProxyType - -from email import utils -from email import errors -from email import _header_value_parser as parser - -class Address: - - def __init__(self, display_name='', username='', domain='', addr_spec=None): - """Create an object represeting a full email address. - - An address can have a 'display_name', a 'username', and a 'domain'. In - addition to specifying the username and domain separately, they may be - specified together by using the addr_spec keyword *instead of* the - username and domain keywords. If an addr_spec string is specified it - must be properly quoted according to RFC 5322 rules; an error will be - raised if it is not. - - An Address object has display_name, username, domain, and addr_spec - attributes, all of which are read-only. The addr_spec and the string - value of the object are both quoted according to RFC5322 rules, but - without any Content Transfer Encoding. - - """ - # This clause with its potential 'raise' may only happen when an - # application program creates an Address object using an addr_spec - # keyword. The email library code itself must always supply username - # and domain. - if addr_spec is not None: - if username or domain: - raise TypeError("addrspec specified when username and/or " - "domain also specified") - a_s, rest = parser.get_addr_spec(addr_spec) - if rest: - raise ValueError("Invalid addr_spec; only '{}' " - "could be parsed from '{}'".format( - a_s, addr_spec)) - if a_s.all_defects: - raise a_s.all_defects[0] - username = a_s.local_part - domain = a_s.domain - self._display_name = display_name - self._username = username - self._domain = domain - - @property - def display_name(self): - return self._display_name - - @property - def username(self): - return self._username - - @property - def domain(self): - return self._domain - - @property - def addr_spec(self): - """The addr_spec (username@domain) portion of the address, quoted - according to RFC 5322 rules, but with no Content Transfer Encoding. - """ - nameset = set(self.username) - if len(nameset) > len(nameset-parser.DOT_ATOM_ENDS): - lp = parser.quote_string(self.username) - else: - lp = self.username - if self.domain: - return lp + '@' + self.domain - if not lp: - return '<>' - return lp - - def __repr__(self): - return "{}(display_name={!r}, username={!r}, domain={!r})".format( - self.__class__.__name__, - self.display_name, self.username, self.domain) - - def __str__(self): - nameset = set(self.display_name) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(self.display_name) - else: - disp = self.display_name - if disp: - addr_spec = '' if self.addr_spec=='<>' else self.addr_spec - return "{} <{}>".format(disp, addr_spec) - return self.addr_spec - - def __eq__(self, other): - if type(other) != type(self): - return False - return (self.display_name == other.display_name and - self.username == other.username and - self.domain == other.domain) - - -class Group: - - def __init__(self, display_name=None, addresses=None): - """Create an object representing an address group. - - An address group consists of a display_name followed by colon and an - list of addresses (see Address) terminated by a semi-colon. The Group - is created by specifying a display_name and a possibly empty list of - Address objects. A Group can also be used to represent a single - address that is not in a group, which is convenient when manipulating - lists that are a combination of Groups and individual Addresses. In - this case the display_name should be set to None. In particular, the - string representation of a Group whose display_name is None is the same - as the Address object, if there is one and only one Address object in - the addresses list. - - """ - self._display_name = display_name - self._addresses = tuple(addresses) if addresses else tuple() - - @property - def display_name(self): - return self._display_name - - @property - def addresses(self): - return self._addresses - - def __repr__(self): - return "{}(display_name={!r}, addresses={!r}".format( - self.__class__.__name__, - self.display_name, self.addresses) - - def __str__(self): - if self.display_name is None and len(self.addresses)==1: - return str(self.addresses[0]) - disp = self.display_name - if disp is not None: - nameset = set(disp) - if len(nameset) > len(nameset-parser.SPECIALS): - disp = parser.quote_string(disp) - adrstr = ", ".join(str(x) for x in self.addresses) - adrstr = ' ' + adrstr if adrstr else adrstr - return "{}:{};".format(disp, adrstr) - - def __eq__(self, other): - if type(other) != type(self): - return False - return (self.display_name == other.display_name and - self.addresses == other.addresses) - - -# Header Classes # - -class BaseHeader(str): - - """Base class for message headers. - - Implements generic behavior and provides tools for subclasses. - - A subclass must define a classmethod named 'parse' that takes an unfolded - value string and a dictionary as its arguments. The dictionary will - contain one key, 'defects', initialized to an empty list. After the call - the dictionary must contain two additional keys: parse_tree, set to the - parse tree obtained from parsing the header, and 'decoded', set to the - string value of the idealized representation of the data from the value. - (That is, encoded words are decoded, and values that have canonical - representations are so represented.) - - The defects key is intended to collect parsing defects, which the message - parser will subsequently dispose of as appropriate. The parser should not, - insofar as practical, raise any errors. Defects should be added to the - list instead. The standard header parsers register defects for RFC - compliance issues, for obsolete RFC syntax, and for unrecoverable parsing - errors. - - The parse method may add additional keys to the dictionary. In this case - the subclass must define an 'init' method, which will be passed the - dictionary as its keyword arguments. The method should use (usually by - setting them as the value of similarly named attributes) and remove all the - extra keys added by its parse method, and then use super to call its parent - class with the remaining arguments and keywords. - - The subclass should also make sure that a 'max_count' attribute is defined - that is either None or 1. XXX: need to better define this API. - - """ - - def __new__(cls, name, value): - kwds = {'defects': []} - cls.parse(value, kwds) - if utils._has_surrogates(kwds['decoded']): - kwds['decoded'] = utils._sanitize(kwds['decoded']) - self = str.__new__(cls, kwds['decoded']) - del kwds['decoded'] - self.init(name, **kwds) - return self - - def init(self, name, *, parse_tree, defects): - self._name = name - self._parse_tree = parse_tree - self._defects = defects - - @property - def name(self): - return self._name - - @property - def defects(self): - return tuple(self._defects) - - def __reduce__(self): - return ( - _reconstruct_header, - ( - self.__class__.__name__, - self.__class__.__bases__, - str(self), - ), - self.__dict__) - - @classmethod - def _reconstruct(cls, value): - return str.__new__(cls, value) - - def fold(self, *, policy): - """Fold header according to policy. - - The parsed representation of the header is folded according to - RFC5322 rules, as modified by the policy. If the parse tree - contains surrogateescaped bytes, the bytes are CTE encoded using - the charset 'unknown-8bit". - - Any non-ASCII characters in the parse tree are CTE encoded using - charset utf-8. XXX: make this a policy setting. - - The returned value is an ASCII-only string possibly containing linesep - characters, and ending with a linesep character. The string includes - the header name and the ': ' separator. - - """ - # At some point we need to only put fws here if it was in the source. - header = parser.Header([ - parser.HeaderLabel([ - parser.ValueTerminal(self.name, 'header-name'), - parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) - return header.fold(policy=policy) - - -def _reconstruct_header(cls_name, bases, value): - return type(cls_name, bases, {})._reconstruct(value) - - -class UnstructuredHeader: - - max_count = None - value_parser = staticmethod(parser.get_unstructured) - - @classmethod - def parse(cls, value, kwds): - kwds['parse_tree'] = cls.value_parser(value) - kwds['decoded'] = str(kwds['parse_tree']) - - -class UniqueUnstructuredHeader(UnstructuredHeader): - - max_count = 1 - - -class DateHeader: - - """Header whose value consists of a single timestamp. - - Provides an additional attribute, datetime, which is either an aware - datetime using a timezone, or a naive datetime if the timezone - in the input string is -0000. Also accepts a datetime as input. - The 'value' attribute is the normalized form of the timestamp, - which means it is the output of format_datetime on the datetime. - """ - - max_count = None - - # This is used only for folding, not for creating 'decoded'. - value_parser = staticmethod(parser.get_unstructured) - - @classmethod - def parse(cls, value, kwds): - if not value: - kwds['defects'].append(errors.HeaderMissingRequiredValue()) - kwds['datetime'] = None - kwds['decoded'] = '' - kwds['parse_tree'] = parser.TokenList() - return - if isinstance(value, str): - value = utils.parsedate_to_datetime(value) - kwds['datetime'] = value - kwds['decoded'] = utils.format_datetime(kwds['datetime']) - kwds['parse_tree'] = cls.value_parser(kwds['decoded']) - - def init(self, *args, **kw): - self._datetime = kw.pop('datetime') - super().init(*args, **kw) - - @property - def datetime(self): - return self._datetime - - -class UniqueDateHeader(DateHeader): - - max_count = 1 - - -class AddressHeader: - - max_count = None - - @staticmethod - def value_parser(value): - address_list, value = parser.get_address_list(value) - assert not value, 'this should not happen' - return address_list - - @classmethod - def parse(cls, value, kwds): - if isinstance(value, str): - # We are translating here from the RFC language (address/mailbox) - # to our API language (group/address). - kwds['parse_tree'] = address_list = cls.value_parser(value) - groups = [] - for addr in address_list.addresses: - groups.append(Group(addr.display_name, - [Address(mb.display_name or '', - mb.local_part or '', - mb.domain or '') - for mb in addr.all_mailboxes])) - defects = list(address_list.all_defects) - else: - # Assume it is Address/Group stuff - if not hasattr(value, '__iter__'): - value = [value] - groups = [Group(None, [item]) if not hasattr(item, 'addresses') - else item - for item in value] - defects = [] - kwds['groups'] = groups - kwds['defects'] = defects - kwds['decoded'] = ', '.join([str(item) for item in groups]) - if 'parse_tree' not in kwds: - kwds['parse_tree'] = cls.value_parser(kwds['decoded']) - - def init(self, *args, **kw): - self._groups = tuple(kw.pop('groups')) - self._addresses = None - super().init(*args, **kw) - - @property - def groups(self): - return self._groups - - @property - def addresses(self): - if self._addresses is None: - self._addresses = tuple([address for group in self._groups - for address in group.addresses]) - return self._addresses - - -class UniqueAddressHeader(AddressHeader): - - max_count = 1 - - -class SingleAddressHeader(AddressHeader): - - @property - def address(self): - if len(self.addresses)!=1: - raise ValueError(("value of single address header {} is not " - "a single address").format(self.name)) - return self.addresses[0] - - -class UniqueSingleAddressHeader(SingleAddressHeader): - - max_count = 1 - - -class MIMEVersionHeader: - - max_count = 1 - - value_parser = staticmethod(parser.parse_mime_version) - - @classmethod - def parse(cls, value, kwds): - kwds['parse_tree'] = parse_tree = cls.value_parser(value) - kwds['decoded'] = str(parse_tree) - kwds['defects'].extend(parse_tree.all_defects) - kwds['major'] = None if parse_tree.minor is None else parse_tree.major - kwds['minor'] = parse_tree.minor - if parse_tree.minor is not None: - kwds['version'] = '{}.{}'.format(kwds['major'], kwds['minor']) - else: - kwds['version'] = None - - def init(self, *args, **kw): - self._version = kw.pop('version') - self._major = kw.pop('major') - self._minor = kw.pop('minor') - super().init(*args, **kw) - - @property - def major(self): - return self._major - - @property - def minor(self): - return self._minor - - @property - def version(self): - return self._version - - -class ParameterizedMIMEHeader: - - # Mixin that handles the params dict. Must be subclassed and - # a property value_parser for the specific header provided. - - max_count = 1 - - @classmethod - def parse(cls, value, kwds): - kwds['parse_tree'] = parse_tree = cls.value_parser(value) - kwds['decoded'] = str(parse_tree) - kwds['defects'].extend(parse_tree.all_defects) - if parse_tree.params is None: - kwds['params'] = {} - else: - # The MIME RFCs specify that parameter ordering is arbitrary. - kwds['params'] = {utils._sanitize(name).lower(): - utils._sanitize(value) - for name, value in parse_tree.params} - - def init(self, *args, **kw): - self._params = kw.pop('params') - super().init(*args, **kw) - - @property - def params(self): - return MappingProxyType(self._params) - - -class ContentTypeHeader(ParameterizedMIMEHeader): - - value_parser = staticmethod(parser.parse_content_type_header) - - def init(self, *args, **kw): - super().init(*args, **kw) - self._maintype = utils._sanitize(self._parse_tree.maintype) - self._subtype = utils._sanitize(self._parse_tree.subtype) - - @property - def maintype(self): - return self._maintype - - @property - def subtype(self): - return self._subtype - - @property - def content_type(self): - return self.maintype + '/' + self.subtype - - -class ContentDispositionHeader(ParameterizedMIMEHeader): - - value_parser = staticmethod(parser.parse_content_disposition_header) - - def init(self, *args, **kw): - super().init(*args, **kw) - cd = self._parse_tree.content_disposition - self._content_disposition = cd if cd is None else utils._sanitize(cd) - - @property - def content_disposition(self): - return self._content_disposition - - -class ContentTransferEncodingHeader: - - max_count = 1 - - value_parser = staticmethod(parser.parse_content_transfer_encoding_header) - - @classmethod - def parse(cls, value, kwds): - kwds['parse_tree'] = parse_tree = cls.value_parser(value) - kwds['decoded'] = str(parse_tree) - kwds['defects'].extend(parse_tree.all_defects) - - def init(self, *args, **kw): - super().init(*args, **kw) - self._cte = utils._sanitize(self._parse_tree.cte) - - @property - def cte(self): - return self._cte - - -# The header factory # - -_default_header_map = { - 'subject': UniqueUnstructuredHeader, - 'date': UniqueDateHeader, - 'resent-date': DateHeader, - 'orig-date': UniqueDateHeader, - 'sender': UniqueSingleAddressHeader, - 'resent-sender': SingleAddressHeader, - 'to': UniqueAddressHeader, - 'resent-to': AddressHeader, - 'cc': UniqueAddressHeader, - 'resent-cc': AddressHeader, - 'bcc': UniqueAddressHeader, - 'resent-bcc': AddressHeader, - 'from': UniqueAddressHeader, - 'resent-from': AddressHeader, - 'reply-to': UniqueAddressHeader, - 'mime-version': MIMEVersionHeader, - 'content-type': ContentTypeHeader, - 'content-disposition': ContentDispositionHeader, - 'content-transfer-encoding': ContentTransferEncodingHeader, - } - -class HeaderRegistry: - - """A header_factory and header registry.""" - - def __init__(self, base_class=BaseHeader, default_class=UnstructuredHeader, - use_default_map=True): - """Create a header_factory that works with the Policy API. - - base_class is the class that will be the last class in the created - header class's __bases__ list. default_class is the class that will be - used if "name" (see __call__) does not appear in the registry. - use_default_map controls whether or not the default mapping of names to - specialized classes is copied in to the registry when the factory is - created. The default is True. - - """ - self.registry = {} - self.base_class = base_class - self.default_class = default_class - if use_default_map: - self.registry.update(_default_header_map) - - def map_to_type(self, name, cls): - """Register cls as the specialized class for handling "name" headers. - - """ - self.registry[name.lower()] = cls - - def __getitem__(self, name): - cls = self.registry.get(name.lower(), self.default_class) - return type('_'+cls.__name__, (cls, self.base_class), {}) - - def __call__(self, name, value): - """Create a header instance for header 'name' from 'value'. - - Creates a header instance by creating a specialized class for parsing - and representing the specified header by combining the factory - base_class with a specialized class from the registry or the - default_class, and passing the name and value to the constructed - class's constructor. - - """ - return self[name](name, value) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/iterators.py --- a/Lib/email/iterators.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/iterators.py Mon Jan 25 17:05:13 2016 +0100 @@ -26,7 +26,8 @@ yield self if self.is_multipart(): for subpart in self.get_payload(): - yield from subpart.walk() + for subsubpart in subpart.walk(): + yield subsubpart @@ -39,7 +40,8 @@ for subpart in msg.walk(): payload = subpart.get_payload(decode=decode) if isinstance(payload, str): - yield from StringIO(payload) + for line in StringIO(payload): + yield line def typed_subpart_iterator(msg, maintype='text', subtype=None): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/message.py --- a/Lib/email/message.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/message.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,16 +8,16 @@ import re import uu -import quopri +import base64 +import binascii import warnings from io import BytesIO, StringIO # Intrapackage imports from email import utils from email import errors -from email._policybase import compat32 +from email import header from email import charset as _charset -from email._encoded_words import decode_b Charset = _charset.Charset SEMISPACE = '; ' @@ -26,6 +26,24 @@ # existence of which force quoting of the parameter value. tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') +# How to figure out if we are processing strings that come from a byte +# source with undecodable characters. +_has_surrogates = re.compile( + '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search + + +# Helper functions +def _sanitize_header(name, value): + # If the header value contains surrogates, return a Header using + # the unknown-8bit charset to encode the bytes as encoded words. + if not isinstance(value, str): + # Assume it is already a header object + return value + if _has_surrogates(value): + return header.Header(value, charset=_charset.UNKNOWN8BIT, + header_name=name) + else: + return value def _splitparam(param): # Split header parameters. BAW: this may be too simple. It isn't @@ -118,8 +136,7 @@ you must use the explicit API to set or get all the headers. Not all of the mapping methods are implemented. """ - def __init__(self, policy=compat32): - self.policy = policy + def __init__(self): self._headers = [] self._unixfrom = None self._payload = None @@ -132,50 +149,22 @@ def __str__(self): """Return the entire formatted message as a string. + This includes the headers, body, and envelope header. """ return self.as_string() - def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): + def as_string(self, unixfrom=False, maxheaderlen=0): """Return the entire formatted message as a string. + Optional `unixfrom' when True, means include the Unix From_ envelope + header. - Optional 'unixfrom', when true, means include the Unix From_ envelope - header. For backward compatibility reasons, if maxheaderlen is - not specified it defaults to 0, so you must override it explicitly - if you want a different maxheaderlen. 'policy' is passed to the - Generator instance used to serialize the mesasge; if it is not - specified the policy associated with the message instance is used. - - If the message object contains binary data that is not encoded - according to RFC standards, the non-compliant data will be replaced by - unicode "unknown character" code points. + This is a convenience method and may not generate the message exactly + as you intend. For more flexibility, use the flatten() method of a + Generator instance. """ from email.generator import Generator - policy = self.policy if policy is None else policy fp = StringIO() - g = Generator(fp, - mangle_from_=False, - maxheaderlen=maxheaderlen, - policy=policy) - g.flatten(self, unixfrom=unixfrom) - return fp.getvalue() - - def __bytes__(self): - """Return the entire formatted message as a bytes object. - """ - return self.as_bytes() - - def as_bytes(self, unixfrom=False, policy=None): - """Return the entire formatted message as a bytes object. - - Optional 'unixfrom', when true, means include the Unix From_ envelope - header. 'policy' is passed to the BytesGenerator instance used to - serialize the message; if not specified the policy associated with - the message instance is used. - """ - from email.generator import BytesGenerator - policy = self.policy if policy is None else policy - fp = BytesIO() - g = BytesGenerator(fp, mangle_from_=False, policy=policy) + g = Generator(fp, mangle_from_=False, maxheaderlen=maxheaderlen) g.flatten(self, unixfrom=unixfrom) return fp.getvalue() @@ -205,11 +194,7 @@ if self._payload is None: self._payload = [payload] else: - try: - self._payload.append(payload) - except AttributeError: - raise TypeError("Attach is not valid on a message with a" - " non-multipart payload") + self._payload.append(payload) def get_payload(self, i=None, decode=False): """Return a reference to the payload. @@ -261,7 +246,7 @@ cte = str(self.get('content-transfer-encoding', '')).lower() # payload may be bytes here. if isinstance(payload, str): - if utils._has_surrogates(payload): + if _has_surrogates(payload): bpayload = payload.encode('ascii', 'surrogateescape') if not decode: try: @@ -273,21 +258,20 @@ bpayload = payload.encode('ascii') except UnicodeError: # This won't happen for RFC compliant messages (messages - # containing only ASCII code points in the unicode input). + # containing only ASCII codepoints in the unicode input). # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') if not decode: return payload if cte == 'quoted-printable': - return quopri.decodestring(bpayload) + return utils._qdecode(bpayload) elif cte == 'base64': - # XXX: this is a bit of a hack; decode_b should probably be factored - # out somewhere, but I haven't figured out where yet. - value, defects = decode_b(b''.join(bpayload.splitlines())) - for defect in defects: - self.policy.handle_defect(self, defect) - return value + try: + return base64.b64decode(bpayload) + except binascii.Error: + # Incorrect padding + return bpayload elif cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): in_file = BytesIO(bpayload) out_file = BytesIO() @@ -307,17 +291,7 @@ Optional charset sets the message's default character set. See set_charset() for details. """ - if hasattr(payload, 'encode'): - if charset is None: - self._payload = payload - return - if not isinstance(charset, Charset): - charset = Charset(charset) - payload = payload.encode(charset.output_charset) - if hasattr(payload, 'decode'): - self._payload = payload.decode('ascii', 'surrogateescape') - else: - self._payload = payload + self._payload = payload if charset is not None: self.set_charset(charset) @@ -356,16 +330,7 @@ try: cte(self) except TypeError: - # This 'if' is for backward compatibility, it allows unicode - # through even though that won't work correctly if the - # message is serialized. - payload = self._payload - if payload: - try: - payload = payload.encode('ascii', 'surrogateescape') - except UnicodeError: - payload = payload.encode(charset.output_charset) - self._payload = charset.body_encode(payload) + self._payload = charset.body_encode(self._payload) self.add_header('Content-Transfer-Encoding', cte) def get_charset(self): @@ -397,17 +362,7 @@ Note: this does not overwrite an existing header with the same field name. Use __delitem__() first to delete any existing headers. """ - max_count = self.policy.header_max_count(name) - if max_count: - lname = name.lower() - found = 0 - for k, v in self._headers: - if k.lower() == lname: - found += 1 - if found >= max_count: - raise ValueError("There may be at most {} {} headers " - "in a message".format(max_count, name)) - self._headers.append(self.policy.header_store_parse(name, val)) + self._headers.append((name, val)) def __delitem__(self, name): """Delete all occurrences of a header, if present. @@ -446,8 +401,7 @@ Any fields deleted and re-inserted are always appended to the header list. """ - return [self.policy.header_fetch_parse(k, v) - for k, v in self._headers] + return [_sanitize_header(k, v) for k, v in self._headers] def items(self): """Get all the message's header fields and values. @@ -457,8 +411,7 @@ Any fields deleted and re-inserted are always appended to the header list. """ - return [(k, self.policy.header_fetch_parse(k, v)) - for k, v in self._headers] + return [(k, _sanitize_header(k, v)) for k, v in self._headers] def get(self, name, failobj=None): """Get a header value. @@ -469,29 +422,10 @@ name = name.lower() for k, v in self._headers: if k.lower() == name: - return self.policy.header_fetch_parse(k, v) + return _sanitize_header(k, v) return failobj # - # "Internal" methods (public API, but only intended for use by a parser - # or generator, not normal application code. - # - - def set_raw(self, name, value): - """Store name and value in the model without modification. - - This is an "internal" API, intended only for use by a parser. - """ - self._headers.append((name, value)) - - def raw_items(self): - """Return the (name, value) header pairs without modification. - - This is an "internal" API, intended only for use by a generator. - """ - return iter(self._headers.copy()) - - # # Additional useful stuff # @@ -508,7 +442,7 @@ name = name.lower() for k, v in self._headers: if k.lower() == name: - values.append(self.policy.header_fetch_parse(k, v)) + values.append(_sanitize_header(k, v)) if not values: return failobj return values @@ -541,7 +475,7 @@ parts.append(_formatparam(k.replace('_', '-'), v)) if _value is not None: parts.insert(0, _value) - self[_name] = SEMISPACE.join(parts) + self._headers.append((_name, SEMISPACE.join(parts))) def replace_header(self, _name, _value): """Replace a header. @@ -553,7 +487,7 @@ _name = _name.lower() for i, (k, v) in zip(range(len(self._headers)), self._headers): if k.lower() == _name: - self._headers[i] = self.policy.header_store_parse(k, _value) + self._headers[i] = (k, _value) break else: raise KeyError(_name) @@ -679,15 +613,17 @@ the form (CHARSET, LANGUAGE, VALUE). Note that both CHARSET and LANGUAGE can be None, in which case you should consider VALUE to be encoded in the us-ascii charset. You can usually ignore LANGUAGE. - The parameter value (either the returned string, or the VALUE item in - the 3-tuple) is always unquoted, unless unquote is set to False. - If your application doesn't care whether the parameter was RFC 2231 - encoded, it can turn the return value into a string as follows: + Your application should be prepared to deal with 3-tuple return + values, and can convert the parameter to a Unicode string like so: - rawparam = msg.get_param('foo') - param = email.utils.collapse_rfc2231_value(rawparam) + param = msg.get_param('foo') + if isinstance(param, tuple): + param = unicode(param[2], param[0] or 'us-ascii') + In any case, the parameter value (either the returned string, or the + VALUE item in the 3-tuple) is always unquoted, unless unquote is set + to False. """ if header not in self: return failobj @@ -700,7 +636,7 @@ return failobj def set_param(self, param, value, header='Content-Type', requote=True, - charset=None, language='', replace=False): + charset=None, language=''): """Set a parameter in the Content-Type header. If the parameter already exists in the header, its value will be @@ -744,11 +680,8 @@ else: ctype = SEMISPACE.join([ctype, append_param]) if ctype != self.get(header): - if replace: - self.replace_header(header, ctype) - else: - del self[header] - self[header] = ctype + del self[header] + self[header] = ctype def del_param(self, param, header='content-type', requote=True): """Remove the given parameter completely from the Content-Type header. @@ -872,8 +805,7 @@ parts.append(k) else: parts.append('%s=%s' % (k, v)) - val = SEMISPACE.join(parts) - newheaders.append(self.policy.header_store_parse(h, val)) + newheaders.append((h, SEMISPACE.join(parts))) else: newheaders.append((h, v)) @@ -927,219 +859,5 @@ """ return [part.get_content_charset(failobj) for part in self.walk()] - def get_content_disposition(self): - """Return the message's content-disposition if it exists, or None. - - The return values can be either 'inline', 'attachment' or None - according to the rfc2183. - """ - value = self.get('content-disposition') - if value is None: - return None - c_d = _splitparam(value)[0].lower() - return c_d - # I.e. def walk(self): ... from email.iterators import walk - - -class MIMEPart(Message): - - def __init__(self, policy=None): - if policy is None: - from email.policy import default - policy = default - Message.__init__(self, policy) - - def is_attachment(self): - c_d = self.get('content-disposition') - return False if c_d is None else c_d.content_disposition == 'attachment' - - def _find_body(self, part, preferencelist): - if part.is_attachment(): - return - maintype, subtype = part.get_content_type().split('/') - if maintype == 'text': - if subtype in preferencelist: - yield (preferencelist.index(subtype), part) - return - if maintype != 'multipart': - return - if subtype != 'related': - for subpart in part.iter_parts(): - yield from self._find_body(subpart, preferencelist) - return - if 'related' in preferencelist: - yield (preferencelist.index('related'), part) - candidate = None - start = part.get_param('start') - if start: - for subpart in part.iter_parts(): - if subpart['content-id'] == start: - candidate = subpart - break - if candidate is None: - subparts = part.get_payload() - candidate = subparts[0] if subparts else None - if candidate is not None: - yield from self._find_body(candidate, preferencelist) - - def get_body(self, preferencelist=('related', 'html', 'plain')): - """Return best candidate mime part for display as 'body' of message. - - Do a depth first search, starting with self, looking for the first part - matching each of the items in preferencelist, and return the part - corresponding to the first item that has a match, or None if no items - have a match. If 'related' is not included in preferencelist, consider - the root part of any multipart/related encountered as a candidate - match. Ignore parts with 'Content-Disposition: attachment'. - """ - best_prio = len(preferencelist) - body = None - for prio, part in self._find_body(self, preferencelist): - if prio < best_prio: - best_prio = prio - body = part - if prio == 0: - break - return body - - _body_types = {('text', 'plain'), - ('text', 'html'), - ('multipart', 'related'), - ('multipart', 'alternative')} - def iter_attachments(self): - """Return an iterator over the non-main parts of a multipart. - - Skip the first of each occurrence of text/plain, text/html, - multipart/related, or multipart/alternative in the multipart (unless - they have a 'Content-Disposition: attachment' header) and include all - remaining subparts in the returned iterator. When applied to a - multipart/related, return all parts except the root part. Return an - empty iterator when applied to a multipart/alternative or a - non-multipart. - """ - maintype, subtype = self.get_content_type().split('/') - if maintype != 'multipart' or subtype == 'alternative': - return - parts = self.get_payload() - if maintype == 'multipart' and subtype == 'related': - # For related, we treat everything but the root as an attachment. - # The root may be indicated by 'start'; if there's no start or we - # can't find the named start, treat the first subpart as the root. - start = self.get_param('start') - if start: - found = False - attachments = [] - for part in parts: - if part.get('content-id') == start: - found = True - else: - attachments.append(part) - if found: - yield from attachments - return - parts.pop(0) - yield from parts - return - # Otherwise we more or less invert the remaining logic in get_body. - # This only really works in edge cases (ex: non-text relateds or - # alternatives) if the sending agent sets content-disposition. - seen = [] # Only skip the first example of each candidate type. - for part in parts: - maintype, subtype = part.get_content_type().split('/') - if ((maintype, subtype) in self._body_types and - not part.is_attachment() and subtype not in seen): - seen.append(subtype) - continue - yield part - - def iter_parts(self): - """Return an iterator over all immediate subparts of a multipart. - - Return an empty iterator for a non-multipart. - """ - if self.get_content_maintype() == 'multipart': - yield from self.get_payload() - - def get_content(self, *args, content_manager=None, **kw): - if content_manager is None: - content_manager = self.policy.content_manager - return content_manager.get_content(self, *args, **kw) - - def set_content(self, *args, content_manager=None, **kw): - if content_manager is None: - content_manager = self.policy.content_manager - content_manager.set_content(self, *args, **kw) - - def _make_multipart(self, subtype, disallowed_subtypes, boundary): - if self.get_content_maintype() == 'multipart': - existing_subtype = self.get_content_subtype() - disallowed_subtypes = disallowed_subtypes + (subtype,) - if existing_subtype in disallowed_subtypes: - raise ValueError("Cannot convert {} to {}".format( - existing_subtype, subtype)) - keep_headers = [] - part_headers = [] - for name, value in self._headers: - if name.lower().startswith('content-'): - part_headers.append((name, value)) - else: - keep_headers.append((name, value)) - if part_headers: - # There is existing content, move it to the first subpart. - part = type(self)(policy=self.policy) - part._headers = part_headers - part._payload = self._payload - self._payload = [part] - else: - self._payload = [] - self._headers = keep_headers - self['Content-Type'] = 'multipart/' + subtype - if boundary is not None: - self.set_param('boundary', boundary) - - def make_related(self, boundary=None): - self._make_multipart('related', ('alternative', 'mixed'), boundary) - - def make_alternative(self, boundary=None): - self._make_multipart('alternative', ('mixed',), boundary) - - def make_mixed(self, boundary=None): - self._make_multipart('mixed', (), boundary) - - def _add_multipart(self, _subtype, *args, _disp=None, **kw): - if (self.get_content_maintype() != 'multipart' or - self.get_content_subtype() != _subtype): - getattr(self, 'make_' + _subtype)() - part = type(self)(policy=self.policy) - part.set_content(*args, **kw) - if _disp and 'content-disposition' not in part: - part['Content-Disposition'] = _disp - self.attach(part) - - def add_related(self, *args, **kw): - self._add_multipart('related', *args, _disp='inline', **kw) - - def add_alternative(self, *args, **kw): - self._add_multipart('alternative', *args, **kw) - - def add_attachment(self, *args, **kw): - self._add_multipart('mixed', *args, _disp='attachment', **kw) - - def clear(self): - self._headers = [] - self._payload = None - - def clear_content(self): - self._headers = [(n, v) for n, v in self._headers - if not n.lower().startswith('content-')] - self._payload = None - - -class EmailMessage(MIMEPart): - - def set_content(self, *args, **kw): - super().set_content(*args, **kw) - if 'MIME-Version' not in self: - self['MIME-Version'] = '1.0' diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/mime/nonmultipart.py --- a/Lib/email/mime/nonmultipart.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/mime/nonmultipart.py Mon Jan 25 17:05:13 2016 +0100 @@ -12,7 +12,7 @@ class MIMENonMultipart(MIMEBase): - """Base class for MIME non-multipart type messages.""" + """Base class for MIME multipart/* type messages.""" def attach(self, payload): # The public API prohibits attaching multiple subparts to MIMEBase diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/mime/text.py --- a/Lib/email/mime/text.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/mime/text.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,7 @@ __all__ = ['MIMEText'] -from email.charset import Charset +from email.encoders import encode_7or8bit from email.mime.nonmultipart import MIMENonMultipart @@ -26,7 +26,7 @@ Content-Transfer-Encoding header will also be set. """ - # If no _charset was specified, check to see if there are non-ascii + # If no _charset was specified, check to see see if there are non-ascii # characters present. If not, use 'us-ascii', otherwise use utf-8. # XXX: This can be removed once #7304 is fixed. if _charset is None: @@ -35,8 +35,6 @@ _charset = 'us-ascii' except UnicodeEncodeError: _charset = 'utf-8' - if isinstance(_charset, Charset): - _charset = str(_charset) MIMENonMultipart.__init__(self, 'text', _subtype, **{'charset': _charset}) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/parser.py --- a/Lib/email/parser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/parser.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,18 +4,19 @@ """A parser of RFC 2822 and MIME email messages.""" -__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser', - 'FeedParser', 'BytesFeedParser'] +__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser'] +import warnings from io import StringIO, TextIOWrapper -from email.feedparser import FeedParser, BytesFeedParser -from email._policybase import compat32 +from email.feedparser import FeedParser +from email.message import Message +from email import policy class Parser: - def __init__(self, _class=None, *, policy=compat32): + def __init__(self, _class=Message, *, policy=policy.default): """Parser of RFC 2822 and MIME email messages. Creates an in-memory object tree representing the email message, which @@ -106,10 +107,8 @@ meaning it parses the entire contents of the file. """ fp = TextIOWrapper(fp, encoding='ascii', errors='surrogateescape') - try: + with fp: return self.parser.parse(fp, headersonly) - finally: - fp.detach() def parsebytes(self, text, headersonly=False): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/policy.py --- a/Lib/email/policy.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/policy.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,214 +1,174 @@ -"""This will be the home for the policy that hooks in the new -code that adds all the email6 features. +"""Policy framework for the email package. + +Allows fine grained feature control of how the package parses and emits data. """ -from email._policybase import Policy, Compat32, compat32, _extend_docstrings -from email.utils import _has_surrogates -from email.headerregistry import HeaderRegistry as HeaderRegistry -from email.contentmanager import raw_data_manager - __all__ = [ - 'Compat32', - 'compat32', 'Policy', - 'EmailPolicy', 'default', 'strict', 'SMTP', 'HTTP', ] -@_extend_docstrings -class EmailPolicy(Policy): - """+ - PROVISIONAL +class _PolicyBase: - The API extensions enabled by this policy are currently provisional. - Refer to the documentation for details. + """Policy Object basic framework. - This policy adds new header parsing and folding algorithms. Instead of - simple strings, headers are custom objects with custom attributes - depending on the type of the field. The folding algorithm fully - implements RFCs 2047 and 5322. + This class is useless unless subclassed. A subclass should define + class attributes with defaults for any values that are to be + managed by the Policy object. The constructor will then allow + non-default values to be set for these attributes at instance + creation time. The instance will be callable, taking these same + attributes keyword arguments, and returning a new instance + identical to the called instance except for those values changed + by the keyword arguments. Instances may be added, yielding new + instances with any non-default values from the right hand + operand overriding those in the left hand operand. That is, - In addition to the settable attributes listed above that apply to - all Policies, this policy adds the following additional attributes: + A + B == A() - utf8 -- if False (the default) message headers will be - serialized as ASCII, using encoded words to encode - any non-ASCII characters in the source strings. If - True, the message headers will be serialized using - utf8 and will not contain encoded words (see RFC - 6532 for more on this serialization format). - - refold_source -- if the value for a header in the Message object - came from the parsing of some source, this attribute - indicates whether or not a generator should refold - that value when transforming the message back into - stream form. The possible values are: - - none -- all source values use original folding - long -- source values that have any line that is - longer than max_line_length will be - refolded - all -- all values are refolded. - - The default is 'long'. - - header_factory -- a callable that takes two arguments, 'name' and - 'value', where 'name' is a header field name and - 'value' is an unfolded header field value, and - returns a string-like object that represents that - header. A default header_factory is provided that - understands some of the RFC5322 header field types. - (Currently address fields and date fields have - special treatment, while all other fields are - treated as unstructured. This list will be - completed before the extension is marked stable.) - - content_manager -- an object with at least two methods: get_content - and set_content. When the get_content or - set_content method of a Message object is called, - it calls the corresponding method of this object, - passing it the message object as its first argument, - and any arguments or keywords that were passed to - it as additional arguments. The default - content_manager is - :data:`~email.contentmanager.raw_data_manager`. + The repr of an instance can be used to reconstruct the object + if and only if the repr of the values can be used to reconstruct + those values. """ - utf8 = False - refold_source = 'long' - header_factory = HeaderRegistry() - content_manager = raw_data_manager + def __init__(self, **kw): + """Create new Policy, possibly overriding some defaults. - def __init__(self, **kw): - # Ensure that each new instance gets a unique header factory - # (as opposed to clones, which share the factory). - if 'header_factory' not in kw: - object.__setattr__(self, 'header_factory', HeaderRegistry()) - super().__init__(**kw) - - def header_max_count(self, name): - """+ - The implementation for this class returns the max_count attribute from - the specialized header class that would be used to construct a header - of type 'name'. - """ - return self.header_factory[name].max_count - - # The logic of the next three methods is chosen such that it is possible to - # switch a Message object between a Compat32 policy and a policy derived - # from this class and have the results stay consistent. This allows a - # Message object constructed with this policy to be passed to a library - # that only handles Compat32 objects, or to receive such an object and - # convert it to use the newer style by just changing its policy. It is - # also chosen because it postpones the relatively expensive full rfc5322 - # parse until as late as possible when parsing from source, since in many - # applications only a few headers will actually be inspected. - - def header_source_parse(self, sourcelines): - """+ - The name is parsed as everything up to the ':' and returned unmodified. - The value is determined by stripping leading whitespace off the - remainder of the first line, joining all subsequent lines together, and - stripping any trailing carriage return or linefeed characters. (This - is the same as Compat32). + See class docstring for a list of overridable attributes. """ - name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) - return (name, value.rstrip('\r\n')) + for name, value in kw.items(): + if hasattr(self, name): + super(_PolicyBase,self).__setattr__(name, value) + else: + raise TypeError( + "{!r} is an invalid keyword argument for {}".format( + name, self.__class__.__name__)) - def header_store_parse(self, name, value): - """+ - The name is returned unchanged. If the input value has a 'name' - attribute and it matches the name ignoring case, the value is returned - unchanged. Otherwise the name and value are passed to header_factory - method, and the resulting custom header object is returned as the - value. In this case a ValueError is raised if the input value contains - CR or LF characters. + def __repr__(self): + args = [ "{}={!r}".format(name, value) + for name, value in self.__dict__.items() ] + return "{}({})".format(self.__class__.__name__, ', '.join(args)) + + def clone(self, **kw): + """Return a new instance with specified attributes changed. + + The new instance has the same attribute values as the current object, + except for the changes passed in as keyword arguments. """ - if hasattr(value, 'name') and value.name.lower() == name.lower(): - return (name, value) - if isinstance(value, str) and len(value.splitlines())>1: - raise ValueError("Header values may not contain linefeed " - "or carriage return characters") - return (name, self.header_factory(name, value)) + for attr, value in self.__dict__.items(): + if attr not in kw: + kw[attr] = value + return self.__class__(**kw) - def header_fetch_parse(self, name, value): - """+ - If the value has a 'name' attribute, it is returned to unmodified. - Otherwise the name and the value with any linesep characters removed - are passed to the header_factory method, and the resulting custom - header object is returned. Any surrogateescaped bytes get turned - into the unicode unknown-character glyph. + def __setattr__(self, name, value): + if hasattr(self, name): + msg = "{!r} object attribute {!r} is read-only" + else: + msg = "{!r} object has no attribute {!r}" + raise AttributeError(msg.format(self.__class__.__name__, name)) + + def __add__(self, other): + """Non-default values from right operand override those from left. + + The object returned is a new instance of the subclass. """ - if hasattr(value, 'name'): - return value - return self.header_factory(name, ''.join(value.splitlines())) + return self.clone(**other.__dict__) - def fold(self, name, value): - """+ - Header folding is controlled by the refold_source policy setting. A - value is considered to be a 'source value' if and only if it does not - have a 'name' attribute (having a 'name' attribute means it is a header - object of some sort). If a source value needs to be refolded according - to the policy, it is converted into a custom header object by passing - the name and the value with any linesep characters removed to the - header_factory method. Folding of a custom header object is done by - calling its fold method with the current policy. - Source values are split into lines using splitlines. If the value is - not to be refolded, the lines are rejoined using the linesep from the - policy and returned. The exception is lines containing non-ascii - binary data. In that case the value is refolded regardless of the - refold_source setting, which causes the binary data to be CTE encoded - using the unknown-8bit charset. +class Policy(_PolicyBase): + + """Controls for how messages are interpreted and formatted. + + Most of the classes and many of the methods in the email package + accept Policy objects as parameters. A Policy object contains a set + of values and functions that control how input is interpreted and how + output is rendered. For example, the parameter 'raise_on_defect' + controls whether or not an RFC violation throws an error or not, + while 'max_line_length' controls the maximum length of output lines + when a Message is serialized. + + Any valid attribute may be overridden when a Policy is created by + passing it as a keyword argument to the constructor. Policy + objects are immutable, but a new Policy object can be created + with only certain values changed by calling the Policy instance + with keyword arguments. Policy objects can also be added, + producing a new Policy object in which the non-default attributes + set in the right hand operand overwrite those specified in the + left operand. + + Settable attributes: + + raise_on_defect -- If true, then defects should be raised + as errors. Default False. + + linesep -- string containing the value to use as + separation between output lines. Default '\n'. + + must_be_7bit -- output must contain only 7bit clean data. + Default False. + + max_line_length -- maximum length of lines, excluding 'linesep', + during serialization. None means no line + wrapping is done. Default is 78. + + Methods: + + register_defect(obj, defect) + defect is a Defect instance. The default implementation appends defect + to the objs 'defects' attribute. + + handle_defect(obj, defect) + intended to be called by parser code that finds a defect. If + raise_on_defect is True, defect is raised as an error, otherwise + register_defect is called. + + """ + + raise_on_defect = False + linesep = '\n' + must_be_7bit = False + max_line_length = 78 + + def handle_defect(self, obj, defect): + """Based on policy, either raise defect or call register_defect. + + handle_defect(obj, defect) + + defect should be a Defect subclass, but in any case must be an + Exception subclass. obj is the object on which the defect should be + registered if it is not raised. If the raise_on_defect is True, the + defect is raised as an error, otherwise the object and the defect are + passed to register_defect. + + This class is intended to be called by parsers that discover defects, + and will not be called from code using the library unless that code is + implementing an alternate parser. """ - return self._fold(name, value, refold_binary=True) + if self.raise_on_defect: + raise defect + self.register_defect(obj, defect) - def fold_binary(self, name, value): - """+ - The same as fold if cte_type is 7bit, except that the returned value is - bytes. + def register_defect(self, obj, defect): + """Record 'defect' on 'obj'. - If cte_type is 8bit, non-ASCII binary data is converted back into - bytes. Headers with binary data are not refolded, regardless of the - refold_header setting, since there is no way to know whether the binary - data consists of single byte characters or multibyte characters. - - If utf8 is true, headers are encoded to utf8, otherwise to ascii with - non-ASCII unicode rendered as encoded words. + Called by handle_defect if raise_on_defect is False. This method is + part of the Policy API so that Policy subclasses can implement custom + defect handling. The default implementation calls the append method + of the defects attribute of obj. """ - folded = self._fold(name, value, refold_binary=self.cte_type=='7bit') - charset = 'utf8' if self.utf8 else 'ascii' - return folded.encode(charset, 'surrogateescape') + obj.defects.append(defect) - def _fold(self, name, value, refold_binary=False): - if hasattr(value, 'name'): - return value.fold(policy=self) - maxlen = self.max_line_length if self.max_line_length else float('inf') - lines = value.splitlines() - refold = (self.refold_source == 'all' or - self.refold_source == 'long' and - (lines and len(lines[0])+len(name)+2 > maxlen or - any(len(x) > maxlen for x in lines[1:]))) - if refold or refold_binary and _has_surrogates(value): - return self.header_factory(name, ''.join(lines)).fold(policy=self) - return name + ': ' + self.linesep.join(lines) + self.linesep - -default = EmailPolicy() -# Make the default policy use the class default header_factory -del default.header_factory +default = Policy() strict = default.clone(raise_on_defect=True) SMTP = default.clone(linesep='\r\n') HTTP = default.clone(linesep='\r\n', max_line_length=None) -SMTPUTF8 = SMTP.clone(utf8=True) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/quoprimime.py --- a/Lib/email/quoprimime.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/quoprimime.py Mon Jan 25 17:05:13 2016 +0100 @@ -40,6 +40,7 @@ ] import re +import io from string import ascii_letters, digits, hexdigits @@ -52,9 +53,8 @@ # space-wise. Remember that headers and bodies have different sets of safe # characters. Initialize both maps with the full expansion, and then override # the safe bytes with the more compact form. -_QUOPRI_MAP = ['=%02X' % c for c in range(256)] -_QUOPRI_HEADER_MAP = _QUOPRI_MAP[:] -_QUOPRI_BODY_MAP = _QUOPRI_MAP[:] +_QUOPRI_HEADER_MAP = dict((c, '=%02X' % c) for c in range(256)) +_QUOPRI_BODY_MAP = _QUOPRI_HEADER_MAP.copy() # Safe header bytes which need no encoding. for c in b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii'): @@ -121,7 +121,8 @@ def quote(c): - return _QUOPRI_MAP[ord(c)] + return '=%02X' % ord(c) + def header_encode(header_bytes, charset='iso-8859-1'): @@ -139,15 +140,67 @@ if not header_bytes: return '' # Iterate over every byte, encoding if necessary. - encoded = header_bytes.decode('latin1').translate(_QUOPRI_HEADER_MAP) + encoded = [] + for octet in header_bytes: + encoded.append(_QUOPRI_HEADER_MAP[octet]) # Now add the RFC chrome to each encoded chunk and glue the chunks # together. - return '=?%s?q?%s?=' % (charset, encoded) + return '=?%s?q?%s?=' % (charset, EMPTYSTRING.join(encoded)) -_QUOPRI_BODY_ENCODE_MAP = _QUOPRI_BODY_MAP[:] -for c in b'\r\n': - _QUOPRI_BODY_ENCODE_MAP[c] = chr(c) +class _body_accumulator(io.StringIO): + + def __init__(self, maxlinelen, eol, *args, **kw): + super().__init__(*args, **kw) + self.eol = eol + self.maxlinelen = self.room = maxlinelen + + def write_str(self, s): + """Add string s to the accumulated body.""" + self.write(s) + self.room -= len(s) + + def newline(self): + """Write eol, then start new line.""" + self.write_str(self.eol) + self.room = self.maxlinelen + + def write_soft_break(self): + """Write a soft break, then start a new line.""" + self.write_str('=') + self.newline() + + def write_wrapped(self, s, extra_room=0): + """Add a soft line break if needed, then write s.""" + if self.room < len(s) + extra_room: + self.write_soft_break() + self.write_str(s) + + def write_char(self, c, is_last_char): + if not is_last_char: + # Another character follows on this line, so we must leave + # extra room, either for it or a soft break, and whitespace + # need not be quoted. + self.write_wrapped(c, extra_room=1) + elif c not in ' \t': + # For this and remaining cases, no more characters follow, + # so there is no need to reserve extra room (since a hard + # break will immediately follow). + self.write_wrapped(c) + elif self.room >= 3: + # It's a whitespace character at end-of-line, and we have room + # for the three-character quoted encoding. + self.write(quote(c)) + elif self.room == 2: + # There's room for the whitespace character and a soft break. + self.write(c) + self.write_soft_break() + else: + # There's room only for a soft break. The quoted whitespace + # will be the only content on the subsequent line. + self.write_soft_break() + self.write(quote(c)) + def body_encode(body, maxlinelen=76, eol=NL): """Encode with quoted-printable, wrapping at maxlinelen characters. @@ -173,56 +226,26 @@ if not body: return body - # quote speacial characters - body = body.translate(_QUOPRI_BODY_ENCODE_MAP) + # The last line may or may not end in eol, but all other lines do. + last_has_eol = (body[-1] in '\r\n') - soft_break = '=' + eol - # leave space for the '=' at the end of a line - maxlinelen1 = maxlinelen - 1 + # This accumulator will make it easier to build the encoded body. + encoded_body = _body_accumulator(maxlinelen, eol) - encoded_body = [] - append = encoded_body.append + lines = body.splitlines() + last_line_no = len(lines) - 1 + for line_no, line in enumerate(lines): + last_char_index = len(line) - 1 + for i, c in enumerate(line): + if body_check(ord(c)): + c = quote(c) + encoded_body.write_char(c, i==last_char_index) + # Add an eol if input line had eol. All input lines have eol except + # possibly the last one. + if line_no < last_line_no or last_has_eol: + encoded_body.newline() - for line in body.splitlines(): - # break up the line into pieces no longer than maxlinelen - 1 - start = 0 - laststart = len(line) - 1 - maxlinelen - while start <= laststart: - stop = start + maxlinelen1 - # make sure we don't break up an escape sequence - if line[stop - 2] == '=': - append(line[start:stop - 1]) - start = stop - 2 - elif line[stop - 1] == '=': - append(line[start:stop]) - start = stop - 1 - else: - append(line[start:stop] + '=') - start = stop - - # handle rest of line, special case if line ends in whitespace - if line and line[-1] in ' \t': - room = start - laststart - if room >= 3: - # It's a whitespace character at end-of-line, and we have room - # for the three-character quoted encoding. - q = quote(line[-1]) - elif room == 2: - # There's room for the whitespace character and a soft break. - q = line[-1] + soft_break - else: - # There's room only for a soft break. The quoted whitespace - # will be the only content on the subsequent line. - q = soft_break + quote(line[-1]) - append(line[start:-1] + q) - else: - append(line[start:]) - - # add back final newline if present - if body[-1] in CRLF: - append('') - - return eol.join(encoded_body) + return encoded_body.getvalue() @@ -296,4 +319,4 @@ the high level email.header class for that functionality. """ s = s.replace('_', ' ') - return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, flags=re.ASCII) + return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, re.ASCII) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/email/utils.py --- a/Lib/email/utils.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/email/utils.py Mon Jan 25 17:05:13 2016 +0100 @@ -25,18 +25,27 @@ import os import re import time +import base64 import random import socket import datetime import urllib.parse +import warnings +from io import StringIO from email._parseaddr import quote from email._parseaddr import AddressList as _AddressList from email._parseaddr import mktime_tz -from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz +# We need wormarounds for bugs in these methods in older Pythons (see below) +from email._parseaddr import parsedate as _parsedate +from email._parseaddr import parsedate_tz as _parsedate_tz +from email._parseaddr import _parsedate_tz as __parsedate_tz + +from quopri import decodestring as _qdecode # Intrapackage imports +from email.encoders import _bencode, _qencode from email.charset import Charset COMMASPACE = ', ' @@ -48,27 +57,6 @@ specialsre = re.compile(r'[][\\()<>@,:;".]') escapesre = re.compile(r'[\\"]') -def _has_surrogates(s): - """Return True if s contains surrogate-escaped binary data.""" - # This check is based on the fact that unless there are surrogates, utf8 - # (Python's default encoding) can encode any string. This is the fastest - # way to check for surrogates, see issue 11454 for timings. - try: - s.encode() - return False - except UnicodeEncodeError: - return True - -# How to deal with a string containing bytes before handing it to the -# application through the 'normal' interface. -def _sanitize(string): - # Turn any escaped bytes into unicode 'unknown' char. If the escaped - # bytes happen to be utf-8 they will instead get decoded, even if they - # were invalid in the charset the source was supposed to be in. This - # seems like it is not a bad thing; a defect was still registered. - original_bytes = string.encode('utf-8', 'surrogateescape') - return original_bytes.decode('utf-8', 'replace') - # Helpers @@ -87,7 +75,7 @@ 'utf-8'. """ name, address = pair - # The address MUST (per RFC) be ascii, so raise an UnicodeError if it isn't. + # The address MUST (per RFC) be ascii, so throw a UnicodeError if it isn't. address.encode('ascii') if name: try: @@ -155,14 +143,30 @@ # 2822 requires that day and month names be the English abbreviations. if timeval is None: timeval = time.time() - if localtime or usegmt: - dt = datetime.datetime.fromtimestamp(timeval, datetime.timezone.utc) + if localtime: + now = time.localtime(timeval) + # Calculate timezone offset, based on whether the local zone has + # daylight savings time, and whether DST is in effect. + if time.daylight and now[-1]: + offset = time.altzone + else: + offset = time.timezone + hours, minutes = divmod(abs(offset), 3600) + # Remember offset is in seconds west of UTC, but the timezone is in + # minutes east of UTC, so the signs differ. + if offset > 0: + sign = '-' + else: + sign = '+' + zone = '%s%02d%02d' % (sign, hours, minutes // 60) else: - dt = datetime.datetime.utcfromtimestamp(timeval) - if localtime: - dt = dt.astimezone() - usegmt = False - return format_datetime(dt, usegmt) + now = time.gmtime(timeval) + # Timezone offset is always -0000 + if usegmt: + zone = 'GMT' + else: + zone = '-0000' + return _format_timetuple_and_zone(now, zone) def format_datetime(dt, usegmt=False): """Turn a datetime into a date string as specified in RFC 2822. @@ -186,28 +190,46 @@ def make_msgid(idstring=None, domain=None): """Returns a string suitable for RFC 2822 compliant Message-ID, e.g: - <142480216486.20800.16526388040877946887@nightshade.la.mastaler.com> + <20020201195627.33539.96671@nightshade.la.mastaler.com> Optional idstring if given is a string used to strengthen the uniqueness of the message id. Optional domain if given provides the portion of the message id after the '@'. It defaults to the locally defined hostname. """ - timeval = int(time.time()*100) + timeval = time.time() + utcdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(timeval)) pid = os.getpid() - randint = random.getrandbits(64) + randint = random.randrange(100000) if idstring is None: idstring = '' else: idstring = '.' + idstring if domain is None: domain = socket.getfqdn() - msgid = '<%d.%d.%d%s@%s>' % (timeval, pid, randint, idstring, domain) + msgid = '<%s.%s.%s%s@%s>' % (utcdate, pid, randint, idstring, domain) return msgid + +# These functions are in the standalone mimelib version only because they've +# subsequently been fixed in the latest Python versions. We use this to worm +# around broken older Pythons. +def parsedate(data): + if not data: + return None + return _parsedate(data) + + +def parsedate_tz(data): + if not data: + return None + return _parsedate_tz(data) + def parsedate_to_datetime(data): - *dtuple, tz = _parsedate_tz(data) + if not data: + return None + *dtuple, tz = __parsedate_tz(data) if tz is None: return datetime.datetime(*dtuple[:6]) return datetime.datetime(*dtuple[:6], @@ -324,59 +346,9 @@ # object. We do not want bytes() normal utf-8 decoder, we want a straight # interpretation of the string as character bytes. charset, language, text = value - if charset is None: - # Issue 17369: if charset/lang is None, decode_rfc2231 couldn't parse - # the value, so use the fallback_charset. - charset = fallback_charset rawbytes = bytes(text, 'raw-unicode-escape') try: return str(rawbytes, charset, errors) except LookupError: # charset is not a known codec. return unquote(text) - - -# -# datetime doesn't provide a localtime function yet, so provide one. Code -# adapted from the patch in issue 9527. This may not be perfect, but it is -# better than not having it. -# - -def localtime(dt=None, isdst=-1): - """Return local time as an aware datetime object. - - If called without arguments, return current time. Otherwise *dt* - argument should be a datetime instance, and it is converted to the - local time zone according to the system time zone database. If *dt* is - naive (that is, dt.tzinfo is None), it is assumed to be in local time. - In this case, a positive or zero value for *isdst* causes localtime to - presume initially that summer time (for example, Daylight Saving Time) - is or is not (respectively) in effect for the specified time. A - negative value for *isdst* causes the localtime() function to attempt - to divine whether summer time is in effect for the specified time. - - """ - if dt is None: - return datetime.datetime.now(datetime.timezone.utc).astimezone() - if dt.tzinfo is not None: - return dt.astimezone() - # We have a naive datetime. Convert to a (localtime) timetuple and pass to - # system mktime together with the isdst hint. System mktime will return - # seconds since epoch. - tm = dt.timetuple()[:-1] + (isdst,) - seconds = time.mktime(tm) - localtm = time.localtime(seconds) - try: - delta = datetime.timedelta(seconds=localtm.tm_gmtoff) - tz = datetime.timezone(delta, localtm.tm_zone) - except AttributeError: - # Compute UTC offset and compare with the value implied by tm_isdst. - # If the values match, use the zone name implied by tm_isdst. - delta = dt - datetime.datetime(*time.gmtime(seconds)[:6]) - dst = time.daylight and localtm.tm_isdst > 0 - gmtoff = -(time.altzone if dst else time.timezone) - if delta == datetime.timedelta(seconds=gmtoff): - tz = datetime.timezone(delta, time.tzname[dst]) - else: - tz = datetime.timezone(delta) - return dt.replace(tzinfo=tz) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/aliases.py --- a/Lib/encodings/aliases.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/aliases.py Mon Jan 25 17:05:13 2016 +0100 @@ -33,9 +33,9 @@ 'us' : 'ascii', 'us_ascii' : 'ascii', - # base64_codec codec - 'base64' : 'base64_codec', - 'base_64' : 'base64_codec', + ## base64_codec codec + #'base64' : 'base64_codec', + #'base_64' : 'base64_codec', # big5 codec 'big5_tw' : 'big5', @@ -45,8 +45,8 @@ 'big5_hkscs' : 'big5hkscs', 'hkscs' : 'big5hkscs', - # bz2_codec codec - 'bz2' : 'bz2_codec', + ## bz2_codec codec + #'bz2' : 'bz2_codec', # cp037 codec '037' : 'cp037', @@ -63,12 +63,6 @@ 'csibm1026' : 'cp1026', 'ibm1026' : 'cp1026', - # cp1125 codec - '1125' : 'cp1125', - 'ibm1125' : 'cp1125', - 'cp866u' : 'cp1125', - 'ruscii' : 'cp1125', - # cp1140 codec '1140' : 'cp1140', 'ibm1140' : 'cp1140', @@ -109,11 +103,6 @@ '1258' : 'cp1258', 'windows_1258' : 'cp1258', - # cp273 codec - '273' : 'cp273', - 'ibm273' : 'cp273', - 'csibm273' : 'cp273', - # cp424 codec '424' : 'cp424', 'csibm424' : 'cp424', @@ -259,8 +248,8 @@ 'cp936' : 'gbk', 'ms936' : 'gbk', - # hex_codec codec - 'hex' : 'hex_codec', + ## hex_codec codec + #'hex' : 'hex_codec', # hp_roman8 codec 'roman8' : 'hp_roman8', @@ -412,11 +401,6 @@ # koi8_r codec 'cskoi8r' : 'koi8_r', - # kz1048 codec - 'kz_1048' : 'kz1048', - 'rk1048' : 'kz1048', - 'strk1048_2002' : 'kz1048', - # latin_1 codec # # Note that the latin_1 codec is implemented internally in C and a @@ -466,13 +450,13 @@ 'cp154' : 'ptcp154', 'cyrillic_asian' : 'ptcp154', - # quopri_codec codec - 'quopri' : 'quopri_codec', - 'quoted_printable' : 'quopri_codec', - 'quotedprintable' : 'quopri_codec', + ## quopri_codec codec + #'quopri' : 'quopri_codec', + #'quoted_printable' : 'quopri_codec', + #'quotedprintable' : 'quopri_codec', - # rot_13 codec - 'rot13' : 'rot_13', + ## rot_13 codec + #'rot13' : 'rot_13', # shift_jis codec 'csshiftjis' : 'shift_jis', @@ -534,12 +518,12 @@ 'utf8_ucs2' : 'utf_8', 'utf8_ucs4' : 'utf_8', - # uu_codec codec - 'uu' : 'uu_codec', + ## uu_codec codec + #'uu' : 'uu_codec', - # zlib_codec codec - 'zip' : 'zlib_codec', - 'zlib' : 'zlib_codec', + ## zlib_codec codec + #'zip' : 'zlib_codec', + #'zlib' : 'zlib_codec', # temporary mac CJK aliases, will be replaced by proper codecs in 3.1 'x_mac_japanese' : 'shift_jis', diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/base64_codec.py --- a/Lib/encodings/base64_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/base64_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Python 'base64_codec' Codec - base64 content transfer encoding. -This codec de/encodes from bytes to bytes. +This codec de/encodes from bytes to bytes and is therefore usable with +bytes.transform() and bytes.untransform(). Written by Marc-Andre Lemburg (mal@lemburg.com). """ @@ -51,5 +52,4 @@ incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/bz2_codec.py --- a/Lib/encodings/bz2_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/bz2_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -74,5 +74,4 @@ incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/cp1125.py --- a/Lib/encodings/cp1125.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,698 +0,0 @@ -""" Python Character Mapping Codec for CP1125 - -"""#" - -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_map) - - def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_map)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] - -class StreamWriter(Codec,codecs.StreamWriter): - pass - -class StreamReader(Codec,codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='cp1125', - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - -### Decoding Map - -decoding_map = codecs.make_identity_dict(range(256)) -decoding_map.update({ - 0x0080: 0x0410, # CYRILLIC CAPITAL LETTER A - 0x0081: 0x0411, # CYRILLIC CAPITAL LETTER BE - 0x0082: 0x0412, # CYRILLIC CAPITAL LETTER VE - 0x0083: 0x0413, # CYRILLIC CAPITAL LETTER GHE - 0x0084: 0x0414, # CYRILLIC CAPITAL LETTER DE - 0x0085: 0x0415, # CYRILLIC CAPITAL LETTER IE - 0x0086: 0x0416, # CYRILLIC CAPITAL LETTER ZHE - 0x0087: 0x0417, # CYRILLIC CAPITAL LETTER ZE - 0x0088: 0x0418, # CYRILLIC CAPITAL LETTER I - 0x0089: 0x0419, # CYRILLIC CAPITAL LETTER SHORT I - 0x008a: 0x041a, # CYRILLIC CAPITAL LETTER KA - 0x008b: 0x041b, # CYRILLIC CAPITAL LETTER EL - 0x008c: 0x041c, # CYRILLIC CAPITAL LETTER EM - 0x008d: 0x041d, # CYRILLIC CAPITAL LETTER EN - 0x008e: 0x041e, # CYRILLIC CAPITAL LETTER O - 0x008f: 0x041f, # CYRILLIC CAPITAL LETTER PE - 0x0090: 0x0420, # CYRILLIC CAPITAL LETTER ER - 0x0091: 0x0421, # CYRILLIC CAPITAL LETTER ES - 0x0092: 0x0422, # CYRILLIC CAPITAL LETTER TE - 0x0093: 0x0423, # CYRILLIC CAPITAL LETTER U - 0x0094: 0x0424, # CYRILLIC CAPITAL LETTER EF - 0x0095: 0x0425, # CYRILLIC CAPITAL LETTER HA - 0x0096: 0x0426, # CYRILLIC CAPITAL LETTER TSE - 0x0097: 0x0427, # CYRILLIC CAPITAL LETTER CHE - 0x0098: 0x0428, # CYRILLIC CAPITAL LETTER SHA - 0x0099: 0x0429, # CYRILLIC CAPITAL LETTER SHCHA - 0x009a: 0x042a, # CYRILLIC CAPITAL LETTER HARD SIGN - 0x009b: 0x042b, # CYRILLIC CAPITAL LETTER YERU - 0x009c: 0x042c, # CYRILLIC CAPITAL LETTER SOFT SIGN - 0x009d: 0x042d, # CYRILLIC CAPITAL LETTER E - 0x009e: 0x042e, # CYRILLIC CAPITAL LETTER YU - 0x009f: 0x042f, # CYRILLIC CAPITAL LETTER YA - 0x00a0: 0x0430, # CYRILLIC SMALL LETTER A - 0x00a1: 0x0431, # CYRILLIC SMALL LETTER BE - 0x00a2: 0x0432, # CYRILLIC SMALL LETTER VE - 0x00a3: 0x0433, # CYRILLIC SMALL LETTER GHE - 0x00a4: 0x0434, # CYRILLIC SMALL LETTER DE - 0x00a5: 0x0435, # CYRILLIC SMALL LETTER IE - 0x00a6: 0x0436, # CYRILLIC SMALL LETTER ZHE - 0x00a7: 0x0437, # CYRILLIC SMALL LETTER ZE - 0x00a8: 0x0438, # CYRILLIC SMALL LETTER I - 0x00a9: 0x0439, # CYRILLIC SMALL LETTER SHORT I - 0x00aa: 0x043a, # CYRILLIC SMALL LETTER KA - 0x00ab: 0x043b, # CYRILLIC SMALL LETTER EL - 0x00ac: 0x043c, # CYRILLIC SMALL LETTER EM - 0x00ad: 0x043d, # CYRILLIC SMALL LETTER EN - 0x00ae: 0x043e, # CYRILLIC SMALL LETTER O - 0x00af: 0x043f, # CYRILLIC SMALL LETTER PE - 0x00b0: 0x2591, # LIGHT SHADE - 0x00b1: 0x2592, # MEDIUM SHADE - 0x00b2: 0x2593, # DARK SHADE - 0x00b3: 0x2502, # BOX DRAWINGS LIGHT VERTICAL - 0x00b4: 0x2524, # BOX DRAWINGS LIGHT VERTICAL AND LEFT - 0x00b5: 0x2561, # BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE - 0x00b6: 0x2562, # BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE - 0x00b7: 0x2556, # BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE - 0x00b8: 0x2555, # BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE - 0x00b9: 0x2563, # BOX DRAWINGS DOUBLE VERTICAL AND LEFT - 0x00ba: 0x2551, # BOX DRAWINGS DOUBLE VERTICAL - 0x00bb: 0x2557, # BOX DRAWINGS DOUBLE DOWN AND LEFT - 0x00bc: 0x255d, # BOX DRAWINGS DOUBLE UP AND LEFT - 0x00bd: 0x255c, # BOX DRAWINGS UP DOUBLE AND LEFT SINGLE - 0x00be: 0x255b, # BOX DRAWINGS UP SINGLE AND LEFT DOUBLE - 0x00bf: 0x2510, # BOX DRAWINGS LIGHT DOWN AND LEFT - 0x00c0: 0x2514, # BOX DRAWINGS LIGHT UP AND RIGHT - 0x00c1: 0x2534, # BOX DRAWINGS LIGHT UP AND HORIZONTAL - 0x00c2: 0x252c, # BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - 0x00c3: 0x251c, # BOX DRAWINGS LIGHT VERTICAL AND RIGHT - 0x00c4: 0x2500, # BOX DRAWINGS LIGHT HORIZONTAL - 0x00c5: 0x253c, # BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - 0x00c6: 0x255e, # BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE - 0x00c7: 0x255f, # BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE - 0x00c8: 0x255a, # BOX DRAWINGS DOUBLE UP AND RIGHT - 0x00c9: 0x2554, # BOX DRAWINGS DOUBLE DOWN AND RIGHT - 0x00ca: 0x2569, # BOX DRAWINGS DOUBLE UP AND HORIZONTAL - 0x00cb: 0x2566, # BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL - 0x00cc: 0x2560, # BOX DRAWINGS DOUBLE VERTICAL AND RIGHT - 0x00cd: 0x2550, # BOX DRAWINGS DOUBLE HORIZONTAL - 0x00ce: 0x256c, # BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL - 0x00cf: 0x2567, # BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE - 0x00d0: 0x2568, # BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE - 0x00d1: 0x2564, # BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE - 0x00d2: 0x2565, # BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE - 0x00d3: 0x2559, # BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE - 0x00d4: 0x2558, # BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE - 0x00d5: 0x2552, # BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE - 0x00d6: 0x2553, # BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE - 0x00d7: 0x256b, # BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE - 0x00d8: 0x256a, # BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE - 0x00d9: 0x2518, # BOX DRAWINGS LIGHT UP AND LEFT - 0x00da: 0x250c, # BOX DRAWINGS LIGHT DOWN AND RIGHT - 0x00db: 0x2588, # FULL BLOCK - 0x00dc: 0x2584, # LOWER HALF BLOCK - 0x00dd: 0x258c, # LEFT HALF BLOCK - 0x00de: 0x2590, # RIGHT HALF BLOCK - 0x00df: 0x2580, # UPPER HALF BLOCK - 0x00e0: 0x0440, # CYRILLIC SMALL LETTER ER - 0x00e1: 0x0441, # CYRILLIC SMALL LETTER ES - 0x00e2: 0x0442, # CYRILLIC SMALL LETTER TE - 0x00e3: 0x0443, # CYRILLIC SMALL LETTER U - 0x00e4: 0x0444, # CYRILLIC SMALL LETTER EF - 0x00e5: 0x0445, # CYRILLIC SMALL LETTER HA - 0x00e6: 0x0446, # CYRILLIC SMALL LETTER TSE - 0x00e7: 0x0447, # CYRILLIC SMALL LETTER CHE - 0x00e8: 0x0448, # CYRILLIC SMALL LETTER SHA - 0x00e9: 0x0449, # CYRILLIC SMALL LETTER SHCHA - 0x00ea: 0x044a, # CYRILLIC SMALL LETTER HARD SIGN - 0x00eb: 0x044b, # CYRILLIC SMALL LETTER YERU - 0x00ec: 0x044c, # CYRILLIC SMALL LETTER SOFT SIGN - 0x00ed: 0x044d, # CYRILLIC SMALL LETTER E - 0x00ee: 0x044e, # CYRILLIC SMALL LETTER YU - 0x00ef: 0x044f, # CYRILLIC SMALL LETTER YA - 0x00f0: 0x0401, # CYRILLIC CAPITAL LETTER IO - 0x00f1: 0x0451, # CYRILLIC SMALL LETTER IO - 0x00f2: 0x0490, # CYRILLIC CAPITAL LETTER GHE WITH UPTURN - 0x00f3: 0x0491, # CYRILLIC SMALL LETTER GHE WITH UPTURN - 0x00f4: 0x0404, # CYRILLIC CAPITAL LETTER UKRAINIAN IE - 0x00f5: 0x0454, # CYRILLIC SMALL LETTER UKRAINIAN IE - 0x00f6: 0x0406, # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - 0x00f7: 0x0456, # CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I - 0x00f8: 0x0407, # CYRILLIC CAPITAL LETTER YI - 0x00f9: 0x0457, # CYRILLIC SMALL LETTER YI - 0x00fa: 0x00b7, # MIDDLE DOT - 0x00fb: 0x221a, # SQUARE ROOT - 0x00fc: 0x2116, # NUMERO SIGN - 0x00fd: 0x00a4, # CURRENCY SIGN - 0x00fe: 0x25a0, # BLACK SQUARE - 0x00ff: 0x00a0, # NO-BREAK SPACE -}) - -### Decoding Table - -decoding_table = ( - '\x00' # 0x0000 -> NULL - '\x01' # 0x0001 -> START OF HEADING - '\x02' # 0x0002 -> START OF TEXT - '\x03' # 0x0003 -> END OF TEXT - '\x04' # 0x0004 -> END OF TRANSMISSION - '\x05' # 0x0005 -> ENQUIRY - '\x06' # 0x0006 -> ACKNOWLEDGE - '\x07' # 0x0007 -> BELL - '\x08' # 0x0008 -> BACKSPACE - '\t' # 0x0009 -> HORIZONTAL TABULATION - '\n' # 0x000a -> LINE FEED - '\x0b' # 0x000b -> VERTICAL TABULATION - '\x0c' # 0x000c -> FORM FEED - '\r' # 0x000d -> CARRIAGE RETURN - '\x0e' # 0x000e -> SHIFT OUT - '\x0f' # 0x000f -> SHIFT IN - '\x10' # 0x0010 -> DATA LINK ESCAPE - '\x11' # 0x0011 -> DEVICE CONTROL ONE - '\x12' # 0x0012 -> DEVICE CONTROL TWO - '\x13' # 0x0013 -> DEVICE CONTROL THREE - '\x14' # 0x0014 -> DEVICE CONTROL FOUR - '\x15' # 0x0015 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x0016 -> SYNCHRONOUS IDLE - '\x17' # 0x0017 -> END OF TRANSMISSION BLOCK - '\x18' # 0x0018 -> CANCEL - '\x19' # 0x0019 -> END OF MEDIUM - '\x1a' # 0x001a -> SUBSTITUTE - '\x1b' # 0x001b -> ESCAPE - '\x1c' # 0x001c -> FILE SEPARATOR - '\x1d' # 0x001d -> GROUP SEPARATOR - '\x1e' # 0x001e -> RECORD SEPARATOR - '\x1f' # 0x001f -> UNIT SEPARATOR - ' ' # 0x0020 -> SPACE - '!' # 0x0021 -> EXCLAMATION MARK - '"' # 0x0022 -> QUOTATION MARK - '#' # 0x0023 -> NUMBER SIGN - '$' # 0x0024 -> DOLLAR SIGN - '%' # 0x0025 -> PERCENT SIGN - '&' # 0x0026 -> AMPERSAND - "'" # 0x0027 -> APOSTROPHE - '(' # 0x0028 -> LEFT PARENTHESIS - ')' # 0x0029 -> RIGHT PARENTHESIS - '*' # 0x002a -> ASTERISK - '+' # 0x002b -> PLUS SIGN - ',' # 0x002c -> COMMA - '-' # 0x002d -> HYPHEN-MINUS - '.' # 0x002e -> FULL STOP - '/' # 0x002f -> SOLIDUS - '0' # 0x0030 -> DIGIT ZERO - '1' # 0x0031 -> DIGIT ONE - '2' # 0x0032 -> DIGIT TWO - '3' # 0x0033 -> DIGIT THREE - '4' # 0x0034 -> DIGIT FOUR - '5' # 0x0035 -> DIGIT FIVE - '6' # 0x0036 -> DIGIT SIX - '7' # 0x0037 -> DIGIT SEVEN - '8' # 0x0038 -> DIGIT EIGHT - '9' # 0x0039 -> DIGIT NINE - ':' # 0x003a -> COLON - ';' # 0x003b -> SEMICOLON - '<' # 0x003c -> LESS-THAN SIGN - '=' # 0x003d -> EQUALS SIGN - '>' # 0x003e -> GREATER-THAN SIGN - '?' # 0x003f -> QUESTION MARK - '@' # 0x0040 -> COMMERCIAL AT - 'A' # 0x0041 -> LATIN CAPITAL LETTER A - 'B' # 0x0042 -> LATIN CAPITAL LETTER B - 'C' # 0x0043 -> LATIN CAPITAL LETTER C - 'D' # 0x0044 -> LATIN CAPITAL LETTER D - 'E' # 0x0045 -> LATIN CAPITAL LETTER E - 'F' # 0x0046 -> LATIN CAPITAL LETTER F - 'G' # 0x0047 -> LATIN CAPITAL LETTER G - 'H' # 0x0048 -> LATIN CAPITAL LETTER H - 'I' # 0x0049 -> LATIN CAPITAL LETTER I - 'J' # 0x004a -> LATIN CAPITAL LETTER J - 'K' # 0x004b -> LATIN CAPITAL LETTER K - 'L' # 0x004c -> LATIN CAPITAL LETTER L - 'M' # 0x004d -> LATIN CAPITAL LETTER M - 'N' # 0x004e -> LATIN CAPITAL LETTER N - 'O' # 0x004f -> LATIN CAPITAL LETTER O - 'P' # 0x0050 -> LATIN CAPITAL LETTER P - 'Q' # 0x0051 -> LATIN CAPITAL LETTER Q - 'R' # 0x0052 -> LATIN CAPITAL LETTER R - 'S' # 0x0053 -> LATIN CAPITAL LETTER S - 'T' # 0x0054 -> LATIN CAPITAL LETTER T - 'U' # 0x0055 -> LATIN CAPITAL LETTER U - 'V' # 0x0056 -> LATIN CAPITAL LETTER V - 'W' # 0x0057 -> LATIN CAPITAL LETTER W - 'X' # 0x0058 -> LATIN CAPITAL LETTER X - 'Y' # 0x0059 -> LATIN CAPITAL LETTER Y - 'Z' # 0x005a -> LATIN CAPITAL LETTER Z - '[' # 0x005b -> LEFT SQUARE BRACKET - '\\' # 0x005c -> REVERSE SOLIDUS - ']' # 0x005d -> RIGHT SQUARE BRACKET - '^' # 0x005e -> CIRCUMFLEX ACCENT - '_' # 0x005f -> LOW LINE - '`' # 0x0060 -> GRAVE ACCENT - 'a' # 0x0061 -> LATIN SMALL LETTER A - 'b' # 0x0062 -> LATIN SMALL LETTER B - 'c' # 0x0063 -> LATIN SMALL LETTER C - 'd' # 0x0064 -> LATIN SMALL LETTER D - 'e' # 0x0065 -> LATIN SMALL LETTER E - 'f' # 0x0066 -> LATIN SMALL LETTER F - 'g' # 0x0067 -> LATIN SMALL LETTER G - 'h' # 0x0068 -> LATIN SMALL LETTER H - 'i' # 0x0069 -> LATIN SMALL LETTER I - 'j' # 0x006a -> LATIN SMALL LETTER J - 'k' # 0x006b -> LATIN SMALL LETTER K - 'l' # 0x006c -> LATIN SMALL LETTER L - 'm' # 0x006d -> LATIN SMALL LETTER M - 'n' # 0x006e -> LATIN SMALL LETTER N - 'o' # 0x006f -> LATIN SMALL LETTER O - 'p' # 0x0070 -> LATIN SMALL LETTER P - 'q' # 0x0071 -> LATIN SMALL LETTER Q - 'r' # 0x0072 -> LATIN SMALL LETTER R - 's' # 0x0073 -> LATIN SMALL LETTER S - 't' # 0x0074 -> LATIN SMALL LETTER T - 'u' # 0x0075 -> LATIN SMALL LETTER U - 'v' # 0x0076 -> LATIN SMALL LETTER V - 'w' # 0x0077 -> LATIN SMALL LETTER W - 'x' # 0x0078 -> LATIN SMALL LETTER X - 'y' # 0x0079 -> LATIN SMALL LETTER Y - 'z' # 0x007a -> LATIN SMALL LETTER Z - '{' # 0x007b -> LEFT CURLY BRACKET - '|' # 0x007c -> VERTICAL LINE - '}' # 0x007d -> RIGHT CURLY BRACKET - '~' # 0x007e -> TILDE - '\x7f' # 0x007f -> DELETE - '\u0410' # 0x0080 -> CYRILLIC CAPITAL LETTER A - '\u0411' # 0x0081 -> CYRILLIC CAPITAL LETTER BE - '\u0412' # 0x0082 -> CYRILLIC CAPITAL LETTER VE - '\u0413' # 0x0083 -> CYRILLIC CAPITAL LETTER GHE - '\u0414' # 0x0084 -> CYRILLIC CAPITAL LETTER DE - '\u0415' # 0x0085 -> CYRILLIC CAPITAL LETTER IE - '\u0416' # 0x0086 -> CYRILLIC CAPITAL LETTER ZHE - '\u0417' # 0x0087 -> CYRILLIC CAPITAL LETTER ZE - '\u0418' # 0x0088 -> CYRILLIC CAPITAL LETTER I - '\u0419' # 0x0089 -> CYRILLIC CAPITAL LETTER SHORT I - '\u041a' # 0x008a -> CYRILLIC CAPITAL LETTER KA - '\u041b' # 0x008b -> CYRILLIC CAPITAL LETTER EL - '\u041c' # 0x008c -> CYRILLIC CAPITAL LETTER EM - '\u041d' # 0x008d -> CYRILLIC CAPITAL LETTER EN - '\u041e' # 0x008e -> CYRILLIC CAPITAL LETTER O - '\u041f' # 0x008f -> CYRILLIC CAPITAL LETTER PE - '\u0420' # 0x0090 -> CYRILLIC CAPITAL LETTER ER - '\u0421' # 0x0091 -> CYRILLIC CAPITAL LETTER ES - '\u0422' # 0x0092 -> CYRILLIC CAPITAL LETTER TE - '\u0423' # 0x0093 -> CYRILLIC CAPITAL LETTER U - '\u0424' # 0x0094 -> CYRILLIC CAPITAL LETTER EF - '\u0425' # 0x0095 -> CYRILLIC CAPITAL LETTER HA - '\u0426' # 0x0096 -> CYRILLIC CAPITAL LETTER TSE - '\u0427' # 0x0097 -> CYRILLIC CAPITAL LETTER CHE - '\u0428' # 0x0098 -> CYRILLIC CAPITAL LETTER SHA - '\u0429' # 0x0099 -> CYRILLIC CAPITAL LETTER SHCHA - '\u042a' # 0x009a -> CYRILLIC CAPITAL LETTER HARD SIGN - '\u042b' # 0x009b -> CYRILLIC CAPITAL LETTER YERU - '\u042c' # 0x009c -> CYRILLIC CAPITAL LETTER SOFT SIGN - '\u042d' # 0x009d -> CYRILLIC CAPITAL LETTER E - '\u042e' # 0x009e -> CYRILLIC CAPITAL LETTER YU - '\u042f' # 0x009f -> CYRILLIC CAPITAL LETTER YA - '\u0430' # 0x00a0 -> CYRILLIC SMALL LETTER A - '\u0431' # 0x00a1 -> CYRILLIC SMALL LETTER BE - '\u0432' # 0x00a2 -> CYRILLIC SMALL LETTER VE - '\u0433' # 0x00a3 -> CYRILLIC SMALL LETTER GHE - '\u0434' # 0x00a4 -> CYRILLIC SMALL LETTER DE - '\u0435' # 0x00a5 -> CYRILLIC SMALL LETTER IE - '\u0436' # 0x00a6 -> CYRILLIC SMALL LETTER ZHE - '\u0437' # 0x00a7 -> CYRILLIC SMALL LETTER ZE - '\u0438' # 0x00a8 -> CYRILLIC SMALL LETTER I - '\u0439' # 0x00a9 -> CYRILLIC SMALL LETTER SHORT I - '\u043a' # 0x00aa -> CYRILLIC SMALL LETTER KA - '\u043b' # 0x00ab -> CYRILLIC SMALL LETTER EL - '\u043c' # 0x00ac -> CYRILLIC SMALL LETTER EM - '\u043d' # 0x00ad -> CYRILLIC SMALL LETTER EN - '\u043e' # 0x00ae -> CYRILLIC SMALL LETTER O - '\u043f' # 0x00af -> CYRILLIC SMALL LETTER PE - '\u2591' # 0x00b0 -> LIGHT SHADE - '\u2592' # 0x00b1 -> MEDIUM SHADE - '\u2593' # 0x00b2 -> DARK SHADE - '\u2502' # 0x00b3 -> BOX DRAWINGS LIGHT VERTICAL - '\u2524' # 0x00b4 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT - '\u2561' # 0x00b5 -> BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE - '\u2562' # 0x00b6 -> BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE - '\u2556' # 0x00b7 -> BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE - '\u2555' # 0x00b8 -> BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE - '\u2563' # 0x00b9 -> BOX DRAWINGS DOUBLE VERTICAL AND LEFT - '\u2551' # 0x00ba -> BOX DRAWINGS DOUBLE VERTICAL - '\u2557' # 0x00bb -> BOX DRAWINGS DOUBLE DOWN AND LEFT - '\u255d' # 0x00bc -> BOX DRAWINGS DOUBLE UP AND LEFT - '\u255c' # 0x00bd -> BOX DRAWINGS UP DOUBLE AND LEFT SINGLE - '\u255b' # 0x00be -> BOX DRAWINGS UP SINGLE AND LEFT DOUBLE - '\u2510' # 0x00bf -> BOX DRAWINGS LIGHT DOWN AND LEFT - '\u2514' # 0x00c0 -> BOX DRAWINGS LIGHT UP AND RIGHT - '\u2534' # 0x00c1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL - '\u252c' # 0x00c2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - '\u251c' # 0x00c3 -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT - '\u2500' # 0x00c4 -> BOX DRAWINGS LIGHT HORIZONTAL - '\u253c' # 0x00c5 -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - '\u255e' # 0x00c6 -> BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE - '\u255f' # 0x00c7 -> BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE - '\u255a' # 0x00c8 -> BOX DRAWINGS DOUBLE UP AND RIGHT - '\u2554' # 0x00c9 -> BOX DRAWINGS DOUBLE DOWN AND RIGHT - '\u2569' # 0x00ca -> BOX DRAWINGS DOUBLE UP AND HORIZONTAL - '\u2566' # 0x00cb -> BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL - '\u2560' # 0x00cc -> BOX DRAWINGS DOUBLE VERTICAL AND RIGHT - '\u2550' # 0x00cd -> BOX DRAWINGS DOUBLE HORIZONTAL - '\u256c' # 0x00ce -> BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL - '\u2567' # 0x00cf -> BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE - '\u2568' # 0x00d0 -> BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE - '\u2564' # 0x00d1 -> BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE - '\u2565' # 0x00d2 -> BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE - '\u2559' # 0x00d3 -> BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE - '\u2558' # 0x00d4 -> BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE - '\u2552' # 0x00d5 -> BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE - '\u2553' # 0x00d6 -> BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE - '\u256b' # 0x00d7 -> BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE - '\u256a' # 0x00d8 -> BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE - '\u2518' # 0x00d9 -> BOX DRAWINGS LIGHT UP AND LEFT - '\u250c' # 0x00da -> BOX DRAWINGS LIGHT DOWN AND RIGHT - '\u2588' # 0x00db -> FULL BLOCK - '\u2584' # 0x00dc -> LOWER HALF BLOCK - '\u258c' # 0x00dd -> LEFT HALF BLOCK - '\u2590' # 0x00de -> RIGHT HALF BLOCK - '\u2580' # 0x00df -> UPPER HALF BLOCK - '\u0440' # 0x00e0 -> CYRILLIC SMALL LETTER ER - '\u0441' # 0x00e1 -> CYRILLIC SMALL LETTER ES - '\u0442' # 0x00e2 -> CYRILLIC SMALL LETTER TE - '\u0443' # 0x00e3 -> CYRILLIC SMALL LETTER U - '\u0444' # 0x00e4 -> CYRILLIC SMALL LETTER EF - '\u0445' # 0x00e5 -> CYRILLIC SMALL LETTER HA - '\u0446' # 0x00e6 -> CYRILLIC SMALL LETTER TSE - '\u0447' # 0x00e7 -> CYRILLIC SMALL LETTER CHE - '\u0448' # 0x00e8 -> CYRILLIC SMALL LETTER SHA - '\u0449' # 0x00e9 -> CYRILLIC SMALL LETTER SHCHA - '\u044a' # 0x00ea -> CYRILLIC SMALL LETTER HARD SIGN - '\u044b' # 0x00eb -> CYRILLIC SMALL LETTER YERU - '\u044c' # 0x00ec -> CYRILLIC SMALL LETTER SOFT SIGN - '\u044d' # 0x00ed -> CYRILLIC SMALL LETTER E - '\u044e' # 0x00ee -> CYRILLIC SMALL LETTER YU - '\u044f' # 0x00ef -> CYRILLIC SMALL LETTER YA - '\u0401' # 0x00f0 -> CYRILLIC CAPITAL LETTER IO - '\u0451' # 0x00f1 -> CYRILLIC SMALL LETTER IO - '\u0490' # 0x00f2 -> CYRILLIC CAPITAL LETTER GHE WITH UPTURN - '\u0491' # 0x00f3 -> CYRILLIC SMALL LETTER GHE WITH UPTURN - '\u0404' # 0x00f4 -> CYRILLIC CAPITAL LETTER UKRAINIAN IE - '\u0454' # 0x00f5 -> CYRILLIC SMALL LETTER UKRAINIAN IE - '\u0406' # 0x00f6 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - '\u0456' # 0x00f7 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I - '\u0407' # 0x00f8 -> CYRILLIC CAPITAL LETTER YI - '\u0457' # 0x00f9 -> CYRILLIC SMALL LETTER YI - '\xb7' # 0x00fa -> MIDDLE DOT - '\u221a' # 0x00fb -> SQUARE ROOT - '\u2116' # 0x00fc -> NUMERO SIGN - '\xa4' # 0x00fd -> CURRENCY SIGN - '\u25a0' # 0x00fe -> BLACK SQUARE - '\xa0' # 0x00ff -> NO-BREAK SPACE -) - -### Encoding Map - -encoding_map = { - 0x0000: 0x0000, # NULL - 0x0001: 0x0001, # START OF HEADING - 0x0002: 0x0002, # START OF TEXT - 0x0003: 0x0003, # END OF TEXT - 0x0004: 0x0004, # END OF TRANSMISSION - 0x0005: 0x0005, # ENQUIRY - 0x0006: 0x0006, # ACKNOWLEDGE - 0x0007: 0x0007, # BELL - 0x0008: 0x0008, # BACKSPACE - 0x0009: 0x0009, # HORIZONTAL TABULATION - 0x000a: 0x000a, # LINE FEED - 0x000b: 0x000b, # VERTICAL TABULATION - 0x000c: 0x000c, # FORM FEED - 0x000d: 0x000d, # CARRIAGE RETURN - 0x000e: 0x000e, # SHIFT OUT - 0x000f: 0x000f, # SHIFT IN - 0x0010: 0x0010, # DATA LINK ESCAPE - 0x0011: 0x0011, # DEVICE CONTROL ONE - 0x0012: 0x0012, # DEVICE CONTROL TWO - 0x0013: 0x0013, # DEVICE CONTROL THREE - 0x0014: 0x0014, # DEVICE CONTROL FOUR - 0x0015: 0x0015, # NEGATIVE ACKNOWLEDGE - 0x0016: 0x0016, # SYNCHRONOUS IDLE - 0x0017: 0x0017, # END OF TRANSMISSION BLOCK - 0x0018: 0x0018, # CANCEL - 0x0019: 0x0019, # END OF MEDIUM - 0x001a: 0x001a, # SUBSTITUTE - 0x001b: 0x001b, # ESCAPE - 0x001c: 0x001c, # FILE SEPARATOR - 0x001d: 0x001d, # GROUP SEPARATOR - 0x001e: 0x001e, # RECORD SEPARATOR - 0x001f: 0x001f, # UNIT SEPARATOR - 0x0020: 0x0020, # SPACE - 0x0021: 0x0021, # EXCLAMATION MARK - 0x0022: 0x0022, # QUOTATION MARK - 0x0023: 0x0023, # NUMBER SIGN - 0x0024: 0x0024, # DOLLAR SIGN - 0x0025: 0x0025, # PERCENT SIGN - 0x0026: 0x0026, # AMPERSAND - 0x0027: 0x0027, # APOSTROPHE - 0x0028: 0x0028, # LEFT PARENTHESIS - 0x0029: 0x0029, # RIGHT PARENTHESIS - 0x002a: 0x002a, # ASTERISK - 0x002b: 0x002b, # PLUS SIGN - 0x002c: 0x002c, # COMMA - 0x002d: 0x002d, # HYPHEN-MINUS - 0x002e: 0x002e, # FULL STOP - 0x002f: 0x002f, # SOLIDUS - 0x0030: 0x0030, # DIGIT ZERO - 0x0031: 0x0031, # DIGIT ONE - 0x0032: 0x0032, # DIGIT TWO - 0x0033: 0x0033, # DIGIT THREE - 0x0034: 0x0034, # DIGIT FOUR - 0x0035: 0x0035, # DIGIT FIVE - 0x0036: 0x0036, # DIGIT SIX - 0x0037: 0x0037, # DIGIT SEVEN - 0x0038: 0x0038, # DIGIT EIGHT - 0x0039: 0x0039, # DIGIT NINE - 0x003a: 0x003a, # COLON - 0x003b: 0x003b, # SEMICOLON - 0x003c: 0x003c, # LESS-THAN SIGN - 0x003d: 0x003d, # EQUALS SIGN - 0x003e: 0x003e, # GREATER-THAN SIGN - 0x003f: 0x003f, # QUESTION MARK - 0x0040: 0x0040, # COMMERCIAL AT - 0x0041: 0x0041, # LATIN CAPITAL LETTER A - 0x0042: 0x0042, # LATIN CAPITAL LETTER B - 0x0043: 0x0043, # LATIN CAPITAL LETTER C - 0x0044: 0x0044, # LATIN CAPITAL LETTER D - 0x0045: 0x0045, # LATIN CAPITAL LETTER E - 0x0046: 0x0046, # LATIN CAPITAL LETTER F - 0x0047: 0x0047, # LATIN CAPITAL LETTER G - 0x0048: 0x0048, # LATIN CAPITAL LETTER H - 0x0049: 0x0049, # LATIN CAPITAL LETTER I - 0x004a: 0x004a, # LATIN CAPITAL LETTER J - 0x004b: 0x004b, # LATIN CAPITAL LETTER K - 0x004c: 0x004c, # LATIN CAPITAL LETTER L - 0x004d: 0x004d, # LATIN CAPITAL LETTER M - 0x004e: 0x004e, # LATIN CAPITAL LETTER N - 0x004f: 0x004f, # LATIN CAPITAL LETTER O - 0x0050: 0x0050, # LATIN CAPITAL LETTER P - 0x0051: 0x0051, # LATIN CAPITAL LETTER Q - 0x0052: 0x0052, # LATIN CAPITAL LETTER R - 0x0053: 0x0053, # LATIN CAPITAL LETTER S - 0x0054: 0x0054, # LATIN CAPITAL LETTER T - 0x0055: 0x0055, # LATIN CAPITAL LETTER U - 0x0056: 0x0056, # LATIN CAPITAL LETTER V - 0x0057: 0x0057, # LATIN CAPITAL LETTER W - 0x0058: 0x0058, # LATIN CAPITAL LETTER X - 0x0059: 0x0059, # LATIN CAPITAL LETTER Y - 0x005a: 0x005a, # LATIN CAPITAL LETTER Z - 0x005b: 0x005b, # LEFT SQUARE BRACKET - 0x005c: 0x005c, # REVERSE SOLIDUS - 0x005d: 0x005d, # RIGHT SQUARE BRACKET - 0x005e: 0x005e, # CIRCUMFLEX ACCENT - 0x005f: 0x005f, # LOW LINE - 0x0060: 0x0060, # GRAVE ACCENT - 0x0061: 0x0061, # LATIN SMALL LETTER A - 0x0062: 0x0062, # LATIN SMALL LETTER B - 0x0063: 0x0063, # LATIN SMALL LETTER C - 0x0064: 0x0064, # LATIN SMALL LETTER D - 0x0065: 0x0065, # LATIN SMALL LETTER E - 0x0066: 0x0066, # LATIN SMALL LETTER F - 0x0067: 0x0067, # LATIN SMALL LETTER G - 0x0068: 0x0068, # LATIN SMALL LETTER H - 0x0069: 0x0069, # LATIN SMALL LETTER I - 0x006a: 0x006a, # LATIN SMALL LETTER J - 0x006b: 0x006b, # LATIN SMALL LETTER K - 0x006c: 0x006c, # LATIN SMALL LETTER L - 0x006d: 0x006d, # LATIN SMALL LETTER M - 0x006e: 0x006e, # LATIN SMALL LETTER N - 0x006f: 0x006f, # LATIN SMALL LETTER O - 0x0070: 0x0070, # LATIN SMALL LETTER P - 0x0071: 0x0071, # LATIN SMALL LETTER Q - 0x0072: 0x0072, # LATIN SMALL LETTER R - 0x0073: 0x0073, # LATIN SMALL LETTER S - 0x0074: 0x0074, # LATIN SMALL LETTER T - 0x0075: 0x0075, # LATIN SMALL LETTER U - 0x0076: 0x0076, # LATIN SMALL LETTER V - 0x0077: 0x0077, # LATIN SMALL LETTER W - 0x0078: 0x0078, # LATIN SMALL LETTER X - 0x0079: 0x0079, # LATIN SMALL LETTER Y - 0x007a: 0x007a, # LATIN SMALL LETTER Z - 0x007b: 0x007b, # LEFT CURLY BRACKET - 0x007c: 0x007c, # VERTICAL LINE - 0x007d: 0x007d, # RIGHT CURLY BRACKET - 0x007e: 0x007e, # TILDE - 0x007f: 0x007f, # DELETE - 0x00a0: 0x00ff, # NO-BREAK SPACE - 0x00a4: 0x00fd, # CURRENCY SIGN - 0x00b7: 0x00fa, # MIDDLE DOT - 0x0401: 0x00f0, # CYRILLIC CAPITAL LETTER IO - 0x0404: 0x00f4, # CYRILLIC CAPITAL LETTER UKRAINIAN IE - 0x0406: 0x00f6, # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - 0x0407: 0x00f8, # CYRILLIC CAPITAL LETTER YI - 0x0410: 0x0080, # CYRILLIC CAPITAL LETTER A - 0x0411: 0x0081, # CYRILLIC CAPITAL LETTER BE - 0x0412: 0x0082, # CYRILLIC CAPITAL LETTER VE - 0x0413: 0x0083, # CYRILLIC CAPITAL LETTER GHE - 0x0414: 0x0084, # CYRILLIC CAPITAL LETTER DE - 0x0415: 0x0085, # CYRILLIC CAPITAL LETTER IE - 0x0416: 0x0086, # CYRILLIC CAPITAL LETTER ZHE - 0x0417: 0x0087, # CYRILLIC CAPITAL LETTER ZE - 0x0418: 0x0088, # CYRILLIC CAPITAL LETTER I - 0x0419: 0x0089, # CYRILLIC CAPITAL LETTER SHORT I - 0x041a: 0x008a, # CYRILLIC CAPITAL LETTER KA - 0x041b: 0x008b, # CYRILLIC CAPITAL LETTER EL - 0x041c: 0x008c, # CYRILLIC CAPITAL LETTER EM - 0x041d: 0x008d, # CYRILLIC CAPITAL LETTER EN - 0x041e: 0x008e, # CYRILLIC CAPITAL LETTER O - 0x041f: 0x008f, # CYRILLIC CAPITAL LETTER PE - 0x0420: 0x0090, # CYRILLIC CAPITAL LETTER ER - 0x0421: 0x0091, # CYRILLIC CAPITAL LETTER ES - 0x0422: 0x0092, # CYRILLIC CAPITAL LETTER TE - 0x0423: 0x0093, # CYRILLIC CAPITAL LETTER U - 0x0424: 0x0094, # CYRILLIC CAPITAL LETTER EF - 0x0425: 0x0095, # CYRILLIC CAPITAL LETTER HA - 0x0426: 0x0096, # CYRILLIC CAPITAL LETTER TSE - 0x0427: 0x0097, # CYRILLIC CAPITAL LETTER CHE - 0x0428: 0x0098, # CYRILLIC CAPITAL LETTER SHA - 0x0429: 0x0099, # CYRILLIC CAPITAL LETTER SHCHA - 0x042a: 0x009a, # CYRILLIC CAPITAL LETTER HARD SIGN - 0x042b: 0x009b, # CYRILLIC CAPITAL LETTER YERU - 0x042c: 0x009c, # CYRILLIC CAPITAL LETTER SOFT SIGN - 0x042d: 0x009d, # CYRILLIC CAPITAL LETTER E - 0x042e: 0x009e, # CYRILLIC CAPITAL LETTER YU - 0x042f: 0x009f, # CYRILLIC CAPITAL LETTER YA - 0x0430: 0x00a0, # CYRILLIC SMALL LETTER A - 0x0431: 0x00a1, # CYRILLIC SMALL LETTER BE - 0x0432: 0x00a2, # CYRILLIC SMALL LETTER VE - 0x0433: 0x00a3, # CYRILLIC SMALL LETTER GHE - 0x0434: 0x00a4, # CYRILLIC SMALL LETTER DE - 0x0435: 0x00a5, # CYRILLIC SMALL LETTER IE - 0x0436: 0x00a6, # CYRILLIC SMALL LETTER ZHE - 0x0437: 0x00a7, # CYRILLIC SMALL LETTER ZE - 0x0438: 0x00a8, # CYRILLIC SMALL LETTER I - 0x0439: 0x00a9, # CYRILLIC SMALL LETTER SHORT I - 0x043a: 0x00aa, # CYRILLIC SMALL LETTER KA - 0x043b: 0x00ab, # CYRILLIC SMALL LETTER EL - 0x043c: 0x00ac, # CYRILLIC SMALL LETTER EM - 0x043d: 0x00ad, # CYRILLIC SMALL LETTER EN - 0x043e: 0x00ae, # CYRILLIC SMALL LETTER O - 0x043f: 0x00af, # CYRILLIC SMALL LETTER PE - 0x0440: 0x00e0, # CYRILLIC SMALL LETTER ER - 0x0441: 0x00e1, # CYRILLIC SMALL LETTER ES - 0x0442: 0x00e2, # CYRILLIC SMALL LETTER TE - 0x0443: 0x00e3, # CYRILLIC SMALL LETTER U - 0x0444: 0x00e4, # CYRILLIC SMALL LETTER EF - 0x0445: 0x00e5, # CYRILLIC SMALL LETTER HA - 0x0446: 0x00e6, # CYRILLIC SMALL LETTER TSE - 0x0447: 0x00e7, # CYRILLIC SMALL LETTER CHE - 0x0448: 0x00e8, # CYRILLIC SMALL LETTER SHA - 0x0449: 0x00e9, # CYRILLIC SMALL LETTER SHCHA - 0x044a: 0x00ea, # CYRILLIC SMALL LETTER HARD SIGN - 0x044b: 0x00eb, # CYRILLIC SMALL LETTER YERU - 0x044c: 0x00ec, # CYRILLIC SMALL LETTER SOFT SIGN - 0x044d: 0x00ed, # CYRILLIC SMALL LETTER E - 0x044e: 0x00ee, # CYRILLIC SMALL LETTER YU - 0x044f: 0x00ef, # CYRILLIC SMALL LETTER YA - 0x0451: 0x00f1, # CYRILLIC SMALL LETTER IO - 0x0454: 0x00f5, # CYRILLIC SMALL LETTER UKRAINIAN IE - 0x0456: 0x00f7, # CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I - 0x0457: 0x00f9, # CYRILLIC SMALL LETTER YI - 0x0490: 0x00f2, # CYRILLIC CAPITAL LETTER GHE WITH UPTURN - 0x0491: 0x00f3, # CYRILLIC SMALL LETTER GHE WITH UPTURN - 0x2116: 0x00fc, # NUMERO SIGN - 0x221a: 0x00fb, # SQUARE ROOT - 0x2500: 0x00c4, # BOX DRAWINGS LIGHT HORIZONTAL - 0x2502: 0x00b3, # BOX DRAWINGS LIGHT VERTICAL - 0x250c: 0x00da, # BOX DRAWINGS LIGHT DOWN AND RIGHT - 0x2510: 0x00bf, # BOX DRAWINGS LIGHT DOWN AND LEFT - 0x2514: 0x00c0, # BOX DRAWINGS LIGHT UP AND RIGHT - 0x2518: 0x00d9, # BOX DRAWINGS LIGHT UP AND LEFT - 0x251c: 0x00c3, # BOX DRAWINGS LIGHT VERTICAL AND RIGHT - 0x2524: 0x00b4, # BOX DRAWINGS LIGHT VERTICAL AND LEFT - 0x252c: 0x00c2, # BOX DRAWINGS LIGHT DOWN AND HORIZONTAL - 0x2534: 0x00c1, # BOX DRAWINGS LIGHT UP AND HORIZONTAL - 0x253c: 0x00c5, # BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL - 0x2550: 0x00cd, # BOX DRAWINGS DOUBLE HORIZONTAL - 0x2551: 0x00ba, # BOX DRAWINGS DOUBLE VERTICAL - 0x2552: 0x00d5, # BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE - 0x2553: 0x00d6, # BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE - 0x2554: 0x00c9, # BOX DRAWINGS DOUBLE DOWN AND RIGHT - 0x2555: 0x00b8, # BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE - 0x2556: 0x00b7, # BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE - 0x2557: 0x00bb, # BOX DRAWINGS DOUBLE DOWN AND LEFT - 0x2558: 0x00d4, # BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE - 0x2559: 0x00d3, # BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE - 0x255a: 0x00c8, # BOX DRAWINGS DOUBLE UP AND RIGHT - 0x255b: 0x00be, # BOX DRAWINGS UP SINGLE AND LEFT DOUBLE - 0x255c: 0x00bd, # BOX DRAWINGS UP DOUBLE AND LEFT SINGLE - 0x255d: 0x00bc, # BOX DRAWINGS DOUBLE UP AND LEFT - 0x255e: 0x00c6, # BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE - 0x255f: 0x00c7, # BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE - 0x2560: 0x00cc, # BOX DRAWINGS DOUBLE VERTICAL AND RIGHT - 0x2561: 0x00b5, # BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE - 0x2562: 0x00b6, # BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE - 0x2563: 0x00b9, # BOX DRAWINGS DOUBLE VERTICAL AND LEFT - 0x2564: 0x00d1, # BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE - 0x2565: 0x00d2, # BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE - 0x2566: 0x00cb, # BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL - 0x2567: 0x00cf, # BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE - 0x2568: 0x00d0, # BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE - 0x2569: 0x00ca, # BOX DRAWINGS DOUBLE UP AND HORIZONTAL - 0x256a: 0x00d8, # BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE - 0x256b: 0x00d7, # BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE - 0x256c: 0x00ce, # BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL - 0x2580: 0x00df, # UPPER HALF BLOCK - 0x2584: 0x00dc, # LOWER HALF BLOCK - 0x2588: 0x00db, # FULL BLOCK - 0x258c: 0x00dd, # LEFT HALF BLOCK - 0x2590: 0x00de, # RIGHT HALF BLOCK - 0x2591: 0x00b0, # LIGHT SHADE - 0x2592: 0x00b1, # MEDIUM SHADE - 0x2593: 0x00b2, # DARK SHADE - 0x25a0: 0x00fe, # BLACK SQUARE -} diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/cp273.py --- a/Lib/encodings/cp273.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,307 +0,0 @@ -""" Python Character Mapping Codec cp273 generated from 'python-mappings/CP273.TXT' with gencodec.py. - -"""#" - -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) - - def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] - -class StreamWriter(Codec,codecs.StreamWriter): - pass - -class StreamReader(Codec,codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='cp273', - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - - -### Decoding Table - -decoding_table = ( - '\x00' # 0x00 -> NULL (NUL) - '\x01' # 0x01 -> START OF HEADING (SOH) - '\x02' # 0x02 -> START OF TEXT (STX) - '\x03' # 0x03 -> END OF TEXT (ETX) - '\x9c' # 0x04 -> STRING TERMINATOR (ST) - '\t' # 0x05 -> CHARACTER TABULATION (HT) - '\x86' # 0x06 -> START OF SELECTED AREA (SSA) - '\x7f' # 0x07 -> DELETE (DEL) - '\x97' # 0x08 -> END OF GUARDED AREA (EPA) - '\x8d' # 0x09 -> REVERSE LINE FEED (RI) - '\x8e' # 0x0A -> SINGLE-SHIFT TWO (SS2) - '\x0b' # 0x0B -> LINE TABULATION (VT) - '\x0c' # 0x0C -> FORM FEED (FF) - '\r' # 0x0D -> CARRIAGE RETURN (CR) - '\x0e' # 0x0E -> SHIFT OUT (SO) - '\x0f' # 0x0F -> SHIFT IN (SI) - '\x10' # 0x10 -> DATALINK ESCAPE (DLE) - '\x11' # 0x11 -> DEVICE CONTROL ONE (DC1) - '\x12' # 0x12 -> DEVICE CONTROL TWO (DC2) - '\x13' # 0x13 -> DEVICE CONTROL THREE (DC3) - '\x9d' # 0x14 -> OPERATING SYSTEM COMMAND (OSC) - '\x85' # 0x15 -> NEXT LINE (NEL) - '\x08' # 0x16 -> BACKSPACE (BS) - '\x87' # 0x17 -> END OF SELECTED AREA (ESA) - '\x18' # 0x18 -> CANCEL (CAN) - '\x19' # 0x19 -> END OF MEDIUM (EM) - '\x92' # 0x1A -> PRIVATE USE TWO (PU2) - '\x8f' # 0x1B -> SINGLE-SHIFT THREE (SS3) - '\x1c' # 0x1C -> FILE SEPARATOR (IS4) - '\x1d' # 0x1D -> GROUP SEPARATOR (IS3) - '\x1e' # 0x1E -> RECORD SEPARATOR (IS2) - '\x1f' # 0x1F -> UNIT SEPARATOR (IS1) - '\x80' # 0x20 -> PADDING CHARACTER (PAD) - '\x81' # 0x21 -> HIGH OCTET PRESET (HOP) - '\x82' # 0x22 -> BREAK PERMITTED HERE (BPH) - '\x83' # 0x23 -> NO BREAK HERE (NBH) - '\x84' # 0x24 -> INDEX (IND) - '\n' # 0x25 -> LINE FEED (LF) - '\x17' # 0x26 -> END OF TRANSMISSION BLOCK (ETB) - '\x1b' # 0x27 -> ESCAPE (ESC) - '\x88' # 0x28 -> CHARACTER TABULATION SET (HTS) - '\x89' # 0x29 -> CHARACTER TABULATION WITH JUSTIFICATION (HTJ) - '\x8a' # 0x2A -> LINE TABULATION SET (VTS) - '\x8b' # 0x2B -> PARTIAL LINE FORWARD (PLD) - '\x8c' # 0x2C -> PARTIAL LINE BACKWARD (PLU) - '\x05' # 0x2D -> ENQUIRY (ENQ) - '\x06' # 0x2E -> ACKNOWLEDGE (ACK) - '\x07' # 0x2F -> BELL (BEL) - '\x90' # 0x30 -> DEVICE CONTROL STRING (DCS) - '\x91' # 0x31 -> PRIVATE USE ONE (PU1) - '\x16' # 0x32 -> SYNCHRONOUS IDLE (SYN) - '\x93' # 0x33 -> SET TRANSMIT STATE (STS) - '\x94' # 0x34 -> CANCEL CHARACTER (CCH) - '\x95' # 0x35 -> MESSAGE WAITING (MW) - '\x96' # 0x36 -> START OF GUARDED AREA (SPA) - '\x04' # 0x37 -> END OF TRANSMISSION (EOT) - '\x98' # 0x38 -> START OF STRING (SOS) - '\x99' # 0x39 -> SINGLE GRAPHIC CHARACTER INTRODUCER (SGCI) - '\x9a' # 0x3A -> SINGLE CHARACTER INTRODUCER (SCI) - '\x9b' # 0x3B -> CONTROL SEQUENCE INTRODUCER (CSI) - '\x14' # 0x3C -> DEVICE CONTROL FOUR (DC4) - '\x15' # 0x3D -> NEGATIVE ACKNOWLEDGE (NAK) - '\x9e' # 0x3E -> PRIVACY MESSAGE (PM) - '\x1a' # 0x3F -> SUBSTITUTE (SUB) - ' ' # 0x40 -> SPACE - '\xa0' # 0x41 -> NO-BREAK SPACE - '\xe2' # 0x42 -> LATIN SMALL LETTER A WITH CIRCUMFLEX - '{' # 0x43 -> LEFT CURLY BRACKET - '\xe0' # 0x44 -> LATIN SMALL LETTER A WITH GRAVE - '\xe1' # 0x45 -> LATIN SMALL LETTER A WITH ACUTE - '\xe3' # 0x46 -> LATIN SMALL LETTER A WITH TILDE - '\xe5' # 0x47 -> LATIN SMALL LETTER A WITH RING ABOVE - '\xe7' # 0x48 -> LATIN SMALL LETTER C WITH CEDILLA - '\xf1' # 0x49 -> LATIN SMALL LETTER N WITH TILDE - '\xc4' # 0x4A -> LATIN CAPITAL LETTER A WITH DIAERESIS - '.' # 0x4B -> FULL STOP - '<' # 0x4C -> LESS-THAN SIGN - '(' # 0x4D -> LEFT PARENTHESIS - '+' # 0x4E -> PLUS SIGN - '!' # 0x4F -> EXCLAMATION MARK - '&' # 0x50 -> AMPERSAND - '\xe9' # 0x51 -> LATIN SMALL LETTER E WITH ACUTE - '\xea' # 0x52 -> LATIN SMALL LETTER E WITH CIRCUMFLEX - '\xeb' # 0x53 -> LATIN SMALL LETTER E WITH DIAERESIS - '\xe8' # 0x54 -> LATIN SMALL LETTER E WITH GRAVE - '\xed' # 0x55 -> LATIN SMALL LETTER I WITH ACUTE - '\xee' # 0x56 -> LATIN SMALL LETTER I WITH CIRCUMFLEX - '\xef' # 0x57 -> LATIN SMALL LETTER I WITH DIAERESIS - '\xec' # 0x58 -> LATIN SMALL LETTER I WITH GRAVE - '~' # 0x59 -> TILDE - '\xdc' # 0x5A -> LATIN CAPITAL LETTER U WITH DIAERESIS - '$' # 0x5B -> DOLLAR SIGN - '*' # 0x5C -> ASTERISK - ')' # 0x5D -> RIGHT PARENTHESIS - ';' # 0x5E -> SEMICOLON - '^' # 0x5F -> CIRCUMFLEX ACCENT - '-' # 0x60 -> HYPHEN-MINUS - '/' # 0x61 -> SOLIDUS - '\xc2' # 0x62 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX - '[' # 0x63 -> LEFT SQUARE BRACKET - '\xc0' # 0x64 -> LATIN CAPITAL LETTER A WITH GRAVE - '\xc1' # 0x65 -> LATIN CAPITAL LETTER A WITH ACUTE - '\xc3' # 0x66 -> LATIN CAPITAL LETTER A WITH TILDE - '\xc5' # 0x67 -> LATIN CAPITAL LETTER A WITH RING ABOVE - '\xc7' # 0x68 -> LATIN CAPITAL LETTER C WITH CEDILLA - '\xd1' # 0x69 -> LATIN CAPITAL LETTER N WITH TILDE - '\xf6' # 0x6A -> LATIN SMALL LETTER O WITH DIAERESIS - ',' # 0x6B -> COMMA - '%' # 0x6C -> PERCENT SIGN - '_' # 0x6D -> LOW LINE - '>' # 0x6E -> GREATER-THAN SIGN - '?' # 0x6F -> QUESTION MARK - '\xf8' # 0x70 -> LATIN SMALL LETTER O WITH STROKE - '\xc9' # 0x71 -> LATIN CAPITAL LETTER E WITH ACUTE - '\xca' # 0x72 -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX - '\xcb' # 0x73 -> LATIN CAPITAL LETTER E WITH DIAERESIS - '\xc8' # 0x74 -> LATIN CAPITAL LETTER E WITH GRAVE - '\xcd' # 0x75 -> LATIN CAPITAL LETTER I WITH ACUTE - '\xce' # 0x76 -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX - '\xcf' # 0x77 -> LATIN CAPITAL LETTER I WITH DIAERESIS - '\xcc' # 0x78 -> LATIN CAPITAL LETTER I WITH GRAVE - '`' # 0x79 -> GRAVE ACCENT - ':' # 0x7A -> COLON - '#' # 0x7B -> NUMBER SIGN - '\xa7' # 0x7C -> SECTION SIGN - "'" # 0x7D -> APOSTROPHE - '=' # 0x7E -> EQUALS SIGN - '"' # 0x7F -> QUOTATION MARK - '\xd8' # 0x80 -> LATIN CAPITAL LETTER O WITH STROKE - 'a' # 0x81 -> LATIN SMALL LETTER A - 'b' # 0x82 -> LATIN SMALL LETTER B - 'c' # 0x83 -> LATIN SMALL LETTER C - 'd' # 0x84 -> LATIN SMALL LETTER D - 'e' # 0x85 -> LATIN SMALL LETTER E - 'f' # 0x86 -> LATIN SMALL LETTER F - 'g' # 0x87 -> LATIN SMALL LETTER G - 'h' # 0x88 -> LATIN SMALL LETTER H - 'i' # 0x89 -> LATIN SMALL LETTER I - '\xab' # 0x8A -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xbb' # 0x8B -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xf0' # 0x8C -> LATIN SMALL LETTER ETH (Icelandic) - '\xfd' # 0x8D -> LATIN SMALL LETTER Y WITH ACUTE - '\xfe' # 0x8E -> LATIN SMALL LETTER THORN (Icelandic) - '\xb1' # 0x8F -> PLUS-MINUS SIGN - '\xb0' # 0x90 -> DEGREE SIGN - 'j' # 0x91 -> LATIN SMALL LETTER J - 'k' # 0x92 -> LATIN SMALL LETTER K - 'l' # 0x93 -> LATIN SMALL LETTER L - 'm' # 0x94 -> LATIN SMALL LETTER M - 'n' # 0x95 -> LATIN SMALL LETTER N - 'o' # 0x96 -> LATIN SMALL LETTER O - 'p' # 0x97 -> LATIN SMALL LETTER P - 'q' # 0x98 -> LATIN SMALL LETTER Q - 'r' # 0x99 -> LATIN SMALL LETTER R - '\xaa' # 0x9A -> FEMININE ORDINAL INDICATOR - '\xba' # 0x9B -> MASCULINE ORDINAL INDICATOR - '\xe6' # 0x9C -> LATIN SMALL LETTER AE - '\xb8' # 0x9D -> CEDILLA - '\xc6' # 0x9E -> LATIN CAPITAL LETTER AE - '\xa4' # 0x9F -> CURRENCY SIGN - '\xb5' # 0xA0 -> MICRO SIGN - '\xdf' # 0xA1 -> LATIN SMALL LETTER SHARP S (German) - 's' # 0xA2 -> LATIN SMALL LETTER S - 't' # 0xA3 -> LATIN SMALL LETTER T - 'u' # 0xA4 -> LATIN SMALL LETTER U - 'v' # 0xA5 -> LATIN SMALL LETTER V - 'w' # 0xA6 -> LATIN SMALL LETTER W - 'x' # 0xA7 -> LATIN SMALL LETTER X - 'y' # 0xA8 -> LATIN SMALL LETTER Y - 'z' # 0xA9 -> LATIN SMALL LETTER Z - '\xa1' # 0xAA -> INVERTED EXCLAMATION MARK - '\xbf' # 0xAB -> INVERTED QUESTION MARK - '\xd0' # 0xAC -> LATIN CAPITAL LETTER ETH (Icelandic) - '\xdd' # 0xAD -> LATIN CAPITAL LETTER Y WITH ACUTE - '\xde' # 0xAE -> LATIN CAPITAL LETTER THORN (Icelandic) - '\xae' # 0xAF -> REGISTERED SIGN - '\xa2' # 0xB0 -> CENT SIGN - '\xa3' # 0xB1 -> POUND SIGN - '\xa5' # 0xB2 -> YEN SIGN - '\xb7' # 0xB3 -> MIDDLE DOT - '\xa9' # 0xB4 -> COPYRIGHT SIGN - '@' # 0xB5 -> COMMERCIAL AT - '\xb6' # 0xB6 -> PILCROW SIGN - '\xbc' # 0xB7 -> VULGAR FRACTION ONE QUARTER - '\xbd' # 0xB8 -> VULGAR FRACTION ONE HALF - '\xbe' # 0xB9 -> VULGAR FRACTION THREE QUARTERS - '\xac' # 0xBA -> NOT SIGN - '|' # 0xBB -> VERTICAL LINE - '\u203e' # 0xBC -> OVERLINE - '\xa8' # 0xBD -> DIAERESIS - '\xb4' # 0xBE -> ACUTE ACCENT - '\xd7' # 0xBF -> MULTIPLICATION SIGN - '\xe4' # 0xC0 -> LATIN SMALL LETTER A WITH DIAERESIS - 'A' # 0xC1 -> LATIN CAPITAL LETTER A - 'B' # 0xC2 -> LATIN CAPITAL LETTER B - 'C' # 0xC3 -> LATIN CAPITAL LETTER C - 'D' # 0xC4 -> LATIN CAPITAL LETTER D - 'E' # 0xC5 -> LATIN CAPITAL LETTER E - 'F' # 0xC6 -> LATIN CAPITAL LETTER F - 'G' # 0xC7 -> LATIN CAPITAL LETTER G - 'H' # 0xC8 -> LATIN CAPITAL LETTER H - 'I' # 0xC9 -> LATIN CAPITAL LETTER I - '\xad' # 0xCA -> SOFT HYPHEN - '\xf4' # 0xCB -> LATIN SMALL LETTER O WITH CIRCUMFLEX - '\xa6' # 0xCC -> BROKEN BAR - '\xf2' # 0xCD -> LATIN SMALL LETTER O WITH GRAVE - '\xf3' # 0xCE -> LATIN SMALL LETTER O WITH ACUTE - '\xf5' # 0xCF -> LATIN SMALL LETTER O WITH TILDE - '\xfc' # 0xD0 -> LATIN SMALL LETTER U WITH DIAERESIS - 'J' # 0xD1 -> LATIN CAPITAL LETTER J - 'K' # 0xD2 -> LATIN CAPITAL LETTER K - 'L' # 0xD3 -> LATIN CAPITAL LETTER L - 'M' # 0xD4 -> LATIN CAPITAL LETTER M - 'N' # 0xD5 -> LATIN CAPITAL LETTER N - 'O' # 0xD6 -> LATIN CAPITAL LETTER O - 'P' # 0xD7 -> LATIN CAPITAL LETTER P - 'Q' # 0xD8 -> LATIN CAPITAL LETTER Q - 'R' # 0xD9 -> LATIN CAPITAL LETTER R - '\xb9' # 0xDA -> SUPERSCRIPT ONE - '\xfb' # 0xDB -> LATIN SMALL LETTER U WITH CIRCUMFLEX - '}' # 0xDC -> RIGHT CURLY BRACKET - '\xf9' # 0xDD -> LATIN SMALL LETTER U WITH GRAVE - '\xfa' # 0xDE -> LATIN SMALL LETTER U WITH ACUTE - '\xff' # 0xDF -> LATIN SMALL LETTER Y WITH DIAERESIS - '\xd6' # 0xE0 -> LATIN CAPITAL LETTER O WITH DIAERESIS - '\xf7' # 0xE1 -> DIVISION SIGN - 'S' # 0xE2 -> LATIN CAPITAL LETTER S - 'T' # 0xE3 -> LATIN CAPITAL LETTER T - 'U' # 0xE4 -> LATIN CAPITAL LETTER U - 'V' # 0xE5 -> LATIN CAPITAL LETTER V - 'W' # 0xE6 -> LATIN CAPITAL LETTER W - 'X' # 0xE7 -> LATIN CAPITAL LETTER X - 'Y' # 0xE8 -> LATIN CAPITAL LETTER Y - 'Z' # 0xE9 -> LATIN CAPITAL LETTER Z - '\xb2' # 0xEA -> SUPERSCRIPT TWO - '\xd4' # 0xEB -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\\' # 0xEC -> REVERSE SOLIDUS - '\xd2' # 0xED -> LATIN CAPITAL LETTER O WITH GRAVE - '\xd3' # 0xEE -> LATIN CAPITAL LETTER O WITH ACUTE - '\xd5' # 0xEF -> LATIN CAPITAL LETTER O WITH TILDE - '0' # 0xF0 -> DIGIT ZERO - '1' # 0xF1 -> DIGIT ONE - '2' # 0xF2 -> DIGIT TWO - '3' # 0xF3 -> DIGIT THREE - '4' # 0xF4 -> DIGIT FOUR - '5' # 0xF5 -> DIGIT FIVE - '6' # 0xF6 -> DIGIT SIX - '7' # 0xF7 -> DIGIT SEVEN - '8' # 0xF8 -> DIGIT EIGHT - '9' # 0xF9 -> DIGIT NINE - '\xb3' # 0xFA -> SUPERSCRIPT THREE - '\xdb' # 0xFB -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX - ']' # 0xFC -> RIGHT SQUARE BRACKET - '\xd9' # 0xFD -> LATIN CAPITAL LETTER U WITH GRAVE - '\xda' # 0xFE -> LATIN CAPITAL LETTER U WITH ACUTE - '\x9f' # 0xFF -> APPLICATION PROGRAM COMMAND (APC) -) - -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/cp65001.py --- a/Lib/encodings/cp65001.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/cp65001.py Mon Jan 25 17:05:13 2016 +0100 @@ -11,23 +11,20 @@ ### Codec APIs encode = functools.partial(codecs.code_page_encode, 65001) -_decode = functools.partial(codecs.code_page_decode, 65001) - -def decode(input, errors='strict'): - return codecs.code_page_decode(65001, input, errors, True) +decode = functools.partial(codecs.code_page_decode, 65001) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return encode(input, self.errors)[0] class IncrementalDecoder(codecs.BufferedIncrementalDecoder): - _buffer_decode = _decode + _buffer_decode = decode class StreamWriter(codecs.StreamWriter): encode = encode class StreamReader(codecs.StreamReader): - decode = _decode + decode = decode ### encodings module API diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/hex_codec.py --- a/Lib/encodings/hex_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/hex_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Python 'hex_codec' Codec - 2-digit hex content transfer encoding. -This codec de/encodes from bytes to bytes. +This codec de/encodes from bytes to bytes and is therefore usable with +bytes.transform() and bytes.untransform(). Written by Marc-Andre Lemburg (mal@lemburg.com). """ @@ -51,5 +52,4 @@ incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/hp_roman8.py --- a/Lib/encodings/hp_roman8.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/hp_roman8.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,8 +5,6 @@ Original source: LaserJet IIP Printer User's Manual HP part no 33471-90901, Hewlet-Packard, June 1989. - (Used with permission) - """#" import codecs @@ -16,18 +14,18 @@ class Codec(codecs.Codec): def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) + return codecs.charmap_encode(input,errors,encoding_map) def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) + return codecs.charmap_decode(input,errors,decoding_map) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] + return codecs.charmap_encode(input,self.errors,encoding_map)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] + return codecs.charmap_decode(input,self.errors,decoding_map)[0] class StreamWriter(Codec,codecs.StreamWriter): pass @@ -48,267 +46,107 @@ streamreader=StreamReader, ) +### Decoding Map -### Decoding Table +decoding_map = codecs.make_identity_dict(range(256)) +decoding_map.update({ + 0x00a1: 0x00c0, # LATIN CAPITAL LETTER A WITH GRAVE + 0x00a2: 0x00c2, # LATIN CAPITAL LETTER A WITH CIRCUMFLEX + 0x00a3: 0x00c8, # LATIN CAPITAL LETTER E WITH GRAVE + 0x00a4: 0x00ca, # LATIN CAPITAL LETTER E WITH CIRCUMFLEX + 0x00a5: 0x00cb, # LATIN CAPITAL LETTER E WITH DIAERESIS + 0x00a6: 0x00ce, # LATIN CAPITAL LETTER I WITH CIRCUMFLEX + 0x00a7: 0x00cf, # LATIN CAPITAL LETTER I WITH DIAERESIS + 0x00a8: 0x00b4, # ACUTE ACCENT + 0x00a9: 0x02cb, # MODIFIER LETTER GRAVE ACCENT (Mandarin Chinese fourth tone) + 0x00aa: 0x02c6, # MODIFIER LETTER CIRCUMFLEX ACCENT + 0x00ab: 0x00a8, # DIAERESIS + 0x00ac: 0x02dc, # SMALL TILDE + 0x00ad: 0x00d9, # LATIN CAPITAL LETTER U WITH GRAVE + 0x00ae: 0x00db, # LATIN CAPITAL LETTER U WITH CIRCUMFLEX + 0x00af: 0x20a4, # LIRA SIGN + 0x00b0: 0x00af, # MACRON + 0x00b1: 0x00dd, # LATIN CAPITAL LETTER Y WITH ACUTE + 0x00b2: 0x00fd, # LATIN SMALL LETTER Y WITH ACUTE + 0x00b3: 0x00b0, # DEGREE SIGN + 0x00b4: 0x00c7, # LATIN CAPITAL LETTER C WITH CEDILLA + 0x00b5: 0x00e7, # LATIN SMALL LETTER C WITH CEDILLA + 0x00b6: 0x00d1, # LATIN CAPITAL LETTER N WITH TILDE + 0x00b7: 0x00f1, # LATIN SMALL LETTER N WITH TILDE + 0x00b8: 0x00a1, # INVERTED EXCLAMATION MARK + 0x00b9: 0x00bf, # INVERTED QUESTION MARK + 0x00ba: 0x00a4, # CURRENCY SIGN + 0x00bb: 0x00a3, # POUND SIGN + 0x00bc: 0x00a5, # YEN SIGN + 0x00bd: 0x00a7, # SECTION SIGN + 0x00be: 0x0192, # LATIN SMALL LETTER F WITH HOOK + 0x00bf: 0x00a2, # CENT SIGN + 0x00c0: 0x00e2, # LATIN SMALL LETTER A WITH CIRCUMFLEX + 0x00c1: 0x00ea, # LATIN SMALL LETTER E WITH CIRCUMFLEX + 0x00c2: 0x00f4, # LATIN SMALL LETTER O WITH CIRCUMFLEX + 0x00c3: 0x00fb, # LATIN SMALL LETTER U WITH CIRCUMFLEX + 0x00c4: 0x00e1, # LATIN SMALL LETTER A WITH ACUTE + 0x00c5: 0x00e9, # LATIN SMALL LETTER E WITH ACUTE + 0x00c6: 0x00f3, # LATIN SMALL LETTER O WITH ACUTE + 0x00c7: 0x00fa, # LATIN SMALL LETTER U WITH ACUTE + 0x00c8: 0x00e0, # LATIN SMALL LETTER A WITH GRAVE + 0x00c9: 0x00e8, # LATIN SMALL LETTER E WITH GRAVE + 0x00ca: 0x00f2, # LATIN SMALL LETTER O WITH GRAVE + 0x00cb: 0x00f9, # LATIN SMALL LETTER U WITH GRAVE + 0x00cc: 0x00e4, # LATIN SMALL LETTER A WITH DIAERESIS + 0x00cd: 0x00eb, # LATIN SMALL LETTER E WITH DIAERESIS + 0x00ce: 0x00f6, # LATIN SMALL LETTER O WITH DIAERESIS + 0x00cf: 0x00fc, # LATIN SMALL LETTER U WITH DIAERESIS + 0x00d0: 0x00c5, # LATIN CAPITAL LETTER A WITH RING ABOVE + 0x00d1: 0x00ee, # LATIN SMALL LETTER I WITH CIRCUMFLEX + 0x00d2: 0x00d8, # LATIN CAPITAL LETTER O WITH STROKE + 0x00d3: 0x00c6, # LATIN CAPITAL LETTER AE + 0x00d4: 0x00e5, # LATIN SMALL LETTER A WITH RING ABOVE + 0x00d5: 0x00ed, # LATIN SMALL LETTER I WITH ACUTE + 0x00d6: 0x00f8, # LATIN SMALL LETTER O WITH STROKE + 0x00d7: 0x00e6, # LATIN SMALL LETTER AE + 0x00d8: 0x00c4, # LATIN CAPITAL LETTER A WITH DIAERESIS + 0x00d9: 0x00ec, # LATIN SMALL LETTER I WITH GRAVE + 0x00da: 0x00d6, # LATIN CAPITAL LETTER O WITH DIAERESIS + 0x00db: 0x00dc, # LATIN CAPITAL LETTER U WITH DIAERESIS + 0x00dc: 0x00c9, # LATIN CAPITAL LETTER E WITH ACUTE + 0x00dd: 0x00ef, # LATIN SMALL LETTER I WITH DIAERESIS + 0x00de: 0x00df, # LATIN SMALL LETTER SHARP S (German) + 0x00df: 0x00d4, # LATIN CAPITAL LETTER O WITH CIRCUMFLEX + 0x00e0: 0x00c1, # LATIN CAPITAL LETTER A WITH ACUTE + 0x00e1: 0x00c3, # LATIN CAPITAL LETTER A WITH TILDE + 0x00e2: 0x00e3, # LATIN SMALL LETTER A WITH TILDE + 0x00e3: 0x00d0, # LATIN CAPITAL LETTER ETH (Icelandic) + 0x00e4: 0x00f0, # LATIN SMALL LETTER ETH (Icelandic) + 0x00e5: 0x00cd, # LATIN CAPITAL LETTER I WITH ACUTE + 0x00e6: 0x00cc, # LATIN CAPITAL LETTER I WITH GRAVE + 0x00e7: 0x00d3, # LATIN CAPITAL LETTER O WITH ACUTE + 0x00e8: 0x00d2, # LATIN CAPITAL LETTER O WITH GRAVE + 0x00e9: 0x00d5, # LATIN CAPITAL LETTER O WITH TILDE + 0x00ea: 0x00f5, # LATIN SMALL LETTER O WITH TILDE + 0x00eb: 0x0160, # LATIN CAPITAL LETTER S WITH CARON + 0x00ec: 0x0161, # LATIN SMALL LETTER S WITH CARON + 0x00ed: 0x00da, # LATIN CAPITAL LETTER U WITH ACUTE + 0x00ee: 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS + 0x00ef: 0x00ff, # LATIN SMALL LETTER Y WITH DIAERESIS + 0x00f0: 0x00de, # LATIN CAPITAL LETTER THORN (Icelandic) + 0x00f1: 0x00fe, # LATIN SMALL LETTER THORN (Icelandic) + 0x00f2: 0x00b7, # MIDDLE DOT + 0x00f3: 0x00b5, # MICRO SIGN + 0x00f4: 0x00b6, # PILCROW SIGN + 0x00f5: 0x00be, # VULGAR FRACTION THREE QUARTERS + 0x00f6: 0x2014, # EM DASH + 0x00f7: 0x00bc, # VULGAR FRACTION ONE QUARTER + 0x00f8: 0x00bd, # VULGAR FRACTION ONE HALF + 0x00f9: 0x00aa, # FEMININE ORDINAL INDICATOR + 0x00fa: 0x00ba, # MASCULINE ORDINAL INDICATOR + 0x00fb: 0x00ab, # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + 0x00fc: 0x25a0, # BLACK SQUARE + 0x00fd: 0x00bb, # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + 0x00fe: 0x00b1, # PLUS-MINUS SIGN + 0x00ff: None, +}) -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE - '\x80' # 0x80 -> - '\x81' # 0x81 -> - '\x82' # 0x82 -> - '\x83' # 0x83 -> - '\x84' # 0x84 -> - '\x85' # 0x85 -> - '\x86' # 0x86 -> - '\x87' # 0x87 -> - '\x88' # 0x88 -> - '\x89' # 0x89 -> - '\x8a' # 0x8A -> - '\x8b' # 0x8B -> - '\x8c' # 0x8C -> - '\x8d' # 0x8D -> - '\x8e' # 0x8E -> - '\x8f' # 0x8F -> - '\x90' # 0x90 -> - '\x91' # 0x91 -> - '\x92' # 0x92 -> - '\x93' # 0x93 -> - '\x94' # 0x94 -> - '\x95' # 0x95 -> - '\x96' # 0x96 -> - '\x97' # 0x97 -> - '\x98' # 0x98 -> - '\x99' # 0x99 -> - '\x9a' # 0x9A -> - '\x9b' # 0x9B -> - '\x9c' # 0x9C -> - '\x9d' # 0x9D -> - '\x9e' # 0x9E -> - '\x9f' # 0x9F -> - '\xa0' # 0xA0 -> NO-BREAK SPACE - '\xc0' # 0xA1 -> LATIN CAPITAL LETTER A WITH GRAVE - '\xc2' # 0xA2 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX - '\xc8' # 0xA3 -> LATIN CAPITAL LETTER E WITH GRAVE - '\xca' # 0xA4 -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX - '\xcb' # 0xA5 -> LATIN CAPITAL LETTER E WITH DIAERESIS - '\xce' # 0xA6 -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX - '\xcf' # 0xA7 -> LATIN CAPITAL LETTER I WITH DIAERESIS - '\xb4' # 0xA8 -> ACUTE ACCENT - '\u02cb' # 0xA9 -> MODIFIER LETTER GRAVE ACCENT (MANDARIN CHINESE FOURTH TONE) - '\u02c6' # 0xAA -> MODIFIER LETTER CIRCUMFLEX ACCENT - '\xa8' # 0xAB -> DIAERESIS - '\u02dc' # 0xAC -> SMALL TILDE - '\xd9' # 0xAD -> LATIN CAPITAL LETTER U WITH GRAVE - '\xdb' # 0xAE -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX - '\u20a4' # 0xAF -> LIRA SIGN - '\xaf' # 0xB0 -> MACRON - '\xdd' # 0xB1 -> LATIN CAPITAL LETTER Y WITH ACUTE - '\xfd' # 0xB2 -> LATIN SMALL LETTER Y WITH ACUTE - '\xb0' # 0xB3 -> DEGREE SIGN - '\xc7' # 0xB4 -> LATIN CAPITAL LETTER C WITH CEDILLA - '\xe7' # 0xB5 -> LATIN SMALL LETTER C WITH CEDILLA - '\xd1' # 0xB6 -> LATIN CAPITAL LETTER N WITH TILDE - '\xf1' # 0xB7 -> LATIN SMALL LETTER N WITH TILDE - '\xa1' # 0xB8 -> INVERTED EXCLAMATION MARK - '\xbf' # 0xB9 -> INVERTED QUESTION MARK - '\xa4' # 0xBA -> CURRENCY SIGN - '\xa3' # 0xBB -> POUND SIGN - '\xa5' # 0xBC -> YEN SIGN - '\xa7' # 0xBD -> SECTION SIGN - '\u0192' # 0xBE -> LATIN SMALL LETTER F WITH HOOK - '\xa2' # 0xBF -> CENT SIGN - '\xe2' # 0xC0 -> LATIN SMALL LETTER A WITH CIRCUMFLEX - '\xea' # 0xC1 -> LATIN SMALL LETTER E WITH CIRCUMFLEX - '\xf4' # 0xC2 -> LATIN SMALL LETTER O WITH CIRCUMFLEX - '\xfb' # 0xC3 -> LATIN SMALL LETTER U WITH CIRCUMFLEX - '\xe1' # 0xC4 -> LATIN SMALL LETTER A WITH ACUTE - '\xe9' # 0xC5 -> LATIN SMALL LETTER E WITH ACUTE - '\xf3' # 0xC6 -> LATIN SMALL LETTER O WITH ACUTE - '\xfa' # 0xC7 -> LATIN SMALL LETTER U WITH ACUTE - '\xe0' # 0xC8 -> LATIN SMALL LETTER A WITH GRAVE - '\xe8' # 0xC9 -> LATIN SMALL LETTER E WITH GRAVE - '\xf2' # 0xCA -> LATIN SMALL LETTER O WITH GRAVE - '\xf9' # 0xCB -> LATIN SMALL LETTER U WITH GRAVE - '\xe4' # 0xCC -> LATIN SMALL LETTER A WITH DIAERESIS - '\xeb' # 0xCD -> LATIN SMALL LETTER E WITH DIAERESIS - '\xf6' # 0xCE -> LATIN SMALL LETTER O WITH DIAERESIS - '\xfc' # 0xCF -> LATIN SMALL LETTER U WITH DIAERESIS - '\xc5' # 0xD0 -> LATIN CAPITAL LETTER A WITH RING ABOVE - '\xee' # 0xD1 -> LATIN SMALL LETTER I WITH CIRCUMFLEX - '\xd8' # 0xD2 -> LATIN CAPITAL LETTER O WITH STROKE - '\xc6' # 0xD3 -> LATIN CAPITAL LETTER AE - '\xe5' # 0xD4 -> LATIN SMALL LETTER A WITH RING ABOVE - '\xed' # 0xD5 -> LATIN SMALL LETTER I WITH ACUTE - '\xf8' # 0xD6 -> LATIN SMALL LETTER O WITH STROKE - '\xe6' # 0xD7 -> LATIN SMALL LETTER AE - '\xc4' # 0xD8 -> LATIN CAPITAL LETTER A WITH DIAERESIS - '\xec' # 0xD9 -> LATIN SMALL LETTER I WITH GRAVE - '\xd6' # 0xDA -> LATIN CAPITAL LETTER O WITH DIAERESIS - '\xdc' # 0xDB -> LATIN CAPITAL LETTER U WITH DIAERESIS - '\xc9' # 0xDC -> LATIN CAPITAL LETTER E WITH ACUTE - '\xef' # 0xDD -> LATIN SMALL LETTER I WITH DIAERESIS - '\xdf' # 0xDE -> LATIN SMALL LETTER SHARP S (GERMAN) - '\xd4' # 0xDF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\xc1' # 0xE0 -> LATIN CAPITAL LETTER A WITH ACUTE - '\xc3' # 0xE1 -> LATIN CAPITAL LETTER A WITH TILDE - '\xe3' # 0xE2 -> LATIN SMALL LETTER A WITH TILDE - '\xd0' # 0xE3 -> LATIN CAPITAL LETTER ETH (ICELANDIC) - '\xf0' # 0xE4 -> LATIN SMALL LETTER ETH (ICELANDIC) - '\xcd' # 0xE5 -> LATIN CAPITAL LETTER I WITH ACUTE - '\xcc' # 0xE6 -> LATIN CAPITAL LETTER I WITH GRAVE - '\xd3' # 0xE7 -> LATIN CAPITAL LETTER O WITH ACUTE - '\xd2' # 0xE8 -> LATIN CAPITAL LETTER O WITH GRAVE - '\xd5' # 0xE9 -> LATIN CAPITAL LETTER O WITH TILDE - '\xf5' # 0xEA -> LATIN SMALL LETTER O WITH TILDE - '\u0160' # 0xEB -> LATIN CAPITAL LETTER S WITH CARON - '\u0161' # 0xEC -> LATIN SMALL LETTER S WITH CARON - '\xda' # 0xED -> LATIN CAPITAL LETTER U WITH ACUTE - '\u0178' # 0xEE -> LATIN CAPITAL LETTER Y WITH DIAERESIS - '\xff' # 0xEF -> LATIN SMALL LETTER Y WITH DIAERESIS - '\xde' # 0xF0 -> LATIN CAPITAL LETTER THORN (ICELANDIC) - '\xfe' # 0xF1 -> LATIN SMALL LETTER THORN (ICELANDIC) - '\xb7' # 0xF2 -> MIDDLE DOT - '\xb5' # 0xF3 -> MICRO SIGN - '\xb6' # 0xF4 -> PILCROW SIGN - '\xbe' # 0xF5 -> VULGAR FRACTION THREE QUARTERS - '\u2014' # 0xF6 -> EM DASH - '\xbc' # 0xF7 -> VULGAR FRACTION ONE QUARTER - '\xbd' # 0xF8 -> VULGAR FRACTION ONE HALF - '\xaa' # 0xF9 -> FEMININE ORDINAL INDICATOR - '\xba' # 0xFA -> MASCULINE ORDINAL INDICATOR - '\xab' # 0xFB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\u25a0' # 0xFC -> BLACK SQUARE - '\xbb' # 0xFD -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xb1' # 0xFE -> PLUS-MINUS SIGN - '\ufffe' -) +### Encoding Map -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) +encoding_map = codecs.make_encoding_map(decoding_map) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/koi8_t.py --- a/Lib/encodings/koi8_t.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,308 +0,0 @@ -""" Python Character Mapping Codec koi8_t -""" -# http://ru.wikipedia.org/wiki/КОИ-8 -# http://www.opensource.apple.com/source/libiconv/libiconv-4/libiconv/tests/KOI8-T.TXT - -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) - - def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] - -class StreamWriter(Codec,codecs.StreamWriter): - pass - -class StreamReader(Codec,codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='koi8-t', - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - - -### Decoding Table - -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE - '\u049b' # 0x80 -> CYRILLIC SMALL LETTER KA WITH DESCENDER - '\u0493' # 0x81 -> CYRILLIC SMALL LETTER GHE WITH STROKE - '\u201a' # 0x82 -> SINGLE LOW-9 QUOTATION MARK - '\u0492' # 0x83 -> CYRILLIC CAPITAL LETTER GHE WITH STROKE - '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK - '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS - '\u2020' # 0x86 -> DAGGER - '\u2021' # 0x87 -> DOUBLE DAGGER - '\ufffe' # 0x88 -> UNDEFINED - '\u2030' # 0x89 -> PER MILLE SIGN - '\u04b3' # 0x8A -> CYRILLIC SMALL LETTER HA WITH DESCENDER - '\u2039' # 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK - '\u04b2' # 0x8C -> CYRILLIC CAPITAL LETTER HA WITH DESCENDER - '\u04b7' # 0x8D -> CYRILLIC SMALL LETTER CHE WITH DESCENDER - '\u04b6' # 0x8E -> CYRILLIC CAPITAL LETTER CHE WITH DESCENDER - '\ufffe' # 0x8F -> UNDEFINED - '\u049a' # 0x90 -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER - '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK - '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK - '\u2022' # 0x95 -> BULLET - '\u2013' # 0x96 -> EN DASH - '\u2014' # 0x97 -> EM DASH - '\ufffe' # 0x98 -> UNDEFINED - '\u2122' # 0x99 -> TRADE MARK SIGN - '\ufffe' # 0x9A -> UNDEFINED - '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - '\ufffe' # 0x9C -> UNDEFINED - '\ufffe' # 0x9D -> UNDEFINED - '\ufffe' # 0x9E -> UNDEFINED - '\ufffe' # 0x9F -> UNDEFINED - '\ufffe' # 0xA0 -> UNDEFINED - '\u04ef' # 0xA1 -> CYRILLIC SMALL LETTER U WITH MACRON - '\u04ee' # 0xA2 -> CYRILLIC CAPITAL LETTER U WITH MACRON - '\u0451' # 0xA3 -> CYRILLIC SMALL LETTER IO - '\xa4' # 0xA4 -> CURRENCY SIGN - '\u04e3' # 0xA5 -> CYRILLIC SMALL LETTER I WITH MACRON - '\xa6' # 0xA6 -> BROKEN BAR - '\xa7' # 0xA7 -> SECTION SIGN - '\ufffe' # 0xA8 -> UNDEFINED - '\ufffe' # 0xA9 -> UNDEFINED - '\ufffe' # 0xAA -> UNDEFINED - '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xac' # 0xAC -> NOT SIGN - '\xad' # 0xAD -> SOFT HYPHEN - '\xae' # 0xAE -> REGISTERED SIGN - '\ufffe' # 0xAF -> UNDEFINED - '\xb0' # 0xB0 -> DEGREE SIGN - '\xb1' # 0xB1 -> PLUS-MINUS SIGN - '\xb2' # 0xB2 -> SUPERSCRIPT TWO - '\u0401' # 0xB3 -> CYRILLIC CAPITAL LETTER IO - '\ufffe' # 0xB4 -> UNDEFINED - '\u04e2' # 0xB5 -> CYRILLIC CAPITAL LETTER I WITH MACRON - '\xb6' # 0xB6 -> PILCROW SIGN - '\xb7' # 0xB7 -> MIDDLE DOT - '\ufffe' # 0xB8 -> UNDEFINED - '\u2116' # 0xB9 -> NUMERO SIGN - '\ufffe' # 0xBA -> UNDEFINED - '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\ufffe' # 0xBC -> UNDEFINED - '\ufffe' # 0xBD -> UNDEFINED - '\ufffe' # 0xBE -> UNDEFINED - '\xa9' # 0xBF -> COPYRIGHT SIGN - '\u044e' # 0xC0 -> CYRILLIC SMALL LETTER YU - '\u0430' # 0xC1 -> CYRILLIC SMALL LETTER A - '\u0431' # 0xC2 -> CYRILLIC SMALL LETTER BE - '\u0446' # 0xC3 -> CYRILLIC SMALL LETTER TSE - '\u0434' # 0xC4 -> CYRILLIC SMALL LETTER DE - '\u0435' # 0xC5 -> CYRILLIC SMALL LETTER IE - '\u0444' # 0xC6 -> CYRILLIC SMALL LETTER EF - '\u0433' # 0xC7 -> CYRILLIC SMALL LETTER GHE - '\u0445' # 0xC8 -> CYRILLIC SMALL LETTER HA - '\u0438' # 0xC9 -> CYRILLIC SMALL LETTER I - '\u0439' # 0xCA -> CYRILLIC SMALL LETTER SHORT I - '\u043a' # 0xCB -> CYRILLIC SMALL LETTER KA - '\u043b' # 0xCC -> CYRILLIC SMALL LETTER EL - '\u043c' # 0xCD -> CYRILLIC SMALL LETTER EM - '\u043d' # 0xCE -> CYRILLIC SMALL LETTER EN - '\u043e' # 0xCF -> CYRILLIC SMALL LETTER O - '\u043f' # 0xD0 -> CYRILLIC SMALL LETTER PE - '\u044f' # 0xD1 -> CYRILLIC SMALL LETTER YA - '\u0440' # 0xD2 -> CYRILLIC SMALL LETTER ER - '\u0441' # 0xD3 -> CYRILLIC SMALL LETTER ES - '\u0442' # 0xD4 -> CYRILLIC SMALL LETTER TE - '\u0443' # 0xD5 -> CYRILLIC SMALL LETTER U - '\u0436' # 0xD6 -> CYRILLIC SMALL LETTER ZHE - '\u0432' # 0xD7 -> CYRILLIC SMALL LETTER VE - '\u044c' # 0xD8 -> CYRILLIC SMALL LETTER SOFT SIGN - '\u044b' # 0xD9 -> CYRILLIC SMALL LETTER YERU - '\u0437' # 0xDA -> CYRILLIC SMALL LETTER ZE - '\u0448' # 0xDB -> CYRILLIC SMALL LETTER SHA - '\u044d' # 0xDC -> CYRILLIC SMALL LETTER E - '\u0449' # 0xDD -> CYRILLIC SMALL LETTER SHCHA - '\u0447' # 0xDE -> CYRILLIC SMALL LETTER CHE - '\u044a' # 0xDF -> CYRILLIC SMALL LETTER HARD SIGN - '\u042e' # 0xE0 -> CYRILLIC CAPITAL LETTER YU - '\u0410' # 0xE1 -> CYRILLIC CAPITAL LETTER A - '\u0411' # 0xE2 -> CYRILLIC CAPITAL LETTER BE - '\u0426' # 0xE3 -> CYRILLIC CAPITAL LETTER TSE - '\u0414' # 0xE4 -> CYRILLIC CAPITAL LETTER DE - '\u0415' # 0xE5 -> CYRILLIC CAPITAL LETTER IE - '\u0424' # 0xE6 -> CYRILLIC CAPITAL LETTER EF - '\u0413' # 0xE7 -> CYRILLIC CAPITAL LETTER GHE - '\u0425' # 0xE8 -> CYRILLIC CAPITAL LETTER HA - '\u0418' # 0xE9 -> CYRILLIC CAPITAL LETTER I - '\u0419' # 0xEA -> CYRILLIC CAPITAL LETTER SHORT I - '\u041a' # 0xEB -> CYRILLIC CAPITAL LETTER KA - '\u041b' # 0xEC -> CYRILLIC CAPITAL LETTER EL - '\u041c' # 0xED -> CYRILLIC CAPITAL LETTER EM - '\u041d' # 0xEE -> CYRILLIC CAPITAL LETTER EN - '\u041e' # 0xEF -> CYRILLIC CAPITAL LETTER O - '\u041f' # 0xF0 -> CYRILLIC CAPITAL LETTER PE - '\u042f' # 0xF1 -> CYRILLIC CAPITAL LETTER YA - '\u0420' # 0xF2 -> CYRILLIC CAPITAL LETTER ER - '\u0421' # 0xF3 -> CYRILLIC CAPITAL LETTER ES - '\u0422' # 0xF4 -> CYRILLIC CAPITAL LETTER TE - '\u0423' # 0xF5 -> CYRILLIC CAPITAL LETTER U - '\u0416' # 0xF6 -> CYRILLIC CAPITAL LETTER ZHE - '\u0412' # 0xF7 -> CYRILLIC CAPITAL LETTER VE - '\u042c' # 0xF8 -> CYRILLIC CAPITAL LETTER SOFT SIGN - '\u042b' # 0xF9 -> CYRILLIC CAPITAL LETTER YERU - '\u0417' # 0xFA -> CYRILLIC CAPITAL LETTER ZE - '\u0428' # 0xFB -> CYRILLIC CAPITAL LETTER SHA - '\u042d' # 0xFC -> CYRILLIC CAPITAL LETTER E - '\u0429' # 0xFD -> CYRILLIC CAPITAL LETTER SHCHA - '\u0427' # 0xFE -> CYRILLIC CAPITAL LETTER CHE - '\u042a' # 0xFF -> CYRILLIC CAPITAL LETTER HARD SIGN -) - -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/kz1048.py --- a/Lib/encodings/kz1048.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,307 +0,0 @@ -""" Python Character Mapping Codec kz1048 generated from 'MAPPINGS/VENDORS/MISC/KZ1048.TXT' with gencodec.py. - -"""#" - -import codecs - -### Codec APIs - -class Codec(codecs.Codec): - - def encode(self, input, errors='strict'): - return codecs.charmap_encode(input, errors, encoding_table) - - def decode(self, input, errors='strict'): - return codecs.charmap_decode(input, errors, decoding_table) - -class IncrementalEncoder(codecs.IncrementalEncoder): - def encode(self, input, final=False): - return codecs.charmap_encode(input, self.errors, encoding_table)[0] - -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.charmap_decode(input, self.errors, decoding_table)[0] - -class StreamWriter(Codec, codecs.StreamWriter): - pass - -class StreamReader(Codec, codecs.StreamReader): - pass - -### encodings module API - -def getregentry(): - return codecs.CodecInfo( - name='kz1048', - encode=Codec().encode, - decode=Codec().decode, - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamreader=StreamReader, - streamwriter=StreamWriter, - ) - - -### Decoding Table - -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE - '\u0402' # 0x80 -> CYRILLIC CAPITAL LETTER DJE - '\u0403' # 0x81 -> CYRILLIC CAPITAL LETTER GJE - '\u201a' # 0x82 -> SINGLE LOW-9 QUOTATION MARK - '\u0453' # 0x83 -> CYRILLIC SMALL LETTER GJE - '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK - '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS - '\u2020' # 0x86 -> DAGGER - '\u2021' # 0x87 -> DOUBLE DAGGER - '\u20ac' # 0x88 -> EURO SIGN - '\u2030' # 0x89 -> PER MILLE SIGN - '\u0409' # 0x8A -> CYRILLIC CAPITAL LETTER LJE - '\u2039' # 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK - '\u040a' # 0x8C -> CYRILLIC CAPITAL LETTER NJE - '\u049a' # 0x8D -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER - '\u04ba' # 0x8E -> CYRILLIC CAPITAL LETTER SHHA - '\u040f' # 0x8F -> CYRILLIC CAPITAL LETTER DZHE - '\u0452' # 0x90 -> CYRILLIC SMALL LETTER DJE - '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK - '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK - '\u2022' # 0x95 -> BULLET - '\u2013' # 0x96 -> EN DASH - '\u2014' # 0x97 -> EM DASH - '\ufffe' # 0x98 -> UNDEFINED - '\u2122' # 0x99 -> TRADE MARK SIGN - '\u0459' # 0x9A -> CYRILLIC SMALL LETTER LJE - '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - '\u045a' # 0x9C -> CYRILLIC SMALL LETTER NJE - '\u049b' # 0x9D -> CYRILLIC SMALL LETTER KA WITH DESCENDER - '\u04bb' # 0x9E -> CYRILLIC SMALL LETTER SHHA - '\u045f' # 0x9F -> CYRILLIC SMALL LETTER DZHE - '\xa0' # 0xA0 -> NO-BREAK SPACE - '\u04b0' # 0xA1 -> CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE - '\u04b1' # 0xA2 -> CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE - '\u04d8' # 0xA3 -> CYRILLIC CAPITAL LETTER SCHWA - '\xa4' # 0xA4 -> CURRENCY SIGN - '\u04e8' # 0xA5 -> CYRILLIC CAPITAL LETTER BARRED O - '\xa6' # 0xA6 -> BROKEN BAR - '\xa7' # 0xA7 -> SECTION SIGN - '\u0401' # 0xA8 -> CYRILLIC CAPITAL LETTER IO - '\xa9' # 0xA9 -> COPYRIGHT SIGN - '\u0492' # 0xAA -> CYRILLIC CAPITAL LETTER GHE WITH STROKE - '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xac' # 0xAC -> NOT SIGN - '\xad' # 0xAD -> SOFT HYPHEN - '\xae' # 0xAE -> REGISTERED SIGN - '\u04ae' # 0xAF -> CYRILLIC CAPITAL LETTER STRAIGHT U - '\xb0' # 0xB0 -> DEGREE SIGN - '\xb1' # 0xB1 -> PLUS-MINUS SIGN - '\u0406' # 0xB2 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - '\u0456' # 0xB3 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I - '\u04e9' # 0xB4 -> CYRILLIC SMALL LETTER BARRED O - '\xb5' # 0xB5 -> MICRO SIGN - '\xb6' # 0xB6 -> PILCROW SIGN - '\xb7' # 0xB7 -> MIDDLE DOT - '\u0451' # 0xB8 -> CYRILLIC SMALL LETTER IO - '\u2116' # 0xB9 -> NUMERO SIGN - '\u0493' # 0xBA -> CYRILLIC SMALL LETTER GHE WITH STROKE - '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\u04d9' # 0xBC -> CYRILLIC SMALL LETTER SCHWA - '\u04a2' # 0xBD -> CYRILLIC CAPITAL LETTER EN WITH DESCENDER - '\u04a3' # 0xBE -> CYRILLIC SMALL LETTER EN WITH DESCENDER - '\u04af' # 0xBF -> CYRILLIC SMALL LETTER STRAIGHT U - '\u0410' # 0xC0 -> CYRILLIC CAPITAL LETTER A - '\u0411' # 0xC1 -> CYRILLIC CAPITAL LETTER BE - '\u0412' # 0xC2 -> CYRILLIC CAPITAL LETTER VE - '\u0413' # 0xC3 -> CYRILLIC CAPITAL LETTER GHE - '\u0414' # 0xC4 -> CYRILLIC CAPITAL LETTER DE - '\u0415' # 0xC5 -> CYRILLIC CAPITAL LETTER IE - '\u0416' # 0xC6 -> CYRILLIC CAPITAL LETTER ZHE - '\u0417' # 0xC7 -> CYRILLIC CAPITAL LETTER ZE - '\u0418' # 0xC8 -> CYRILLIC CAPITAL LETTER I - '\u0419' # 0xC9 -> CYRILLIC CAPITAL LETTER SHORT I - '\u041a' # 0xCA -> CYRILLIC CAPITAL LETTER KA - '\u041b' # 0xCB -> CYRILLIC CAPITAL LETTER EL - '\u041c' # 0xCC -> CYRILLIC CAPITAL LETTER EM - '\u041d' # 0xCD -> CYRILLIC CAPITAL LETTER EN - '\u041e' # 0xCE -> CYRILLIC CAPITAL LETTER O - '\u041f' # 0xCF -> CYRILLIC CAPITAL LETTER PE - '\u0420' # 0xD0 -> CYRILLIC CAPITAL LETTER ER - '\u0421' # 0xD1 -> CYRILLIC CAPITAL LETTER ES - '\u0422' # 0xD2 -> CYRILLIC CAPITAL LETTER TE - '\u0423' # 0xD3 -> CYRILLIC CAPITAL LETTER U - '\u0424' # 0xD4 -> CYRILLIC CAPITAL LETTER EF - '\u0425' # 0xD5 -> CYRILLIC CAPITAL LETTER HA - '\u0426' # 0xD6 -> CYRILLIC CAPITAL LETTER TSE - '\u0427' # 0xD7 -> CYRILLIC CAPITAL LETTER CHE - '\u0428' # 0xD8 -> CYRILLIC CAPITAL LETTER SHA - '\u0429' # 0xD9 -> CYRILLIC CAPITAL LETTER SHCHA - '\u042a' # 0xDA -> CYRILLIC CAPITAL LETTER HARD SIGN - '\u042b' # 0xDB -> CYRILLIC CAPITAL LETTER YERU - '\u042c' # 0xDC -> CYRILLIC CAPITAL LETTER SOFT SIGN - '\u042d' # 0xDD -> CYRILLIC CAPITAL LETTER E - '\u042e' # 0xDE -> CYRILLIC CAPITAL LETTER YU - '\u042f' # 0xDF -> CYRILLIC CAPITAL LETTER YA - '\u0430' # 0xE0 -> CYRILLIC SMALL LETTER A - '\u0431' # 0xE1 -> CYRILLIC SMALL LETTER BE - '\u0432' # 0xE2 -> CYRILLIC SMALL LETTER VE - '\u0433' # 0xE3 -> CYRILLIC SMALL LETTER GHE - '\u0434' # 0xE4 -> CYRILLIC SMALL LETTER DE - '\u0435' # 0xE5 -> CYRILLIC SMALL LETTER IE - '\u0436' # 0xE6 -> CYRILLIC SMALL LETTER ZHE - '\u0437' # 0xE7 -> CYRILLIC SMALL LETTER ZE - '\u0438' # 0xE8 -> CYRILLIC SMALL LETTER I - '\u0439' # 0xE9 -> CYRILLIC SMALL LETTER SHORT I - '\u043a' # 0xEA -> CYRILLIC SMALL LETTER KA - '\u043b' # 0xEB -> CYRILLIC SMALL LETTER EL - '\u043c' # 0xEC -> CYRILLIC SMALL LETTER EM - '\u043d' # 0xED -> CYRILLIC SMALL LETTER EN - '\u043e' # 0xEE -> CYRILLIC SMALL LETTER O - '\u043f' # 0xEF -> CYRILLIC SMALL LETTER PE - '\u0440' # 0xF0 -> CYRILLIC SMALL LETTER ER - '\u0441' # 0xF1 -> CYRILLIC SMALL LETTER ES - '\u0442' # 0xF2 -> CYRILLIC SMALL LETTER TE - '\u0443' # 0xF3 -> CYRILLIC SMALL LETTER U - '\u0444' # 0xF4 -> CYRILLIC SMALL LETTER EF - '\u0445' # 0xF5 -> CYRILLIC SMALL LETTER HA - '\u0446' # 0xF6 -> CYRILLIC SMALL LETTER TSE - '\u0447' # 0xF7 -> CYRILLIC SMALL LETTER CHE - '\u0448' # 0xF8 -> CYRILLIC SMALL LETTER SHA - '\u0449' # 0xF9 -> CYRILLIC SMALL LETTER SHCHA - '\u044a' # 0xFA -> CYRILLIC SMALL LETTER HARD SIGN - '\u044b' # 0xFB -> CYRILLIC SMALL LETTER YERU - '\u044c' # 0xFC -> CYRILLIC SMALL LETTER SOFT SIGN - '\u044d' # 0xFD -> CYRILLIC SMALL LETTER E - '\u044e' # 0xFE -> CYRILLIC SMALL LETTER YU - '\u044f' # 0xFF -> CYRILLIC SMALL LETTER YA -) - -### Encoding table -encoding_table = codecs.charmap_build(decoding_table) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/mac_latin2.py --- a/Lib/encodings/mac_latin2.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/mac_latin2.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -""" Python Character Mapping Codec mac_latin2 generated from 'MAPPINGS/VENDORS/MICSFT/MAC/LATIN2.TXT' with gencodec.py. +""" Python Character Mapping Codec generated from 'LATIN2.TXT' with gencodec.py. Written by Marc-Andre Lemburg (mal@lemburg.com). @@ -14,18 +14,18 @@ class Codec(codecs.Codec): def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) + return codecs.charmap_encode(input,errors,encoding_map) def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) + return codecs.charmap_decode(input,errors,decoding_map) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] + return codecs.charmap_encode(input,self.errors,encoding_map)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] + return codecs.charmap_decode(input,self.errors,decoding_map)[0] class StreamWriter(Codec,codecs.StreamWriter): pass @@ -46,267 +46,138 @@ streamwriter=StreamWriter, ) +### Decoding Map -### Decoding Table +decoding_map = codecs.make_identity_dict(range(256)) +decoding_map.update({ + 0x0080: 0x00c4, # LATIN CAPITAL LETTER A WITH DIAERESIS + 0x0081: 0x0100, # LATIN CAPITAL LETTER A WITH MACRON + 0x0082: 0x0101, # LATIN SMALL LETTER A WITH MACRON + 0x0083: 0x00c9, # LATIN CAPITAL LETTER E WITH ACUTE + 0x0084: 0x0104, # LATIN CAPITAL LETTER A WITH OGONEK + 0x0085: 0x00d6, # LATIN CAPITAL LETTER O WITH DIAERESIS + 0x0086: 0x00dc, # LATIN CAPITAL LETTER U WITH DIAERESIS + 0x0087: 0x00e1, # LATIN SMALL LETTER A WITH ACUTE + 0x0088: 0x0105, # LATIN SMALL LETTER A WITH OGONEK + 0x0089: 0x010c, # LATIN CAPITAL LETTER C WITH CARON + 0x008a: 0x00e4, # LATIN SMALL LETTER A WITH DIAERESIS + 0x008b: 0x010d, # LATIN SMALL LETTER C WITH CARON + 0x008c: 0x0106, # LATIN CAPITAL LETTER C WITH ACUTE + 0x008d: 0x0107, # LATIN SMALL LETTER C WITH ACUTE + 0x008e: 0x00e9, # LATIN SMALL LETTER E WITH ACUTE + 0x008f: 0x0179, # LATIN CAPITAL LETTER Z WITH ACUTE + 0x0090: 0x017a, # LATIN SMALL LETTER Z WITH ACUTE + 0x0091: 0x010e, # LATIN CAPITAL LETTER D WITH CARON + 0x0092: 0x00ed, # LATIN SMALL LETTER I WITH ACUTE + 0x0093: 0x010f, # LATIN SMALL LETTER D WITH CARON + 0x0094: 0x0112, # LATIN CAPITAL LETTER E WITH MACRON + 0x0095: 0x0113, # LATIN SMALL LETTER E WITH MACRON + 0x0096: 0x0116, # LATIN CAPITAL LETTER E WITH DOT ABOVE + 0x0097: 0x00f3, # LATIN SMALL LETTER O WITH ACUTE + 0x0098: 0x0117, # LATIN SMALL LETTER E WITH DOT ABOVE + 0x0099: 0x00f4, # LATIN SMALL LETTER O WITH CIRCUMFLEX + 0x009a: 0x00f6, # LATIN SMALL LETTER O WITH DIAERESIS + 0x009b: 0x00f5, # LATIN SMALL LETTER O WITH TILDE + 0x009c: 0x00fa, # LATIN SMALL LETTER U WITH ACUTE + 0x009d: 0x011a, # LATIN CAPITAL LETTER E WITH CARON + 0x009e: 0x011b, # LATIN SMALL LETTER E WITH CARON + 0x009f: 0x00fc, # LATIN SMALL LETTER U WITH DIAERESIS + 0x00a0: 0x2020, # DAGGER + 0x00a1: 0x00b0, # DEGREE SIGN + 0x00a2: 0x0118, # LATIN CAPITAL LETTER E WITH OGONEK + 0x00a4: 0x00a7, # SECTION SIGN + 0x00a5: 0x2022, # BULLET + 0x00a6: 0x00b6, # PILCROW SIGN + 0x00a7: 0x00df, # LATIN SMALL LETTER SHARP S + 0x00a8: 0x00ae, # REGISTERED SIGN + 0x00aa: 0x2122, # TRADE MARK SIGN + 0x00ab: 0x0119, # LATIN SMALL LETTER E WITH OGONEK + 0x00ac: 0x00a8, # DIAERESIS + 0x00ad: 0x2260, # NOT EQUAL TO + 0x00ae: 0x0123, # LATIN SMALL LETTER G WITH CEDILLA + 0x00af: 0x012e, # LATIN CAPITAL LETTER I WITH OGONEK + 0x00b0: 0x012f, # LATIN SMALL LETTER I WITH OGONEK + 0x00b1: 0x012a, # LATIN CAPITAL LETTER I WITH MACRON + 0x00b2: 0x2264, # LESS-THAN OR EQUAL TO + 0x00b3: 0x2265, # GREATER-THAN OR EQUAL TO + 0x00b4: 0x012b, # LATIN SMALL LETTER I WITH MACRON + 0x00b5: 0x0136, # LATIN CAPITAL LETTER K WITH CEDILLA + 0x00b6: 0x2202, # PARTIAL DIFFERENTIAL + 0x00b7: 0x2211, # N-ARY SUMMATION + 0x00b8: 0x0142, # LATIN SMALL LETTER L WITH STROKE + 0x00b9: 0x013b, # LATIN CAPITAL LETTER L WITH CEDILLA + 0x00ba: 0x013c, # LATIN SMALL LETTER L WITH CEDILLA + 0x00bb: 0x013d, # LATIN CAPITAL LETTER L WITH CARON + 0x00bc: 0x013e, # LATIN SMALL LETTER L WITH CARON + 0x00bd: 0x0139, # LATIN CAPITAL LETTER L WITH ACUTE + 0x00be: 0x013a, # LATIN SMALL LETTER L WITH ACUTE + 0x00bf: 0x0145, # LATIN CAPITAL LETTER N WITH CEDILLA + 0x00c0: 0x0146, # LATIN SMALL LETTER N WITH CEDILLA + 0x00c1: 0x0143, # LATIN CAPITAL LETTER N WITH ACUTE + 0x00c2: 0x00ac, # NOT SIGN + 0x00c3: 0x221a, # SQUARE ROOT + 0x00c4: 0x0144, # LATIN SMALL LETTER N WITH ACUTE + 0x00c5: 0x0147, # LATIN CAPITAL LETTER N WITH CARON + 0x00c6: 0x2206, # INCREMENT + 0x00c7: 0x00ab, # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + 0x00c8: 0x00bb, # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + 0x00c9: 0x2026, # HORIZONTAL ELLIPSIS + 0x00ca: 0x00a0, # NO-BREAK SPACE + 0x00cb: 0x0148, # LATIN SMALL LETTER N WITH CARON + 0x00cc: 0x0150, # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + 0x00cd: 0x00d5, # LATIN CAPITAL LETTER O WITH TILDE + 0x00ce: 0x0151, # LATIN SMALL LETTER O WITH DOUBLE ACUTE + 0x00cf: 0x014c, # LATIN CAPITAL LETTER O WITH MACRON + 0x00d0: 0x2013, # EN DASH + 0x00d1: 0x2014, # EM DASH + 0x00d2: 0x201c, # LEFT DOUBLE QUOTATION MARK + 0x00d3: 0x201d, # RIGHT DOUBLE QUOTATION MARK + 0x00d4: 0x2018, # LEFT SINGLE QUOTATION MARK + 0x00d5: 0x2019, # RIGHT SINGLE QUOTATION MARK + 0x00d6: 0x00f7, # DIVISION SIGN + 0x00d7: 0x25ca, # LOZENGE + 0x00d8: 0x014d, # LATIN SMALL LETTER O WITH MACRON + 0x00d9: 0x0154, # LATIN CAPITAL LETTER R WITH ACUTE + 0x00da: 0x0155, # LATIN SMALL LETTER R WITH ACUTE + 0x00db: 0x0158, # LATIN CAPITAL LETTER R WITH CARON + 0x00dc: 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 0x00dd: 0x203a, # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + 0x00de: 0x0159, # LATIN SMALL LETTER R WITH CARON + 0x00df: 0x0156, # LATIN CAPITAL LETTER R WITH CEDILLA + 0x00e0: 0x0157, # LATIN SMALL LETTER R WITH CEDILLA + 0x00e1: 0x0160, # LATIN CAPITAL LETTER S WITH CARON + 0x00e2: 0x201a, # SINGLE LOW-9 QUOTATION MARK + 0x00e3: 0x201e, # DOUBLE LOW-9 QUOTATION MARK + 0x00e4: 0x0161, # LATIN SMALL LETTER S WITH CARON + 0x00e5: 0x015a, # LATIN CAPITAL LETTER S WITH ACUTE + 0x00e6: 0x015b, # LATIN SMALL LETTER S WITH ACUTE + 0x00e7: 0x00c1, # LATIN CAPITAL LETTER A WITH ACUTE + 0x00e8: 0x0164, # LATIN CAPITAL LETTER T WITH CARON + 0x00e9: 0x0165, # LATIN SMALL LETTER T WITH CARON + 0x00ea: 0x00cd, # LATIN CAPITAL LETTER I WITH ACUTE + 0x00eb: 0x017d, # LATIN CAPITAL LETTER Z WITH CARON + 0x00ec: 0x017e, # LATIN SMALL LETTER Z WITH CARON + 0x00ed: 0x016a, # LATIN CAPITAL LETTER U WITH MACRON + 0x00ee: 0x00d3, # LATIN CAPITAL LETTER O WITH ACUTE + 0x00ef: 0x00d4, # LATIN CAPITAL LETTER O WITH CIRCUMFLEX + 0x00f0: 0x016b, # LATIN SMALL LETTER U WITH MACRON + 0x00f1: 0x016e, # LATIN CAPITAL LETTER U WITH RING ABOVE + 0x00f2: 0x00da, # LATIN CAPITAL LETTER U WITH ACUTE + 0x00f3: 0x016f, # LATIN SMALL LETTER U WITH RING ABOVE + 0x00f4: 0x0170, # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + 0x00f5: 0x0171, # LATIN SMALL LETTER U WITH DOUBLE ACUTE + 0x00f6: 0x0172, # LATIN CAPITAL LETTER U WITH OGONEK + 0x00f7: 0x0173, # LATIN SMALL LETTER U WITH OGONEK + 0x00f8: 0x00dd, # LATIN CAPITAL LETTER Y WITH ACUTE + 0x00f9: 0x00fd, # LATIN SMALL LETTER Y WITH ACUTE + 0x00fa: 0x0137, # LATIN SMALL LETTER K WITH CEDILLA + 0x00fb: 0x017b, # LATIN CAPITAL LETTER Z WITH DOT ABOVE + 0x00fc: 0x0141, # LATIN CAPITAL LETTER L WITH STROKE + 0x00fd: 0x017c, # LATIN SMALL LETTER Z WITH DOT ABOVE + 0x00fe: 0x0122, # LATIN CAPITAL LETTER G WITH CEDILLA + 0x00ff: 0x02c7, # CARON +}) -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE - '\xc4' # 0x80 -> LATIN CAPITAL LETTER A WITH DIAERESIS - '\u0100' # 0x81 -> LATIN CAPITAL LETTER A WITH MACRON - '\u0101' # 0x82 -> LATIN SMALL LETTER A WITH MACRON - '\xc9' # 0x83 -> LATIN CAPITAL LETTER E WITH ACUTE - '\u0104' # 0x84 -> LATIN CAPITAL LETTER A WITH OGONEK - '\xd6' # 0x85 -> LATIN CAPITAL LETTER O WITH DIAERESIS - '\xdc' # 0x86 -> LATIN CAPITAL LETTER U WITH DIAERESIS - '\xe1' # 0x87 -> LATIN SMALL LETTER A WITH ACUTE - '\u0105' # 0x88 -> LATIN SMALL LETTER A WITH OGONEK - '\u010c' # 0x89 -> LATIN CAPITAL LETTER C WITH CARON - '\xe4' # 0x8A -> LATIN SMALL LETTER A WITH DIAERESIS - '\u010d' # 0x8B -> LATIN SMALL LETTER C WITH CARON - '\u0106' # 0x8C -> LATIN CAPITAL LETTER C WITH ACUTE - '\u0107' # 0x8D -> LATIN SMALL LETTER C WITH ACUTE - '\xe9' # 0x8E -> LATIN SMALL LETTER E WITH ACUTE - '\u0179' # 0x8F -> LATIN CAPITAL LETTER Z WITH ACUTE - '\u017a' # 0x90 -> LATIN SMALL LETTER Z WITH ACUTE - '\u010e' # 0x91 -> LATIN CAPITAL LETTER D WITH CARON - '\xed' # 0x92 -> LATIN SMALL LETTER I WITH ACUTE - '\u010f' # 0x93 -> LATIN SMALL LETTER D WITH CARON - '\u0112' # 0x94 -> LATIN CAPITAL LETTER E WITH MACRON - '\u0113' # 0x95 -> LATIN SMALL LETTER E WITH MACRON - '\u0116' # 0x96 -> LATIN CAPITAL LETTER E WITH DOT ABOVE - '\xf3' # 0x97 -> LATIN SMALL LETTER O WITH ACUTE - '\u0117' # 0x98 -> LATIN SMALL LETTER E WITH DOT ABOVE - '\xf4' # 0x99 -> LATIN SMALL LETTER O WITH CIRCUMFLEX - '\xf6' # 0x9A -> LATIN SMALL LETTER O WITH DIAERESIS - '\xf5' # 0x9B -> LATIN SMALL LETTER O WITH TILDE - '\xfa' # 0x9C -> LATIN SMALL LETTER U WITH ACUTE - '\u011a' # 0x9D -> LATIN CAPITAL LETTER E WITH CARON - '\u011b' # 0x9E -> LATIN SMALL LETTER E WITH CARON - '\xfc' # 0x9F -> LATIN SMALL LETTER U WITH DIAERESIS - '\u2020' # 0xA0 -> DAGGER - '\xb0' # 0xA1 -> DEGREE SIGN - '\u0118' # 0xA2 -> LATIN CAPITAL LETTER E WITH OGONEK - '\xa3' # 0xA3 -> POUND SIGN - '\xa7' # 0xA4 -> SECTION SIGN - '\u2022' # 0xA5 -> BULLET - '\xb6' # 0xA6 -> PILCROW SIGN - '\xdf' # 0xA7 -> LATIN SMALL LETTER SHARP S - '\xae' # 0xA8 -> REGISTERED SIGN - '\xa9' # 0xA9 -> COPYRIGHT SIGN - '\u2122' # 0xAA -> TRADE MARK SIGN - '\u0119' # 0xAB -> LATIN SMALL LETTER E WITH OGONEK - '\xa8' # 0xAC -> DIAERESIS - '\u2260' # 0xAD -> NOT EQUAL TO - '\u0123' # 0xAE -> LATIN SMALL LETTER G WITH CEDILLA - '\u012e' # 0xAF -> LATIN CAPITAL LETTER I WITH OGONEK - '\u012f' # 0xB0 -> LATIN SMALL LETTER I WITH OGONEK - '\u012a' # 0xB1 -> LATIN CAPITAL LETTER I WITH MACRON - '\u2264' # 0xB2 -> LESS-THAN OR EQUAL TO - '\u2265' # 0xB3 -> GREATER-THAN OR EQUAL TO - '\u012b' # 0xB4 -> LATIN SMALL LETTER I WITH MACRON - '\u0136' # 0xB5 -> LATIN CAPITAL LETTER K WITH CEDILLA - '\u2202' # 0xB6 -> PARTIAL DIFFERENTIAL - '\u2211' # 0xB7 -> N-ARY SUMMATION - '\u0142' # 0xB8 -> LATIN SMALL LETTER L WITH STROKE - '\u013b' # 0xB9 -> LATIN CAPITAL LETTER L WITH CEDILLA - '\u013c' # 0xBA -> LATIN SMALL LETTER L WITH CEDILLA - '\u013d' # 0xBB -> LATIN CAPITAL LETTER L WITH CARON - '\u013e' # 0xBC -> LATIN SMALL LETTER L WITH CARON - '\u0139' # 0xBD -> LATIN CAPITAL LETTER L WITH ACUTE - '\u013a' # 0xBE -> LATIN SMALL LETTER L WITH ACUTE - '\u0145' # 0xBF -> LATIN CAPITAL LETTER N WITH CEDILLA - '\u0146' # 0xC0 -> LATIN SMALL LETTER N WITH CEDILLA - '\u0143' # 0xC1 -> LATIN CAPITAL LETTER N WITH ACUTE - '\xac' # 0xC2 -> NOT SIGN - '\u221a' # 0xC3 -> SQUARE ROOT - '\u0144' # 0xC4 -> LATIN SMALL LETTER N WITH ACUTE - '\u0147' # 0xC5 -> LATIN CAPITAL LETTER N WITH CARON - '\u2206' # 0xC6 -> INCREMENT - '\xab' # 0xC7 -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xbb' # 0xC8 -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\u2026' # 0xC9 -> HORIZONTAL ELLIPSIS - '\xa0' # 0xCA -> NO-BREAK SPACE - '\u0148' # 0xCB -> LATIN SMALL LETTER N WITH CARON - '\u0150' # 0xCC -> LATIN CAPITAL LETTER O WITH DOUBLE ACUTE - '\xd5' # 0xCD -> LATIN CAPITAL LETTER O WITH TILDE - '\u0151' # 0xCE -> LATIN SMALL LETTER O WITH DOUBLE ACUTE - '\u014c' # 0xCF -> LATIN CAPITAL LETTER O WITH MACRON - '\u2013' # 0xD0 -> EN DASH - '\u2014' # 0xD1 -> EM DASH - '\u201c' # 0xD2 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0xD3 -> RIGHT DOUBLE QUOTATION MARK - '\u2018' # 0xD4 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0xD5 -> RIGHT SINGLE QUOTATION MARK - '\xf7' # 0xD6 -> DIVISION SIGN - '\u25ca' # 0xD7 -> LOZENGE - '\u014d' # 0xD8 -> LATIN SMALL LETTER O WITH MACRON - '\u0154' # 0xD9 -> LATIN CAPITAL LETTER R WITH ACUTE - '\u0155' # 0xDA -> LATIN SMALL LETTER R WITH ACUTE - '\u0158' # 0xDB -> LATIN CAPITAL LETTER R WITH CARON - '\u2039' # 0xDC -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK - '\u203a' # 0xDD -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - '\u0159' # 0xDE -> LATIN SMALL LETTER R WITH CARON - '\u0156' # 0xDF -> LATIN CAPITAL LETTER R WITH CEDILLA - '\u0157' # 0xE0 -> LATIN SMALL LETTER R WITH CEDILLA - '\u0160' # 0xE1 -> LATIN CAPITAL LETTER S WITH CARON - '\u201a' # 0xE2 -> SINGLE LOW-9 QUOTATION MARK - '\u201e' # 0xE3 -> DOUBLE LOW-9 QUOTATION MARK - '\u0161' # 0xE4 -> LATIN SMALL LETTER S WITH CARON - '\u015a' # 0xE5 -> LATIN CAPITAL LETTER S WITH ACUTE - '\u015b' # 0xE6 -> LATIN SMALL LETTER S WITH ACUTE - '\xc1' # 0xE7 -> LATIN CAPITAL LETTER A WITH ACUTE - '\u0164' # 0xE8 -> LATIN CAPITAL LETTER T WITH CARON - '\u0165' # 0xE9 -> LATIN SMALL LETTER T WITH CARON - '\xcd' # 0xEA -> LATIN CAPITAL LETTER I WITH ACUTE - '\u017d' # 0xEB -> LATIN CAPITAL LETTER Z WITH CARON - '\u017e' # 0xEC -> LATIN SMALL LETTER Z WITH CARON - '\u016a' # 0xED -> LATIN CAPITAL LETTER U WITH MACRON - '\xd3' # 0xEE -> LATIN CAPITAL LETTER O WITH ACUTE - '\xd4' # 0xEF -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\u016b' # 0xF0 -> LATIN SMALL LETTER U WITH MACRON - '\u016e' # 0xF1 -> LATIN CAPITAL LETTER U WITH RING ABOVE - '\xda' # 0xF2 -> LATIN CAPITAL LETTER U WITH ACUTE - '\u016f' # 0xF3 -> LATIN SMALL LETTER U WITH RING ABOVE - '\u0170' # 0xF4 -> LATIN CAPITAL LETTER U WITH DOUBLE ACUTE - '\u0171' # 0xF5 -> LATIN SMALL LETTER U WITH DOUBLE ACUTE - '\u0172' # 0xF6 -> LATIN CAPITAL LETTER U WITH OGONEK - '\u0173' # 0xF7 -> LATIN SMALL LETTER U WITH OGONEK - '\xdd' # 0xF8 -> LATIN CAPITAL LETTER Y WITH ACUTE - '\xfd' # 0xF9 -> LATIN SMALL LETTER Y WITH ACUTE - '\u0137' # 0xFA -> LATIN SMALL LETTER K WITH CEDILLA - '\u017b' # 0xFB -> LATIN CAPITAL LETTER Z WITH DOT ABOVE - '\u0141' # 0xFC -> LATIN CAPITAL LETTER L WITH STROKE - '\u017c' # 0xFD -> LATIN SMALL LETTER Z WITH DOT ABOVE - '\u0122' # 0xFE -> LATIN CAPITAL LETTER G WITH CEDILLA - '\u02c7' # 0xFF -> CARON -) +### Encoding Map -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) +encoding_map = codecs.make_encoding_map(decoding_map) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/palmos.py --- a/Lib/encodings/palmos.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/palmos.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,18 +10,18 @@ class Codec(codecs.Codec): def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) + return codecs.charmap_encode(input,errors,encoding_map) def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) + return codecs.charmap_decode(input,errors,decoding_map) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] + return codecs.charmap_encode(input,self.errors,encoding_map)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] + return codecs.charmap_decode(input,self.errors,decoding_map)[0] class StreamWriter(Codec,codecs.StreamWriter): pass @@ -42,267 +42,42 @@ streamwriter=StreamWriter, ) +### Decoding Map -### Decoding Table +decoding_map = codecs.make_identity_dict(range(256)) -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE - '\u20ac' # 0x80 -> EURO SIGN - '\x81' # 0x81 -> - '\u201a' # 0x82 -> SINGLE LOW-9 QUOTATION MARK - '\u0192' # 0x83 -> LATIN SMALL LETTER F WITH HOOK - '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK - '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS - '\u2020' # 0x86 -> DAGGER - '\u2021' # 0x87 -> DOUBLE DAGGER - '\u02c6' # 0x88 -> MODIFIER LETTER CIRCUMFLEX ACCENT - '\u2030' # 0x89 -> PER MILLE SIGN - '\u0160' # 0x8A -> LATIN CAPITAL LETTER S WITH CARON - '\u2039' # 0x8B -> SINGLE LEFT-POINTING ANGLE QUOTATION MARK - '\u0152' # 0x8C -> LATIN CAPITAL LIGATURE OE - '\u2666' # 0x8D -> BLACK DIAMOND SUIT - '\u2663' # 0x8E -> BLACK CLUB SUIT - '\u2665' # 0x8F -> BLACK HEART SUIT - '\u2660' # 0x90 -> BLACK SPADE SUIT - '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK - '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK - '\u2022' # 0x95 -> BULLET - '\u2013' # 0x96 -> EN DASH - '\u2014' # 0x97 -> EM DASH - '\u02dc' # 0x98 -> SMALL TILDE - '\u2122' # 0x99 -> TRADE MARK SIGN - '\u0161' # 0x9A -> LATIN SMALL LETTER S WITH CARON - '\x9b' # 0x9B -> - '\u0153' # 0x9C -> LATIN SMALL LIGATURE OE - '\x9d' # 0x9D -> - '\x9e' # 0x9E -> - '\u0178' # 0x9F -> LATIN CAPITAL LETTER Y WITH DIAERESIS - '\xa0' # 0xA0 -> NO-BREAK SPACE - '\xa1' # 0xA1 -> INVERTED EXCLAMATION MARK - '\xa2' # 0xA2 -> CENT SIGN - '\xa3' # 0xA3 -> POUND SIGN - '\xa4' # 0xA4 -> CURRENCY SIGN - '\xa5' # 0xA5 -> YEN SIGN - '\xa6' # 0xA6 -> BROKEN BAR - '\xa7' # 0xA7 -> SECTION SIGN - '\xa8' # 0xA8 -> DIAERESIS - '\xa9' # 0xA9 -> COPYRIGHT SIGN - '\xaa' # 0xAA -> FEMININE ORDINAL INDICATOR - '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xac' # 0xAC -> NOT SIGN - '\xad' # 0xAD -> SOFT HYPHEN - '\xae' # 0xAE -> REGISTERED SIGN - '\xaf' # 0xAF -> MACRON - '\xb0' # 0xB0 -> DEGREE SIGN - '\xb1' # 0xB1 -> PLUS-MINUS SIGN - '\xb2' # 0xB2 -> SUPERSCRIPT TWO - '\xb3' # 0xB3 -> SUPERSCRIPT THREE - '\xb4' # 0xB4 -> ACUTE ACCENT - '\xb5' # 0xB5 -> MICRO SIGN - '\xb6' # 0xB6 -> PILCROW SIGN - '\xb7' # 0xB7 -> MIDDLE DOT - '\xb8' # 0xB8 -> CEDILLA - '\xb9' # 0xB9 -> SUPERSCRIPT ONE - '\xba' # 0xBA -> MASCULINE ORDINAL INDICATOR - '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xbc' # 0xBC -> VULGAR FRACTION ONE QUARTER - '\xbd' # 0xBD -> VULGAR FRACTION ONE HALF - '\xbe' # 0xBE -> VULGAR FRACTION THREE QUARTERS - '\xbf' # 0xBF -> INVERTED QUESTION MARK - '\xc0' # 0xC0 -> LATIN CAPITAL LETTER A WITH GRAVE - '\xc1' # 0xC1 -> LATIN CAPITAL LETTER A WITH ACUTE - '\xc2' # 0xC2 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX - '\xc3' # 0xC3 -> LATIN CAPITAL LETTER A WITH TILDE - '\xc4' # 0xC4 -> LATIN CAPITAL LETTER A WITH DIAERESIS - '\xc5' # 0xC5 -> LATIN CAPITAL LETTER A WITH RING ABOVE - '\xc6' # 0xC6 -> LATIN CAPITAL LETTER AE - '\xc7' # 0xC7 -> LATIN CAPITAL LETTER C WITH CEDILLA - '\xc8' # 0xC8 -> LATIN CAPITAL LETTER E WITH GRAVE - '\xc9' # 0xC9 -> LATIN CAPITAL LETTER E WITH ACUTE - '\xca' # 0xCA -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX - '\xcb' # 0xCB -> LATIN CAPITAL LETTER E WITH DIAERESIS - '\xcc' # 0xCC -> LATIN CAPITAL LETTER I WITH GRAVE - '\xcd' # 0xCD -> LATIN CAPITAL LETTER I WITH ACUTE - '\xce' # 0xCE -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX - '\xcf' # 0xCF -> LATIN CAPITAL LETTER I WITH DIAERESIS - '\xd0' # 0xD0 -> LATIN CAPITAL LETTER ETH (Icelandic) - '\xd1' # 0xD1 -> LATIN CAPITAL LETTER N WITH TILDE - '\xd2' # 0xD2 -> LATIN CAPITAL LETTER O WITH GRAVE - '\xd3' # 0xD3 -> LATIN CAPITAL LETTER O WITH ACUTE - '\xd4' # 0xD4 -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX - '\xd5' # 0xD5 -> LATIN CAPITAL LETTER O WITH TILDE - '\xd6' # 0xD6 -> LATIN CAPITAL LETTER O WITH DIAERESIS - '\xd7' # 0xD7 -> MULTIPLICATION SIGN - '\xd8' # 0xD8 -> LATIN CAPITAL LETTER O WITH STROKE - '\xd9' # 0xD9 -> LATIN CAPITAL LETTER U WITH GRAVE - '\xda' # 0xDA -> LATIN CAPITAL LETTER U WITH ACUTE - '\xdb' # 0xDB -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX - '\xdc' # 0xDC -> LATIN CAPITAL LETTER U WITH DIAERESIS - '\xdd' # 0xDD -> LATIN CAPITAL LETTER Y WITH ACUTE - '\xde' # 0xDE -> LATIN CAPITAL LETTER THORN (Icelandic) - '\xdf' # 0xDF -> LATIN SMALL LETTER SHARP S (German) - '\xe0' # 0xE0 -> LATIN SMALL LETTER A WITH GRAVE - '\xe1' # 0xE1 -> LATIN SMALL LETTER A WITH ACUTE - '\xe2' # 0xE2 -> LATIN SMALL LETTER A WITH CIRCUMFLEX - '\xe3' # 0xE3 -> LATIN SMALL LETTER A WITH TILDE - '\xe4' # 0xE4 -> LATIN SMALL LETTER A WITH DIAERESIS - '\xe5' # 0xE5 -> LATIN SMALL LETTER A WITH RING ABOVE - '\xe6' # 0xE6 -> LATIN SMALL LETTER AE - '\xe7' # 0xE7 -> LATIN SMALL LETTER C WITH CEDILLA - '\xe8' # 0xE8 -> LATIN SMALL LETTER E WITH GRAVE - '\xe9' # 0xE9 -> LATIN SMALL LETTER E WITH ACUTE - '\xea' # 0xEA -> LATIN SMALL LETTER E WITH CIRCUMFLEX - '\xeb' # 0xEB -> LATIN SMALL LETTER E WITH DIAERESIS - '\xec' # 0xEC -> LATIN SMALL LETTER I WITH GRAVE - '\xed' # 0xED -> LATIN SMALL LETTER I WITH ACUTE - '\xee' # 0xEE -> LATIN SMALL LETTER I WITH CIRCUMFLEX - '\xef' # 0xEF -> LATIN SMALL LETTER I WITH DIAERESIS - '\xf0' # 0xF0 -> LATIN SMALL LETTER ETH (Icelandic) - '\xf1' # 0xF1 -> LATIN SMALL LETTER N WITH TILDE - '\xf2' # 0xF2 -> LATIN SMALL LETTER O WITH GRAVE - '\xf3' # 0xF3 -> LATIN SMALL LETTER O WITH ACUTE - '\xf4' # 0xF4 -> LATIN SMALL LETTER O WITH CIRCUMFLEX - '\xf5' # 0xF5 -> LATIN SMALL LETTER O WITH TILDE - '\xf6' # 0xF6 -> LATIN SMALL LETTER O WITH DIAERESIS - '\xf7' # 0xF7 -> DIVISION SIGN - '\xf8' # 0xF8 -> LATIN SMALL LETTER O WITH STROKE - '\xf9' # 0xF9 -> LATIN SMALL LETTER U WITH GRAVE - '\xfa' # 0xFA -> LATIN SMALL LETTER U WITH ACUTE - '\xfb' # 0xFB -> LATIN SMALL LETTER U WITH CIRCUMFLEX - '\xfc' # 0xFC -> LATIN SMALL LETTER U WITH DIAERESIS - '\xfd' # 0xFD -> LATIN SMALL LETTER Y WITH ACUTE - '\xfe' # 0xFE -> LATIN SMALL LETTER THORN (Icelandic) - '\xff' # 0xFF -> LATIN SMALL LETTER Y WITH DIAERESIS -) +# The PalmOS character set is mostly iso-8859-1 with some differences. +decoding_map.update({ + 0x0080: 0x20ac, # EURO SIGN + 0x0082: 0x201a, # SINGLE LOW-9 QUOTATION MARK + 0x0083: 0x0192, # LATIN SMALL LETTER F WITH HOOK + 0x0084: 0x201e, # DOUBLE LOW-9 QUOTATION MARK + 0x0085: 0x2026, # HORIZONTAL ELLIPSIS + 0x0086: 0x2020, # DAGGER + 0x0087: 0x2021, # DOUBLE DAGGER + 0x0088: 0x02c6, # MODIFIER LETTER CIRCUMFLEX ACCENT + 0x0089: 0x2030, # PER MILLE SIGN + 0x008a: 0x0160, # LATIN CAPITAL LETTER S WITH CARON + 0x008b: 0x2039, # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + 0x008c: 0x0152, # LATIN CAPITAL LIGATURE OE + 0x008d: 0x2666, # BLACK DIAMOND SUIT + 0x008e: 0x2663, # BLACK CLUB SUIT + 0x008f: 0x2665, # BLACK HEART SUIT + 0x0090: 0x2660, # BLACK SPADE SUIT + 0x0091: 0x2018, # LEFT SINGLE QUOTATION MARK + 0x0092: 0x2019, # RIGHT SINGLE QUOTATION MARK + 0x0093: 0x201c, # LEFT DOUBLE QUOTATION MARK + 0x0094: 0x201d, # RIGHT DOUBLE QUOTATION MARK + 0x0095: 0x2022, # BULLET + 0x0096: 0x2013, # EN DASH + 0x0097: 0x2014, # EM DASH + 0x0098: 0x02dc, # SMALL TILDE + 0x0099: 0x2122, # TRADE MARK SIGN + 0x009a: 0x0161, # LATIN SMALL LETTER S WITH CARON + 0x009c: 0x0153, # LATIN SMALL LIGATURE OE + 0x009f: 0x0178, # LATIN CAPITAL LETTER Y WITH DIAERESIS +}) -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) +### Encoding Map + +encoding_map = codecs.make_encoding_map(decoding_map) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/ptcp154.py --- a/Lib/encodings/ptcp154.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/ptcp154.py Mon Jan 25 17:05:13 2016 +0100 @@ -14,18 +14,18 @@ class Codec(codecs.Codec): def encode(self,input,errors='strict'): - return codecs.charmap_encode(input,errors,encoding_table) + return codecs.charmap_encode(input,errors,encoding_map) def decode(self,input,errors='strict'): - return codecs.charmap_decode(input,errors,decoding_table) + return codecs.charmap_decode(input,errors,decoding_map) class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): - return codecs.charmap_encode(input,self.errors,encoding_table)[0] + return codecs.charmap_encode(input,self.errors,encoding_map)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): - return codecs.charmap_decode(input,self.errors,decoding_table)[0] + return codecs.charmap_decode(input,self.errors,decoding_map)[0] class StreamWriter(Codec,codecs.StreamWriter): pass @@ -46,267 +46,130 @@ streamwriter=StreamWriter, ) +### Decoding Map -### Decoding Table +decoding_map = codecs.make_identity_dict(range(256)) +decoding_map.update({ + 0x0080: 0x0496, # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER + 0x0081: 0x0492, # CYRILLIC CAPITAL LETTER GHE WITH STROKE + 0x0082: 0x04ee, # CYRILLIC CAPITAL LETTER U WITH MACRON + 0x0083: 0x0493, # CYRILLIC SMALL LETTER GHE WITH STROKE + 0x0084: 0x201e, # DOUBLE LOW-9 QUOTATION MARK + 0x0085: 0x2026, # HORIZONTAL ELLIPSIS + 0x0086: 0x04b6, # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER + 0x0087: 0x04ae, # CYRILLIC CAPITAL LETTER STRAIGHT U + 0x0088: 0x04b2, # CYRILLIC CAPITAL LETTER HA WITH DESCENDER + 0x0089: 0x04af, # CYRILLIC SMALL LETTER STRAIGHT U + 0x008a: 0x04a0, # CYRILLIC CAPITAL LETTER BASHKIR KA + 0x008b: 0x04e2, # CYRILLIC CAPITAL LETTER I WITH MACRON + 0x008c: 0x04a2, # CYRILLIC CAPITAL LETTER EN WITH DESCENDER + 0x008d: 0x049a, # CYRILLIC CAPITAL LETTER KA WITH DESCENDER + 0x008e: 0x04ba, # CYRILLIC CAPITAL LETTER SHHA + 0x008f: 0x04b8, # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE + 0x0090: 0x0497, # CYRILLIC SMALL LETTER ZHE WITH DESCENDER + 0x0091: 0x2018, # LEFT SINGLE QUOTATION MARK + 0x0092: 0x2019, # RIGHT SINGLE QUOTATION MARK + 0x0093: 0x201c, # LEFT DOUBLE QUOTATION MARK + 0x0094: 0x201d, # RIGHT DOUBLE QUOTATION MARK + 0x0095: 0x2022, # BULLET + 0x0096: 0x2013, # EN DASH + 0x0097: 0x2014, # EM DASH + 0x0098: 0x04b3, # CYRILLIC SMALL LETTER HA WITH DESCENDER + 0x0099: 0x04b7, # CYRILLIC SMALL LETTER CHE WITH DESCENDER + 0x009a: 0x04a1, # CYRILLIC SMALL LETTER BASHKIR KA + 0x009b: 0x04e3, # CYRILLIC SMALL LETTER I WITH MACRON + 0x009c: 0x04a3, # CYRILLIC SMALL LETTER EN WITH DESCENDER + 0x009d: 0x049b, # CYRILLIC SMALL LETTER KA WITH DESCENDER + 0x009e: 0x04bb, # CYRILLIC SMALL LETTER SHHA + 0x009f: 0x04b9, # CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE + 0x00a1: 0x040e, # CYRILLIC CAPITAL LETTER SHORT U (Byelorussian) + 0x00a2: 0x045e, # CYRILLIC SMALL LETTER SHORT U (Byelorussian) + 0x00a3: 0x0408, # CYRILLIC CAPITAL LETTER JE + 0x00a4: 0x04e8, # CYRILLIC CAPITAL LETTER BARRED O + 0x00a5: 0x0498, # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER + 0x00a6: 0x04b0, # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE + 0x00a8: 0x0401, # CYRILLIC CAPITAL LETTER IO + 0x00aa: 0x04d8, # CYRILLIC CAPITAL LETTER SCHWA + 0x00ad: 0x04ef, # CYRILLIC SMALL LETTER U WITH MACRON + 0x00af: 0x049c, # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE + 0x00b1: 0x04b1, # CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE + 0x00b2: 0x0406, # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + 0x00b3: 0x0456, # CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + 0x00b4: 0x0499, # CYRILLIC SMALL LETTER ZE WITH DESCENDER + 0x00b5: 0x04e9, # CYRILLIC SMALL LETTER BARRED O + 0x00b8: 0x0451, # CYRILLIC SMALL LETTER IO + 0x00b9: 0x2116, # NUMERO SIGN + 0x00ba: 0x04d9, # CYRILLIC SMALL LETTER SCHWA + 0x00bc: 0x0458, # CYRILLIC SMALL LETTER JE + 0x00bd: 0x04aa, # CYRILLIC CAPITAL LETTER ES WITH DESCENDER + 0x00be: 0x04ab, # CYRILLIC SMALL LETTER ES WITH DESCENDER + 0x00bf: 0x049d, # CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE + 0x00c0: 0x0410, # CYRILLIC CAPITAL LETTER A + 0x00c1: 0x0411, # CYRILLIC CAPITAL LETTER BE + 0x00c2: 0x0412, # CYRILLIC CAPITAL LETTER VE + 0x00c3: 0x0413, # CYRILLIC CAPITAL LETTER GHE + 0x00c4: 0x0414, # CYRILLIC CAPITAL LETTER DE + 0x00c5: 0x0415, # CYRILLIC CAPITAL LETTER IE + 0x00c6: 0x0416, # CYRILLIC CAPITAL LETTER ZHE + 0x00c7: 0x0417, # CYRILLIC CAPITAL LETTER ZE + 0x00c8: 0x0418, # CYRILLIC CAPITAL LETTER I + 0x00c9: 0x0419, # CYRILLIC CAPITAL LETTER SHORT I + 0x00ca: 0x041a, # CYRILLIC CAPITAL LETTER KA + 0x00cb: 0x041b, # CYRILLIC CAPITAL LETTER EL + 0x00cc: 0x041c, # CYRILLIC CAPITAL LETTER EM + 0x00cd: 0x041d, # CYRILLIC CAPITAL LETTER EN + 0x00ce: 0x041e, # CYRILLIC CAPITAL LETTER O + 0x00cf: 0x041f, # CYRILLIC CAPITAL LETTER PE + 0x00d0: 0x0420, # CYRILLIC CAPITAL LETTER ER + 0x00d1: 0x0421, # CYRILLIC CAPITAL LETTER ES + 0x00d2: 0x0422, # CYRILLIC CAPITAL LETTER TE + 0x00d3: 0x0423, # CYRILLIC CAPITAL LETTER U + 0x00d4: 0x0424, # CYRILLIC CAPITAL LETTER EF + 0x00d5: 0x0425, # CYRILLIC CAPITAL LETTER HA + 0x00d6: 0x0426, # CYRILLIC CAPITAL LETTER TSE + 0x00d7: 0x0427, # CYRILLIC CAPITAL LETTER CHE + 0x00d8: 0x0428, # CYRILLIC CAPITAL LETTER SHA + 0x00d9: 0x0429, # CYRILLIC CAPITAL LETTER SHCHA + 0x00da: 0x042a, # CYRILLIC CAPITAL LETTER HARD SIGN + 0x00db: 0x042b, # CYRILLIC CAPITAL LETTER YERU + 0x00dc: 0x042c, # CYRILLIC CAPITAL LETTER SOFT SIGN + 0x00dd: 0x042d, # CYRILLIC CAPITAL LETTER E + 0x00de: 0x042e, # CYRILLIC CAPITAL LETTER YU + 0x00df: 0x042f, # CYRILLIC CAPITAL LETTER YA + 0x00e0: 0x0430, # CYRILLIC SMALL LETTER A + 0x00e1: 0x0431, # CYRILLIC SMALL LETTER BE + 0x00e2: 0x0432, # CYRILLIC SMALL LETTER VE + 0x00e3: 0x0433, # CYRILLIC SMALL LETTER GHE + 0x00e4: 0x0434, # CYRILLIC SMALL LETTER DE + 0x00e5: 0x0435, # CYRILLIC SMALL LETTER IE + 0x00e6: 0x0436, # CYRILLIC SMALL LETTER ZHE + 0x00e7: 0x0437, # CYRILLIC SMALL LETTER ZE + 0x00e8: 0x0438, # CYRILLIC SMALL LETTER I + 0x00e9: 0x0439, # CYRILLIC SMALL LETTER SHORT I + 0x00ea: 0x043a, # CYRILLIC SMALL LETTER KA + 0x00eb: 0x043b, # CYRILLIC SMALL LETTER EL + 0x00ec: 0x043c, # CYRILLIC SMALL LETTER EM + 0x00ed: 0x043d, # CYRILLIC SMALL LETTER EN + 0x00ee: 0x043e, # CYRILLIC SMALL LETTER O + 0x00ef: 0x043f, # CYRILLIC SMALL LETTER PE + 0x00f0: 0x0440, # CYRILLIC SMALL LETTER ER + 0x00f1: 0x0441, # CYRILLIC SMALL LETTER ES + 0x00f2: 0x0442, # CYRILLIC SMALL LETTER TE + 0x00f3: 0x0443, # CYRILLIC SMALL LETTER U + 0x00f4: 0x0444, # CYRILLIC SMALL LETTER EF + 0x00f5: 0x0445, # CYRILLIC SMALL LETTER HA + 0x00f6: 0x0446, # CYRILLIC SMALL LETTER TSE + 0x00f7: 0x0447, # CYRILLIC SMALL LETTER CHE + 0x00f8: 0x0448, # CYRILLIC SMALL LETTER SHA + 0x00f9: 0x0449, # CYRILLIC SMALL LETTER SHCHA + 0x00fa: 0x044a, # CYRILLIC SMALL LETTER HARD SIGN + 0x00fb: 0x044b, # CYRILLIC SMALL LETTER YERU + 0x00fc: 0x044c, # CYRILLIC SMALL LETTER SOFT SIGN + 0x00fd: 0x044d, # CYRILLIC SMALL LETTER E + 0x00fe: 0x044e, # CYRILLIC SMALL LETTER YU + 0x00ff: 0x044f, # CYRILLIC SMALL LETTER YA +}) -decoding_table = ( - '\x00' # 0x00 -> NULL - '\x01' # 0x01 -> START OF HEADING - '\x02' # 0x02 -> START OF TEXT - '\x03' # 0x03 -> END OF TEXT - '\x04' # 0x04 -> END OF TRANSMISSION - '\x05' # 0x05 -> ENQUIRY - '\x06' # 0x06 -> ACKNOWLEDGE - '\x07' # 0x07 -> BELL - '\x08' # 0x08 -> BACKSPACE - '\t' # 0x09 -> HORIZONTAL TABULATION - '\n' # 0x0A -> LINE FEED - '\x0b' # 0x0B -> VERTICAL TABULATION - '\x0c' # 0x0C -> FORM FEED - '\r' # 0x0D -> CARRIAGE RETURN - '\x0e' # 0x0E -> SHIFT OUT - '\x0f' # 0x0F -> SHIFT IN - '\x10' # 0x10 -> DATA LINK ESCAPE - '\x11' # 0x11 -> DEVICE CONTROL ONE - '\x12' # 0x12 -> DEVICE CONTROL TWO - '\x13' # 0x13 -> DEVICE CONTROL THREE - '\x14' # 0x14 -> DEVICE CONTROL FOUR - '\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE - '\x16' # 0x16 -> SYNCHRONOUS IDLE - '\x17' # 0x17 -> END OF TRANSMISSION BLOCK - '\x18' # 0x18 -> CANCEL - '\x19' # 0x19 -> END OF MEDIUM - '\x1a' # 0x1A -> SUBSTITUTE - '\x1b' # 0x1B -> ESCAPE - '\x1c' # 0x1C -> FILE SEPARATOR - '\x1d' # 0x1D -> GROUP SEPARATOR - '\x1e' # 0x1E -> RECORD SEPARATOR - '\x1f' # 0x1F -> UNIT SEPARATOR - ' ' # 0x20 -> SPACE - '!' # 0x21 -> EXCLAMATION MARK - '"' # 0x22 -> QUOTATION MARK - '#' # 0x23 -> NUMBER SIGN - '$' # 0x24 -> DOLLAR SIGN - '%' # 0x25 -> PERCENT SIGN - '&' # 0x26 -> AMPERSAND - "'" # 0x27 -> APOSTROPHE - '(' # 0x28 -> LEFT PARENTHESIS - ')' # 0x29 -> RIGHT PARENTHESIS - '*' # 0x2A -> ASTERISK - '+' # 0x2B -> PLUS SIGN - ',' # 0x2C -> COMMA - '-' # 0x2D -> HYPHEN-MINUS - '.' # 0x2E -> FULL STOP - '/' # 0x2F -> SOLIDUS - '0' # 0x30 -> DIGIT ZERO - '1' # 0x31 -> DIGIT ONE - '2' # 0x32 -> DIGIT TWO - '3' # 0x33 -> DIGIT THREE - '4' # 0x34 -> DIGIT FOUR - '5' # 0x35 -> DIGIT FIVE - '6' # 0x36 -> DIGIT SIX - '7' # 0x37 -> DIGIT SEVEN - '8' # 0x38 -> DIGIT EIGHT - '9' # 0x39 -> DIGIT NINE - ':' # 0x3A -> COLON - ';' # 0x3B -> SEMICOLON - '<' # 0x3C -> LESS-THAN SIGN - '=' # 0x3D -> EQUALS SIGN - '>' # 0x3E -> GREATER-THAN SIGN - '?' # 0x3F -> QUESTION MARK - '@' # 0x40 -> COMMERCIAL AT - 'A' # 0x41 -> LATIN CAPITAL LETTER A - 'B' # 0x42 -> LATIN CAPITAL LETTER B - 'C' # 0x43 -> LATIN CAPITAL LETTER C - 'D' # 0x44 -> LATIN CAPITAL LETTER D - 'E' # 0x45 -> LATIN CAPITAL LETTER E - 'F' # 0x46 -> LATIN CAPITAL LETTER F - 'G' # 0x47 -> LATIN CAPITAL LETTER G - 'H' # 0x48 -> LATIN CAPITAL LETTER H - 'I' # 0x49 -> LATIN CAPITAL LETTER I - 'J' # 0x4A -> LATIN CAPITAL LETTER J - 'K' # 0x4B -> LATIN CAPITAL LETTER K - 'L' # 0x4C -> LATIN CAPITAL LETTER L - 'M' # 0x4D -> LATIN CAPITAL LETTER M - 'N' # 0x4E -> LATIN CAPITAL LETTER N - 'O' # 0x4F -> LATIN CAPITAL LETTER O - 'P' # 0x50 -> LATIN CAPITAL LETTER P - 'Q' # 0x51 -> LATIN CAPITAL LETTER Q - 'R' # 0x52 -> LATIN CAPITAL LETTER R - 'S' # 0x53 -> LATIN CAPITAL LETTER S - 'T' # 0x54 -> LATIN CAPITAL LETTER T - 'U' # 0x55 -> LATIN CAPITAL LETTER U - 'V' # 0x56 -> LATIN CAPITAL LETTER V - 'W' # 0x57 -> LATIN CAPITAL LETTER W - 'X' # 0x58 -> LATIN CAPITAL LETTER X - 'Y' # 0x59 -> LATIN CAPITAL LETTER Y - 'Z' # 0x5A -> LATIN CAPITAL LETTER Z - '[' # 0x5B -> LEFT SQUARE BRACKET - '\\' # 0x5C -> REVERSE SOLIDUS - ']' # 0x5D -> RIGHT SQUARE BRACKET - '^' # 0x5E -> CIRCUMFLEX ACCENT - '_' # 0x5F -> LOW LINE - '`' # 0x60 -> GRAVE ACCENT - 'a' # 0x61 -> LATIN SMALL LETTER A - 'b' # 0x62 -> LATIN SMALL LETTER B - 'c' # 0x63 -> LATIN SMALL LETTER C - 'd' # 0x64 -> LATIN SMALL LETTER D - 'e' # 0x65 -> LATIN SMALL LETTER E - 'f' # 0x66 -> LATIN SMALL LETTER F - 'g' # 0x67 -> LATIN SMALL LETTER G - 'h' # 0x68 -> LATIN SMALL LETTER H - 'i' # 0x69 -> LATIN SMALL LETTER I - 'j' # 0x6A -> LATIN SMALL LETTER J - 'k' # 0x6B -> LATIN SMALL LETTER K - 'l' # 0x6C -> LATIN SMALL LETTER L - 'm' # 0x6D -> LATIN SMALL LETTER M - 'n' # 0x6E -> LATIN SMALL LETTER N - 'o' # 0x6F -> LATIN SMALL LETTER O - 'p' # 0x70 -> LATIN SMALL LETTER P - 'q' # 0x71 -> LATIN SMALL LETTER Q - 'r' # 0x72 -> LATIN SMALL LETTER R - 's' # 0x73 -> LATIN SMALL LETTER S - 't' # 0x74 -> LATIN SMALL LETTER T - 'u' # 0x75 -> LATIN SMALL LETTER U - 'v' # 0x76 -> LATIN SMALL LETTER V - 'w' # 0x77 -> LATIN SMALL LETTER W - 'x' # 0x78 -> LATIN SMALL LETTER X - 'y' # 0x79 -> LATIN SMALL LETTER Y - 'z' # 0x7A -> LATIN SMALL LETTER Z - '{' # 0x7B -> LEFT CURLY BRACKET - '|' # 0x7C -> VERTICAL LINE - '}' # 0x7D -> RIGHT CURLY BRACKET - '~' # 0x7E -> TILDE - '\x7f' # 0x7F -> DELETE (DEL) - '\u0496' # 0x80 -> CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER - '\u0492' # 0x81 -> CYRILLIC CAPITAL LETTER GHE WITH STROKE - '\u04ee' # 0x82 -> CYRILLIC CAPITAL LETTER U WITH MACRON - '\u0493' # 0x83 -> CYRILLIC SMALL LETTER GHE WITH STROKE - '\u201e' # 0x84 -> DOUBLE LOW-9 QUOTATION MARK - '\u2026' # 0x85 -> HORIZONTAL ELLIPSIS - '\u04b6' # 0x86 -> CYRILLIC CAPITAL LETTER CHE WITH DESCENDER - '\u04ae' # 0x87 -> CYRILLIC CAPITAL LETTER STRAIGHT U - '\u04b2' # 0x88 -> CYRILLIC CAPITAL LETTER HA WITH DESCENDER - '\u04af' # 0x89 -> CYRILLIC SMALL LETTER STRAIGHT U - '\u04a0' # 0x8A -> CYRILLIC CAPITAL LETTER BASHKIR KA - '\u04e2' # 0x8B -> CYRILLIC CAPITAL LETTER I WITH MACRON - '\u04a2' # 0x8C -> CYRILLIC CAPITAL LETTER EN WITH DESCENDER - '\u049a' # 0x8D -> CYRILLIC CAPITAL LETTER KA WITH DESCENDER - '\u04ba' # 0x8E -> CYRILLIC CAPITAL LETTER SHHA - '\u04b8' # 0x8F -> CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE - '\u0497' # 0x90 -> CYRILLIC SMALL LETTER ZHE WITH DESCENDER - '\u2018' # 0x91 -> LEFT SINGLE QUOTATION MARK - '\u2019' # 0x92 -> RIGHT SINGLE QUOTATION MARK - '\u201c' # 0x93 -> LEFT DOUBLE QUOTATION MARK - '\u201d' # 0x94 -> RIGHT DOUBLE QUOTATION MARK - '\u2022' # 0x95 -> BULLET - '\u2013' # 0x96 -> EN DASH - '\u2014' # 0x97 -> EM DASH - '\u04b3' # 0x98 -> CYRILLIC SMALL LETTER HA WITH DESCENDER - '\u04b7' # 0x99 -> CYRILLIC SMALL LETTER CHE WITH DESCENDER - '\u04a1' # 0x9A -> CYRILLIC SMALL LETTER BASHKIR KA - '\u04e3' # 0x9B -> CYRILLIC SMALL LETTER I WITH MACRON - '\u04a3' # 0x9C -> CYRILLIC SMALL LETTER EN WITH DESCENDER - '\u049b' # 0x9D -> CYRILLIC SMALL LETTER KA WITH DESCENDER - '\u04bb' # 0x9E -> CYRILLIC SMALL LETTER SHHA - '\u04b9' # 0x9F -> CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE - '\xa0' # 0xA0 -> NO-BREAK SPACE - '\u040e' # 0xA1 -> CYRILLIC CAPITAL LETTER SHORT U (Byelorussian) - '\u045e' # 0xA2 -> CYRILLIC SMALL LETTER SHORT U (Byelorussian) - '\u0408' # 0xA3 -> CYRILLIC CAPITAL LETTER JE - '\u04e8' # 0xA4 -> CYRILLIC CAPITAL LETTER BARRED O - '\u0498' # 0xA5 -> CYRILLIC CAPITAL LETTER ZE WITH DESCENDER - '\u04b0' # 0xA6 -> CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE - '\xa7' # 0xA7 -> SECTION SIGN - '\u0401' # 0xA8 -> CYRILLIC CAPITAL LETTER IO - '\xa9' # 0xA9 -> COPYRIGHT SIGN - '\u04d8' # 0xAA -> CYRILLIC CAPITAL LETTER SCHWA - '\xab' # 0xAB -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK - '\xac' # 0xAC -> NOT SIGN - '\u04ef' # 0xAD -> CYRILLIC SMALL LETTER U WITH MACRON - '\xae' # 0xAE -> REGISTERED SIGN - '\u049c' # 0xAF -> CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE - '\xb0' # 0xB0 -> DEGREE SIGN - '\u04b1' # 0xB1 -> CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE - '\u0406' # 0xB2 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I - '\u0456' # 0xB3 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I - '\u0499' # 0xB4 -> CYRILLIC SMALL LETTER ZE WITH DESCENDER - '\u04e9' # 0xB5 -> CYRILLIC SMALL LETTER BARRED O - '\xb6' # 0xB6 -> PILCROW SIGN - '\xb7' # 0xB7 -> MIDDLE DOT - '\u0451' # 0xB8 -> CYRILLIC SMALL LETTER IO - '\u2116' # 0xB9 -> NUMERO SIGN - '\u04d9' # 0xBA -> CYRILLIC SMALL LETTER SCHWA - '\xbb' # 0xBB -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - '\u0458' # 0xBC -> CYRILLIC SMALL LETTER JE - '\u04aa' # 0xBD -> CYRILLIC CAPITAL LETTER ES WITH DESCENDER - '\u04ab' # 0xBE -> CYRILLIC SMALL LETTER ES WITH DESCENDER - '\u049d' # 0xBF -> CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE - '\u0410' # 0xC0 -> CYRILLIC CAPITAL LETTER A - '\u0411' # 0xC1 -> CYRILLIC CAPITAL LETTER BE - '\u0412' # 0xC2 -> CYRILLIC CAPITAL LETTER VE - '\u0413' # 0xC3 -> CYRILLIC CAPITAL LETTER GHE - '\u0414' # 0xC4 -> CYRILLIC CAPITAL LETTER DE - '\u0415' # 0xC5 -> CYRILLIC CAPITAL LETTER IE - '\u0416' # 0xC6 -> CYRILLIC CAPITAL LETTER ZHE - '\u0417' # 0xC7 -> CYRILLIC CAPITAL LETTER ZE - '\u0418' # 0xC8 -> CYRILLIC CAPITAL LETTER I - '\u0419' # 0xC9 -> CYRILLIC CAPITAL LETTER SHORT I - '\u041a' # 0xCA -> CYRILLIC CAPITAL LETTER KA - '\u041b' # 0xCB -> CYRILLIC CAPITAL LETTER EL - '\u041c' # 0xCC -> CYRILLIC CAPITAL LETTER EM - '\u041d' # 0xCD -> CYRILLIC CAPITAL LETTER EN - '\u041e' # 0xCE -> CYRILLIC CAPITAL LETTER O - '\u041f' # 0xCF -> CYRILLIC CAPITAL LETTER PE - '\u0420' # 0xD0 -> CYRILLIC CAPITAL LETTER ER - '\u0421' # 0xD1 -> CYRILLIC CAPITAL LETTER ES - '\u0422' # 0xD2 -> CYRILLIC CAPITAL LETTER TE - '\u0423' # 0xD3 -> CYRILLIC CAPITAL LETTER U - '\u0424' # 0xD4 -> CYRILLIC CAPITAL LETTER EF - '\u0425' # 0xD5 -> CYRILLIC CAPITAL LETTER HA - '\u0426' # 0xD6 -> CYRILLIC CAPITAL LETTER TSE - '\u0427' # 0xD7 -> CYRILLIC CAPITAL LETTER CHE - '\u0428' # 0xD8 -> CYRILLIC CAPITAL LETTER SHA - '\u0429' # 0xD9 -> CYRILLIC CAPITAL LETTER SHCHA - '\u042a' # 0xDA -> CYRILLIC CAPITAL LETTER HARD SIGN - '\u042b' # 0xDB -> CYRILLIC CAPITAL LETTER YERU - '\u042c' # 0xDC -> CYRILLIC CAPITAL LETTER SOFT SIGN - '\u042d' # 0xDD -> CYRILLIC CAPITAL LETTER E - '\u042e' # 0xDE -> CYRILLIC CAPITAL LETTER YU - '\u042f' # 0xDF -> CYRILLIC CAPITAL LETTER YA - '\u0430' # 0xE0 -> CYRILLIC SMALL LETTER A - '\u0431' # 0xE1 -> CYRILLIC SMALL LETTER BE - '\u0432' # 0xE2 -> CYRILLIC SMALL LETTER VE - '\u0433' # 0xE3 -> CYRILLIC SMALL LETTER GHE - '\u0434' # 0xE4 -> CYRILLIC SMALL LETTER DE - '\u0435' # 0xE5 -> CYRILLIC SMALL LETTER IE - '\u0436' # 0xE6 -> CYRILLIC SMALL LETTER ZHE - '\u0437' # 0xE7 -> CYRILLIC SMALL LETTER ZE - '\u0438' # 0xE8 -> CYRILLIC SMALL LETTER I - '\u0439' # 0xE9 -> CYRILLIC SMALL LETTER SHORT I - '\u043a' # 0xEA -> CYRILLIC SMALL LETTER KA - '\u043b' # 0xEB -> CYRILLIC SMALL LETTER EL - '\u043c' # 0xEC -> CYRILLIC SMALL LETTER EM - '\u043d' # 0xED -> CYRILLIC SMALL LETTER EN - '\u043e' # 0xEE -> CYRILLIC SMALL LETTER O - '\u043f' # 0xEF -> CYRILLIC SMALL LETTER PE - '\u0440' # 0xF0 -> CYRILLIC SMALL LETTER ER - '\u0441' # 0xF1 -> CYRILLIC SMALL LETTER ES - '\u0442' # 0xF2 -> CYRILLIC SMALL LETTER TE - '\u0443' # 0xF3 -> CYRILLIC SMALL LETTER U - '\u0444' # 0xF4 -> CYRILLIC SMALL LETTER EF - '\u0445' # 0xF5 -> CYRILLIC SMALL LETTER HA - '\u0446' # 0xF6 -> CYRILLIC SMALL LETTER TSE - '\u0447' # 0xF7 -> CYRILLIC SMALL LETTER CHE - '\u0448' # 0xF8 -> CYRILLIC SMALL LETTER SHA - '\u0449' # 0xF9 -> CYRILLIC SMALL LETTER SHCHA - '\u044a' # 0xFA -> CYRILLIC SMALL LETTER HARD SIGN - '\u044b' # 0xFB -> CYRILLIC SMALL LETTER YERU - '\u044c' # 0xFC -> CYRILLIC SMALL LETTER SOFT SIGN - '\u044d' # 0xFD -> CYRILLIC SMALL LETTER E - '\u044e' # 0xFE -> CYRILLIC SMALL LETTER YU - '\u044f' # 0xFF -> CYRILLIC SMALL LETTER YA -) +### Encoding Map -### Encoding table -encoding_table=codecs.charmap_build(decoding_table) +encoding_map = codecs.make_encoding_map(decoding_map) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/quopri_codec.py --- a/Lib/encodings/quopri_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/quopri_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Codec for quoted-printable encoding. -This codec de/encodes from bytes to bytes. +This codec de/encodes from bytes to bytes and is therefore usable with +bytes.transform() and bytes.untransform(). """ import codecs @@ -11,7 +12,7 @@ assert errors == 'strict' f = BytesIO(input) g = BytesIO() - quopri.encode(f, g, quotetabs=True) + quopri.encode(f, g, 1) return (g.getvalue(), len(input)) def quopri_decode(input, errors='strict'): @@ -52,5 +53,4 @@ incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/rot_13.py --- a/Lib/encodings/rot_13.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/rot_13.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,7 +1,8 @@ #!/usr/bin/env python """ Python Character Mapping Codec for ROT13. -This codec de/encodes from str to str. +This codec de/encodes from str to str and is therefore usable with +str.transform() and str.untransform(). Written by Marc-Andre Lemburg (mal@lemburg.com). """ @@ -42,7 +43,6 @@ incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, - _is_text_encoding=False, ) ### Map @@ -106,7 +106,7 @@ ### Filter API def rot13(infile, outfile): - outfile.write(codecs.encode(infile.read(), 'rot-13')) + outfile.write(infile.read().encode('rot-13')) if __name__ == '__main__': import sys diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/uu_codec.py --- a/Lib/encodings/uu_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/uu_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Python 'uu_codec' Codec - UU content transfer encoding. -This codec de/encodes from bytes to bytes. +This codec de/encodes from bytes to bytes and is therefore usable with +bytes.transform() and bytes.untransform(). Written by Marc-Andre Lemburg (mal@lemburg.com). Some details were adapted from uu.py which was written by Lance Ellinghouse and @@ -54,7 +55,7 @@ data = binascii.a2b_uu(s) except binascii.Error as v: # Workaround for broken uuencoders by /Fredrik Lundh - nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 + nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3 data = binascii.a2b_uu(s[:nbytes]) #sys.stderr.write("Warning: %s\n" % str(v)) write(data) @@ -95,5 +96,4 @@ incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/encodings/zlib_codec.py --- a/Lib/encodings/zlib_codec.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/encodings/zlib_codec.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,7 @@ """Python 'zlib_codec' Codec - zlib compression encoding. -This codec de/encodes from bytes to bytes. +This codec de/encodes from bytes to bytes and is therefore usable with +bytes.transform() and bytes.untransform(). Written by Marc-Andre Lemburg (mal@lemburg.com). """ @@ -73,5 +74,4 @@ incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, - _is_text_encoding=False, ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ensurepip/__init__.py --- a/Lib/ensurepip/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,210 +0,0 @@ -import os -import os.path -import pkgutil -import sys -import tempfile - - -__all__ = ["version", "bootstrap"] - - -_SETUPTOOLS_VERSION = "19.4" - -_PIP_VERSION = "8.0.2" - -# pip currently requires ssl support, so we try to provide a nicer -# error message when that is missing (http://bugs.python.org/issue19744) -_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION)) -try: - import ssl -except ImportError: - ssl = None - def _require_ssl_for_pip(): - raise RuntimeError(_MISSING_SSL_MESSAGE) -else: - def _require_ssl_for_pip(): - pass - -_PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION), - ("pip", _PIP_VERSION), -] - - -def _run_pip(args, additional_paths=None): - # Add our bundled software to the sys.path so we can import it - if additional_paths is not None: - sys.path = additional_paths + sys.path - - # Install the bundled software - import pip - pip.main(args) - - -def version(): - """ - Returns a string specifying the bundled version of pip. - """ - return _PIP_VERSION - -def _disable_pip_configuration_settings(): - # We deliberately ignore all pip environment variables - # when invoking pip - # See http://bugs.python.org/issue19734 for details - keys_to_remove = [k for k in os.environ if k.startswith("PIP_")] - for k in keys_to_remove: - del os.environ[k] - # We also ignore the settings in the default pip configuration file - # See http://bugs.python.org/issue20053 for details - os.environ['PIP_CONFIG_FILE'] = os.devnull - - -def bootstrap(*, root=None, upgrade=False, user=False, - altinstall=False, default_pip=False, - verbosity=0): - """ - Bootstrap pip into the current Python installation (or the given root - directory). - - Note that calling this function will alter both sys.path and os.environ. - """ - if altinstall and default_pip: - raise ValueError("Cannot use altinstall and default_pip together") - - _require_ssl_for_pip() - _disable_pip_configuration_settings() - - # By default, installing pip and setuptools installs all of the - # following scripts (X.Y == running Python version): - # - # pip, pipX, pipX.Y, easy_install, easy_install-X.Y - # - # pip 1.5+ allows ensurepip to request that some of those be left out - if altinstall: - # omit pip, pipX and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "altinstall" - elif not default_pip: - # omit pip and easy_install - os.environ["ENSUREPIP_OPTIONS"] = "install" - - with tempfile.TemporaryDirectory() as tmpdir: - # Put our bundled wheels into a temporary directory and construct the - # additional paths that need added to sys.path - additional_paths = [] - for project, version in _PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) - whl = pkgutil.get_data( - "ensurepip", - "_bundled/{}".format(wheel_name), - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: - fp.write(whl) - - additional_paths.append(os.path.join(tmpdir, wheel_name)) - - # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] - if root: - args += ["--root", root] - if upgrade: - args += ["--upgrade"] - if user: - args += ["--user"] - if verbosity: - args += ["-" + "v" * verbosity] - - _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) - -def _uninstall_helper(*, verbosity=0): - """Helper to support a clean default uninstall process on Windows - - Note that calling this function may alter os.environ. - """ - # Nothing to do if pip was never installed, or has been removed - try: - import pip - except ImportError: - return - - # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) - return - - _require_ssl_for_pip() - _disable_pip_configuration_settings() - - # Construct the arguments to be passed to the pip command - args = ["uninstall", "-y", "--disable-pip-version-check"] - if verbosity: - args += ["-" + "v" * verbosity] - - _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) - - -def _main(argv=None): - if ssl is None: - print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE), - file=sys.stderr) - return - - import argparse - parser = argparse.ArgumentParser(prog="python -m ensurepip") - parser.add_argument( - "--version", - action="version", - version="pip {}".format(version()), - help="Show the version of pip that is bundled with this Python.", - ) - parser.add_argument( - "-v", "--verbose", - action="count", - default=0, - dest="verbosity", - help=("Give more output. Option is additive, and can be used up to 3 " - "times."), - ) - parser.add_argument( - "-U", "--upgrade", - action="store_true", - default=False, - help="Upgrade pip and dependencies, even if already installed.", - ) - parser.add_argument( - "--user", - action="store_true", - default=False, - help="Install using the user scheme.", - ) - parser.add_argument( - "--root", - default=None, - help="Install everything relative to this alternate root directory.", - ) - parser.add_argument( - "--altinstall", - action="store_true", - default=False, - help=("Make an alternate install, installing only the X.Y versioned" - "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"), - ) - parser.add_argument( - "--default-pip", - action="store_true", - default=False, - help=("Make a default pip install, installing the unqualified pip " - "and easy_install in addition to the versioned scripts"), - ) - - args = parser.parse_args(argv) - - bootstrap( - root=args.root, - upgrade=args.upgrade, - user=args.user, - verbosity=args.verbosity, - altinstall=args.altinstall, - default_pip=args.default_pip, - ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ensurepip/__main__.py --- a/Lib/ensurepip/__main__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -import ensurepip - -if __name__ == "__main__": - ensurepip._main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ensurepip/_bundled/pip-8.0.2-py2.py3-none-any.whl Binary file Lib/ensurepip/_bundled/pip-8.0.2-py2.py3-none-any.whl has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ensurepip/_bundled/setuptools-19.4-py2.py3-none-any.whl Binary file Lib/ensurepip/_bundled/setuptools-19.4-py2.py3-none-any.whl has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ensurepip/_uninstall.py --- a/Lib/ensurepip/_uninstall.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,30 +0,0 @@ -"""Basic pip uninstallation support, helper for the Windows uninstaller""" - -import argparse -import ensurepip - - -def _main(argv=None): - parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall") - parser.add_argument( - "--version", - action="version", - version="pip {}".format(ensurepip.version()), - help="Show the version of pip this will attempt to uninstall.", - ) - parser.add_argument( - "-v", "--verbose", - action="count", - default=0, - dest="verbosity", - help=("Give more output. Option is additive, and can be used up to 3 " - "times."), - ) - - args = parser.parse_args(argv) - - ensurepip._uninstall_helper(verbosity=args.verbosity) - - -if __name__ == "__main__": - _main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/enum.py --- a/Lib/enum.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,574 +0,0 @@ -import sys -from types import MappingProxyType, DynamicClassAttribute - -# try _collections first to reduce startup cost -try: - from _collections import OrderedDict -except ImportError: - from collections import OrderedDict - - -__all__ = ['EnumMeta', 'Enum', 'IntEnum', 'unique'] - - -def _is_descriptor(obj): - """Returns True if obj is a descriptor, False otherwise.""" - return ( - hasattr(obj, '__get__') or - hasattr(obj, '__set__') or - hasattr(obj, '__delete__')) - - -def _is_dunder(name): - """Returns True if a __dunder__ name, False otherwise.""" - return (name[:2] == name[-2:] == '__' and - name[2:3] != '_' and - name[-3:-2] != '_' and - len(name) > 4) - - -def _is_sunder(name): - """Returns True if a _sunder_ name, False otherwise.""" - return (name[0] == name[-1] == '_' and - name[1:2] != '_' and - name[-2:-1] != '_' and - len(name) > 2) - - -def _make_class_unpicklable(cls): - """Make the given class un-picklable.""" - def _break_on_call_reduce(self, proto): - raise TypeError('%r cannot be pickled' % self) - cls.__reduce_ex__ = _break_on_call_reduce - cls.__module__ = '' - - -class _EnumDict(dict): - """Track enum member order and ensure member names are not reused. - - EnumMeta will use the names found in self._member_names as the - enumeration member names. - - """ - def __init__(self): - super().__init__() - self._member_names = [] - - def __setitem__(self, key, value): - """Changes anything not dundered or not a descriptor. - - If an enum member name is used twice, an error is raised; duplicate - values are not checked for. - - Single underscore (sunder) names are reserved. - - """ - if _is_sunder(key): - raise ValueError('_names_ are reserved for future Enum use') - elif _is_dunder(key): - pass - elif key in self._member_names: - # descriptor overwriting an enum? - raise TypeError('Attempted to reuse key: %r' % key) - elif not _is_descriptor(value): - if key in self: - # enum overwriting a descriptor? - raise TypeError('Key already defined as: %r' % self[key]) - self._member_names.append(key) - super().__setitem__(key, value) - - - -# Dummy value for Enum as EnumMeta explicitly checks for it, but of course -# until EnumMeta finishes running the first time the Enum class doesn't exist. -# This is also why there are checks in EnumMeta like `if Enum is not None` -Enum = None - - -class EnumMeta(type): - """Metaclass for Enum""" - @classmethod - def __prepare__(metacls, cls, bases): - return _EnumDict() - - def __new__(metacls, cls, bases, classdict): - # an Enum class is final once enumeration items have been defined; it - # cannot be mixed with other types (int, float, etc.) if it has an - # inherited __new__ unless a new __new__ is defined (or the resulting - # class will fail). - member_type, first_enum = metacls._get_mixins_(bases) - __new__, save_new, use_args = metacls._find_new_(classdict, member_type, - first_enum) - - # save enum items into separate mapping so they don't get baked into - # the new class - members = {k: classdict[k] for k in classdict._member_names} - for name in classdict._member_names: - del classdict[name] - - # check for illegal enum names (any others?) - invalid_names = set(members) & {'mro', } - if invalid_names: - raise ValueError('Invalid enum member name: {0}'.format( - ','.join(invalid_names))) - - # create a default docstring if one has not been provided - if '__doc__' not in classdict: - classdict['__doc__'] = 'An enumeration.' - - # create our new Enum type - enum_class = super().__new__(metacls, cls, bases, classdict) - enum_class._member_names_ = [] # names in definition order - enum_class._member_map_ = OrderedDict() # name->value map - enum_class._member_type_ = member_type - - # save attributes from super classes so we know if we can take - # the shortcut of storing members in the class dict - base_attributes = {a for b in bases for a in b.__dict__} - - # Reverse value->name map for hashable values. - enum_class._value2member_map_ = {} - - # If a custom type is mixed into the Enum, and it does not know how - # to pickle itself, pickle.dumps will succeed but pickle.loads will - # fail. Rather than have the error show up later and possibly far - # from the source, sabotage the pickle protocol for this class so - # that pickle.dumps also fails. - # - # However, if the new class implements its own __reduce_ex__, do not - # sabotage -- it's on them to make sure it works correctly. We use - # __reduce_ex__ instead of any of the others as it is preferred by - # pickle over __reduce__, and it handles all pickle protocols. - if '__reduce_ex__' not in classdict: - if member_type is not object: - methods = ('__getnewargs_ex__', '__getnewargs__', - '__reduce_ex__', '__reduce__') - if not any(m in member_type.__dict__ for m in methods): - _make_class_unpicklable(enum_class) - - # instantiate them, checking for duplicates as we go - # we instantiate first instead of checking for duplicates first in case - # a custom __new__ is doing something funky with the values -- such as - # auto-numbering ;) - for member_name in classdict._member_names: - value = members[member_name] - if not isinstance(value, tuple): - args = (value, ) - else: - args = value - if member_type is tuple: # special case for tuple enums - args = (args, ) # wrap it one more time - if not use_args: - enum_member = __new__(enum_class) - if not hasattr(enum_member, '_value_'): - enum_member._value_ = value - else: - enum_member = __new__(enum_class, *args) - if not hasattr(enum_member, '_value_'): - enum_member._value_ = member_type(*args) - value = enum_member._value_ - enum_member._name_ = member_name - enum_member.__objclass__ = enum_class - enum_member.__init__(*args) - # If another member with the same value was already defined, the - # new member becomes an alias to the existing one. - for name, canonical_member in enum_class._member_map_.items(): - if canonical_member._value_ == enum_member._value_: - enum_member = canonical_member - break - else: - # Aliases don't appear in member names (only in __members__). - enum_class._member_names_.append(member_name) - # performance boost for any member that would not shadow - # a DynamicClassAttribute - if member_name not in base_attributes: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ - enum_class._member_map_[member_name] = enum_member - try: - # This may fail if value is not hashable. We can't add the value - # to the map, and by-value lookups for this value will be - # linear. - enum_class._value2member_map_[value] = enum_member - except TypeError: - pass - - # double check that repr and friends are not the mixin's or various - # things break (such as pickle) - for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): - class_method = getattr(enum_class, name) - obj_method = getattr(member_type, name, None) - enum_method = getattr(first_enum, name, None) - if obj_method is not None and obj_method is class_method: - setattr(enum_class, name, enum_method) - - # replace any other __new__ with our own (as long as Enum is not None, - # anyway) -- again, this is to support pickle - if Enum is not None: - # if the user defined their own __new__, save it before it gets - # clobbered in case they subclass later - if save_new: - enum_class.__new_member__ = __new__ - enum_class.__new__ = Enum.__new__ - return enum_class - - def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): - """Either returns an existing member, or creates a new enum class. - - This method is used both when an enum class is given a value to match - to an enumeration member (i.e. Color(3)) and for the functional API - (i.e. Color = Enum('Color', names='red green blue')). - - When used for the functional API: - - `value` will be the name of the new class. - - `names` should be either a string of white-space/comma delimited names - (values will start at `start`), or an iterator/mapping of name, value pairs. - - `module` should be set to the module this class is being created in; - if it is not set, an attempt to find that module will be made, but if - it fails the class will not be picklable. - - `qualname` should be set to the actual location this class can be found - at in its module; by default it is set to the global scope. If this is - not correct, unpickling will fail in some circumstances. - - `type`, if set, will be mixed in as the first base class. - - """ - if names is None: # simple value lookup - return cls.__new__(cls, value) - # otherwise, functional API: we're creating a new Enum type - return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start) - - def __contains__(cls, member): - return isinstance(member, cls) and member._name_ in cls._member_map_ - - def __delattr__(cls, attr): - # nicer error message when someone tries to delete an attribute - # (see issue19025). - if attr in cls._member_map_: - raise AttributeError( - "%s: cannot delete Enum member." % cls.__name__) - super().__delattr__(attr) - - def __dir__(self): - return (['__class__', '__doc__', '__members__', '__module__'] + - self._member_names_) - - def __getattr__(cls, name): - """Return the enum member matching `name` - - We use __getattr__ instead of descriptors or inserting into the enum - class' __dict__ in order to support `name` and `value` being both - properties for enum members (which live in the class' __dict__) and - enum members themselves. - - """ - if _is_dunder(name): - raise AttributeError(name) - try: - return cls._member_map_[name] - except KeyError: - raise AttributeError(name) from None - - def __getitem__(cls, name): - return cls._member_map_[name] - - def __iter__(cls): - return (cls._member_map_[name] for name in cls._member_names_) - - def __len__(cls): - return len(cls._member_names_) - - @property - def __members__(cls): - """Returns a mapping of member name->value. - - This mapping lists all enum members, including aliases. Note that this - is a read-only view of the internal mapping. - - """ - return MappingProxyType(cls._member_map_) - - def __repr__(cls): - return "" % cls.__name__ - - def __reversed__(cls): - return (cls._member_map_[name] for name in reversed(cls._member_names_)) - - def __setattr__(cls, name, value): - """Block attempts to reassign Enum members. - - A simple assignment to the class namespace only changes one of the - several possible ways to get an Enum member from the Enum class, - resulting in an inconsistent Enumeration. - - """ - member_map = cls.__dict__.get('_member_map_', {}) - if name in member_map: - raise AttributeError('Cannot reassign members.') - super().__setattr__(name, value) - - def _create_(cls, class_name, names=None, *, module=None, qualname=None, type=None, start=1): - """Convenience method to create a new Enum class. - - `names` can be: - - * A string containing member names, separated either with spaces or - commas. Values are incremented by 1 from `start`. - * An iterable of member names. Values are incremented by 1 from `start`. - * An iterable of (member name, value) pairs. - * A mapping of member name -> value pairs. - - """ - metacls = cls.__class__ - bases = (cls, ) if type is None else (type, cls) - classdict = metacls.__prepare__(class_name, bases) - - # special processing needed for names? - if isinstance(names, str): - names = names.replace(',', ' ').split() - if isinstance(names, (tuple, list)) and isinstance(names[0], str): - names = [(e, i) for (i, e) in enumerate(names, start)] - - # Here, names is either an iterable of (name, value) or a mapping. - for item in names: - if isinstance(item, str): - member_name, member_value = item, names[item] - else: - member_name, member_value = item - classdict[member_name] = member_value - enum_class = metacls.__new__(metacls, class_name, bases, classdict) - - # TODO: replace the frame hack if a blessed way to know the calling - # module is ever developed - if module is None: - try: - module = sys._getframe(2).f_globals['__name__'] - except (AttributeError, ValueError) as exc: - pass - if module is None: - _make_class_unpicklable(enum_class) - else: - enum_class.__module__ = module - if qualname is not None: - enum_class.__qualname__ = qualname - - return enum_class - - @staticmethod - def _get_mixins_(bases): - """Returns the type for creating enum members, and the first inherited - enum class. - - bases: the tuple of bases that was given to __new__ - - """ - if not bases: - return object, Enum - - # double check that we are not subclassing a class with existing - # enumeration members; while we're at it, see if any other data - # type has been mixed in so we can use the correct __new__ - member_type = first_enum = None - for base in bases: - if (base is not Enum and - issubclass(base, Enum) and - base._member_names_): - raise TypeError("Cannot extend enumerations") - # base is now the last base in bases - if not issubclass(base, Enum): - raise TypeError("new enumerations must be created as " - "`ClassName([mixin_type,] enum_type)`") - - # get correct mix-in type (either mix-in type of Enum subclass, or - # first base if last base is Enum) - if not issubclass(bases[0], Enum): - member_type = bases[0] # first data type - first_enum = bases[-1] # enum type - else: - for base in bases[0].__mro__: - # most common: (IntEnum, int, Enum, object) - # possible: (, , - # , , - # ) - if issubclass(base, Enum): - if first_enum is None: - first_enum = base - else: - if member_type is None: - member_type = base - - return member_type, first_enum - - @staticmethod - def _find_new_(classdict, member_type, first_enum): - """Returns the __new__ to be used for creating the enum members. - - classdict: the class dictionary given to __new__ - member_type: the data type whose __new__ will be used by default - first_enum: enumeration to check for an overriding __new__ - - """ - # now find the correct __new__, checking to see of one was defined - # by the user; also check earlier enum classes in case a __new__ was - # saved as __new_member__ - __new__ = classdict.get('__new__', None) - - # should __new__ be saved as __new_member__ later? - save_new = __new__ is not None - - if __new__ is None: - # check all possibles for __new_member__ before falling back to - # __new__ - for method in ('__new_member__', '__new__'): - for possible in (member_type, first_enum): - target = getattr(possible, method, None) - if target not in { - None, - None.__new__, - object.__new__, - Enum.__new__, - }: - __new__ = target - break - if __new__ is not None: - break - else: - __new__ = object.__new__ - - # if a non-object.__new__ is used then whatever value/tuple was - # assigned to the enum member name will be passed to __new__ and to the - # new enum member's __init__ - if __new__ is object.__new__: - use_args = False - else: - use_args = True - - return __new__, save_new, use_args - - -class Enum(metaclass=EnumMeta): - """Generic enumeration. - - Derive from this class to define new enumerations. - - """ - def __new__(cls, value): - # all enum instances are actually created during class construction - # without calling this method; this method is called by the metaclass' - # __call__ (i.e. Color(3) ), and by pickle - if type(value) is cls: - # For lookups like Color(Color.red) - return value - # by-value search for a matching enum member - # see if it's in the reverse mapping (for hashable values) - try: - if value in cls._value2member_map_: - return cls._value2member_map_[value] - except TypeError: - # not there, now do long search -- O(n) behavior - for member in cls._member_map_.values(): - if member._value_ == value: - return member - raise ValueError("%r is not a valid %s" % (value, cls.__name__)) - - def __repr__(self): - return "<%s.%s: %r>" % ( - self.__class__.__name__, self._name_, self._value_) - - def __str__(self): - return "%s.%s" % (self.__class__.__name__, self._name_) - - def __dir__(self): - added_behavior = [ - m - for cls in self.__class__.mro() - for m in cls.__dict__ - if m[0] != '_' and m not in self._member_map_ - ] - return (['__class__', '__doc__', '__module__'] + added_behavior) - - def __format__(self, format_spec): - # mixed-in Enums should use the mixed-in type's __format__, otherwise - # we can get strange results with the Enum name showing up instead of - # the value - - # pure Enum branch - if self._member_type_ is object: - cls = str - val = str(self) - # mix-in branch - else: - cls = self._member_type_ - val = self._value_ - return cls.__format__(val, format_spec) - - def __hash__(self): - return hash(self._name_) - - def __reduce_ex__(self, proto): - return self.__class__, (self._value_, ) - - # DynamicClassAttribute is used to provide access to the `name` and - # `value` properties of enum members while keeping some measure of - # protection from modification, while still allowing for an enumeration - # to have members named `name` and `value`. This works because enumeration - # members are not set directly on the enum class -- __getattr__ is - # used to look them up. - - @DynamicClassAttribute - def name(self): - """The name of the Enum member.""" - return self._name_ - - @DynamicClassAttribute - def value(self): - """The value of the Enum member.""" - return self._value_ - - @classmethod - def _convert(cls, name, module, filter, source=None): - """ - Create a new Enum subclass that replaces a collection of global constants - """ - # convert all constants from source (or module) that pass filter() to - # a new Enum called name, and export the enum and its members back to - # module; - # also, replace the __reduce_ex__ method so unpickling works in - # previous Python versions - module_globals = vars(sys.modules[module]) - if source: - source = vars(source) - else: - source = module_globals - members = {name: value for name, value in source.items() - if filter(name)} - cls = cls(name, members, module=module) - cls.__reduce_ex__ = _reduce_ex_by_name - module_globals.update(cls.__members__) - module_globals[name] = cls - return cls - - -class IntEnum(int, Enum): - """Enum where members are also (and must be) ints""" - - -def _reduce_ex_by_name(self, proto): - return self.name - -def unique(enumeration): - """Class decorator for enumerations ensuring unique member values.""" - duplicates = [] - for name, member in enumeration.__members__.items(): - if name != member.name: - duplicates.append((name, member.name)) - if duplicates: - alias_details = ', '.join( - ["%s -> %s" % (alias, name) for (alias, name) in duplicates]) - raise ValueError('duplicate values found in %r: %s' % - (enumeration, alias_details)) - return enumeration diff -r 6db40a9955dc -r 0d413f60cc23 Lib/filecmp.py --- a/Lib/filecmp.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/filecmp.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,7 +6,6 @@ Functions: cmp(f1, f2, shallow=True) -> int cmpfiles(a, b, common) -> ([], [], []) - clear_cache() """ @@ -14,18 +13,11 @@ import stat from itertools import filterfalse -__all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES'] +__all__ = ["cmp", "dircmp", "cmpfiles"] _cache = {} BUFSIZE = 8*1024 -DEFAULT_IGNORES = [ - 'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__'] - -def clear_cache(): - """Clear the filecmp cache.""" - _cache.clear() - def cmp(f1, f2, shallow=True): """Compare two files. @@ -36,15 +28,14 @@ f2 -- Second file name shallow -- Just check stat signature (do not read the files). - defaults to True. + defaults to 1. Return value: True if the files are the same, False otherwise. This function uses a cache for past comparisons and the results, - with cache entries invalidated if their stat information - changes. The cache may be cleared by calling clear_cache(). + with a cache invalidation mechanism relying on stale signatures. """ @@ -61,7 +52,7 @@ if outcome is None: outcome = _do_cmp(f1, f2) if len(_cache) > 100: # limit the maximum size of the cache - clear_cache() + _cache.clear() _cache[f1, f2, s1, s2] = outcome return outcome @@ -89,7 +80,7 @@ dircmp(a, b, ignore=None, hide=None) A and B are directories. IGNORE is a list of names to ignore, - defaults to DEFAULT_IGNORES. + defaults to ['RCS', 'CVS', 'tags']. HIDE is a list of names to hide, defaults to [os.curdir, os.pardir]. @@ -125,7 +116,7 @@ else: self.hide = hide if ignore is None: - self.ignore = DEFAULT_IGNORES + self.ignore = ['RCS', 'CVS', 'tags'] # Names ignored in comparison else: self.ignore = ignore @@ -156,12 +147,12 @@ ok = 1 try: a_stat = os.stat(a_path) - except OSError as why: + except os.error as why: # print('Can\'t stat', a_path, ':', why.args[1]) ok = 0 try: b_stat = os.stat(b_path) - except OSError as why: + except os.error as why: # print('Can\'t stat', b_path, ':', why.args[1]) ok = 0 @@ -277,7 +268,7 @@ def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except OSError: + except os.error: return 2 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/fileinput.py --- a/Lib/fileinput.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/fileinput.py Mon Jan 25 17:05:13 2016 +0100 @@ -30,7 +30,7 @@ All files are opened in text mode by default, you can override this by setting the mode parameter to input() or FileInput.__init__(). -If an I/O error occurs during opening or reading a file, the OSError +If an I/O error occurs during opening or reading a file, the IOError exception is raised. If sys.stdin is used more than once, the second and further use will @@ -82,8 +82,7 @@ import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", - "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", - "hook_encoded"] + "isfirstline", "isstdin", "FileInput"] _state = None @@ -91,11 +90,13 @@ def input(files=None, inplace=False, backup="", bufsize=0, mode="r", openhook=None): - """Return an instance of the FileInput class, which can be iterated. + """input(files=None, inplace=False, backup="", bufsize=0, \ +mode="r", openhook=None) - The parameters are passed to the constructor of the FileInput class. - The returned instance, in addition to being an iterator, - keeps global state for the functions of this module,. + Create an instance of the FileInput class. The instance will be used + as global state for the functions of this module, and is also returned + to use during iteration. The parameters to this function will be passed + along to the constructor of the FileInput class. """ global _state if _state and _state._file: @@ -182,7 +183,7 @@ return _state.isstdin() class FileInput: - """FileInput([files[, inplace[, backup[, bufsize, [, mode[, openhook]]]]]]) + """class FileInput([files[, inplace[, backup[, mode[, openhook]]]]]) Class FileInput is the implementation of the module; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(), @@ -223,10 +224,6 @@ if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " "'r', 'rU', 'U' and 'rb'") - if 'U' in mode: - import warnings - warnings.warn("'U' mode is deprecated", - DeprecationWarning, 2) self._mode = mode if openhook: if inplace: @@ -239,10 +236,8 @@ self.close() def close(self): - try: - self.nextfile() - finally: - self._files = () + self.nextfile() + self._files = () def __enter__(self): return self @@ -278,31 +273,29 @@ def nextfile(self): savestdout = self._savestdout - self._savestdout = None + self._savestdout = 0 if savestdout: sys.stdout = savestdout output = self._output - self._output = None - try: - if output: - output.close() - finally: - file = self._file - self._file = None - try: - if file and not self._isstdin: - file.close() - finally: - backupfilename = self._backupfilename - self._backupfilename = None - if backupfilename and not self._backup: - try: os.unlink(backupfilename) - except OSError: pass + self._output = 0 + if output: + output.close() - self._isstdin = False - self._buffer = [] - self._bufindex = 0 + file = self._file + self._file = 0 + if file and not self._isstdin: + file.close() + + backupfilename = self._backupfilename + self._backupfilename = 0 + if backupfilename and not self._backup: + try: os.unlink(backupfilename) + except OSError: pass + + self._isstdin = False + self._buffer = [] + self._bufindex = 0 def readline(self): try: @@ -316,10 +309,7 @@ return line if not self._file: if not self._files: - if 'b' in self._mode: - return b'' - else: - return '' + return "" self._filename = self._files[0] self._files = self._files[1:] self._filelineno = 0 @@ -328,20 +318,15 @@ self._backupfilename = 0 if self._filename == '-': self._filename = '' - if 'b' in self._mode: - self._file = getattr(sys.stdin, 'buffer', sys.stdin) - else: - self._file = sys.stdin + self._file = sys.stdin self._isstdin = True else: if self._inplace: self._backupfilename = ( self._filename + (self._backup or ".bak")) - try: - os.unlink(self._backupfilename) - except OSError: - pass - # The next few lines may raise OSError + try: os.unlink(self._backupfilename) + except os.error: pass + # The next few lines may raise IOError os.rename(self._filename, self._backupfilename) self._file = open(self._backupfilename, self._mode) try: @@ -363,7 +348,7 @@ self._savestdout = sys.stdout sys.stdout = self._output else: - # This may raise OSError + # This may raise IOError if self._openhook: self._file = self._openhook(self._filename, self._mode) else: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/fnmatch.py --- a/Lib/fnmatch.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/fnmatch.py Mon Jan 25 17:05:13 2016 +0100 @@ -35,7 +35,7 @@ pat = os.path.normcase(pat) return fnmatchcase(name, pat) -@functools.lru_cache(maxsize=256, typed=True) +@functools.lru_cache(maxsize=250, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): pat_str = str(pat, 'ISO-8859-1') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/formatter.py --- a/Lib/formatter.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/formatter.py Mon Jan 25 17:05:13 2016 +0100 @@ -19,9 +19,6 @@ """ import sys -import warnings -warnings.warn('the formatter module is deprecated', DeprecationWarning, - stacklevel=2) AS_IS = None @@ -436,15 +433,11 @@ fp = open(sys.argv[1]) else: fp = sys.stdin - try: - for line in fp: - if line == '\n': - f.end_paragraph(1) - else: - f.add_flowing_data(line) - finally: - if fp is not sys.stdin: - fp.close() + for line in fp: + if line == '\n': + f.end_paragraph(1) + else: + f.add_flowing_data(line) f.end_paragraph(0) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/fractions.py --- a/Lib/fractions.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/fractions.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,17 +20,6 @@ Unless b==0, the result will have the same sign as b (so that when b is divided by it, the result comes out positive). """ - import warnings - warnings.warn('fractions.gcd() is deprecated. Use math.gcd() instead.', - DeprecationWarning, 2) - if type(a) is int is type(b): - if (b or a) < 0: - return -math.gcd(a, b) - return math.gcd(a, b) - return _gcd(a, b) - -def _gcd(a, b): - # Supports non-integers for backward compatibility. while b: a, b = b, a%b return a @@ -81,7 +70,7 @@ __slots__ = ('_numerator', '_denominator') # We're immutable, so use __new__ not __init__ - def __new__(cls, numerator=0, denominator=None, _normalize=True): + def __new__(cls, numerator=0, denominator=None): """Constructs a Rational. Takes a string like '3/2' or '1.5', another Rational instance, a @@ -115,19 +104,22 @@ self = super(Fraction, cls).__new__(cls) if denominator is None: - if type(numerator) is int: - self._numerator = numerator - self._denominator = 1 - return self - - elif isinstance(numerator, numbers.Rational): + if isinstance(numerator, numbers.Rational): self._numerator = numerator.numerator self._denominator = numerator.denominator return self - elif isinstance(numerator, (float, Decimal)): - # Exact conversion - self._numerator, self._denominator = numerator.as_integer_ratio() + elif isinstance(numerator, float): + # Exact conversion from float + value = Fraction.from_float(numerator) + self._numerator = value._numerator + self._denominator = value._denominator + return self + + elif isinstance(numerator, Decimal): + value = Fraction.from_decimal(numerator) + self._numerator = value._numerator + self._denominator = value._denominator return self elif isinstance(numerator, str): @@ -161,9 +153,6 @@ raise TypeError("argument should be a string " "or a Rational instance") - elif type(numerator) is int is type(denominator): - pass # *very* normal case - elif (isinstance(numerator, numbers.Rational) and isinstance(denominator, numbers.Rational)): numerator, denominator = ( @@ -176,18 +165,9 @@ if denominator == 0: raise ZeroDivisionError('Fraction(%s, 0)' % numerator) - if _normalize: - if type(numerator) is int is type(denominator): - # *very* normal case - g = math.gcd(numerator, denominator) - if denominator < 0: - g = -g - else: - g = _gcd(numerator, denominator) - numerator //= g - denominator //= g - self._numerator = numerator - self._denominator = denominator + g = gcd(numerator, denominator) + self._numerator = numerator // g + self._denominator = denominator // g return self @classmethod @@ -202,6 +182,8 @@ elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) + if math.isnan(f) or math.isinf(f): + raise TypeError("Cannot convert %r to %s." % (f, cls.__name__)) return cls(*f.as_integer_ratio()) @classmethod @@ -214,7 +196,17 @@ raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - return cls(*dec.as_integer_ratio()) + if not dec.is_finite(): + # Catches infinities and nans. + raise TypeError("Cannot convert %s to %s." % (dec, cls.__name__)) + sign, digits, exp = dec.as_tuple() + digits = int(''.join(map(str, digits))) + if sign: + digits = -digits + if exp >= 0: + return cls(digits * 10 ** exp) + else: + return cls(digits, 10 ** -exp) def limit_denominator(self, max_denominator=1000000): """Closest Fraction to self with denominator at most max_denominator. @@ -281,8 +273,7 @@ def __repr__(self): """repr(self)""" - return '%s(%s, %s)' % (self.__class__.__name__, - self._numerator, self._denominator) + return ('Fraction(%s, %s)' % (self._numerator, self._denominator)) def __str__(self): """str(self)""" @@ -400,17 +391,17 @@ def _add(a, b): """a + b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db + b.numerator * da, - da * db) + return Fraction(a.numerator * b.denominator + + b.numerator * a.denominator, + a.denominator * b.denominator) __add__, __radd__ = _operator_fallbacks(_add, operator.add) def _sub(a, b): """a - b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db - b.numerator * da, - da * db) + return Fraction(a.numerator * b.denominator - + b.numerator * a.denominator, + a.denominator * b.denominator) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) @@ -458,12 +449,10 @@ power = b.numerator if power >= 0: return Fraction(a._numerator ** power, - a._denominator ** power, - _normalize=False) + a._denominator ** power) else: return Fraction(a._denominator ** -power, - a._numerator ** -power, - _normalize=False) + a._numerator ** -power) else: # A fractional power will generally produce an # irrational number. @@ -487,15 +476,15 @@ def __pos__(a): """+a: Coerces a subclass instance to Fraction""" - return Fraction(a._numerator, a._denominator, _normalize=False) + return Fraction(a._numerator, a._denominator) def __neg__(a): """-a""" - return Fraction(-a._numerator, a._denominator, _normalize=False) + return Fraction(-a._numerator, a._denominator) def __abs__(a): """abs(a)""" - return Fraction(abs(a._numerator), a._denominator, _normalize=False) + return Fraction(abs(a._numerator), a._denominator) def __trunc__(a): """trunc(a)""" @@ -562,8 +551,6 @@ def __eq__(a, b): """a == b""" - if type(b) is int: - return a._numerator == b and a._denominator == 1 if isinstance(b, numbers.Rational): return (a._numerator == b.numerator and a._denominator == b.denominator) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ftplib.py --- a/Lib/ftplib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ftplib.py Mon Jan 25 17:05:13 2016 +0100 @@ -39,11 +39,9 @@ import os import sys import socket -import warnings from socket import _GLOBAL_DEFAULT_TIMEOUT -__all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", - "all_errors"] +__all__ = ["FTP","Netrc"] # Magic number from MSG_OOB = 0x1 # Process data out of band @@ -51,8 +49,6 @@ # The standard FTP server control port FTP_PORT = 21 -# The sizehint parameter passed to readline() calls -MAXLINE = 8192 # Exception raised when an error or invalid response is received @@ -65,7 +61,7 @@ # All exceptions (hopefully) that may be raised here and that aren't # (always) programming errors on our side -all_errors = (Error, OSError, EOFError) +all_errors = (Error, IOError, EOFError) # Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) @@ -100,7 +96,6 @@ debugging = 0 host = '' port = FTP_PORT - maxline = MAXLINE sock = None file = None welcome = None @@ -128,7 +123,7 @@ if self.sock is not None: try: self.quit() - except (OSError, EOFError): + except (socket.error, EOFError): pass finally: if self.sock is not None: @@ -138,7 +133,6 @@ '''Connect to host. Arguments are: - host: hostname to connect to (string, default previous host) - port: port to connect to (integer, default previous port) - - timeout: the timeout to set against the ftp socket(s) - source_address: a 2-tuple (host, port) for the socket to bind to as its source address before connecting. ''' @@ -189,8 +183,7 @@ # Internal: send one line to the server, appending CRLF def putline(self, line): line = line + CRLF - if self.debugging > 1: - print('*put*', self.sanitize(line)) + if self.debugging > 1: print('*put*', self.sanitize(line)) self.sock.sendall(line.encode(self.encoding)) # Internal: send one command to the server (through putline()) @@ -201,17 +194,12 @@ # Internal: return one line from the server, stripping CRLF. # Raise EOFError if the connection is closed def getline(self): - line = self.file.readline(self.maxline + 1) - if len(line) > self.maxline: - raise Error("got more than %d bytes" % self.maxline) + line = self.file.readline() if self.debugging > 1: print('*get*', self.sanitize(line)) - if not line: - raise EOFError - if line[-2:] == CRLF: - line = line[:-2] - elif line[-1:] in CRLF: - line = line[:-1] + if not line: raise EOFError + if line[-2:] == CRLF: line = line[:-2] + elif line[-1:] in CRLF: line = line[:-1] return line # Internal: get a response from the server, which may possibly @@ -234,8 +222,7 @@ # Raise various errors if the response indicates an error def getresp(self): resp = self.getmultiline() - if self.debugging: - print('*resp*', self.sanitize(resp)) + if self.debugging: print('*resp*', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in {'1', '2', '3'}: @@ -259,8 +246,7 @@ IP and Synch; that doesn't seem to work with the servers I've tried. Instead, just send the ABOR command as OOB data.''' line = b'ABOR' + B_CRLF - if self.debugging > 1: - print('*put urgent*', self.sanitize(line)) + if self.debugging > 1: print('*put urgent*', self.sanitize(line)) self.sock.sendall(line, MSG_OOB) resp = self.getmultiline() if resp[:3] not in {'426', '225', '226'}: @@ -288,7 +274,7 @@ return self.voidcmd(cmd) def sendeprt(self, host, port): - '''Send an EPRT command with the current host and the given port number.''' + '''Send a EPRT command with the current host and the given port number.''' af = 0 if self.af == socket.AF_INET: af = 1 @@ -302,25 +288,21 @@ def makeport(self): '''Create a new socket and send a PORT command for it.''' - err = None + msg = "getaddrinfo returns an empty list" sock = None for res in socket.getaddrinfo(None, 0, self.af, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): af, socktype, proto, canonname, sa = res try: sock = socket.socket(af, socktype, proto) sock.bind(sa) - except OSError as _: - err = _ + except socket.error as msg: if sock: sock.close() sock = None continue break - if sock is None: - if err is not None: - raise err - else: - raise OSError("getaddrinfo returns an empty list") + if not sock: + raise socket.error(msg) sock.listen(1) port = sock.getsockname()[1] # Get proper port host = self.sock.getsockname()[0] # Get proper host @@ -400,12 +382,9 @@ def login(self, user = '', passwd = '', acct = ''): '''Login, default anonymous.''' - if not user: - user = 'anonymous' - if not passwd: - passwd = '' - if not acct: - acct = '' + if not user: user = 'anonymous' + if not passwd: passwd = '' + if not acct: acct = '' if user == 'anonymous' and passwd in {'', '-'}: # If there is no anonymous ftp password specified # then we'll just use anonymous@ @@ -416,10 +395,8 @@ # host or country. passwd = passwd + 'anonymous@' resp = self.sendcmd('USER ' + user) - if resp[0] == '3': - resp = self.sendcmd('PASS ' + passwd) - if resp[0] == '3': - resp = self.sendcmd('ACCT ' + acct) + if resp[0] == '3': resp = self.sendcmd('PASS ' + passwd) + if resp[0] == '3': resp = self.sendcmd('ACCT ' + acct) if resp[0] != '2': raise error_reply(resp) return resp @@ -445,9 +422,6 @@ if not data: break callback(data) - # shutdown ssl layer - if _SSLSocket is not None and isinstance(conn, _SSLSocket): - conn.unwrap() return self.voidresp() def retrlines(self, cmd, callback = None): @@ -462,17 +436,13 @@ Returns: The response code. """ - if callback is None: - callback = print_line + if callback is None: callback = print_line resp = self.sendcmd('TYPE A') with self.transfercmd(cmd) as conn, \ conn.makefile('r', encoding=self.encoding) as fp: while 1: - line = fp.readline(self.maxline + 1) - if len(line) > self.maxline: - raise Error("got more than %d bytes" % self.maxline) - if self.debugging > 2: - print('*retr*', repr(line)) + line = fp.readline() + if self.debugging > 2: print('*retr*', repr(line)) if not line: break if line[-2:] == CRLF: @@ -480,9 +450,6 @@ elif line[-1:] == '\n': line = line[:-1] callback(line) - # shutdown ssl layer - if _SSLSocket is not None and isinstance(conn, _SSLSocket): - conn.unwrap() return self.voidresp() def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None): @@ -494,7 +461,7 @@ blocksize: The maximum data size to read from fp and send over the connection at once. [default: 8192] callback: An optional single parameter callable that is called on - each block of data after it is sent. [default: None] + on each block of data after it is sent. [default: None] rest: Passed to transfercmd(). [default: None] Returns: @@ -504,14 +471,9 @@ with self.transfercmd(cmd, rest) as conn: while 1: buf = fp.read(blocksize) - if not buf: - break + if not buf: break conn.sendall(buf) - if callback: - callback(buf) - # shutdown ssl layer - if _SSLSocket is not None and isinstance(conn, _SSLSocket): - conn.unwrap() + if callback: callback(buf) return self.voidresp() def storlines(self, cmd, fp, callback=None): @@ -521,7 +483,7 @@ cmd: A STOR command. fp: A file-like object with a readline() method. callback: An optional single parameter callable that is called on - each line after it is sent. [default: None] + on each line after it is sent. [default: None] Returns: The response code. @@ -529,20 +491,13 @@ self.voidcmd('TYPE A') with self.transfercmd(cmd) as conn: while 1: - buf = fp.readline(self.maxline + 1) - if len(buf) > self.maxline: - raise Error("got more than %d bytes" % self.maxline) - if not buf: - break + buf = fp.readline() + if not buf: break if buf[-2:] != B_CRLF: if buf[-1] in B_CRLF: buf = buf[:-1] buf = buf + B_CRLF conn.sendall(buf) - if callback: - callback(buf) - # shutdown ssl layer - if _SSLSocket is not None and isinstance(conn, _SSLSocket): - conn.unwrap() + if callback: callback(buf) return self.voidresp() def acct(self, password): @@ -668,24 +623,17 @@ def close(self): '''Close the connection without assuming anything about it.''' - try: - file = self.file - self.file = None - if file is not None: - file.close() - finally: - sock = self.sock - self.sock = None - if sock is not None: - sock.close() + if self.file is not None: + self.file.close() + if self.sock is not None: + self.sock.close() + self.file = self.sock = None try: import ssl except ImportError: - _SSLSocket = None + pass else: - _SSLSocket = ssl.SSLSocket - class FTP_TLS(FTP): '''A FTP subclass which adds TLS support to FTP as described in RFC-4217. @@ -719,7 +667,7 @@ '221 Goodbye.' >>> ''' - ssl_version = ssl.PROTOCOL_SSLv23 + ssl_version = ssl.PROTOCOL_TLSv1 def __init__(self, host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, @@ -732,10 +680,6 @@ "exclusive") self.keyfile = keyfile self.certfile = certfile - if context is None: - context = ssl._create_stdlib_context(self.ssl_version, - certfile=certfile, - keyfile=keyfile) self.context = context self._prot_p = False FTP.__init__(self, host, user, passwd, acct, timeout, source_address) @@ -749,12 +693,16 @@ '''Set up secure control connection by using TLS/SSL.''' if isinstance(self.sock, ssl.SSLSocket): raise ValueError("Already using TLS") - if self.ssl_version >= ssl.PROTOCOL_SSLv23: + if self.ssl_version == ssl.PROTOCOL_TLSv1: resp = self.voidcmd('AUTH TLS') else: resp = self.voidcmd('AUTH SSL') - self.sock = self.context.wrap_socket(self.sock, - server_hostname=self.host) + if self.context is not None: + self.sock = self.context.wrap_socket(self.sock) + else: + self.sock = ssl.wrap_socket(self.sock, self.keyfile, + self.certfile, + ssl_version=self.ssl_version) self.file = self.sock.makefile(mode='r', encoding=self.encoding) return resp @@ -793,10 +741,76 @@ def ntransfercmd(self, cmd, rest=None): conn, size = FTP.ntransfercmd(self, cmd, rest) if self._prot_p: - conn = self.context.wrap_socket(conn, - server_hostname=self.host) + if self.context is not None: + conn = self.context.wrap_socket(conn) + else: + conn = ssl.wrap_socket(conn, self.keyfile, self.certfile, + ssl_version=self.ssl_version) return conn, size + def retrbinary(self, cmd, callback, blocksize=8192, rest=None): + self.voidcmd('TYPE I') + with self.transfercmd(cmd, rest) as conn: + while 1: + data = conn.recv(blocksize) + if not data: + break + callback(data) + # shutdown ssl layer + if isinstance(conn, ssl.SSLSocket): + conn.unwrap() + return self.voidresp() + + def retrlines(self, cmd, callback = None): + if callback is None: callback = print_line + resp = self.sendcmd('TYPE A') + conn = self.transfercmd(cmd) + fp = conn.makefile('r', encoding=self.encoding) + with fp, conn: + while 1: + line = fp.readline() + if self.debugging > 2: print('*retr*', repr(line)) + if not line: + break + if line[-2:] == CRLF: + line = line[:-2] + elif line[-1:] == '\n': + line = line[:-1] + callback(line) + # shutdown ssl layer + if isinstance(conn, ssl.SSLSocket): + conn.unwrap() + return self.voidresp() + + def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None): + self.voidcmd('TYPE I') + with self.transfercmd(cmd, rest) as conn: + while 1: + buf = fp.read(blocksize) + if not buf: break + conn.sendall(buf) + if callback: callback(buf) + # shutdown ssl layer + if isinstance(conn, ssl.SSLSocket): + conn.unwrap() + return self.voidresp() + + def storlines(self, cmd, fp, callback=None): + self.voidcmd('TYPE A') + with self.transfercmd(cmd) as conn: + while 1: + buf = fp.readline() + if not buf: break + if buf[-2:] != B_CRLF: + if buf[-1] in B_CRLF: buf = buf[:-1] + buf = buf + B_CRLF + conn.sendall(buf) + if callback: callback(buf) + # shutdown ssl layer + if isinstance(conn, ssl.SSLSocket): + conn.unwrap() + return self.voidresp() + def abort(self): # overridden as we can't pass MSG_OOB flag to sendall() line = b'ABOR' + B_CRLF @@ -807,7 +821,7 @@ return resp __all__.append('FTP_TLS') - all_errors = (Error, OSError, EOFError, ssl.SSLError) + all_errors = (Error, IOError, EOFError, ssl.SSLError) _150_re = None @@ -853,7 +867,7 @@ def parse229(resp, peer): - '''Parse the '229' response for an EPSV request. + '''Parse the '229' response for a EPSV request. Raises error_proto if it does not contain '(|||port|)' Return ('host.addr.as.numbers', port#) tuple.''' @@ -904,8 +918,7 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): '''Copy file from one FTP-instance to another.''' - if not targetname: - targetname = sourcename + if not targetname: targetname = sourcename type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) @@ -915,15 +928,119 @@ # transfer request. # So: STOR before RETR, because here the target is a "user". treply = target.sendcmd('STOR ' + targetname) - if treply[:3] not in {'125', '150'}: - raise error_proto # RFC 959 + if treply[:3] not in {'125', '150'}: raise error_proto # RFC 959 sreply = source.sendcmd('RETR ' + sourcename) - if sreply[:3] not in {'125', '150'}: - raise error_proto # RFC 959 + if sreply[:3] not in {'125', '150'}: raise error_proto # RFC 959 source.voidresp() target.voidresp() +class Netrc: + """Class to parse & provide access to 'netrc' format files. + + See the netrc(4) man page for information on the file format. + + WARNING: This class is obsolete -- use module netrc instead. + + """ + __defuser = None + __defpasswd = None + __defacct = None + + def __init__(self, filename=None): + if filename is None: + if "HOME" in os.environ: + filename = os.path.join(os.environ["HOME"], + ".netrc") + else: + raise IOError("specify file to load or set $HOME") + self.__hosts = {} + self.__macros = {} + fp = open(filename, "r") + in_macro = 0 + while 1: + line = fp.readline() + if not line: break + if in_macro and line.strip(): + macro_lines.append(line) + continue + elif in_macro: + self.__macros[macro_name] = tuple(macro_lines) + in_macro = 0 + words = line.split() + host = user = passwd = acct = None + default = 0 + i = 0 + while i < len(words): + w1 = words[i] + if i+1 < len(words): + w2 = words[i + 1] + else: + w2 = None + if w1 == 'default': + default = 1 + elif w1 == 'machine' and w2: + host = w2.lower() + i = i + 1 + elif w1 == 'login' and w2: + user = w2 + i = i + 1 + elif w1 == 'password' and w2: + passwd = w2 + i = i + 1 + elif w1 == 'account' and w2: + acct = w2 + i = i + 1 + elif w1 == 'macdef' and w2: + macro_name = w2 + macro_lines = [] + in_macro = 1 + break + i = i + 1 + if default: + self.__defuser = user or self.__defuser + self.__defpasswd = passwd or self.__defpasswd + self.__defacct = acct or self.__defacct + if host: + if host in self.__hosts: + ouser, opasswd, oacct = \ + self.__hosts[host] + user = user or ouser + passwd = passwd or opasswd + acct = acct or oacct + self.__hosts[host] = user, passwd, acct + fp.close() + + def get_hosts(self): + """Return a list of hosts mentioned in the .netrc file.""" + return self.__hosts.keys() + + def get_account(self, host): + """Returns login information for the named host. + + The return value is a triple containing userid, + password, and the accounting field. + + """ + host = host.lower() + user = passwd = acct = None + if host in self.__hosts: + user, passwd, acct = self.__hosts[host] + user = user or self.__defuser + passwd = passwd or self.__defpasswd + acct = acct or self.__defacct + return user, passwd, acct + + def get_macros(self): + """Return a list of all defined macro names.""" + return self.__macros.keys() + + def get_macro(self, macro): + """Return a sequence of lines which define a named macro.""" + return self.__macros[macro] + + + def test(): '''Test program. Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... @@ -937,8 +1054,6 @@ print(test.__doc__) sys.exit(0) - import netrc - debugging = 0 rcfile = None while sys.argv[1] == '-d': @@ -953,14 +1068,14 @@ ftp.set_debuglevel(debugging) userid = passwd = acct = '' try: - netrcobj = netrc.netrc(rcfile) - except OSError: + netrc = Netrc(rcfile) + except IOError: if rcfile is not None: sys.stderr.write("Could not open account file" " -- using anonymous login.") else: try: - userid, acct, passwd = netrcobj.authenticators(host) + userid, passwd, acct = netrc.get_account(host) except KeyError: # no account for host sys.stderr.write( diff -r 6db40a9955dc -r 0d413f60cc23 Lib/functools.py --- a/Lib/functools.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/functools.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,31 +3,20 @@ # Python module wrapper for _functools C module # to allow utilities written in Python to be added # to the functools module. -# Written by Nick Coghlan , -# Raymond Hettinger , -# and Łukasz Langa . -# Copyright (C) 2006-2013 Python Software Foundation. +# Written by Nick Coghlan +# and Raymond Hettinger +# Copyright (C) 2006-2010 Python Software Foundation. # See C source code for _functools credits/copyright __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', - 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', - 'partialmethod', 'singledispatch'] + 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial'] +from _functools import partial, reduce +from collections import namedtuple try: - from _functools import reduce -except ImportError: - pass -from abc import get_cache_token -from collections import namedtuple -from types import MappingProxyType -from weakref import WeakKeyDictionary -try: - from _thread import RLock -except ImportError: - class RLock: - 'Dummy reentrant lock for builds without threads' - def __enter__(self): pass - def __exit__(self, exctype, excinst, exctb): pass + from _thread import allocate_lock as Lock +except: + from _dummy_thread import allocate_lock as Lock ################################################################################ @@ -55,6 +44,7 @@ are updated with the corresponding attribute from the wrapped function (defaults to functools.WRAPPER_UPDATES) """ + wrapper.__wrapped__ = wrapped for attr in assigned: try: value = getattr(wrapped, attr) @@ -64,9 +54,6 @@ setattr(wrapper, attr, value) for attr in updated: getattr(wrapper, attr).update(getattr(wrapped, attr, {})) - # Issue #17482: set __wrapped__ last so we don't inadvertently copy it - # from the wrapped function when updating __dict__ - wrapper.__wrapped__ = wrapped # Return the wrapper so this can be used as a decorator via partial() return wrapper @@ -89,116 +76,31 @@ ### total_ordering class decorator ################################################################################ -# The total ordering functions all invoke the root magic method directly -# rather than using the corresponding operator. This avoids possible -# infinite recursion that could occur when the operator dispatch logic -# detects a NotImplemented result and then calls a reflected method. - -def _gt_from_lt(self, other, NotImplemented=NotImplemented): - 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' - op_result = self.__lt__(other) - if op_result is NotImplemented: - return op_result - return not op_result and self != other - -def _le_from_lt(self, other, NotImplemented=NotImplemented): - 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' - op_result = self.__lt__(other) - return op_result or self == other - -def _ge_from_lt(self, other, NotImplemented=NotImplemented): - 'Return a >= b. Computed by @total_ordering from (not a < b).' - op_result = self.__lt__(other) - if op_result is NotImplemented: - return op_result - return not op_result - -def _ge_from_le(self, other, NotImplemented=NotImplemented): - 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' - op_result = self.__le__(other) - if op_result is NotImplemented: - return op_result - return not op_result or self == other - -def _lt_from_le(self, other, NotImplemented=NotImplemented): - 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' - op_result = self.__le__(other) - if op_result is NotImplemented: - return op_result - return op_result and self != other - -def _gt_from_le(self, other, NotImplemented=NotImplemented): - 'Return a > b. Computed by @total_ordering from (not a <= b).' - op_result = self.__le__(other) - if op_result is NotImplemented: - return op_result - return not op_result - -def _lt_from_gt(self, other, NotImplemented=NotImplemented): - 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' - op_result = self.__gt__(other) - if op_result is NotImplemented: - return op_result - return not op_result and self != other - -def _ge_from_gt(self, other, NotImplemented=NotImplemented): - 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' - op_result = self.__gt__(other) - return op_result or self == other - -def _le_from_gt(self, other, NotImplemented=NotImplemented): - 'Return a <= b. Computed by @total_ordering from (not a > b).' - op_result = self.__gt__(other) - if op_result is NotImplemented: - return op_result - return not op_result - -def _le_from_ge(self, other, NotImplemented=NotImplemented): - 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' - op_result = self.__ge__(other) - if op_result is NotImplemented: - return op_result - return not op_result or self == other - -def _gt_from_ge(self, other, NotImplemented=NotImplemented): - 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' - op_result = self.__ge__(other) - if op_result is NotImplemented: - return op_result - return op_result and self != other - -def _lt_from_ge(self, other, NotImplemented=NotImplemented): - 'Return a < b. Computed by @total_ordering from (not a >= b).' - op_result = self.__ge__(other) - if op_result is NotImplemented: - return op_result - return not op_result - -_convert = { - '__lt__': [('__gt__', _gt_from_lt), - ('__le__', _le_from_lt), - ('__ge__', _ge_from_lt)], - '__le__': [('__ge__', _ge_from_le), - ('__lt__', _lt_from_le), - ('__gt__', _gt_from_le)], - '__gt__': [('__lt__', _lt_from_gt), - ('__ge__', _ge_from_gt), - ('__le__', _le_from_gt)], - '__ge__': [('__le__', _le_from_ge), - ('__gt__', _gt_from_ge), - ('__lt__', _lt_from_ge)] -} - def total_ordering(cls): """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), + ('__ge__', lambda self, other: not self < other)], + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), + ('__gt__', lambda self, other: not self <= other)], + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), + ('__le__', lambda self, other: not self > other)], + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), + ('__lt__', lambda self, other: not self >= other)] + } # Find user-defined comparisons (not those inherited from object). - roots = [op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)] + roots = [op for op in convert if getattr(cls, op, None) is not getattr(object, op, None)] if not roots: raise ValueError('must define at least one ordering operation: < > <= >=') root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ - for opname, opfunc in _convert[root]: + for opname, opfunc in convert[root]: if opname not in roots: opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ setattr(cls, opname, opfunc) return cls @@ -223,6 +125,8 @@ return mycmp(self.obj, other.obj) <= 0 def __ge__(self, other): return mycmp(self.obj, other.obj) >= 0 + def __ne__(self, other): + return mycmp(self.obj, other.obj) != 0 __hash__ = None return K @@ -233,162 +137,12 @@ ################################################################################ -### partial() argument application -################################################################################ - -# Purely functional, no descriptor behaviour -def partial(func, *args, **keywords): - """New function with partial application of the given arguments - and keywords. - """ - if hasattr(func, 'func'): - args = func.args + args - tmpkw = func.keywords.copy() - tmpkw.update(keywords) - keywords = tmpkw - del tmpkw - func = func.func - - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc - -try: - from _functools import partial -except ImportError: - pass - -# Descriptor version -class partialmethod(object): - """Method descriptor with partial application of the given arguments - and keywords. - - Supports wrapping existing descriptors and handles non-descriptor - callables as instance methods. - """ - - def __init__(self, func, *args, **keywords): - if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) - - # func could be a descriptor like classmethod which isn't callable, - # so we can't inherit from partial (it verifies func is callable) - if isinstance(func, partialmethod): - # flattening is mandatory in order to place cls/self before all - # other arguments - # it's also more efficient since only one function will be called - self.func = func.func - self.args = func.args + args - self.keywords = func.keywords.copy() - self.keywords.update(keywords) - else: - self.func = func - self.args = args - self.keywords = keywords - - def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) - - def _make_unbound_method(self): - def _method(*args, **keywords): - call_keywords = self.keywords.copy() - call_keywords.update(keywords) - cls_or_self, *rest = args - call_args = (cls_or_self,) + self.args + tuple(rest) - return self.func(*call_args, **call_keywords) - _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self - return _method - - def __get__(self, obj, cls): - get = getattr(self.func, "__get__", None) - result = None - if get is not None: - new_func = get(obj, cls) - if new_func is not self.func: - # Assume __get__ returning something new indicates the - # creation of an appropriate callable - result = partial(new_func, *self.args, **self.keywords) - try: - result.__self__ = new_func.__self__ - except AttributeError: - pass - if result is None: - # If the underlying descriptor didn't do anything, treat this - # like an instance method - result = self._make_unbound_method().__get__(obj, cls) - return result - - @property - def __isabstractmethod__(self): - return getattr(self.func, "__isabstractmethod__", False) - - -################################################################################ ### LRU Cache function decorator ################################################################################ _CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) -class _HashedSeq(list): - """ This class guarantees that hash() will be called no more than once - per element. This is important because the lru_cache() will hash - the key multiple times on a cache miss. - - """ - - __slots__ = 'hashvalue' - - def __init__(self, tup, hash=hash): - self[:] = tup - self.hashvalue = hash(tup) - - def __hash__(self): - return self.hashvalue - -def _make_key(args, kwds, typed, - kwd_mark = (object(),), - fasttypes = {int, str, frozenset, type(None)}, - sorted=sorted, tuple=tuple, type=type, len=len): - """Make a cache key from optionally typed positional and keyword arguments - - The key is constructed in a way that is flat as possible rather than - as a nested structure that would take more memory. - - If there is only a single argument and its data type is known to cache - its hash value, then that argument is returned without a wrapper. This - saves space and improves lookup speed. - - """ - key = args - if kwds: - sorted_items = sorted(kwds.items()) - key += kwd_mark - for item in sorted_items: - key += item - if typed: - key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for k, v in sorted_items) - elif len(key) == 1 and type(key[0]) in fasttypes: - return key[0] - return _HashedSeq(key) - -def lru_cache(maxsize=128, typed=False): +def lru_cache(maxsize=100, typed=False): """Least-recently-used cache decorator. If *maxsize* is set to None, the LRU features are disabled and the cache @@ -400,352 +154,117 @@ Arguments to the cached function must be hashable. - View the cache statistics named tuple (hits, misses, maxsize, currsize) - with f.cache_info(). Clear the cache and statistics with f.cache_clear(). + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). Access the underlying function with f.__wrapped__. See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used """ - # Users should only access the lru_cache through its public API: # cache_info, cache_clear, and f.__wrapped__ # The internals of the lru_cache are encapsulated for thread safety and # to allow the implementation to change (including a possible C version). - # Early detection of an erroneous call to @lru_cache without any arguments - # resulting in the inner function being passed to maxsize instead of an - # integer or None. - if maxsize is not None and not isinstance(maxsize, int): - raise TypeError('Expected maxsize to be an integer or None') + def decorating_function(user_function): - def decorating_function(user_function): - wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) - return update_wrapper(wrapper, user_function) + cache = dict() + hits = misses = 0 + kwd_mark = (object(),) # separate positional and keyword args + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = Lock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + def make_key(args, kwds, typed, tuple=tuple, sorted=sorted, type=type): + # helper function to build a cache key from positional and keyword args + key = args + if kwds: + sorted_items = tuple(sorted(kwds.items())) + key += kwd_mark + sorted_items + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + return key + + if maxsize == 0: + + @wraps(user_function) + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + nonlocal misses + result = user_function(*args, **kwds) + misses += 1 + return result + + elif maxsize is None: + + @wraps(user_function) + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + nonlocal hits, misses + key = make_key(args, kwds, typed) if kwds or typed else args + result = cache_get(key, root) # root used here as a unique not-found sentinel + if result is not root: + hits += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + misses += 1 + return result + + else: + + @wraps(user_function) + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + nonlocal hits, misses + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it to the front of the list + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + hits += 1 + return result + result = user_function(*args, **kwds) + with lock: + last = root[PREV] + link = [last, root, key, result] + cache[key] = last[NEXT] = root[PREV] = link + if _len(cache) > maxsize: + # purge least recently used cache entry + old_prev, old_next, old_key, old_result = root[NEXT] + root[NEXT] = old_next + old_next[PREV] = root + del cache[old_key] + misses += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(hits, misses, maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + nonlocal hits, misses + with lock: + cache.clear() + root[:] = [root, root, None, None] + hits = misses = 0 + + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return wrapper return decorating_function - -def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): - # Constants shared by all lru cache instances: - sentinel = object() # unique object used to signal cache misses - make_key = _make_key # build a key from the function arguments - PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields - - cache = {} - hits = misses = 0 - full = False - cache_get = cache.get # bound method to lookup a key or return None - lock = RLock() # because linkedlist updates aren't threadsafe - root = [] # root of the circular doubly linked list - root[:] = [root, root, None, None] # initialize by pointing to self - - if maxsize == 0: - - def wrapper(*args, **kwds): - # No caching -- just a statistics update after a successful call - nonlocal misses - result = user_function(*args, **kwds) - misses += 1 - return result - - elif maxsize is None: - - def wrapper(*args, **kwds): - # Simple caching without ordering or size limit - nonlocal hits, misses - key = make_key(args, kwds, typed) - result = cache_get(key, sentinel) - if result is not sentinel: - hits += 1 - return result - result = user_function(*args, **kwds) - cache[key] = result - misses += 1 - return result - - else: - - def wrapper(*args, **kwds): - # Size limited caching that tracks accesses by recency - nonlocal root, hits, misses, full - key = make_key(args, kwds, typed) - with lock: - link = cache_get(key) - if link is not None: - # Move the link to the front of the circular queue - link_prev, link_next, _key, result = link - link_prev[NEXT] = link_next - link_next[PREV] = link_prev - last = root[PREV] - last[NEXT] = root[PREV] = link - link[PREV] = last - link[NEXT] = root - hits += 1 - return result - result = user_function(*args, **kwds) - with lock: - if key in cache: - # Getting here means that this same key was added to the - # cache while the lock was released. Since the link - # update is already done, we need only return the - # computed result and update the count of misses. - pass - elif full: - # Use the old root to store the new key and result. - oldroot = root - oldroot[KEY] = key - oldroot[RESULT] = result - # Empty the oldest link and make it the new root. - # Keep a reference to the old key and old result to - # prevent their ref counts from going to zero during the - # update. That will prevent potentially arbitrary object - # clean-up code (i.e. __del__) from running while we're - # still adjusting the links. - root = oldroot[NEXT] - oldkey = root[KEY] - oldresult = root[RESULT] - root[KEY] = root[RESULT] = None - # Now update the cache dictionary. - del cache[oldkey] - # Save the potentially reentrant cache[key] assignment - # for last, after the root and links have been put in - # a consistent state. - cache[key] = oldroot - else: - # Put result in a new link at the front of the queue. - last = root[PREV] - link = [last, root, key, result] - last[NEXT] = root[PREV] = cache[key] = link - full = (len(cache) >= maxsize) - misses += 1 - return result - - def cache_info(): - """Report cache statistics""" - with lock: - return _CacheInfo(hits, misses, maxsize, len(cache)) - - def cache_clear(): - """Clear the cache and cache statistics""" - nonlocal hits, misses, full - with lock: - cache.clear() - root[:] = [root, root, None, None] - hits = misses = 0 - full = False - - wrapper.cache_info = cache_info - wrapper.cache_clear = cache_clear - return wrapper - -try: - from _functools import _lru_cache_wrapper -except ImportError: - pass - - -################################################################################ -### singledispatch() - single-dispatch generic function decorator -################################################################################ - -def _c3_merge(sequences): - """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - - Adapted from http://www.python.org/download/releases/2.3/mro/. - - """ - result = [] - while True: - sequences = [s for s in sequences if s] # purge empty sequences - if not sequences: - return result - for s1 in sequences: # find merge candidates among seq heads - candidate = s1[0] - for s2 in sequences: - if candidate in s2[1:]: - candidate = None - break # reject the current head, it appears later - else: - break - if candidate is None: - raise RuntimeError("Inconsistent hierarchy") - result.append(candidate) - # remove the chosen candidate - for seq in sequences: - if seq[0] == candidate: - del seq[0] - -def _c3_mro(cls, abcs=None): - """Computes the method resolution order using extended C3 linearization. - - If no *abcs* are given, the algorithm works exactly like the built-in C3 - linearization used for method resolution. - - If given, *abcs* is a list of abstract base classes that should be inserted - into the resulting MRO. Unrelated ABCs are ignored and don't end up in the - result. The algorithm inserts ABCs where their functionality is introduced, - i.e. issubclass(cls, abc) returns True for the class itself but returns - False for all its direct base classes. Implicit ABCs for a given class - (either registered or inferred from the presence of a special method like - __len__) are inserted directly after the last ABC explicitly listed in the - MRO of said class. If two implicit ABCs end up next to each other in the - resulting MRO, their ordering depends on the order of types in *abcs*. - - """ - for i, base in enumerate(reversed(cls.__bases__)): - if hasattr(base, '__abstractmethods__'): - boundary = len(cls.__bases__) - i - break # Bases up to the last explicit ABC are considered first. - else: - boundary = 0 - abcs = list(abcs) if abcs else [] - explicit_bases = list(cls.__bases__[:boundary]) - abstract_bases = [] - other_bases = list(cls.__bases__[boundary:]) - for base in abcs: - if issubclass(cls, base) and not any( - issubclass(b, base) for b in cls.__bases__ - ): - # If *cls* is the class that introduces behaviour described by - # an ABC *base*, insert said ABC to its MRO. - abstract_bases.append(base) - for base in abstract_bases: - abcs.remove(base) - explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases] - abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases] - other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases] - return _c3_merge( - [[cls]] + - explicit_c3_mros + abstract_c3_mros + other_c3_mros + - [explicit_bases] + [abstract_bases] + [other_bases] - ) - -def _compose_mro(cls, types): - """Calculates the method resolution order for a given class *cls*. - - Includes relevant abstract base classes (with their respective bases) from - the *types* iterable. Uses a modified C3 linearization algorithm. - - """ - bases = set(cls.__mro__) - # Remove entries which are already present in the __mro__ or unrelated. - def is_related(typ): - return (typ not in bases and hasattr(typ, '__mro__') - and issubclass(cls, typ)) - types = [n for n in types if is_related(n)] - # Remove entries which are strict bases of other entries (they will end up - # in the MRO anyway. - def is_strict_base(typ): - for other in types: - if typ != other and typ in other.__mro__: - return True - return False - types = [n for n in types if not is_strict_base(n)] - # Subclasses of the ABCs in *types* which are also implemented by - # *cls* can be used to stabilize ABC ordering. - type_set = set(types) - mro = [] - for typ in types: - found = [] - for sub in typ.__subclasses__(): - if sub not in bases and issubclass(cls, sub): - found.append([s for s in sub.__mro__ if s in type_set]) - if not found: - mro.append(typ) - continue - # Favor subclasses with the biggest number of useful bases - found.sort(key=len, reverse=True) - for sub in found: - for subcls in sub: - if subcls not in mro: - mro.append(subcls) - return _c3_mro(cls, abcs=mro) - -def _find_impl(cls, registry): - """Returns the best matching implementation from *registry* for type *cls*. - - Where there is no registered implementation for a specific type, its method - resolution order is used to find a more generic implementation. - - Note: if *registry* does not contain an implementation for the base - *object* type, this function may return None. - - """ - mro = _compose_mro(cls, registry.keys()) - match = None - for t in mro: - if match is not None: - # If *match* is an implicit ABC but there is another unrelated, - # equally matching implicit ABC, refuse the temptation to guess. - if (t in registry and t not in cls.__mro__ - and match not in cls.__mro__ - and not issubclass(match, t)): - raise RuntimeError("Ambiguous dispatch: {} or {}".format( - match, t)) - break - if t in registry: - match = t - return registry.get(match) - -def singledispatch(func): - """Single-dispatch generic function decorator. - - Transforms a function into a generic function, which can have different - behaviours depending upon the type of its first argument. The decorated - function acts as the default implementation, and additional - implementations can be registered using the register() attribute of the - generic function. - - """ - registry = {} - dispatch_cache = WeakKeyDictionary() - cache_token = None - - def dispatch(cls): - """generic_func.dispatch(cls) -> - - Runs the dispatch algorithm to return the best available implementation - for the given *cls* registered on *generic_func*. - - """ - nonlocal cache_token - if cache_token is not None: - current_token = get_cache_token() - if cache_token != current_token: - dispatch_cache.clear() - cache_token = current_token - try: - impl = dispatch_cache[cls] - except KeyError: - try: - impl = registry[cls] - except KeyError: - impl = _find_impl(cls, registry) - dispatch_cache[cls] = impl - return impl - - def register(cls, func=None): - """generic_func.register(cls, func) -> func - - Registers a new implementation for the given *cls* on a *generic_func*. - - """ - nonlocal cache_token - if func is None: - return lambda f: register(cls, f) - registry[cls] = func - if cache_token is None and hasattr(cls, '__abstractmethods__'): - cache_token = get_cache_token() - dispatch_cache.clear() - return func - - def wrapper(*args, **kw): - return dispatch(args[0].__class__)(*args, **kw) - - registry[object] = func - wrapper.register = register - wrapper.dispatch = dispatch - wrapper.registry = MappingProxyType(registry) - wrapper._clear_cache = dispatch_cache.clear - update_wrapper(wrapper, func) - return wrapper diff -r 6db40a9955dc -r 0d413f60cc23 Lib/genericpath.py --- a/Lib/genericpath.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/genericpath.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,8 +7,7 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdir', 'isfile'] # Does a path exist? @@ -17,18 +16,18 @@ """Test whether a path exists. Returns False for broken symbolic links""" try: os.stat(path) - except OSError: + except os.error: return False return True # This follows symbolic links, so both islink() and isdir() can be true -# for the same path on systems that support symlinks +# for the same path ono systems that support symlinks def isfile(path): """Test whether a path is a regular file""" try: st = os.stat(path) - except OSError: + except os.error: return False return stat.S_ISREG(st.st_mode) @@ -40,7 +39,7 @@ """Return true if the pathname refers to an existing directory.""" try: st = os.stat(s) - except OSError: + except os.error: return False return stat.S_ISDIR(st.st_mode) @@ -76,31 +75,6 @@ return s1[:i] return s1 -# Are two stat buffers (obtained from stat, fstat or lstat) -# describing the same file? -def samestat(s1, s2): - """Test whether two stat buffers reference the same file""" - return (s1.st_ino == s2.st_ino and - s1.st_dev == s2.st_dev) - - -# Are two filenames really pointing to the same file? -def samefile(f1, f2): - """Test whether two pathnames reference the same actual file""" - s1 = os.stat(f1) - s2 = os.stat(f2) - return samestat(s1, s2) - - -# Are two open files really referencing the same file? -# (Not necessarily the same file descriptor!) -def sameopenfile(fp1, fp2): - """Test whether two open file objects reference the same file""" - s1 = os.fstat(fp1) - s2 = os.fstat(fp2) - return samestat(s1, s2) - - # Split a path in root and extension. # The extension is everything starting at the last dot in the last # pathname component; the root is everything before that. @@ -130,16 +104,3 @@ filenameIndex += 1 return p, p[:0] - -def _check_arg_types(funcname, *args): - hasstr = hasbytes = False - for s in args: - if isinstance(s, str): - hasstr = True - elif isinstance(s, bytes): - hasbytes = True - else: - raise TypeError('%s() argument must be str or bytes, not %r' % - (funcname, s.__class__.__name__)) from None - if hasstr and hasbytes: - raise TypeError("Can't mix strings and bytes in path components") from None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/getopt.py --- a/Lib/getopt.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/getopt.py Mon Jan 25 17:05:13 2016 +0100 @@ -28,7 +28,7 @@ # - RETURN_IN_ORDER option # - GNU extension with '-' as first character of option string # - optional arguments, specified by double colons -# - an option string with a W followed by semicolon should +# - a option string with a W followed by semicolon should # treat "-W foo" as "--foo" __all__ = ["GetoptError","error","getopt","gnu_getopt"] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/getpass.py --- a/Lib/getpass.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/getpass.py Mon Jan 25 17:05:13 2016 +0100 @@ -15,11 +15,7 @@ # Guido van Rossum (Windows support and cleanup) # Gregory P. Smith (tty support & GetPassWarning) -import contextlib -import io -import os -import sys -import warnings +import os, sys, warnings __all__ = ["getpass","getuser","GetPassWarning"] @@ -42,64 +38,59 @@ Always restores terminal settings before returning. """ - passwd = None - with contextlib.ExitStack() as stack: + fd = None + tty = None + try: + # Always try reading and writing directly on the tty first. + fd = os.open('/dev/tty', os.O_RDWR|os.O_NOCTTY) + tty = os.fdopen(fd, 'w+', 1) + input = tty + if not stream: + stream = tty + except EnvironmentError as e: + # If that fails, see if stdin can be controlled. try: - # Always try reading and writing directly on the tty first. - fd = os.open('/dev/tty', os.O_RDWR|os.O_NOCTTY) - tty = io.FileIO(fd, 'w+') - stack.enter_context(tty) - input = io.TextIOWrapper(tty) - stack.enter_context(input) - if not stream: - stream = input - except OSError as e: - # If that fails, see if stdin can be controlled. - stack.close() + fd = sys.stdin.fileno() + except (AttributeError, ValueError): + passwd = fallback_getpass(prompt, stream) + input = sys.stdin + if not stream: + stream = sys.stderr + + if fd is not None: + passwd = None + try: + old = termios.tcgetattr(fd) # a copy to save + new = old[:] + new[3] &= ~termios.ECHO # 3 == 'lflags' + tcsetattr_flags = termios.TCSAFLUSH + if hasattr(termios, 'TCSASOFT'): + tcsetattr_flags |= termios.TCSASOFT try: - fd = sys.stdin.fileno() - except (AttributeError, ValueError): - fd = None - passwd = fallback_getpass(prompt, stream) - input = sys.stdin - if not stream: - stream = sys.stderr + termios.tcsetattr(fd, tcsetattr_flags, new) + passwd = _raw_input(prompt, stream, input=input) + finally: + termios.tcsetattr(fd, tcsetattr_flags, old) + stream.flush() # issue7208 + except termios.error: + if passwd is not None: + # _raw_input succeeded. The final tcsetattr failed. Reraise + # instead of leaving the terminal in an unknown state. + raise + # We can't control the tty or stdin. Give up and use normal IO. + # fallback_getpass() raises an appropriate warning. + del input, tty # clean up unused file objects before blocking + passwd = fallback_getpass(prompt, stream) - if fd is not None: - try: - old = termios.tcgetattr(fd) # a copy to save - new = old[:] - new[3] &= ~termios.ECHO # 3 == 'lflags' - tcsetattr_flags = termios.TCSAFLUSH - if hasattr(termios, 'TCSASOFT'): - tcsetattr_flags |= termios.TCSASOFT - try: - termios.tcsetattr(fd, tcsetattr_flags, new) - passwd = _raw_input(prompt, stream, input=input) - finally: - termios.tcsetattr(fd, tcsetattr_flags, old) - stream.flush() # issue7208 - except termios.error: - if passwd is not None: - # _raw_input succeeded. The final tcsetattr failed. Reraise - # instead of leaving the terminal in an unknown state. - raise - # We can't control the tty or stdin. Give up and use normal IO. - # fallback_getpass() raises an appropriate warning. - if stream is not input: - # clean up unused file objects before blocking - stack.close() - passwd = fallback_getpass(prompt, stream) - - stream.write('\n') - return passwd + stream.write('\n') + return passwd def win_getpass(prompt='Password: ', stream=None): """Prompt for password with echo off, using Windows getch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) - + import msvcrt for c in prompt: msvcrt.putwch(c) pw = "" @@ -135,13 +126,7 @@ input = sys.stdin prompt = str(prompt) if prompt: - try: - stream.write(prompt) - except UnicodeEncodeError: - # Use replace error handler to get as much as possible printed. - prompt = prompt.encode(stream.encoding, 'replace') - prompt = prompt.decode(stream.encoding) - stream.write(prompt) + stream.write(prompt) stream.flush() # NOTE: The Python C API calls flockfile() (and unlock) during readline. line = input.readline() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/gettext.py --- a/Lib/gettext.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/gettext.py Mon Jan 25 17:05:13 2016 +0100 @@ -52,12 +52,10 @@ __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog', 'find', 'translation', 'install', 'textdomain', 'bindtextdomain', - 'bind_textdomain_codeset', - 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext', - 'ldngettext', 'lngettext', 'ngettext', + 'dgettext', 'dngettext', 'gettext', 'ngettext', ] -_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale') +_default_localedir = os.path.join(sys.prefix, 'share', 'locale') def c2py(plural): @@ -227,13 +225,6 @@ LE_MAGIC = 0x950412de BE_MAGIC = 0xde120495 - # Acceptable .mo versions - VERSIONS = (0, 1) - - def _get_versions(self, version): - """Returns a tuple of major version, minor version""" - return (version >> 16, version & 0xffff) - def _parse(self, fp): """Override this method to support alternative .mo formats.""" unpack = struct.unpack @@ -253,13 +244,7 @@ version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20]) ii = '>II' else: - raise OSError(0, 'Bad magic number', filename) - - major_version, minor_version = self._get_versions(version) - - if major_version not in self.VERSIONS: - raise OSError(0, 'Bad version number ' + str(major_version), filename) - + raise IOError(0, 'Bad magic number', filename) # Now put all messages from the .mo file buffer into the catalog # dictionary. for i in range(0, msgcount): @@ -271,16 +256,15 @@ msg = buf[moff:mend] tmsg = buf[toff:tend] else: - raise OSError(0, 'File is corrupt', filename) + raise IOError(0, 'File is corrupt', filename) # See if we're looking at GNU .mo conventions for metadata if mlen == 0: # Catalog description - lastk = None + lastk = k = None for b_item in tmsg.split('\n'.encode("ascii")): item = b_item.decode().strip() if not item: continue - k = v = None if ':' in item: k, v = item.split(':', 1) k = k.strip().lower() @@ -414,7 +398,7 @@ if not mofiles: if fallback: return NullTranslations() - raise OSError(ENOENT, 'No translation file found for domain', domain) + raise IOError(ENOENT, 'No translation file found for domain', domain) # Avoid opening, reading, and parsing the .mo file after it's been done # once. result = None @@ -476,7 +460,7 @@ try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) - except OSError: + except IOError: return message return t.gettext(message) @@ -484,7 +468,7 @@ try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) - except OSError: + except IOError: return message return t.lgettext(message) @@ -492,7 +476,7 @@ try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) - except OSError: + except IOError: if n == 1: return msgid1 else: @@ -503,7 +487,7 @@ try: t = translation(domain, _localedirs.get(domain, None), codeset=_localecodesets.get(domain)) - except OSError: + except IOError: if n == 1: return msgid1 else: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/glob.py --- a/Lib/glob.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/glob.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,67 +4,37 @@ import re import fnmatch -__all__ = ["glob", "iglob", "escape"] +__all__ = ["glob", "iglob"] -def glob(pathname, *, recursive=False): +def glob(pathname): """Return a list of paths matching a pathname pattern. - The pattern may contain simple shell-style wildcards a la - fnmatch. However, unlike fnmatch, filenames starting with a - dot are special cases that are not matched by '*' and '?' - patterns. + The pattern may contain simple shell-style wildcards a la fnmatch. - If recursive is true, the pattern '**' will match any files and - zero or more directories and subdirectories. """ - return list(iglob(pathname, recursive=recursive)) + return list(iglob(pathname)) -def iglob(pathname, *, recursive=False): +def iglob(pathname): """Return an iterator which yields the paths matching a pathname pattern. - The pattern may contain simple shell-style wildcards a la - fnmatch. However, unlike fnmatch, filenames starting with a - dot are special cases that are not matched by '*' and '?' - patterns. + The pattern may contain simple shell-style wildcards a la fnmatch. - If recursive is true, the pattern '**' will match any files and - zero or more directories and subdirectories. """ - it = _iglob(pathname, recursive) - if recursive and _isrecursive(pathname): - s = next(it) # skip empty string - assert not s - return it - -def _iglob(pathname, recursive): + if not has_magic(pathname): + if os.path.lexists(pathname): + yield pathname + return dirname, basename = os.path.split(pathname) - if not has_magic(pathname): - if basename: - if os.path.lexists(pathname): - yield pathname - else: - # Patterns ending with a slash should match only directories - if os.path.isdir(dirname): - yield pathname + if not dirname: + for name in glob1(None, basename): + yield name return - if not dirname: - if recursive and _isrecursive(basename): - yield from glob2(dirname, basename) - else: - yield from glob1(dirname, basename) - return - # `os.path.split()` returns the argument itself as a dirname if it is a - # drive or UNC path. Prevent an infinite recursion if a drive or UNC path - # contains magic characters (i.e. r'\\?\C:'). - if dirname != pathname and has_magic(dirname): - dirs = _iglob(dirname, recursive) + if has_magic(dirname): + dirs = iglob(dirname) else: dirs = [dirname] if has_magic(basename): - if recursive and _isrecursive(basename): - glob_in_dir = glob2 - else: - glob_in_dir = glob1 + glob_in_dir = glob1 else: glob_in_dir = glob0 for dirname in dirs: @@ -83,14 +53,14 @@ dirname = os.curdir try: names = os.listdir(dirname) - except OSError: + except os.error: return [] - if not _ishidden(pattern): - names = [x for x in names if not _ishidden(x)] + if pattern[0] != '.': + names = [x for x in names if x[0] != '.'] return fnmatch.filter(names, pattern) def glob0(dirname, basename): - if not basename: + if basename == '': # `os.path.split()` returns an empty basename for paths ending with a # directory separator. 'q*x/' should match only directories. if os.path.isdir(dirname): @@ -100,35 +70,9 @@ return [basename] return [] -# This helper function recursively yields relative pathnames inside a literal -# directory. -def glob2(dirname, pattern): - assert _isrecursive(pattern) - yield pattern[:0] - yield from _rlistdir(dirname) - -# Recursively yields relative pathnames inside a literal directory. -def _rlistdir(dirname): - if not dirname: - if isinstance(dirname, bytes): - dirname = bytes(os.curdir, 'ASCII') - else: - dirname = os.curdir - try: - names = os.listdir(dirname) - except os.error: - return - for x in names: - if not _ishidden(x): - yield x - path = os.path.join(dirname, x) if dirname else x - for y in _rlistdir(path): - yield os.path.join(x, y) - - -magic_check = re.compile('([*?[])') -magic_check_bytes = re.compile(b'([*?[])') +magic_check = re.compile('[*?[]') +magic_check_bytes = re.compile(b'[*?[]') def has_magic(s): if isinstance(s, bytes): @@ -136,24 +80,3 @@ else: match = magic_check.search(s) return match is not None - -def _ishidden(path): - return path[0] in ('.', b'.'[0]) - -def _isrecursive(pattern): - if isinstance(pattern, bytes): - return pattern == b'**' - else: - return pattern == '**' - -def escape(pathname): - """Escape all special characters. - """ - # Escaping is done by wrapping any of "*?[" between square brackets. - # Metacharacters do not work in the drive part and shouldn't be escaped. - drive, pathname = os.path.splitdrive(pathname) - if isinstance(pathname, bytes): - pathname = magic_check_bytes.sub(br'[\1]', pathname) - else: - pathname = magic_check.sub(r'[\1]', pathname) - return drive + pathname diff -r 6db40a9955dc -r 0d413f60cc23 Lib/gzip.py --- a/Lib/gzip.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/gzip.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,7 +9,6 @@ import zlib import builtins import io -import _compression __all__ = ["GzipFile", "open", "compress", "decompress"] @@ -17,55 +16,23 @@ READ, WRITE = 1, 2 -def open(filename, mode="rb", compresslevel=9, - encoding=None, errors=None, newline=None): - """Open a gzip-compressed file in binary or text mode. - - The filename argument can be an actual filename (a str or bytes object), or - an existing file object to read from or write to. - - The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for - binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is - "rb", and the default compresslevel is 9. - - For binary mode, this function is equivalent to the GzipFile constructor: - GzipFile(filename, mode, compresslevel). In this case, the encoding, errors - and newline arguments must not be provided. - - For text mode, a GzipFile object is created, and wrapped in an - io.TextIOWrapper instance with the specified encoding, error handling - behavior, and line ending(s). - - """ - if "t" in mode: - if "b" in mode: - raise ValueError("Invalid mode: %r" % (mode,)) - else: - if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") - if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") - if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") - - gz_mode = mode.replace("t", "") - if isinstance(filename, (str, bytes)): - binary_file = GzipFile(filename, gz_mode, compresslevel) - elif hasattr(filename, "read") or hasattr(filename, "write"): - binary_file = GzipFile(None, gz_mode, compresslevel, filename) - else: - raise TypeError("filename must be a str or bytes object, or a file") - - if "t" in mode: - return io.TextIOWrapper(binary_file, encoding, errors, newline) - else: - return binary_file - def write32u(output, value): # The L format writes the bit pattern correctly whether signed # or unsigned. output.write(struct.pack("' - def __repr__(self): - s = repr(self.fileobj) - return '' + def _check_closed(self): + """Raises a ValueError if the underlying file object has been closed. + + """ + if self.closed: + raise ValueError('I/O operation on closed file.') def _init_write(self, filename): self.name = filename - self.crc = zlib.crc32(b"") + self.crc = zlib.crc32(b"") & 0xffffffff self.size = 0 self.writebuf = [] self.bufsize = 0 - self.offset = 0 # Current file offset for seek(), tell(), etc def _write_gzip_header(self): self.fileobj.write(b'\037\213') # magic header @@ -223,8 +223,7 @@ # RFC 1952 requires the FNAME field to be Latin-1. Do not # include filenames that cannot be represented that way. fname = os.path.basename(self.name) - if not isinstance(fname, bytes): - fname = fname.encode('latin-1') + fname = fname.encode('latin-1') if fname.endswith(b'.gz'): fname = fname[:-3] except UnicodeEncodeError: @@ -233,7 +232,7 @@ if fname: flags = FNAME self.fileobj.write(chr(flags).encode('latin-1')) - mtime = self._write_mtime + mtime = self.mtime if mtime is None: mtime = time.time() write32u(self.fileobj, int(mtime)) @@ -242,82 +241,246 @@ if fname: self.fileobj.write(fname + b'\000') + def _init_read(self): + self.crc = zlib.crc32(b"") & 0xffffffff + self.size = 0 + + def _read_gzip_header(self): + magic = self.fileobj.read(2) + if magic == b'': + raise EOFError("Reached EOF") + + if magic != b'\037\213': + raise IOError('Not a gzipped file') + method = ord( self.fileobj.read(1) ) + if method != 8: + raise IOError('Unknown compression method') + flag = ord( self.fileobj.read(1) ) + self.mtime = read32(self.fileobj) + # extraflag = self.fileobj.read(1) + # os = self.fileobj.read(1) + self.fileobj.read(2) + + if flag & FEXTRA: + # Read & discard the extra field, if present + xlen = ord(self.fileobj.read(1)) + xlen = xlen + 256*ord(self.fileobj.read(1)) + self.fileobj.read(xlen) + if flag & FNAME: + # Read and discard a null-terminated string containing the filename + while True: + s = self.fileobj.read(1) + if not s or s==b'\000': + break + if flag & FCOMMENT: + # Read and discard a null-terminated string containing a comment + while True: + s = self.fileobj.read(1) + if not s or s==b'\000': + break + if flag & FHCRC: + self.fileobj.read(2) # Read & discard the 16-bit header CRC + + unused = self.fileobj.unused() + if unused: + uncompress = self.decompress.decompress(unused) + self._add_read_data(uncompress) + def write(self,data): - self._check_not_closed() + self._check_closed() if self.mode != WRITE: import errno - raise OSError(errno.EBADF, "write() on read-only GzipFile object") + raise IOError(errno.EBADF, "write() on read-only GzipFile object") if self.fileobj is None: raise ValueError("write() on closed GzipFile object") - if isinstance(data, bytes): - length = len(data) - else: - # accept any data that supports the buffer protocol - data = memoryview(data) - length = data.nbytes + # Convert data type if called by io.BufferedWriter. + if isinstance(data, memoryview): + data = data.tobytes() - if length > 0: - self.fileobj.write(self.compress.compress(data)) - self.size += length - self.crc = zlib.crc32(data, self.crc) - self.offset += length + if len(data) > 0: + self.size = self.size + len(data) + self.crc = zlib.crc32(data, self.crc) & 0xffffffff + self.fileobj.write( self.compress.compress(data) ) + self.offset += len(data) - return length + return len(data) def read(self, size=-1): - self._check_not_closed() + self._check_closed() if self.mode != READ: import errno - raise OSError(errno.EBADF, "read() on write-only GzipFile object") - return self._buffer.read(size) + raise IOError(errno.EBADF, "read() on write-only GzipFile object") + + if self.extrasize <= 0 and self.fileobj is None: + return b'' + + readsize = 1024 + if size < 0: # get the whole thing + try: + while True: + self._read(readsize) + readsize = min(self.max_read_chunk, readsize * 2) + except EOFError: + size = self.extrasize + else: # just get some more of it + try: + while size > self.extrasize: + self._read(readsize) + readsize = min(self.max_read_chunk, readsize * 2) + except EOFError: + if size > self.extrasize: + size = self.extrasize + + offset = self.offset - self.extrastart + chunk = self.extrabuf[offset: offset + size] + self.extrasize = self.extrasize - size + + self.offset += size + return chunk def read1(self, size=-1): - """Implements BufferedIOBase.read1() - - Reads up to a buffer's worth of data is size is negative.""" - self._check_not_closed() + self._check_closed() if self.mode != READ: import errno - raise OSError(errno.EBADF, "read1() on write-only GzipFile object") + raise IOError(errno.EBADF, "read1() on write-only GzipFile object") - if size < 0: - size = io.DEFAULT_BUFFER_SIZE - return self._buffer.read1(size) + if self.extrasize <= 0 and self.fileobj is None: + return b'' + + try: + self._read() + except EOFError: + pass + if size < 0 or size > self.extrasize: + size = self.extrasize + + offset = self.offset - self.extrastart + chunk = self.extrabuf[offset: offset + size] + self.extrasize -= size + self.offset += size + return chunk def peek(self, n): - self._check_not_closed() if self.mode != READ: import errno - raise OSError(errno.EBADF, "peek() on write-only GzipFile object") - return self._buffer.peek(n) + raise IOError(errno.EBADF, "peek() on write-only GzipFile object") + + # Do not return ridiculously small buffers, for one common idiom + # is to call peek(1) and expect more bytes in return. + if n < 100: + n = 100 + if self.extrasize == 0: + if self.fileobj is None: + return b'' + try: + # 1024 is the same buffering heuristic used in read() + self._read(max(n, 1024)) + except EOFError: + pass + offset = self.offset - self.extrastart + remaining = self.extrasize + assert remaining == len(self.extrabuf) - offset + return self.extrabuf[offset:offset + n] + + def _unread(self, buf): + self.extrasize = len(buf) + self.extrasize + self.offset -= len(buf) + + def _read(self, size=1024): + if self.fileobj is None: + raise EOFError("Reached EOF") + + if self._new_member: + # If the _new_member flag is set, we have to + # jump to the next member, if there is one. + self._init_read() + self._read_gzip_header() + self.decompress = zlib.decompressobj(-zlib.MAX_WBITS) + self._new_member = False + + # Read a chunk of data from the file + buf = self.fileobj.read(size) + + # If the EOF has been reached, flush the decompression object + # and mark this object as finished. + + if buf == b"": + uncompress = self.decompress.flush() + # Prepend the already read bytes to the fileobj to they can be + # seen by _read_eof() + self.fileobj.prepend(self.decompress.unused_data, True) + self._read_eof() + self._add_read_data( uncompress ) + raise EOFError('Reached EOF') + + uncompress = self.decompress.decompress(buf) + self._add_read_data( uncompress ) + + if self.decompress.unused_data != b"": + # Ending case: we've come to the end of a member in the file, + # so seek back to the start of the unused data, finish up + # this member, and read a new gzip header. + # Prepend the already read bytes to the fileobj to they can be + # seen by _read_eof() and _read_gzip_header() + self.fileobj.prepend(self.decompress.unused_data, True) + # Check the CRC and file size, and set the flag so we read + # a new member on the next call + self._read_eof() + self._new_member = True + + def _add_read_data(self, data): + self.crc = zlib.crc32(data, self.crc) & 0xffffffff + offset = self.offset - self.extrastart + self.extrabuf = self.extrabuf[offset:] + data + self.extrasize = self.extrasize + len(data) + self.extrastart = self.offset + self.size = self.size + len(data) + + def _read_eof(self): + # We've read to the end of the file + # We check the that the computed CRC and size of the + # uncompressed data matches the stored values. Note that the size + # stored is the true file size mod 2**32. + crc32 = read32(self.fileobj) + isize = read32(self.fileobj) # may exceed 2GB + if crc32 != self.crc: + raise IOError("CRC check failed %s != %s" % (hex(crc32), + hex(self.crc))) + elif isize != (self.size & 0xffffffff): + raise IOError("Incorrect length of data produced") + + # Gzip files can be padded with zeroes and still have archives. + # Consume all zero bytes and set the file position to the first + # non-zero byte. See http://www.gzip.org/#faq8 + c = b"\x00" + while c == b"\x00": + c = self.fileobj.read(1) + if c: + self.fileobj.prepend(c, True) @property def closed(self): return self.fileobj is None def close(self): - fileobj = self.fileobj - if fileobj is None: + if self.fileobj is None: return - self.fileobj = None - try: - if self.mode == WRITE: - fileobj.write(self.compress.flush()) - write32u(fileobj, self.crc) - # self.size may exceed 2GB, or even 4GB - write32u(fileobj, self.size & 0xffffffff) - elif self.mode == READ: - self._buffer.close() - finally: - myfileobj = self.myfileobj - if myfileobj: - self.myfileobj = None - myfileobj.close() + if self.mode == WRITE: + self.fileobj.write(self.compress.flush()) + write32u(self.fileobj, self.crc) + # self.size may exceed 2GB, or even 4GB + write32u(self.fileobj, self.size & 0xffffffff) + self.fileobj = None + elif self.mode == READ: + self.fileobj = None + if self.myfileobj: + self.myfileobj.close() + self.myfileobj = None def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): - self._check_not_closed() + self._check_closed() if self.mode == WRITE: # Ensure the compressor's buffer is flushed self.fileobj.write(self.compress.flush(zlib_mode)) @@ -335,8 +498,13 @@ '''Return the uncompressed stream file position indicator to the beginning of the file''' if self.mode != READ: - raise OSError("Can't rewind in write mode") - self._buffer.seek(0) + raise IOError("Can't rewind in write mode") + self.fileobj.seek(0) + self._new_member = True + self.extrabuf = b"" + self.extrasize = 0 + self.extrastart = 0 + self.offset = 0 def readable(self): return self.mode == READ @@ -347,175 +515,74 @@ def seekable(self): return True - def seek(self, offset, whence=io.SEEK_SET): + def seek(self, offset, whence=0): + if whence: + if whence == 1: + offset = self.offset + offset + else: + raise ValueError('Seek from end not supported') if self.mode == WRITE: - if whence != io.SEEK_SET: - if whence == io.SEEK_CUR: - offset = self.offset + offset - else: - raise ValueError('Seek from end not supported') if offset < self.offset: - raise OSError('Negative seek in write mode') + raise IOError('Negative seek in write mode') count = offset - self.offset chunk = bytes(1024) for i in range(count // 1024): self.write(chunk) self.write(bytes(count % 1024)) elif self.mode == READ: - self._check_not_closed() - return self._buffer.seek(offset, whence) + if offset < self.offset: + # for negative seek, rewind and do positive seek + self.rewind() + count = offset - self.offset + for i in range(count // 1024): + self.read(1024) + self.read(count % 1024) return self.offset def readline(self, size=-1): - self._check_not_closed() - return self._buffer.readline(size) + if size < 0: + # Shortcut common case - newline found in buffer. + offset = self.offset - self.extrastart + i = self.extrabuf.find(b'\n', offset) + 1 + if i > 0: + self.extrasize -= i - offset + self.offset += i - offset + return self.extrabuf[offset: i] + size = sys.maxsize + readsize = self.min_readsize + else: + readsize = size + bufs = [] + while size != 0: + c = self.read(readsize) + i = c.find(b'\n') -class _GzipReader(_compression.DecompressReader): - def __init__(self, fp): - super().__init__(_PaddedFile(fp), zlib.decompressobj, - wbits=-zlib.MAX_WBITS) - # Set flag indicating start of a new member - self._new_member = True - self._last_mtime = None + # We set i=size to break out of the loop under two + # conditions: 1) there's no newline, and the chunk is + # larger than size, or 2) there is a newline, but the + # resulting line would be longer than 'size'. + if (size <= i) or (i == -1 and len(c) > size): + i = size - 1 - def _init_read(self): - self._crc = zlib.crc32(b"") - self._stream_size = 0 # Decompressed size of unconcatenated stream + if i >= 0 or c == b'': + bufs.append(c[:i + 1]) # Add portion of last chunk + self._unread(c[i + 1:]) # Push back rest of chunk + break - def _read_exact(self, n): - '''Read exactly *n* bytes from `self._fp` + # Append chunk to list, decrease 'size', + bufs.append(c) + size = size - len(c) + readsize = min(size, readsize * 2) + if readsize > self.min_readsize: + self.min_readsize = min(readsize, self.min_readsize * 2, 512) + return b''.join(bufs) # Return resulting line - This method is required because self._fp may be unbuffered, - i.e. return short reads. - ''' - - data = self._fp.read(n) - while len(data) < n: - b = self._fp.read(n - len(data)) - if not b: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") - data += b - return data - - def _read_gzip_header(self): - magic = self._fp.read(2) - if magic == b'': - return False - - if magic != b'\037\213': - raise OSError('Not a gzipped file (%r)' % magic) - - (method, flag, - self._last_mtime) = struct.unpack(" blocksize: - password = new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - dkey = b'' - loop = 1 - from_bytes = int.from_bytes - while len(dkey) < dklen: - prev = prf(salt + loop.to_bytes(4, 'big')) - # endianess doesn't matter here as long to / from use the same - rkey = int.from_bytes(prev, 'big') - for i in range(iterations - 1): - prev = prf(prev) - # rkey = rkey ^ prev - rkey ^= from_bytes(prev, 'big') - loop += 1 - dkey += rkey.to_bytes(inner.digest_size, 'big') - - return dkey[:dklen] - - for __func_name in __always_supported: # try them all, some may not work due to the OpenSSL # version not supporting that algorithm. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/heapq.py --- a/Lib/heapq.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/heapq.py Mon Jan 25 17:05:13 2016 +0100 @@ -127,6 +127,9 @@ __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', 'nlargest', 'nsmallest', 'heappushpop'] +from itertools import islice, repeat, count, tee, chain +import bisect + def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" heap.append(item) @@ -139,8 +142,9 @@ returnitem = heap[0] heap[0] = lastelt _siftup(heap, 0) - return returnitem - return lastelt + else: + returnitem = lastelt + return returnitem def heapreplace(heap, item): """Pop and return the current smallest value, and add the new item. @@ -176,28 +180,55 @@ for i in reversed(range(n//2)): _siftup(x, i) -def _heappop_max(heap): - """Maxheap version of a heappop.""" - lastelt = heap.pop() # raises appropriate IndexError if heap is empty - if heap: - returnitem = heap[0] - heap[0] = lastelt - _siftup_max(heap, 0) - return returnitem - return lastelt +def nlargest(n, iterable): + """Find the n largest elements in a dataset. -def _heapreplace_max(heap, item): - """Maxheap version of a heappop followed by a heappush.""" - returnitem = heap[0] # raises appropriate IndexError if heap is empty - heap[0] = item - _siftup_max(heap, 0) - return returnitem + Equivalent to: sorted(iterable, reverse=True)[:n] + """ + if n < 0: + return [] + it = iter(iterable) + result = list(islice(it, n)) + if not result: + return result + heapify(result) + _heappushpop = heappushpop + for elem in it: + _heappushpop(result, elem) + result.sort(reverse=True) + return result -def _heapify_max(x): - """Transform list into a maxheap, in-place, in O(len(x)) time.""" - n = len(x) - for i in reversed(range(n//2)): - _siftup_max(x, i) +def nsmallest(n, iterable): + """Find the n smallest elements in a dataset. + + Equivalent to: sorted(iterable)[:n] + """ + if n < 0: + return [] + if hasattr(iterable, '__len__') and n * 10 <= len(iterable): + # For smaller values of n, the bisect method is faster than a minheap. + # It is also memory efficient, consuming only n elements of space. + it = iter(iterable) + result = sorted(islice(it, 0, n)) + if not result: + return result + insort = bisect.insort + pop = result.pop + los = result[-1] # los --> Largest of the nsmallest + for elem in it: + if elem < los: + insort(result, elem) + pop() + los = result[-1] + return result + # An alternative approach manifests the whole iterable in memory but + # saves comparisons by heapifying all at once. Also, saves time + # over bisect.insort() which has O(n) data movement time for every + # insertion. Finding the n smallest of an m length iterable requires + # O(m) + O(n log m) comparisons. + h = list(iterable) + heapify(h) + return list(map(heappop, repeat(h, min(n, len(h))))) # 'heap' is a heap at all indices >= startpos, except possibly for pos. pos # is the index of a leaf with a possibly out-of-order value. Restore the @@ -275,43 +306,13 @@ heap[pos] = newitem _siftdown(heap, startpos, pos) -def _siftdown_max(heap, startpos, pos): - 'Maxheap variant of _siftdown' - newitem = heap[pos] - # Follow the path to the root, moving parents down until finding a place - # newitem fits. - while pos > startpos: - parentpos = (pos - 1) >> 1 - parent = heap[parentpos] - if parent < newitem: - heap[pos] = parent - pos = parentpos - continue - break - heap[pos] = newitem +# If available, use C implementation +try: + from _heapq import * +except ImportError: + pass -def _siftup_max(heap, pos): - 'Maxheap variant of _siftup' - endpos = len(heap) - startpos = pos - newitem = heap[pos] - # Bubble up the larger child until hitting a leaf. - childpos = 2*pos + 1 # leftmost child position - while childpos < endpos: - # Set childpos to index of larger child. - rightpos = childpos + 1 - if rightpos < endpos and not heap[rightpos] < heap[childpos]: - childpos = rightpos - # Move the larger child up. - heap[pos] = heap[childpos] - pos = childpos - childpos = 2*pos + 1 - # The leaf at pos is empty now. Put newitem there, and bubble it up - # to its final resting place (by sifting its parents down). - heap[pos] = newitem - _siftdown_max(heap, startpos, pos) - -def merge(*iterables, key=None, reverse=False): +def merge(*iterables): '''Merge multiple sorted inputs into a single sorted output. Similar to sorted(itertools.chain(*iterables)) but returns a generator, @@ -321,158 +322,47 @@ >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] - If *key* is not None, applies a key function to each element to determine - its sort order. - - >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) - ['dog', 'cat', 'fish', 'horse', 'kangaroo'] - ''' + _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration h = [] h_append = h.append - - if reverse: - _heapify = _heapify_max - _heappop = _heappop_max - _heapreplace = _heapreplace_max - direction = -1 - else: - _heapify = heapify - _heappop = heappop - _heapreplace = heapreplace - direction = 1 - - if key is None: - for order, it in enumerate(map(iter, iterables)): - try: - next = it.__next__ - h_append([next(), order * direction, next]) - except StopIteration: - pass - _heapify(h) - while len(h) > 1: - try: - while True: - value, order, next = s = h[0] - yield value - s[0] = next() # raises StopIteration when exhausted - _heapreplace(h, s) # restore heap condition - except StopIteration: - _heappop(h) # remove empty iterator - if h: - # fast case when only a single iterator remains - value, order, next = h[0] - yield value - yield from next.__self__ - return - - for order, it in enumerate(map(iter, iterables)): + for itnum, it in enumerate(map(iter, iterables)): try: next = it.__next__ - value = next() - h_append([key(value), order * direction, value, next]) - except StopIteration: + h_append([next(), itnum, next]) + except _StopIteration: pass - _heapify(h) - while len(h) > 1: + heapify(h) + + while 1: try: - while True: - key_value, order, value, next = s = h[0] - yield value - value = next() - s[0] = key(value) - s[2] = value - _heapreplace(h, s) - except StopIteration: - _heappop(h) - if h: - key_value, order, value, next = h[0] - yield value - yield from next.__self__ + while 1: + v, itnum, next = s = h[0] # raises IndexError when h is empty + yield v + s[0] = next() # raises StopIteration when exhausted + _heapreplace(h, s) # restore heap condition + except _StopIteration: + _heappop(h) # remove empty iterator + except IndexError: + return - -# Algorithm notes for nlargest() and nsmallest() -# ============================================== -# -# Make a single pass over the data while keeping the k most extreme values -# in a heap. Memory consumption is limited to keeping k values in a list. -# -# Measured performance for random inputs: -# -# number of comparisons -# n inputs k-extreme values (average of 5 trials) % more than min() -# ------------- ---------------- --------------------- ----------------- -# 1,000 100 3,317 231.7% -# 10,000 100 14,046 40.5% -# 100,000 100 105,749 5.7% -# 1,000,000 100 1,007,751 0.8% -# 10,000,000 100 10,009,401 0.1% -# -# Theoretical number of comparisons for k smallest of n random inputs: -# -# Step Comparisons Action -# ---- -------------------------- --------------------------- -# 1 1.66 * k heapify the first k-inputs -# 2 n - k compare remaining elements to top of heap -# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap -# 4 k * lg2(k) - (k/2) final sort of the k most extreme values -# -# Combining and simplifying for a rough estimate gives: -# -# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) -# -# Computing the number of comparisons for step 3: -# ----------------------------------------------- -# * For the i-th new value from the iterable, the probability of being in the -# k most extreme values is k/i. For example, the probability of the 101st -# value seen being in the 100 most extreme values is 100/101. -# * If the value is a new extreme value, the cost of inserting it into the -# heap is 1 + log(k, 2). -# * The probability times the cost gives: -# (k/i) * (1 + log(k, 2)) -# * Summing across the remaining n-k elements gives: -# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) -# * This reduces to: -# (H(n) - H(k)) * k * (1 + log(k, 2)) -# * Where H(n) is the n-th harmonic number estimated by: -# gamma = 0.5772156649 -# H(n) = log(n, e) + gamma + 1 / (2 * n) -# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence -# * Substituting the H(n) formula: -# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) -# -# Worst-case for step 3: -# ---------------------- -# In the worst case, the input data is reversed sorted so that every new element -# must be inserted in the heap: -# -# comparisons = 1.66 * k + log(k, 2) * (n - k) -# -# Alternative Algorithms -# ---------------------- -# Other algorithms were not used because they: -# 1) Took much more auxiliary memory, -# 2) Made multiple passes over the data. -# 3) Made more comparisons in common cases (small k, large n, semi-random input). -# See the more detailed comparison of approach at: -# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest - +# Extend the implementations of nsmallest and nlargest to use a key= argument +_nsmallest = nsmallest def nsmallest(n, iterable, key=None): """Find the n smallest elements in a dataset. Equivalent to: sorted(iterable, key=key)[:n] """ - - # Short-cut for n==1 is to use min() + # Short-cut for n==1 is to use min() when len(iterable)>0 if n == 1: it = iter(iterable) - sentinel = object() + head = list(islice(it, 1)) + if not head: + return [] if key is None: - result = min(it, default=sentinel) - else: - result = min(it, default=sentinel, key=key) - return [] if result is sentinel else [result] + return [min(chain(head, it))] + return [min(chain(head, it), key=key)] # When n>=size, it's faster to use sorted() try: @@ -485,57 +375,32 @@ # When key is none, use simpler decoration if key is None: - it = iter(iterable) - # put the range(n) first so that zip() doesn't - # consume one too many elements from the iterator - result = [(elem, i) for i, elem in zip(range(n), it)] - if not result: - return result - _heapify_max(result) - top = result[0][0] - order = n - _heapreplace = _heapreplace_max - for elem in it: - if elem < top: - _heapreplace(result, (elem, order)) - top = result[0][0] - order += 1 - result.sort() - return [r[0] for r in result] + it = zip(iterable, count()) # decorate + result = _nsmallest(n, it) + return [r[0] for r in result] # undecorate # General case, slowest method - it = iter(iterable) - result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] - if not result: - return result - _heapify_max(result) - top = result[0][0] - order = n - _heapreplace = _heapreplace_max - for elem in it: - k = key(elem) - if k < top: - _heapreplace(result, (k, order, elem)) - top = result[0][0] - order += 1 - result.sort() - return [r[2] for r in result] + in1, in2 = tee(iterable) + it = zip(map(key, in1), count(), in2) # decorate + result = _nsmallest(n, it) + return [r[2] for r in result] # undecorate +_nlargest = nlargest def nlargest(n, iterable, key=None): """Find the n largest elements in a dataset. Equivalent to: sorted(iterable, key=key, reverse=True)[:n] """ - # Short-cut for n==1 is to use max() + # Short-cut for n==1 is to use max() when len(iterable)>0 if n == 1: it = iter(iterable) - sentinel = object() + head = list(islice(it, 1)) + if not head: + return [] if key is None: - result = max(it, default=sentinel) - else: - result = max(it, default=sentinel, key=key) - return [] if result is sentinel else [result] + return [max(chain(head, it))] + return [max(chain(head, it), key=key)] # When n>=size, it's faster to use sorted() try: @@ -548,60 +413,26 @@ # When key is none, use simpler decoration if key is None: - it = iter(iterable) - result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] - if not result: - return result - heapify(result) - top = result[0][0] - order = -n - _heapreplace = heapreplace - for elem in it: - if top < elem: - _heapreplace(result, (elem, order)) - top = result[0][0] - order -= 1 - result.sort(reverse=True) - return [r[0] for r in result] + it = zip(iterable, count(0,-1)) # decorate + result = _nlargest(n, it) + return [r[0] for r in result] # undecorate # General case, slowest method - it = iter(iterable) - result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] - if not result: - return result - heapify(result) - top = result[0][0] - order = -n - _heapreplace = heapreplace - for elem in it: - k = key(elem) - if top < k: - _heapreplace(result, (k, order, elem)) - top = result[0][0] - order -= 1 - result.sort(reverse=True) - return [r[2] for r in result] - -# If available, use C implementation -try: - from _heapq import * -except ImportError: - pass -try: - from _heapq import _heapreplace_max -except ImportError: - pass -try: - from _heapq import _heapify_max -except ImportError: - pass -try: - from _heapq import _heappop_max -except ImportError: - pass - + in1, in2 = tee(iterable) + it = zip(map(key, in1), count(0,-1), in2) # decorate + result = _nlargest(n, it) + return [r[2] for r in result] # undecorate if __name__ == "__main__": + # Simple sanity test + heap = [] + data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] + for item in data: + heappush(heap, item) + sort = [] + while heap: + sort.append(heappop(heap)) + print(sort) import doctest - print(doctest.testmod()) + doctest.testmod() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/hmac.py --- a/Lib/hmac.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/hmac.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,8 +4,6 @@ """ import warnings as _warnings -from _operator import _compare_digest as compare_digest -import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -15,7 +13,6 @@ digest_size = None - class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. @@ -29,27 +26,21 @@ key: key for the keyed hash object. msg: Initial input for the hash, if provided. digestmod: A module supporting PEP 247. *OR* - A hashlib constructor returning a new hash object. *OR* - A hash name suitable for hashlib.new(). + A hashlib constructor returning a new hash object. Defaults to hashlib.md5. - Implicit default to hashlib.md5 is deprecated and will be - removed in Python 3.6. - Note: key and msg must be a bytes or bytearray objects. + Note: key and msg must be bytes objects. """ - if not isinstance(key, (bytes, bytearray)): - raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__) + if not isinstance(key, bytes): + raise TypeError("expected bytes, but got %r" % type(key).__name__) if digestmod is None: - _warnings.warn("HMAC() without an explicit digestmod argument " - "is deprecated.", PendingDeprecationWarning, 2) - digestmod = _hashlib.md5 + import hashlib + digestmod = hashlib.md5 if callable(digestmod): self.digest_cons = digestmod - elif isinstance(digestmod, str): - self.digest_cons = lambda d=b'': _hashlib.new(digestmod, d) else: self.digest_cons = lambda d=b'': digestmod.new(d) @@ -70,10 +61,6 @@ RuntimeWarning, 2) blocksize = self.blocksize - # self.blocksize is the default blocksize. self.block_size is - # effective block size as well as the public API attribute. - self.block_size = blocksize - if len(key) > blocksize: key = self.digest_cons(key).digest() @@ -83,13 +70,11 @@ if msg is not None: self.update(msg) - @property - def name(self): - return "hmac-" + self.inner.name - def update(self, msg): """Update this hashing object with the string msg. """ + if not isinstance(msg, bytes): + raise TypeError("expected bytes, but got %r" % type(msg).__name__) self.inner.update(msg) def copy(self): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/html/__init__.py --- a/Lib/html/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/html/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,12 +2,12 @@ General functions for HTML manipulation. """ -import re as _re -from html.entities import html5 as _html5 +_escape_map = {ord('&'): '&', ord('<'): '<', ord('>'): '>'} +_escape_map_full = {ord('&'): '&', ord('<'): '<', ord('>'): '>', + ord('"'): '"', ord('\''): '''} -__all__ = ['escape', 'unescape'] - +# NB: this is a candidate for a bytes/string polymorphic interface def escape(s, quote=True): """ @@ -16,117 +16,6 @@ characters, both double quote (") and single quote (') characters are also translated. """ - s = s.replace("&", "&") # Must be done first! - s = s.replace("<", "<") - s = s.replace(">", ">") if quote: - s = s.replace('"', """) - s = s.replace('\'', "'") - return s - - -# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references - -_invalid_charrefs = { - 0x00: '\ufffd', # REPLACEMENT CHARACTER - 0x0d: '\r', # CARRIAGE RETURN - 0x80: '\u20ac', # EURO SIGN - 0x81: '\x81', # - 0x82: '\u201a', # SINGLE LOW-9 QUOTATION MARK - 0x83: '\u0192', # LATIN SMALL LETTER F WITH HOOK - 0x84: '\u201e', # DOUBLE LOW-9 QUOTATION MARK - 0x85: '\u2026', # HORIZONTAL ELLIPSIS - 0x86: '\u2020', # DAGGER - 0x87: '\u2021', # DOUBLE DAGGER - 0x88: '\u02c6', # MODIFIER LETTER CIRCUMFLEX ACCENT - 0x89: '\u2030', # PER MILLE SIGN - 0x8a: '\u0160', # LATIN CAPITAL LETTER S WITH CARON - 0x8b: '\u2039', # SINGLE LEFT-POINTING ANGLE QUOTATION MARK - 0x8c: '\u0152', # LATIN CAPITAL LIGATURE OE - 0x8d: '\x8d', # - 0x8e: '\u017d', # LATIN CAPITAL LETTER Z WITH CARON - 0x8f: '\x8f', # - 0x90: '\x90', # - 0x91: '\u2018', # LEFT SINGLE QUOTATION MARK - 0x92: '\u2019', # RIGHT SINGLE QUOTATION MARK - 0x93: '\u201c', # LEFT DOUBLE QUOTATION MARK - 0x94: '\u201d', # RIGHT DOUBLE QUOTATION MARK - 0x95: '\u2022', # BULLET - 0x96: '\u2013', # EN DASH - 0x97: '\u2014', # EM DASH - 0x98: '\u02dc', # SMALL TILDE - 0x99: '\u2122', # TRADE MARK SIGN - 0x9a: '\u0161', # LATIN SMALL LETTER S WITH CARON - 0x9b: '\u203a', # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK - 0x9c: '\u0153', # LATIN SMALL LIGATURE OE - 0x9d: '\x9d', # - 0x9e: '\u017e', # LATIN SMALL LETTER Z WITH CARON - 0x9f: '\u0178', # LATIN CAPITAL LETTER Y WITH DIAERESIS -} - -_invalid_codepoints = { - # 0x0001 to 0x0008 - 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, - # 0x000E to 0x001F - 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, - 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - # 0x007F to 0x009F - 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, - 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, - 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - # 0xFDD0 to 0xFDEF - 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, 0xfdd8, - 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, 0xfde0, 0xfde1, - 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, 0xfde8, 0xfde9, 0xfdea, - 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef, - # others - 0xb, 0xfffe, 0xffff, 0x1fffe, 0x1ffff, 0x2fffe, 0x2ffff, 0x3fffe, 0x3ffff, - 0x4fffe, 0x4ffff, 0x5fffe, 0x5ffff, 0x6fffe, 0x6ffff, 0x7fffe, 0x7ffff, - 0x8fffe, 0x8ffff, 0x9fffe, 0x9ffff, 0xafffe, 0xaffff, 0xbfffe, 0xbffff, - 0xcfffe, 0xcffff, 0xdfffe, 0xdffff, 0xefffe, 0xeffff, 0xffffe, 0xfffff, - 0x10fffe, 0x10ffff -} - - -def _replace_charref(s): - s = s.group(1) - if s[0] == '#': - # numeric charref - if s[1] in 'xX': - num = int(s[2:].rstrip(';'), 16) - else: - num = int(s[1:].rstrip(';')) - if num in _invalid_charrefs: - return _invalid_charrefs[num] - if 0xD800 <= num <= 0xDFFF or num > 0x10FFFF: - return '\uFFFD' - if num in _invalid_codepoints: - return '' - return chr(num) - else: - # named charref - if s in _html5: - return _html5[s] - # find the longest matching name (as defined by the standard) - for x in range(len(s)-1, 1, -1): - if s[:x] in _html5: - return _html5[s[:x]] + s[x:] - else: - return '&' + s - - -_charref = _re.compile(r'&(#[0-9]+;?' - r'|#[xX][0-9a-fA-F]+;?' - r'|[^\t\n\f <&#;]{1,32};?)') - -def unescape(s): - """ - Convert all named and numeric character references (e.g. >, >, - &x3e;) in the string s to the corresponding unicode characters. - This function uses the rules defined by the HTML 5 standard - for both valid and invalid character references, and the list of - HTML 5 named character references defined in html.entities.html5. - """ - if '&' not in s: - return s - return _charref.sub(_replace_charref, s) + return s.translate(_escape_map_full) + return s.translate(_escape_map) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/html/entities.py --- a/Lib/html/entities.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/html/entities.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,6 @@ """HTML character entity references.""" -__all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] - - -# maps the HTML entity name to the Unicode code point +# maps the HTML entity name to the Unicode codepoint name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -224,7 +221,7 @@ 'spades': 0x2660, # black spade suit, U+2660 ISOpub 'sub': 0x2282, # subset of, U+2282 ISOtech 'sube': 0x2286, # subset of or equal to, U+2286 ISOtech - 'sum': 0x2211, # n-ary summation, U+2211 ISOamsb + 'sum': 0x2211, # n-ary sumation, U+2211 ISOamsb 'sup': 0x2283, # superset of, U+2283 ISOtech 'sup1': 0x00b9, # superscript one = superscript digit one, U+00B9 ISOnum 'sup2': 0x00b2, # superscript two = superscript digit two = squared, U+00B2 ISOnum @@ -259,2243 +256,7 @@ 'zwnj': 0x200c, # zero width non-joiner, U+200C NEW RFC 2070 } - -# maps the HTML5 named character references to the equivalent Unicode character(s) -html5 = { - 'Aacute': '\xc1', - 'aacute': '\xe1', - 'Aacute;': '\xc1', - 'aacute;': '\xe1', - 'Abreve;': '\u0102', - 'abreve;': '\u0103', - 'ac;': '\u223e', - 'acd;': '\u223f', - 'acE;': '\u223e\u0333', - 'Acirc': '\xc2', - 'acirc': '\xe2', - 'Acirc;': '\xc2', - 'acirc;': '\xe2', - 'acute': '\xb4', - 'acute;': '\xb4', - 'Acy;': '\u0410', - 'acy;': '\u0430', - 'AElig': '\xc6', - 'aelig': '\xe6', - 'AElig;': '\xc6', - 'aelig;': '\xe6', - 'af;': '\u2061', - 'Afr;': '\U0001d504', - 'afr;': '\U0001d51e', - 'Agrave': '\xc0', - 'agrave': '\xe0', - 'Agrave;': '\xc0', - 'agrave;': '\xe0', - 'alefsym;': '\u2135', - 'aleph;': '\u2135', - 'Alpha;': '\u0391', - 'alpha;': '\u03b1', - 'Amacr;': '\u0100', - 'amacr;': '\u0101', - 'amalg;': '\u2a3f', - 'AMP': '&', - 'amp': '&', - 'AMP;': '&', - 'amp;': '&', - 'And;': '\u2a53', - 'and;': '\u2227', - 'andand;': '\u2a55', - 'andd;': '\u2a5c', - 'andslope;': '\u2a58', - 'andv;': '\u2a5a', - 'ang;': '\u2220', - 'ange;': '\u29a4', - 'angle;': '\u2220', - 'angmsd;': '\u2221', - 'angmsdaa;': '\u29a8', - 'angmsdab;': '\u29a9', - 'angmsdac;': '\u29aa', - 'angmsdad;': '\u29ab', - 'angmsdae;': '\u29ac', - 'angmsdaf;': '\u29ad', - 'angmsdag;': '\u29ae', - 'angmsdah;': '\u29af', - 'angrt;': '\u221f', - 'angrtvb;': '\u22be', - 'angrtvbd;': '\u299d', - 'angsph;': '\u2222', - 'angst;': '\xc5', - 'angzarr;': '\u237c', - 'Aogon;': '\u0104', - 'aogon;': '\u0105', - 'Aopf;': '\U0001d538', - 'aopf;': '\U0001d552', - 'ap;': '\u2248', - 'apacir;': '\u2a6f', - 'apE;': '\u2a70', - 'ape;': '\u224a', - 'apid;': '\u224b', - 'apos;': "'", - 'ApplyFunction;': '\u2061', - 'approx;': '\u2248', - 'approxeq;': '\u224a', - 'Aring': '\xc5', - 'aring': '\xe5', - 'Aring;': '\xc5', - 'aring;': '\xe5', - 'Ascr;': '\U0001d49c', - 'ascr;': '\U0001d4b6', - 'Assign;': '\u2254', - 'ast;': '*', - 'asymp;': '\u2248', - 'asympeq;': '\u224d', - 'Atilde': '\xc3', - 'atilde': '\xe3', - 'Atilde;': '\xc3', - 'atilde;': '\xe3', - 'Auml': '\xc4', - 'auml': '\xe4', - 'Auml;': '\xc4', - 'auml;': '\xe4', - 'awconint;': '\u2233', - 'awint;': '\u2a11', - 'backcong;': '\u224c', - 'backepsilon;': '\u03f6', - 'backprime;': '\u2035', - 'backsim;': '\u223d', - 'backsimeq;': '\u22cd', - 'Backslash;': '\u2216', - 'Barv;': '\u2ae7', - 'barvee;': '\u22bd', - 'Barwed;': '\u2306', - 'barwed;': '\u2305', - 'barwedge;': '\u2305', - 'bbrk;': '\u23b5', - 'bbrktbrk;': '\u23b6', - 'bcong;': '\u224c', - 'Bcy;': '\u0411', - 'bcy;': '\u0431', - 'bdquo;': '\u201e', - 'becaus;': '\u2235', - 'Because;': '\u2235', - 'because;': '\u2235', - 'bemptyv;': '\u29b0', - 'bepsi;': '\u03f6', - 'bernou;': '\u212c', - 'Bernoullis;': '\u212c', - 'Beta;': '\u0392', - 'beta;': '\u03b2', - 'beth;': '\u2136', - 'between;': '\u226c', - 'Bfr;': '\U0001d505', - 'bfr;': '\U0001d51f', - 'bigcap;': '\u22c2', - 'bigcirc;': '\u25ef', - 'bigcup;': '\u22c3', - 'bigodot;': '\u2a00', - 'bigoplus;': '\u2a01', - 'bigotimes;': '\u2a02', - 'bigsqcup;': '\u2a06', - 'bigstar;': '\u2605', - 'bigtriangledown;': '\u25bd', - 'bigtriangleup;': '\u25b3', - 'biguplus;': '\u2a04', - 'bigvee;': '\u22c1', - 'bigwedge;': '\u22c0', - 'bkarow;': '\u290d', - 'blacklozenge;': '\u29eb', - 'blacksquare;': '\u25aa', - 'blacktriangle;': '\u25b4', - 'blacktriangledown;': '\u25be', - 'blacktriangleleft;': '\u25c2', - 'blacktriangleright;': '\u25b8', - 'blank;': '\u2423', - 'blk12;': '\u2592', - 'blk14;': '\u2591', - 'blk34;': '\u2593', - 'block;': '\u2588', - 'bne;': '=\u20e5', - 'bnequiv;': '\u2261\u20e5', - 'bNot;': '\u2aed', - 'bnot;': '\u2310', - 'Bopf;': '\U0001d539', - 'bopf;': '\U0001d553', - 'bot;': '\u22a5', - 'bottom;': '\u22a5', - 'bowtie;': '\u22c8', - 'boxbox;': '\u29c9', - 'boxDL;': '\u2557', - 'boxDl;': '\u2556', - 'boxdL;': '\u2555', - 'boxdl;': '\u2510', - 'boxDR;': '\u2554', - 'boxDr;': '\u2553', - 'boxdR;': '\u2552', - 'boxdr;': '\u250c', - 'boxH;': '\u2550', - 'boxh;': '\u2500', - 'boxHD;': '\u2566', - 'boxHd;': '\u2564', - 'boxhD;': '\u2565', - 'boxhd;': '\u252c', - 'boxHU;': '\u2569', - 'boxHu;': '\u2567', - 'boxhU;': '\u2568', - 'boxhu;': '\u2534', - 'boxminus;': '\u229f', - 'boxplus;': '\u229e', - 'boxtimes;': '\u22a0', - 'boxUL;': '\u255d', - 'boxUl;': '\u255c', - 'boxuL;': '\u255b', - 'boxul;': '\u2518', - 'boxUR;': '\u255a', - 'boxUr;': '\u2559', - 'boxuR;': '\u2558', - 'boxur;': '\u2514', - 'boxV;': '\u2551', - 'boxv;': '\u2502', - 'boxVH;': '\u256c', - 'boxVh;': '\u256b', - 'boxvH;': '\u256a', - 'boxvh;': '\u253c', - 'boxVL;': '\u2563', - 'boxVl;': '\u2562', - 'boxvL;': '\u2561', - 'boxvl;': '\u2524', - 'boxVR;': '\u2560', - 'boxVr;': '\u255f', - 'boxvR;': '\u255e', - 'boxvr;': '\u251c', - 'bprime;': '\u2035', - 'Breve;': '\u02d8', - 'breve;': '\u02d8', - 'brvbar': '\xa6', - 'brvbar;': '\xa6', - 'Bscr;': '\u212c', - 'bscr;': '\U0001d4b7', - 'bsemi;': '\u204f', - 'bsim;': '\u223d', - 'bsime;': '\u22cd', - 'bsol;': '\\', - 'bsolb;': '\u29c5', - 'bsolhsub;': '\u27c8', - 'bull;': '\u2022', - 'bullet;': '\u2022', - 'bump;': '\u224e', - 'bumpE;': '\u2aae', - 'bumpe;': '\u224f', - 'Bumpeq;': '\u224e', - 'bumpeq;': '\u224f', - 'Cacute;': '\u0106', - 'cacute;': '\u0107', - 'Cap;': '\u22d2', - 'cap;': '\u2229', - 'capand;': '\u2a44', - 'capbrcup;': '\u2a49', - 'capcap;': '\u2a4b', - 'capcup;': '\u2a47', - 'capdot;': '\u2a40', - 'CapitalDifferentialD;': '\u2145', - 'caps;': '\u2229\ufe00', - 'caret;': '\u2041', - 'caron;': '\u02c7', - 'Cayleys;': '\u212d', - 'ccaps;': '\u2a4d', - 'Ccaron;': '\u010c', - 'ccaron;': '\u010d', - 'Ccedil': '\xc7', - 'ccedil': '\xe7', - 'Ccedil;': '\xc7', - 'ccedil;': '\xe7', - 'Ccirc;': '\u0108', - 'ccirc;': '\u0109', - 'Cconint;': '\u2230', - 'ccups;': '\u2a4c', - 'ccupssm;': '\u2a50', - 'Cdot;': '\u010a', - 'cdot;': '\u010b', - 'cedil': '\xb8', - 'cedil;': '\xb8', - 'Cedilla;': '\xb8', - 'cemptyv;': '\u29b2', - 'cent': '\xa2', - 'cent;': '\xa2', - 'CenterDot;': '\xb7', - 'centerdot;': '\xb7', - 'Cfr;': '\u212d', - 'cfr;': '\U0001d520', - 'CHcy;': '\u0427', - 'chcy;': '\u0447', - 'check;': '\u2713', - 'checkmark;': '\u2713', - 'Chi;': '\u03a7', - 'chi;': '\u03c7', - 'cir;': '\u25cb', - 'circ;': '\u02c6', - 'circeq;': '\u2257', - 'circlearrowleft;': '\u21ba', - 'circlearrowright;': '\u21bb', - 'circledast;': '\u229b', - 'circledcirc;': '\u229a', - 'circleddash;': '\u229d', - 'CircleDot;': '\u2299', - 'circledR;': '\xae', - 'circledS;': '\u24c8', - 'CircleMinus;': '\u2296', - 'CirclePlus;': '\u2295', - 'CircleTimes;': '\u2297', - 'cirE;': '\u29c3', - 'cire;': '\u2257', - 'cirfnint;': '\u2a10', - 'cirmid;': '\u2aef', - 'cirscir;': '\u29c2', - 'ClockwiseContourIntegral;': '\u2232', - 'CloseCurlyDoubleQuote;': '\u201d', - 'CloseCurlyQuote;': '\u2019', - 'clubs;': '\u2663', - 'clubsuit;': '\u2663', - 'Colon;': '\u2237', - 'colon;': ':', - 'Colone;': '\u2a74', - 'colone;': '\u2254', - 'coloneq;': '\u2254', - 'comma;': ',', - 'commat;': '@', - 'comp;': '\u2201', - 'compfn;': '\u2218', - 'complement;': '\u2201', - 'complexes;': '\u2102', - 'cong;': '\u2245', - 'congdot;': '\u2a6d', - 'Congruent;': '\u2261', - 'Conint;': '\u222f', - 'conint;': '\u222e', - 'ContourIntegral;': '\u222e', - 'Copf;': '\u2102', - 'copf;': '\U0001d554', - 'coprod;': '\u2210', - 'Coproduct;': '\u2210', - 'COPY': '\xa9', - 'copy': '\xa9', - 'COPY;': '\xa9', - 'copy;': '\xa9', - 'copysr;': '\u2117', - 'CounterClockwiseContourIntegral;': '\u2233', - 'crarr;': '\u21b5', - 'Cross;': '\u2a2f', - 'cross;': '\u2717', - 'Cscr;': '\U0001d49e', - 'cscr;': '\U0001d4b8', - 'csub;': '\u2acf', - 'csube;': '\u2ad1', - 'csup;': '\u2ad0', - 'csupe;': '\u2ad2', - 'ctdot;': '\u22ef', - 'cudarrl;': '\u2938', - 'cudarrr;': '\u2935', - 'cuepr;': '\u22de', - 'cuesc;': '\u22df', - 'cularr;': '\u21b6', - 'cularrp;': '\u293d', - 'Cup;': '\u22d3', - 'cup;': '\u222a', - 'cupbrcap;': '\u2a48', - 'CupCap;': '\u224d', - 'cupcap;': '\u2a46', - 'cupcup;': '\u2a4a', - 'cupdot;': '\u228d', - 'cupor;': '\u2a45', - 'cups;': '\u222a\ufe00', - 'curarr;': '\u21b7', - 'curarrm;': '\u293c', - 'curlyeqprec;': '\u22de', - 'curlyeqsucc;': '\u22df', - 'curlyvee;': '\u22ce', - 'curlywedge;': '\u22cf', - 'curren': '\xa4', - 'curren;': '\xa4', - 'curvearrowleft;': '\u21b6', - 'curvearrowright;': '\u21b7', - 'cuvee;': '\u22ce', - 'cuwed;': '\u22cf', - 'cwconint;': '\u2232', - 'cwint;': '\u2231', - 'cylcty;': '\u232d', - 'Dagger;': '\u2021', - 'dagger;': '\u2020', - 'daleth;': '\u2138', - 'Darr;': '\u21a1', - 'dArr;': '\u21d3', - 'darr;': '\u2193', - 'dash;': '\u2010', - 'Dashv;': '\u2ae4', - 'dashv;': '\u22a3', - 'dbkarow;': '\u290f', - 'dblac;': '\u02dd', - 'Dcaron;': '\u010e', - 'dcaron;': '\u010f', - 'Dcy;': '\u0414', - 'dcy;': '\u0434', - 'DD;': '\u2145', - 'dd;': '\u2146', - 'ddagger;': '\u2021', - 'ddarr;': '\u21ca', - 'DDotrahd;': '\u2911', - 'ddotseq;': '\u2a77', - 'deg': '\xb0', - 'deg;': '\xb0', - 'Del;': '\u2207', - 'Delta;': '\u0394', - 'delta;': '\u03b4', - 'demptyv;': '\u29b1', - 'dfisht;': '\u297f', - 'Dfr;': '\U0001d507', - 'dfr;': '\U0001d521', - 'dHar;': '\u2965', - 'dharl;': '\u21c3', - 'dharr;': '\u21c2', - 'DiacriticalAcute;': '\xb4', - 'DiacriticalDot;': '\u02d9', - 'DiacriticalDoubleAcute;': '\u02dd', - 'DiacriticalGrave;': '`', - 'DiacriticalTilde;': '\u02dc', - 'diam;': '\u22c4', - 'Diamond;': '\u22c4', - 'diamond;': '\u22c4', - 'diamondsuit;': '\u2666', - 'diams;': '\u2666', - 'die;': '\xa8', - 'DifferentialD;': '\u2146', - 'digamma;': '\u03dd', - 'disin;': '\u22f2', - 'div;': '\xf7', - 'divide': '\xf7', - 'divide;': '\xf7', - 'divideontimes;': '\u22c7', - 'divonx;': '\u22c7', - 'DJcy;': '\u0402', - 'djcy;': '\u0452', - 'dlcorn;': '\u231e', - 'dlcrop;': '\u230d', - 'dollar;': '$', - 'Dopf;': '\U0001d53b', - 'dopf;': '\U0001d555', - 'Dot;': '\xa8', - 'dot;': '\u02d9', - 'DotDot;': '\u20dc', - 'doteq;': '\u2250', - 'doteqdot;': '\u2251', - 'DotEqual;': '\u2250', - 'dotminus;': '\u2238', - 'dotplus;': '\u2214', - 'dotsquare;': '\u22a1', - 'doublebarwedge;': '\u2306', - 'DoubleContourIntegral;': '\u222f', - 'DoubleDot;': '\xa8', - 'DoubleDownArrow;': '\u21d3', - 'DoubleLeftArrow;': '\u21d0', - 'DoubleLeftRightArrow;': '\u21d4', - 'DoubleLeftTee;': '\u2ae4', - 'DoubleLongLeftArrow;': '\u27f8', - 'DoubleLongLeftRightArrow;': '\u27fa', - 'DoubleLongRightArrow;': '\u27f9', - 'DoubleRightArrow;': '\u21d2', - 'DoubleRightTee;': '\u22a8', - 'DoubleUpArrow;': '\u21d1', - 'DoubleUpDownArrow;': '\u21d5', - 'DoubleVerticalBar;': '\u2225', - 'DownArrow;': '\u2193', - 'Downarrow;': '\u21d3', - 'downarrow;': '\u2193', - 'DownArrowBar;': '\u2913', - 'DownArrowUpArrow;': '\u21f5', - 'DownBreve;': '\u0311', - 'downdownarrows;': '\u21ca', - 'downharpoonleft;': '\u21c3', - 'downharpoonright;': '\u21c2', - 'DownLeftRightVector;': '\u2950', - 'DownLeftTeeVector;': '\u295e', - 'DownLeftVector;': '\u21bd', - 'DownLeftVectorBar;': '\u2956', - 'DownRightTeeVector;': '\u295f', - 'DownRightVector;': '\u21c1', - 'DownRightVectorBar;': '\u2957', - 'DownTee;': '\u22a4', - 'DownTeeArrow;': '\u21a7', - 'drbkarow;': '\u2910', - 'drcorn;': '\u231f', - 'drcrop;': '\u230c', - 'Dscr;': '\U0001d49f', - 'dscr;': '\U0001d4b9', - 'DScy;': '\u0405', - 'dscy;': '\u0455', - 'dsol;': '\u29f6', - 'Dstrok;': '\u0110', - 'dstrok;': '\u0111', - 'dtdot;': '\u22f1', - 'dtri;': '\u25bf', - 'dtrif;': '\u25be', - 'duarr;': '\u21f5', - 'duhar;': '\u296f', - 'dwangle;': '\u29a6', - 'DZcy;': '\u040f', - 'dzcy;': '\u045f', - 'dzigrarr;': '\u27ff', - 'Eacute': '\xc9', - 'eacute': '\xe9', - 'Eacute;': '\xc9', - 'eacute;': '\xe9', - 'easter;': '\u2a6e', - 'Ecaron;': '\u011a', - 'ecaron;': '\u011b', - 'ecir;': '\u2256', - 'Ecirc': '\xca', - 'ecirc': '\xea', - 'Ecirc;': '\xca', - 'ecirc;': '\xea', - 'ecolon;': '\u2255', - 'Ecy;': '\u042d', - 'ecy;': '\u044d', - 'eDDot;': '\u2a77', - 'Edot;': '\u0116', - 'eDot;': '\u2251', - 'edot;': '\u0117', - 'ee;': '\u2147', - 'efDot;': '\u2252', - 'Efr;': '\U0001d508', - 'efr;': '\U0001d522', - 'eg;': '\u2a9a', - 'Egrave': '\xc8', - 'egrave': '\xe8', - 'Egrave;': '\xc8', - 'egrave;': '\xe8', - 'egs;': '\u2a96', - 'egsdot;': '\u2a98', - 'el;': '\u2a99', - 'Element;': '\u2208', - 'elinters;': '\u23e7', - 'ell;': '\u2113', - 'els;': '\u2a95', - 'elsdot;': '\u2a97', - 'Emacr;': '\u0112', - 'emacr;': '\u0113', - 'empty;': '\u2205', - 'emptyset;': '\u2205', - 'EmptySmallSquare;': '\u25fb', - 'emptyv;': '\u2205', - 'EmptyVerySmallSquare;': '\u25ab', - 'emsp13;': '\u2004', - 'emsp14;': '\u2005', - 'emsp;': '\u2003', - 'ENG;': '\u014a', - 'eng;': '\u014b', - 'ensp;': '\u2002', - 'Eogon;': '\u0118', - 'eogon;': '\u0119', - 'Eopf;': '\U0001d53c', - 'eopf;': '\U0001d556', - 'epar;': '\u22d5', - 'eparsl;': '\u29e3', - 'eplus;': '\u2a71', - 'epsi;': '\u03b5', - 'Epsilon;': '\u0395', - 'epsilon;': '\u03b5', - 'epsiv;': '\u03f5', - 'eqcirc;': '\u2256', - 'eqcolon;': '\u2255', - 'eqsim;': '\u2242', - 'eqslantgtr;': '\u2a96', - 'eqslantless;': '\u2a95', - 'Equal;': '\u2a75', - 'equals;': '=', - 'EqualTilde;': '\u2242', - 'equest;': '\u225f', - 'Equilibrium;': '\u21cc', - 'equiv;': '\u2261', - 'equivDD;': '\u2a78', - 'eqvparsl;': '\u29e5', - 'erarr;': '\u2971', - 'erDot;': '\u2253', - 'Escr;': '\u2130', - 'escr;': '\u212f', - 'esdot;': '\u2250', - 'Esim;': '\u2a73', - 'esim;': '\u2242', - 'Eta;': '\u0397', - 'eta;': '\u03b7', - 'ETH': '\xd0', - 'eth': '\xf0', - 'ETH;': '\xd0', - 'eth;': '\xf0', - 'Euml': '\xcb', - 'euml': '\xeb', - 'Euml;': '\xcb', - 'euml;': '\xeb', - 'euro;': '\u20ac', - 'excl;': '!', - 'exist;': '\u2203', - 'Exists;': '\u2203', - 'expectation;': '\u2130', - 'ExponentialE;': '\u2147', - 'exponentiale;': '\u2147', - 'fallingdotseq;': '\u2252', - 'Fcy;': '\u0424', - 'fcy;': '\u0444', - 'female;': '\u2640', - 'ffilig;': '\ufb03', - 'fflig;': '\ufb00', - 'ffllig;': '\ufb04', - 'Ffr;': '\U0001d509', - 'ffr;': '\U0001d523', - 'filig;': '\ufb01', - 'FilledSmallSquare;': '\u25fc', - 'FilledVerySmallSquare;': '\u25aa', - 'fjlig;': 'fj', - 'flat;': '\u266d', - 'fllig;': '\ufb02', - 'fltns;': '\u25b1', - 'fnof;': '\u0192', - 'Fopf;': '\U0001d53d', - 'fopf;': '\U0001d557', - 'ForAll;': '\u2200', - 'forall;': '\u2200', - 'fork;': '\u22d4', - 'forkv;': '\u2ad9', - 'Fouriertrf;': '\u2131', - 'fpartint;': '\u2a0d', - 'frac12': '\xbd', - 'frac12;': '\xbd', - 'frac13;': '\u2153', - 'frac14': '\xbc', - 'frac14;': '\xbc', - 'frac15;': '\u2155', - 'frac16;': '\u2159', - 'frac18;': '\u215b', - 'frac23;': '\u2154', - 'frac25;': '\u2156', - 'frac34': '\xbe', - 'frac34;': '\xbe', - 'frac35;': '\u2157', - 'frac38;': '\u215c', - 'frac45;': '\u2158', - 'frac56;': '\u215a', - 'frac58;': '\u215d', - 'frac78;': '\u215e', - 'frasl;': '\u2044', - 'frown;': '\u2322', - 'Fscr;': '\u2131', - 'fscr;': '\U0001d4bb', - 'gacute;': '\u01f5', - 'Gamma;': '\u0393', - 'gamma;': '\u03b3', - 'Gammad;': '\u03dc', - 'gammad;': '\u03dd', - 'gap;': '\u2a86', - 'Gbreve;': '\u011e', - 'gbreve;': '\u011f', - 'Gcedil;': '\u0122', - 'Gcirc;': '\u011c', - 'gcirc;': '\u011d', - 'Gcy;': '\u0413', - 'gcy;': '\u0433', - 'Gdot;': '\u0120', - 'gdot;': '\u0121', - 'gE;': '\u2267', - 'ge;': '\u2265', - 'gEl;': '\u2a8c', - 'gel;': '\u22db', - 'geq;': '\u2265', - 'geqq;': '\u2267', - 'geqslant;': '\u2a7e', - 'ges;': '\u2a7e', - 'gescc;': '\u2aa9', - 'gesdot;': '\u2a80', - 'gesdoto;': '\u2a82', - 'gesdotol;': '\u2a84', - 'gesl;': '\u22db\ufe00', - 'gesles;': '\u2a94', - 'Gfr;': '\U0001d50a', - 'gfr;': '\U0001d524', - 'Gg;': '\u22d9', - 'gg;': '\u226b', - 'ggg;': '\u22d9', - 'gimel;': '\u2137', - 'GJcy;': '\u0403', - 'gjcy;': '\u0453', - 'gl;': '\u2277', - 'gla;': '\u2aa5', - 'glE;': '\u2a92', - 'glj;': '\u2aa4', - 'gnap;': '\u2a8a', - 'gnapprox;': '\u2a8a', - 'gnE;': '\u2269', - 'gne;': '\u2a88', - 'gneq;': '\u2a88', - 'gneqq;': '\u2269', - 'gnsim;': '\u22e7', - 'Gopf;': '\U0001d53e', - 'gopf;': '\U0001d558', - 'grave;': '`', - 'GreaterEqual;': '\u2265', - 'GreaterEqualLess;': '\u22db', - 'GreaterFullEqual;': '\u2267', - 'GreaterGreater;': '\u2aa2', - 'GreaterLess;': '\u2277', - 'GreaterSlantEqual;': '\u2a7e', - 'GreaterTilde;': '\u2273', - 'Gscr;': '\U0001d4a2', - 'gscr;': '\u210a', - 'gsim;': '\u2273', - 'gsime;': '\u2a8e', - 'gsiml;': '\u2a90', - 'GT': '>', - 'gt': '>', - 'GT;': '>', - 'Gt;': '\u226b', - 'gt;': '>', - 'gtcc;': '\u2aa7', - 'gtcir;': '\u2a7a', - 'gtdot;': '\u22d7', - 'gtlPar;': '\u2995', - 'gtquest;': '\u2a7c', - 'gtrapprox;': '\u2a86', - 'gtrarr;': '\u2978', - 'gtrdot;': '\u22d7', - 'gtreqless;': '\u22db', - 'gtreqqless;': '\u2a8c', - 'gtrless;': '\u2277', - 'gtrsim;': '\u2273', - 'gvertneqq;': '\u2269\ufe00', - 'gvnE;': '\u2269\ufe00', - 'Hacek;': '\u02c7', - 'hairsp;': '\u200a', - 'half;': '\xbd', - 'hamilt;': '\u210b', - 'HARDcy;': '\u042a', - 'hardcy;': '\u044a', - 'hArr;': '\u21d4', - 'harr;': '\u2194', - 'harrcir;': '\u2948', - 'harrw;': '\u21ad', - 'Hat;': '^', - 'hbar;': '\u210f', - 'Hcirc;': '\u0124', - 'hcirc;': '\u0125', - 'hearts;': '\u2665', - 'heartsuit;': '\u2665', - 'hellip;': '\u2026', - 'hercon;': '\u22b9', - 'Hfr;': '\u210c', - 'hfr;': '\U0001d525', - 'HilbertSpace;': '\u210b', - 'hksearow;': '\u2925', - 'hkswarow;': '\u2926', - 'hoarr;': '\u21ff', - 'homtht;': '\u223b', - 'hookleftarrow;': '\u21a9', - 'hookrightarrow;': '\u21aa', - 'Hopf;': '\u210d', - 'hopf;': '\U0001d559', - 'horbar;': '\u2015', - 'HorizontalLine;': '\u2500', - 'Hscr;': '\u210b', - 'hscr;': '\U0001d4bd', - 'hslash;': '\u210f', - 'Hstrok;': '\u0126', - 'hstrok;': '\u0127', - 'HumpDownHump;': '\u224e', - 'HumpEqual;': '\u224f', - 'hybull;': '\u2043', - 'hyphen;': '\u2010', - 'Iacute': '\xcd', - 'iacute': '\xed', - 'Iacute;': '\xcd', - 'iacute;': '\xed', - 'ic;': '\u2063', - 'Icirc': '\xce', - 'icirc': '\xee', - 'Icirc;': '\xce', - 'icirc;': '\xee', - 'Icy;': '\u0418', - 'icy;': '\u0438', - 'Idot;': '\u0130', - 'IEcy;': '\u0415', - 'iecy;': '\u0435', - 'iexcl': '\xa1', - 'iexcl;': '\xa1', - 'iff;': '\u21d4', - 'Ifr;': '\u2111', - 'ifr;': '\U0001d526', - 'Igrave': '\xcc', - 'igrave': '\xec', - 'Igrave;': '\xcc', - 'igrave;': '\xec', - 'ii;': '\u2148', - 'iiiint;': '\u2a0c', - 'iiint;': '\u222d', - 'iinfin;': '\u29dc', - 'iiota;': '\u2129', - 'IJlig;': '\u0132', - 'ijlig;': '\u0133', - 'Im;': '\u2111', - 'Imacr;': '\u012a', - 'imacr;': '\u012b', - 'image;': '\u2111', - 'ImaginaryI;': '\u2148', - 'imagline;': '\u2110', - 'imagpart;': '\u2111', - 'imath;': '\u0131', - 'imof;': '\u22b7', - 'imped;': '\u01b5', - 'Implies;': '\u21d2', - 'in;': '\u2208', - 'incare;': '\u2105', - 'infin;': '\u221e', - 'infintie;': '\u29dd', - 'inodot;': '\u0131', - 'Int;': '\u222c', - 'int;': '\u222b', - 'intcal;': '\u22ba', - 'integers;': '\u2124', - 'Integral;': '\u222b', - 'intercal;': '\u22ba', - 'Intersection;': '\u22c2', - 'intlarhk;': '\u2a17', - 'intprod;': '\u2a3c', - 'InvisibleComma;': '\u2063', - 'InvisibleTimes;': '\u2062', - 'IOcy;': '\u0401', - 'iocy;': '\u0451', - 'Iogon;': '\u012e', - 'iogon;': '\u012f', - 'Iopf;': '\U0001d540', - 'iopf;': '\U0001d55a', - 'Iota;': '\u0399', - 'iota;': '\u03b9', - 'iprod;': '\u2a3c', - 'iquest': '\xbf', - 'iquest;': '\xbf', - 'Iscr;': '\u2110', - 'iscr;': '\U0001d4be', - 'isin;': '\u2208', - 'isindot;': '\u22f5', - 'isinE;': '\u22f9', - 'isins;': '\u22f4', - 'isinsv;': '\u22f3', - 'isinv;': '\u2208', - 'it;': '\u2062', - 'Itilde;': '\u0128', - 'itilde;': '\u0129', - 'Iukcy;': '\u0406', - 'iukcy;': '\u0456', - 'Iuml': '\xcf', - 'iuml': '\xef', - 'Iuml;': '\xcf', - 'iuml;': '\xef', - 'Jcirc;': '\u0134', - 'jcirc;': '\u0135', - 'Jcy;': '\u0419', - 'jcy;': '\u0439', - 'Jfr;': '\U0001d50d', - 'jfr;': '\U0001d527', - 'jmath;': '\u0237', - 'Jopf;': '\U0001d541', - 'jopf;': '\U0001d55b', - 'Jscr;': '\U0001d4a5', - 'jscr;': '\U0001d4bf', - 'Jsercy;': '\u0408', - 'jsercy;': '\u0458', - 'Jukcy;': '\u0404', - 'jukcy;': '\u0454', - 'Kappa;': '\u039a', - 'kappa;': '\u03ba', - 'kappav;': '\u03f0', - 'Kcedil;': '\u0136', - 'kcedil;': '\u0137', - 'Kcy;': '\u041a', - 'kcy;': '\u043a', - 'Kfr;': '\U0001d50e', - 'kfr;': '\U0001d528', - 'kgreen;': '\u0138', - 'KHcy;': '\u0425', - 'khcy;': '\u0445', - 'KJcy;': '\u040c', - 'kjcy;': '\u045c', - 'Kopf;': '\U0001d542', - 'kopf;': '\U0001d55c', - 'Kscr;': '\U0001d4a6', - 'kscr;': '\U0001d4c0', - 'lAarr;': '\u21da', - 'Lacute;': '\u0139', - 'lacute;': '\u013a', - 'laemptyv;': '\u29b4', - 'lagran;': '\u2112', - 'Lambda;': '\u039b', - 'lambda;': '\u03bb', - 'Lang;': '\u27ea', - 'lang;': '\u27e8', - 'langd;': '\u2991', - 'langle;': '\u27e8', - 'lap;': '\u2a85', - 'Laplacetrf;': '\u2112', - 'laquo': '\xab', - 'laquo;': '\xab', - 'Larr;': '\u219e', - 'lArr;': '\u21d0', - 'larr;': '\u2190', - 'larrb;': '\u21e4', - 'larrbfs;': '\u291f', - 'larrfs;': '\u291d', - 'larrhk;': '\u21a9', - 'larrlp;': '\u21ab', - 'larrpl;': '\u2939', - 'larrsim;': '\u2973', - 'larrtl;': '\u21a2', - 'lat;': '\u2aab', - 'lAtail;': '\u291b', - 'latail;': '\u2919', - 'late;': '\u2aad', - 'lates;': '\u2aad\ufe00', - 'lBarr;': '\u290e', - 'lbarr;': '\u290c', - 'lbbrk;': '\u2772', - 'lbrace;': '{', - 'lbrack;': '[', - 'lbrke;': '\u298b', - 'lbrksld;': '\u298f', - 'lbrkslu;': '\u298d', - 'Lcaron;': '\u013d', - 'lcaron;': '\u013e', - 'Lcedil;': '\u013b', - 'lcedil;': '\u013c', - 'lceil;': '\u2308', - 'lcub;': '{', - 'Lcy;': '\u041b', - 'lcy;': '\u043b', - 'ldca;': '\u2936', - 'ldquo;': '\u201c', - 'ldquor;': '\u201e', - 'ldrdhar;': '\u2967', - 'ldrushar;': '\u294b', - 'ldsh;': '\u21b2', - 'lE;': '\u2266', - 'le;': '\u2264', - 'LeftAngleBracket;': '\u27e8', - 'LeftArrow;': '\u2190', - 'Leftarrow;': '\u21d0', - 'leftarrow;': '\u2190', - 'LeftArrowBar;': '\u21e4', - 'LeftArrowRightArrow;': '\u21c6', - 'leftarrowtail;': '\u21a2', - 'LeftCeiling;': '\u2308', - 'LeftDoubleBracket;': '\u27e6', - 'LeftDownTeeVector;': '\u2961', - 'LeftDownVector;': '\u21c3', - 'LeftDownVectorBar;': '\u2959', - 'LeftFloor;': '\u230a', - 'leftharpoondown;': '\u21bd', - 'leftharpoonup;': '\u21bc', - 'leftleftarrows;': '\u21c7', - 'LeftRightArrow;': '\u2194', - 'Leftrightarrow;': '\u21d4', - 'leftrightarrow;': '\u2194', - 'leftrightarrows;': '\u21c6', - 'leftrightharpoons;': '\u21cb', - 'leftrightsquigarrow;': '\u21ad', - 'LeftRightVector;': '\u294e', - 'LeftTee;': '\u22a3', - 'LeftTeeArrow;': '\u21a4', - 'LeftTeeVector;': '\u295a', - 'leftthreetimes;': '\u22cb', - 'LeftTriangle;': '\u22b2', - 'LeftTriangleBar;': '\u29cf', - 'LeftTriangleEqual;': '\u22b4', - 'LeftUpDownVector;': '\u2951', - 'LeftUpTeeVector;': '\u2960', - 'LeftUpVector;': '\u21bf', - 'LeftUpVectorBar;': '\u2958', - 'LeftVector;': '\u21bc', - 'LeftVectorBar;': '\u2952', - 'lEg;': '\u2a8b', - 'leg;': '\u22da', - 'leq;': '\u2264', - 'leqq;': '\u2266', - 'leqslant;': '\u2a7d', - 'les;': '\u2a7d', - 'lescc;': '\u2aa8', - 'lesdot;': '\u2a7f', - 'lesdoto;': '\u2a81', - 'lesdotor;': '\u2a83', - 'lesg;': '\u22da\ufe00', - 'lesges;': '\u2a93', - 'lessapprox;': '\u2a85', - 'lessdot;': '\u22d6', - 'lesseqgtr;': '\u22da', - 'lesseqqgtr;': '\u2a8b', - 'LessEqualGreater;': '\u22da', - 'LessFullEqual;': '\u2266', - 'LessGreater;': '\u2276', - 'lessgtr;': '\u2276', - 'LessLess;': '\u2aa1', - 'lesssim;': '\u2272', - 'LessSlantEqual;': '\u2a7d', - 'LessTilde;': '\u2272', - 'lfisht;': '\u297c', - 'lfloor;': '\u230a', - 'Lfr;': '\U0001d50f', - 'lfr;': '\U0001d529', - 'lg;': '\u2276', - 'lgE;': '\u2a91', - 'lHar;': '\u2962', - 'lhard;': '\u21bd', - 'lharu;': '\u21bc', - 'lharul;': '\u296a', - 'lhblk;': '\u2584', - 'LJcy;': '\u0409', - 'ljcy;': '\u0459', - 'Ll;': '\u22d8', - 'll;': '\u226a', - 'llarr;': '\u21c7', - 'llcorner;': '\u231e', - 'Lleftarrow;': '\u21da', - 'llhard;': '\u296b', - 'lltri;': '\u25fa', - 'Lmidot;': '\u013f', - 'lmidot;': '\u0140', - 'lmoust;': '\u23b0', - 'lmoustache;': '\u23b0', - 'lnap;': '\u2a89', - 'lnapprox;': '\u2a89', - 'lnE;': '\u2268', - 'lne;': '\u2a87', - 'lneq;': '\u2a87', - 'lneqq;': '\u2268', - 'lnsim;': '\u22e6', - 'loang;': '\u27ec', - 'loarr;': '\u21fd', - 'lobrk;': '\u27e6', - 'LongLeftArrow;': '\u27f5', - 'Longleftarrow;': '\u27f8', - 'longleftarrow;': '\u27f5', - 'LongLeftRightArrow;': '\u27f7', - 'Longleftrightarrow;': '\u27fa', - 'longleftrightarrow;': '\u27f7', - 'longmapsto;': '\u27fc', - 'LongRightArrow;': '\u27f6', - 'Longrightarrow;': '\u27f9', - 'longrightarrow;': '\u27f6', - 'looparrowleft;': '\u21ab', - 'looparrowright;': '\u21ac', - 'lopar;': '\u2985', - 'Lopf;': '\U0001d543', - 'lopf;': '\U0001d55d', - 'loplus;': '\u2a2d', - 'lotimes;': '\u2a34', - 'lowast;': '\u2217', - 'lowbar;': '_', - 'LowerLeftArrow;': '\u2199', - 'LowerRightArrow;': '\u2198', - 'loz;': '\u25ca', - 'lozenge;': '\u25ca', - 'lozf;': '\u29eb', - 'lpar;': '(', - 'lparlt;': '\u2993', - 'lrarr;': '\u21c6', - 'lrcorner;': '\u231f', - 'lrhar;': '\u21cb', - 'lrhard;': '\u296d', - 'lrm;': '\u200e', - 'lrtri;': '\u22bf', - 'lsaquo;': '\u2039', - 'Lscr;': '\u2112', - 'lscr;': '\U0001d4c1', - 'Lsh;': '\u21b0', - 'lsh;': '\u21b0', - 'lsim;': '\u2272', - 'lsime;': '\u2a8d', - 'lsimg;': '\u2a8f', - 'lsqb;': '[', - 'lsquo;': '\u2018', - 'lsquor;': '\u201a', - 'Lstrok;': '\u0141', - 'lstrok;': '\u0142', - 'LT': '<', - 'lt': '<', - 'LT;': '<', - 'Lt;': '\u226a', - 'lt;': '<', - 'ltcc;': '\u2aa6', - 'ltcir;': '\u2a79', - 'ltdot;': '\u22d6', - 'lthree;': '\u22cb', - 'ltimes;': '\u22c9', - 'ltlarr;': '\u2976', - 'ltquest;': '\u2a7b', - 'ltri;': '\u25c3', - 'ltrie;': '\u22b4', - 'ltrif;': '\u25c2', - 'ltrPar;': '\u2996', - 'lurdshar;': '\u294a', - 'luruhar;': '\u2966', - 'lvertneqq;': '\u2268\ufe00', - 'lvnE;': '\u2268\ufe00', - 'macr': '\xaf', - 'macr;': '\xaf', - 'male;': '\u2642', - 'malt;': '\u2720', - 'maltese;': '\u2720', - 'Map;': '\u2905', - 'map;': '\u21a6', - 'mapsto;': '\u21a6', - 'mapstodown;': '\u21a7', - 'mapstoleft;': '\u21a4', - 'mapstoup;': '\u21a5', - 'marker;': '\u25ae', - 'mcomma;': '\u2a29', - 'Mcy;': '\u041c', - 'mcy;': '\u043c', - 'mdash;': '\u2014', - 'mDDot;': '\u223a', - 'measuredangle;': '\u2221', - 'MediumSpace;': '\u205f', - 'Mellintrf;': '\u2133', - 'Mfr;': '\U0001d510', - 'mfr;': '\U0001d52a', - 'mho;': '\u2127', - 'micro': '\xb5', - 'micro;': '\xb5', - 'mid;': '\u2223', - 'midast;': '*', - 'midcir;': '\u2af0', - 'middot': '\xb7', - 'middot;': '\xb7', - 'minus;': '\u2212', - 'minusb;': '\u229f', - 'minusd;': '\u2238', - 'minusdu;': '\u2a2a', - 'MinusPlus;': '\u2213', - 'mlcp;': '\u2adb', - 'mldr;': '\u2026', - 'mnplus;': '\u2213', - 'models;': '\u22a7', - 'Mopf;': '\U0001d544', - 'mopf;': '\U0001d55e', - 'mp;': '\u2213', - 'Mscr;': '\u2133', - 'mscr;': '\U0001d4c2', - 'mstpos;': '\u223e', - 'Mu;': '\u039c', - 'mu;': '\u03bc', - 'multimap;': '\u22b8', - 'mumap;': '\u22b8', - 'nabla;': '\u2207', - 'Nacute;': '\u0143', - 'nacute;': '\u0144', - 'nang;': '\u2220\u20d2', - 'nap;': '\u2249', - 'napE;': '\u2a70\u0338', - 'napid;': '\u224b\u0338', - 'napos;': '\u0149', - 'napprox;': '\u2249', - 'natur;': '\u266e', - 'natural;': '\u266e', - 'naturals;': '\u2115', - 'nbsp': '\xa0', - 'nbsp;': '\xa0', - 'nbump;': '\u224e\u0338', - 'nbumpe;': '\u224f\u0338', - 'ncap;': '\u2a43', - 'Ncaron;': '\u0147', - 'ncaron;': '\u0148', - 'Ncedil;': '\u0145', - 'ncedil;': '\u0146', - 'ncong;': '\u2247', - 'ncongdot;': '\u2a6d\u0338', - 'ncup;': '\u2a42', - 'Ncy;': '\u041d', - 'ncy;': '\u043d', - 'ndash;': '\u2013', - 'ne;': '\u2260', - 'nearhk;': '\u2924', - 'neArr;': '\u21d7', - 'nearr;': '\u2197', - 'nearrow;': '\u2197', - 'nedot;': '\u2250\u0338', - 'NegativeMediumSpace;': '\u200b', - 'NegativeThickSpace;': '\u200b', - 'NegativeThinSpace;': '\u200b', - 'NegativeVeryThinSpace;': '\u200b', - 'nequiv;': '\u2262', - 'nesear;': '\u2928', - 'nesim;': '\u2242\u0338', - 'NestedGreaterGreater;': '\u226b', - 'NestedLessLess;': '\u226a', - 'NewLine;': '\n', - 'nexist;': '\u2204', - 'nexists;': '\u2204', - 'Nfr;': '\U0001d511', - 'nfr;': '\U0001d52b', - 'ngE;': '\u2267\u0338', - 'nge;': '\u2271', - 'ngeq;': '\u2271', - 'ngeqq;': '\u2267\u0338', - 'ngeqslant;': '\u2a7e\u0338', - 'nges;': '\u2a7e\u0338', - 'nGg;': '\u22d9\u0338', - 'ngsim;': '\u2275', - 'nGt;': '\u226b\u20d2', - 'ngt;': '\u226f', - 'ngtr;': '\u226f', - 'nGtv;': '\u226b\u0338', - 'nhArr;': '\u21ce', - 'nharr;': '\u21ae', - 'nhpar;': '\u2af2', - 'ni;': '\u220b', - 'nis;': '\u22fc', - 'nisd;': '\u22fa', - 'niv;': '\u220b', - 'NJcy;': '\u040a', - 'njcy;': '\u045a', - 'nlArr;': '\u21cd', - 'nlarr;': '\u219a', - 'nldr;': '\u2025', - 'nlE;': '\u2266\u0338', - 'nle;': '\u2270', - 'nLeftarrow;': '\u21cd', - 'nleftarrow;': '\u219a', - 'nLeftrightarrow;': '\u21ce', - 'nleftrightarrow;': '\u21ae', - 'nleq;': '\u2270', - 'nleqq;': '\u2266\u0338', - 'nleqslant;': '\u2a7d\u0338', - 'nles;': '\u2a7d\u0338', - 'nless;': '\u226e', - 'nLl;': '\u22d8\u0338', - 'nlsim;': '\u2274', - 'nLt;': '\u226a\u20d2', - 'nlt;': '\u226e', - 'nltri;': '\u22ea', - 'nltrie;': '\u22ec', - 'nLtv;': '\u226a\u0338', - 'nmid;': '\u2224', - 'NoBreak;': '\u2060', - 'NonBreakingSpace;': '\xa0', - 'Nopf;': '\u2115', - 'nopf;': '\U0001d55f', - 'not': '\xac', - 'Not;': '\u2aec', - 'not;': '\xac', - 'NotCongruent;': '\u2262', - 'NotCupCap;': '\u226d', - 'NotDoubleVerticalBar;': '\u2226', - 'NotElement;': '\u2209', - 'NotEqual;': '\u2260', - 'NotEqualTilde;': '\u2242\u0338', - 'NotExists;': '\u2204', - 'NotGreater;': '\u226f', - 'NotGreaterEqual;': '\u2271', - 'NotGreaterFullEqual;': '\u2267\u0338', - 'NotGreaterGreater;': '\u226b\u0338', - 'NotGreaterLess;': '\u2279', - 'NotGreaterSlantEqual;': '\u2a7e\u0338', - 'NotGreaterTilde;': '\u2275', - 'NotHumpDownHump;': '\u224e\u0338', - 'NotHumpEqual;': '\u224f\u0338', - 'notin;': '\u2209', - 'notindot;': '\u22f5\u0338', - 'notinE;': '\u22f9\u0338', - 'notinva;': '\u2209', - 'notinvb;': '\u22f7', - 'notinvc;': '\u22f6', - 'NotLeftTriangle;': '\u22ea', - 'NotLeftTriangleBar;': '\u29cf\u0338', - 'NotLeftTriangleEqual;': '\u22ec', - 'NotLess;': '\u226e', - 'NotLessEqual;': '\u2270', - 'NotLessGreater;': '\u2278', - 'NotLessLess;': '\u226a\u0338', - 'NotLessSlantEqual;': '\u2a7d\u0338', - 'NotLessTilde;': '\u2274', - 'NotNestedGreaterGreater;': '\u2aa2\u0338', - 'NotNestedLessLess;': '\u2aa1\u0338', - 'notni;': '\u220c', - 'notniva;': '\u220c', - 'notnivb;': '\u22fe', - 'notnivc;': '\u22fd', - 'NotPrecedes;': '\u2280', - 'NotPrecedesEqual;': '\u2aaf\u0338', - 'NotPrecedesSlantEqual;': '\u22e0', - 'NotReverseElement;': '\u220c', - 'NotRightTriangle;': '\u22eb', - 'NotRightTriangleBar;': '\u29d0\u0338', - 'NotRightTriangleEqual;': '\u22ed', - 'NotSquareSubset;': '\u228f\u0338', - 'NotSquareSubsetEqual;': '\u22e2', - 'NotSquareSuperset;': '\u2290\u0338', - 'NotSquareSupersetEqual;': '\u22e3', - 'NotSubset;': '\u2282\u20d2', - 'NotSubsetEqual;': '\u2288', - 'NotSucceeds;': '\u2281', - 'NotSucceedsEqual;': '\u2ab0\u0338', - 'NotSucceedsSlantEqual;': '\u22e1', - 'NotSucceedsTilde;': '\u227f\u0338', - 'NotSuperset;': '\u2283\u20d2', - 'NotSupersetEqual;': '\u2289', - 'NotTilde;': '\u2241', - 'NotTildeEqual;': '\u2244', - 'NotTildeFullEqual;': '\u2247', - 'NotTildeTilde;': '\u2249', - 'NotVerticalBar;': '\u2224', - 'npar;': '\u2226', - 'nparallel;': '\u2226', - 'nparsl;': '\u2afd\u20e5', - 'npart;': '\u2202\u0338', - 'npolint;': '\u2a14', - 'npr;': '\u2280', - 'nprcue;': '\u22e0', - 'npre;': '\u2aaf\u0338', - 'nprec;': '\u2280', - 'npreceq;': '\u2aaf\u0338', - 'nrArr;': '\u21cf', - 'nrarr;': '\u219b', - 'nrarrc;': '\u2933\u0338', - 'nrarrw;': '\u219d\u0338', - 'nRightarrow;': '\u21cf', - 'nrightarrow;': '\u219b', - 'nrtri;': '\u22eb', - 'nrtrie;': '\u22ed', - 'nsc;': '\u2281', - 'nsccue;': '\u22e1', - 'nsce;': '\u2ab0\u0338', - 'Nscr;': '\U0001d4a9', - 'nscr;': '\U0001d4c3', - 'nshortmid;': '\u2224', - 'nshortparallel;': '\u2226', - 'nsim;': '\u2241', - 'nsime;': '\u2244', - 'nsimeq;': '\u2244', - 'nsmid;': '\u2224', - 'nspar;': '\u2226', - 'nsqsube;': '\u22e2', - 'nsqsupe;': '\u22e3', - 'nsub;': '\u2284', - 'nsubE;': '\u2ac5\u0338', - 'nsube;': '\u2288', - 'nsubset;': '\u2282\u20d2', - 'nsubseteq;': '\u2288', - 'nsubseteqq;': '\u2ac5\u0338', - 'nsucc;': '\u2281', - 'nsucceq;': '\u2ab0\u0338', - 'nsup;': '\u2285', - 'nsupE;': '\u2ac6\u0338', - 'nsupe;': '\u2289', - 'nsupset;': '\u2283\u20d2', - 'nsupseteq;': '\u2289', - 'nsupseteqq;': '\u2ac6\u0338', - 'ntgl;': '\u2279', - 'Ntilde': '\xd1', - 'ntilde': '\xf1', - 'Ntilde;': '\xd1', - 'ntilde;': '\xf1', - 'ntlg;': '\u2278', - 'ntriangleleft;': '\u22ea', - 'ntrianglelefteq;': '\u22ec', - 'ntriangleright;': '\u22eb', - 'ntrianglerighteq;': '\u22ed', - 'Nu;': '\u039d', - 'nu;': '\u03bd', - 'num;': '#', - 'numero;': '\u2116', - 'numsp;': '\u2007', - 'nvap;': '\u224d\u20d2', - 'nVDash;': '\u22af', - 'nVdash;': '\u22ae', - 'nvDash;': '\u22ad', - 'nvdash;': '\u22ac', - 'nvge;': '\u2265\u20d2', - 'nvgt;': '>\u20d2', - 'nvHarr;': '\u2904', - 'nvinfin;': '\u29de', - 'nvlArr;': '\u2902', - 'nvle;': '\u2264\u20d2', - 'nvlt;': '<\u20d2', - 'nvltrie;': '\u22b4\u20d2', - 'nvrArr;': '\u2903', - 'nvrtrie;': '\u22b5\u20d2', - 'nvsim;': '\u223c\u20d2', - 'nwarhk;': '\u2923', - 'nwArr;': '\u21d6', - 'nwarr;': '\u2196', - 'nwarrow;': '\u2196', - 'nwnear;': '\u2927', - 'Oacute': '\xd3', - 'oacute': '\xf3', - 'Oacute;': '\xd3', - 'oacute;': '\xf3', - 'oast;': '\u229b', - 'ocir;': '\u229a', - 'Ocirc': '\xd4', - 'ocirc': '\xf4', - 'Ocirc;': '\xd4', - 'ocirc;': '\xf4', - 'Ocy;': '\u041e', - 'ocy;': '\u043e', - 'odash;': '\u229d', - 'Odblac;': '\u0150', - 'odblac;': '\u0151', - 'odiv;': '\u2a38', - 'odot;': '\u2299', - 'odsold;': '\u29bc', - 'OElig;': '\u0152', - 'oelig;': '\u0153', - 'ofcir;': '\u29bf', - 'Ofr;': '\U0001d512', - 'ofr;': '\U0001d52c', - 'ogon;': '\u02db', - 'Ograve': '\xd2', - 'ograve': '\xf2', - 'Ograve;': '\xd2', - 'ograve;': '\xf2', - 'ogt;': '\u29c1', - 'ohbar;': '\u29b5', - 'ohm;': '\u03a9', - 'oint;': '\u222e', - 'olarr;': '\u21ba', - 'olcir;': '\u29be', - 'olcross;': '\u29bb', - 'oline;': '\u203e', - 'olt;': '\u29c0', - 'Omacr;': '\u014c', - 'omacr;': '\u014d', - 'Omega;': '\u03a9', - 'omega;': '\u03c9', - 'Omicron;': '\u039f', - 'omicron;': '\u03bf', - 'omid;': '\u29b6', - 'ominus;': '\u2296', - 'Oopf;': '\U0001d546', - 'oopf;': '\U0001d560', - 'opar;': '\u29b7', - 'OpenCurlyDoubleQuote;': '\u201c', - 'OpenCurlyQuote;': '\u2018', - 'operp;': '\u29b9', - 'oplus;': '\u2295', - 'Or;': '\u2a54', - 'or;': '\u2228', - 'orarr;': '\u21bb', - 'ord;': '\u2a5d', - 'order;': '\u2134', - 'orderof;': '\u2134', - 'ordf': '\xaa', - 'ordf;': '\xaa', - 'ordm': '\xba', - 'ordm;': '\xba', - 'origof;': '\u22b6', - 'oror;': '\u2a56', - 'orslope;': '\u2a57', - 'orv;': '\u2a5b', - 'oS;': '\u24c8', - 'Oscr;': '\U0001d4aa', - 'oscr;': '\u2134', - 'Oslash': '\xd8', - 'oslash': '\xf8', - 'Oslash;': '\xd8', - 'oslash;': '\xf8', - 'osol;': '\u2298', - 'Otilde': '\xd5', - 'otilde': '\xf5', - 'Otilde;': '\xd5', - 'otilde;': '\xf5', - 'Otimes;': '\u2a37', - 'otimes;': '\u2297', - 'otimesas;': '\u2a36', - 'Ouml': '\xd6', - 'ouml': '\xf6', - 'Ouml;': '\xd6', - 'ouml;': '\xf6', - 'ovbar;': '\u233d', - 'OverBar;': '\u203e', - 'OverBrace;': '\u23de', - 'OverBracket;': '\u23b4', - 'OverParenthesis;': '\u23dc', - 'par;': '\u2225', - 'para': '\xb6', - 'para;': '\xb6', - 'parallel;': '\u2225', - 'parsim;': '\u2af3', - 'parsl;': '\u2afd', - 'part;': '\u2202', - 'PartialD;': '\u2202', - 'Pcy;': '\u041f', - 'pcy;': '\u043f', - 'percnt;': '%', - 'period;': '.', - 'permil;': '\u2030', - 'perp;': '\u22a5', - 'pertenk;': '\u2031', - 'Pfr;': '\U0001d513', - 'pfr;': '\U0001d52d', - 'Phi;': '\u03a6', - 'phi;': '\u03c6', - 'phiv;': '\u03d5', - 'phmmat;': '\u2133', - 'phone;': '\u260e', - 'Pi;': '\u03a0', - 'pi;': '\u03c0', - 'pitchfork;': '\u22d4', - 'piv;': '\u03d6', - 'planck;': '\u210f', - 'planckh;': '\u210e', - 'plankv;': '\u210f', - 'plus;': '+', - 'plusacir;': '\u2a23', - 'plusb;': '\u229e', - 'pluscir;': '\u2a22', - 'plusdo;': '\u2214', - 'plusdu;': '\u2a25', - 'pluse;': '\u2a72', - 'PlusMinus;': '\xb1', - 'plusmn': '\xb1', - 'plusmn;': '\xb1', - 'plussim;': '\u2a26', - 'plustwo;': '\u2a27', - 'pm;': '\xb1', - 'Poincareplane;': '\u210c', - 'pointint;': '\u2a15', - 'Popf;': '\u2119', - 'popf;': '\U0001d561', - 'pound': '\xa3', - 'pound;': '\xa3', - 'Pr;': '\u2abb', - 'pr;': '\u227a', - 'prap;': '\u2ab7', - 'prcue;': '\u227c', - 'prE;': '\u2ab3', - 'pre;': '\u2aaf', - 'prec;': '\u227a', - 'precapprox;': '\u2ab7', - 'preccurlyeq;': '\u227c', - 'Precedes;': '\u227a', - 'PrecedesEqual;': '\u2aaf', - 'PrecedesSlantEqual;': '\u227c', - 'PrecedesTilde;': '\u227e', - 'preceq;': '\u2aaf', - 'precnapprox;': '\u2ab9', - 'precneqq;': '\u2ab5', - 'precnsim;': '\u22e8', - 'precsim;': '\u227e', - 'Prime;': '\u2033', - 'prime;': '\u2032', - 'primes;': '\u2119', - 'prnap;': '\u2ab9', - 'prnE;': '\u2ab5', - 'prnsim;': '\u22e8', - 'prod;': '\u220f', - 'Product;': '\u220f', - 'profalar;': '\u232e', - 'profline;': '\u2312', - 'profsurf;': '\u2313', - 'prop;': '\u221d', - 'Proportion;': '\u2237', - 'Proportional;': '\u221d', - 'propto;': '\u221d', - 'prsim;': '\u227e', - 'prurel;': '\u22b0', - 'Pscr;': '\U0001d4ab', - 'pscr;': '\U0001d4c5', - 'Psi;': '\u03a8', - 'psi;': '\u03c8', - 'puncsp;': '\u2008', - 'Qfr;': '\U0001d514', - 'qfr;': '\U0001d52e', - 'qint;': '\u2a0c', - 'Qopf;': '\u211a', - 'qopf;': '\U0001d562', - 'qprime;': '\u2057', - 'Qscr;': '\U0001d4ac', - 'qscr;': '\U0001d4c6', - 'quaternions;': '\u210d', - 'quatint;': '\u2a16', - 'quest;': '?', - 'questeq;': '\u225f', - 'QUOT': '"', - 'quot': '"', - 'QUOT;': '"', - 'quot;': '"', - 'rAarr;': '\u21db', - 'race;': '\u223d\u0331', - 'Racute;': '\u0154', - 'racute;': '\u0155', - 'radic;': '\u221a', - 'raemptyv;': '\u29b3', - 'Rang;': '\u27eb', - 'rang;': '\u27e9', - 'rangd;': '\u2992', - 'range;': '\u29a5', - 'rangle;': '\u27e9', - 'raquo': '\xbb', - 'raquo;': '\xbb', - 'Rarr;': '\u21a0', - 'rArr;': '\u21d2', - 'rarr;': '\u2192', - 'rarrap;': '\u2975', - 'rarrb;': '\u21e5', - 'rarrbfs;': '\u2920', - 'rarrc;': '\u2933', - 'rarrfs;': '\u291e', - 'rarrhk;': '\u21aa', - 'rarrlp;': '\u21ac', - 'rarrpl;': '\u2945', - 'rarrsim;': '\u2974', - 'Rarrtl;': '\u2916', - 'rarrtl;': '\u21a3', - 'rarrw;': '\u219d', - 'rAtail;': '\u291c', - 'ratail;': '\u291a', - 'ratio;': '\u2236', - 'rationals;': '\u211a', - 'RBarr;': '\u2910', - 'rBarr;': '\u290f', - 'rbarr;': '\u290d', - 'rbbrk;': '\u2773', - 'rbrace;': '}', - 'rbrack;': ']', - 'rbrke;': '\u298c', - 'rbrksld;': '\u298e', - 'rbrkslu;': '\u2990', - 'Rcaron;': '\u0158', - 'rcaron;': '\u0159', - 'Rcedil;': '\u0156', - 'rcedil;': '\u0157', - 'rceil;': '\u2309', - 'rcub;': '}', - 'Rcy;': '\u0420', - 'rcy;': '\u0440', - 'rdca;': '\u2937', - 'rdldhar;': '\u2969', - 'rdquo;': '\u201d', - 'rdquor;': '\u201d', - 'rdsh;': '\u21b3', - 'Re;': '\u211c', - 'real;': '\u211c', - 'realine;': '\u211b', - 'realpart;': '\u211c', - 'reals;': '\u211d', - 'rect;': '\u25ad', - 'REG': '\xae', - 'reg': '\xae', - 'REG;': '\xae', - 'reg;': '\xae', - 'ReverseElement;': '\u220b', - 'ReverseEquilibrium;': '\u21cb', - 'ReverseUpEquilibrium;': '\u296f', - 'rfisht;': '\u297d', - 'rfloor;': '\u230b', - 'Rfr;': '\u211c', - 'rfr;': '\U0001d52f', - 'rHar;': '\u2964', - 'rhard;': '\u21c1', - 'rharu;': '\u21c0', - 'rharul;': '\u296c', - 'Rho;': '\u03a1', - 'rho;': '\u03c1', - 'rhov;': '\u03f1', - 'RightAngleBracket;': '\u27e9', - 'RightArrow;': '\u2192', - 'Rightarrow;': '\u21d2', - 'rightarrow;': '\u2192', - 'RightArrowBar;': '\u21e5', - 'RightArrowLeftArrow;': '\u21c4', - 'rightarrowtail;': '\u21a3', - 'RightCeiling;': '\u2309', - 'RightDoubleBracket;': '\u27e7', - 'RightDownTeeVector;': '\u295d', - 'RightDownVector;': '\u21c2', - 'RightDownVectorBar;': '\u2955', - 'RightFloor;': '\u230b', - 'rightharpoondown;': '\u21c1', - 'rightharpoonup;': '\u21c0', - 'rightleftarrows;': '\u21c4', - 'rightleftharpoons;': '\u21cc', - 'rightrightarrows;': '\u21c9', - 'rightsquigarrow;': '\u219d', - 'RightTee;': '\u22a2', - 'RightTeeArrow;': '\u21a6', - 'RightTeeVector;': '\u295b', - 'rightthreetimes;': '\u22cc', - 'RightTriangle;': '\u22b3', - 'RightTriangleBar;': '\u29d0', - 'RightTriangleEqual;': '\u22b5', - 'RightUpDownVector;': '\u294f', - 'RightUpTeeVector;': '\u295c', - 'RightUpVector;': '\u21be', - 'RightUpVectorBar;': '\u2954', - 'RightVector;': '\u21c0', - 'RightVectorBar;': '\u2953', - 'ring;': '\u02da', - 'risingdotseq;': '\u2253', - 'rlarr;': '\u21c4', - 'rlhar;': '\u21cc', - 'rlm;': '\u200f', - 'rmoust;': '\u23b1', - 'rmoustache;': '\u23b1', - 'rnmid;': '\u2aee', - 'roang;': '\u27ed', - 'roarr;': '\u21fe', - 'robrk;': '\u27e7', - 'ropar;': '\u2986', - 'Ropf;': '\u211d', - 'ropf;': '\U0001d563', - 'roplus;': '\u2a2e', - 'rotimes;': '\u2a35', - 'RoundImplies;': '\u2970', - 'rpar;': ')', - 'rpargt;': '\u2994', - 'rppolint;': '\u2a12', - 'rrarr;': '\u21c9', - 'Rrightarrow;': '\u21db', - 'rsaquo;': '\u203a', - 'Rscr;': '\u211b', - 'rscr;': '\U0001d4c7', - 'Rsh;': '\u21b1', - 'rsh;': '\u21b1', - 'rsqb;': ']', - 'rsquo;': '\u2019', - 'rsquor;': '\u2019', - 'rthree;': '\u22cc', - 'rtimes;': '\u22ca', - 'rtri;': '\u25b9', - 'rtrie;': '\u22b5', - 'rtrif;': '\u25b8', - 'rtriltri;': '\u29ce', - 'RuleDelayed;': '\u29f4', - 'ruluhar;': '\u2968', - 'rx;': '\u211e', - 'Sacute;': '\u015a', - 'sacute;': '\u015b', - 'sbquo;': '\u201a', - 'Sc;': '\u2abc', - 'sc;': '\u227b', - 'scap;': '\u2ab8', - 'Scaron;': '\u0160', - 'scaron;': '\u0161', - 'sccue;': '\u227d', - 'scE;': '\u2ab4', - 'sce;': '\u2ab0', - 'Scedil;': '\u015e', - 'scedil;': '\u015f', - 'Scirc;': '\u015c', - 'scirc;': '\u015d', - 'scnap;': '\u2aba', - 'scnE;': '\u2ab6', - 'scnsim;': '\u22e9', - 'scpolint;': '\u2a13', - 'scsim;': '\u227f', - 'Scy;': '\u0421', - 'scy;': '\u0441', - 'sdot;': '\u22c5', - 'sdotb;': '\u22a1', - 'sdote;': '\u2a66', - 'searhk;': '\u2925', - 'seArr;': '\u21d8', - 'searr;': '\u2198', - 'searrow;': '\u2198', - 'sect': '\xa7', - 'sect;': '\xa7', - 'semi;': ';', - 'seswar;': '\u2929', - 'setminus;': '\u2216', - 'setmn;': '\u2216', - 'sext;': '\u2736', - 'Sfr;': '\U0001d516', - 'sfr;': '\U0001d530', - 'sfrown;': '\u2322', - 'sharp;': '\u266f', - 'SHCHcy;': '\u0429', - 'shchcy;': '\u0449', - 'SHcy;': '\u0428', - 'shcy;': '\u0448', - 'ShortDownArrow;': '\u2193', - 'ShortLeftArrow;': '\u2190', - 'shortmid;': '\u2223', - 'shortparallel;': '\u2225', - 'ShortRightArrow;': '\u2192', - 'ShortUpArrow;': '\u2191', - 'shy': '\xad', - 'shy;': '\xad', - 'Sigma;': '\u03a3', - 'sigma;': '\u03c3', - 'sigmaf;': '\u03c2', - 'sigmav;': '\u03c2', - 'sim;': '\u223c', - 'simdot;': '\u2a6a', - 'sime;': '\u2243', - 'simeq;': '\u2243', - 'simg;': '\u2a9e', - 'simgE;': '\u2aa0', - 'siml;': '\u2a9d', - 'simlE;': '\u2a9f', - 'simne;': '\u2246', - 'simplus;': '\u2a24', - 'simrarr;': '\u2972', - 'slarr;': '\u2190', - 'SmallCircle;': '\u2218', - 'smallsetminus;': '\u2216', - 'smashp;': '\u2a33', - 'smeparsl;': '\u29e4', - 'smid;': '\u2223', - 'smile;': '\u2323', - 'smt;': '\u2aaa', - 'smte;': '\u2aac', - 'smtes;': '\u2aac\ufe00', - 'SOFTcy;': '\u042c', - 'softcy;': '\u044c', - 'sol;': '/', - 'solb;': '\u29c4', - 'solbar;': '\u233f', - 'Sopf;': '\U0001d54a', - 'sopf;': '\U0001d564', - 'spades;': '\u2660', - 'spadesuit;': '\u2660', - 'spar;': '\u2225', - 'sqcap;': '\u2293', - 'sqcaps;': '\u2293\ufe00', - 'sqcup;': '\u2294', - 'sqcups;': '\u2294\ufe00', - 'Sqrt;': '\u221a', - 'sqsub;': '\u228f', - 'sqsube;': '\u2291', - 'sqsubset;': '\u228f', - 'sqsubseteq;': '\u2291', - 'sqsup;': '\u2290', - 'sqsupe;': '\u2292', - 'sqsupset;': '\u2290', - 'sqsupseteq;': '\u2292', - 'squ;': '\u25a1', - 'Square;': '\u25a1', - 'square;': '\u25a1', - 'SquareIntersection;': '\u2293', - 'SquareSubset;': '\u228f', - 'SquareSubsetEqual;': '\u2291', - 'SquareSuperset;': '\u2290', - 'SquareSupersetEqual;': '\u2292', - 'SquareUnion;': '\u2294', - 'squarf;': '\u25aa', - 'squf;': '\u25aa', - 'srarr;': '\u2192', - 'Sscr;': '\U0001d4ae', - 'sscr;': '\U0001d4c8', - 'ssetmn;': '\u2216', - 'ssmile;': '\u2323', - 'sstarf;': '\u22c6', - 'Star;': '\u22c6', - 'star;': '\u2606', - 'starf;': '\u2605', - 'straightepsilon;': '\u03f5', - 'straightphi;': '\u03d5', - 'strns;': '\xaf', - 'Sub;': '\u22d0', - 'sub;': '\u2282', - 'subdot;': '\u2abd', - 'subE;': '\u2ac5', - 'sube;': '\u2286', - 'subedot;': '\u2ac3', - 'submult;': '\u2ac1', - 'subnE;': '\u2acb', - 'subne;': '\u228a', - 'subplus;': '\u2abf', - 'subrarr;': '\u2979', - 'Subset;': '\u22d0', - 'subset;': '\u2282', - 'subseteq;': '\u2286', - 'subseteqq;': '\u2ac5', - 'SubsetEqual;': '\u2286', - 'subsetneq;': '\u228a', - 'subsetneqq;': '\u2acb', - 'subsim;': '\u2ac7', - 'subsub;': '\u2ad5', - 'subsup;': '\u2ad3', - 'succ;': '\u227b', - 'succapprox;': '\u2ab8', - 'succcurlyeq;': '\u227d', - 'Succeeds;': '\u227b', - 'SucceedsEqual;': '\u2ab0', - 'SucceedsSlantEqual;': '\u227d', - 'SucceedsTilde;': '\u227f', - 'succeq;': '\u2ab0', - 'succnapprox;': '\u2aba', - 'succneqq;': '\u2ab6', - 'succnsim;': '\u22e9', - 'succsim;': '\u227f', - 'SuchThat;': '\u220b', - 'Sum;': '\u2211', - 'sum;': '\u2211', - 'sung;': '\u266a', - 'sup1': '\xb9', - 'sup1;': '\xb9', - 'sup2': '\xb2', - 'sup2;': '\xb2', - 'sup3': '\xb3', - 'sup3;': '\xb3', - 'Sup;': '\u22d1', - 'sup;': '\u2283', - 'supdot;': '\u2abe', - 'supdsub;': '\u2ad8', - 'supE;': '\u2ac6', - 'supe;': '\u2287', - 'supedot;': '\u2ac4', - 'Superset;': '\u2283', - 'SupersetEqual;': '\u2287', - 'suphsol;': '\u27c9', - 'suphsub;': '\u2ad7', - 'suplarr;': '\u297b', - 'supmult;': '\u2ac2', - 'supnE;': '\u2acc', - 'supne;': '\u228b', - 'supplus;': '\u2ac0', - 'Supset;': '\u22d1', - 'supset;': '\u2283', - 'supseteq;': '\u2287', - 'supseteqq;': '\u2ac6', - 'supsetneq;': '\u228b', - 'supsetneqq;': '\u2acc', - 'supsim;': '\u2ac8', - 'supsub;': '\u2ad4', - 'supsup;': '\u2ad6', - 'swarhk;': '\u2926', - 'swArr;': '\u21d9', - 'swarr;': '\u2199', - 'swarrow;': '\u2199', - 'swnwar;': '\u292a', - 'szlig': '\xdf', - 'szlig;': '\xdf', - 'Tab;': '\t', - 'target;': '\u2316', - 'Tau;': '\u03a4', - 'tau;': '\u03c4', - 'tbrk;': '\u23b4', - 'Tcaron;': '\u0164', - 'tcaron;': '\u0165', - 'Tcedil;': '\u0162', - 'tcedil;': '\u0163', - 'Tcy;': '\u0422', - 'tcy;': '\u0442', - 'tdot;': '\u20db', - 'telrec;': '\u2315', - 'Tfr;': '\U0001d517', - 'tfr;': '\U0001d531', - 'there4;': '\u2234', - 'Therefore;': '\u2234', - 'therefore;': '\u2234', - 'Theta;': '\u0398', - 'theta;': '\u03b8', - 'thetasym;': '\u03d1', - 'thetav;': '\u03d1', - 'thickapprox;': '\u2248', - 'thicksim;': '\u223c', - 'ThickSpace;': '\u205f\u200a', - 'thinsp;': '\u2009', - 'ThinSpace;': '\u2009', - 'thkap;': '\u2248', - 'thksim;': '\u223c', - 'THORN': '\xde', - 'thorn': '\xfe', - 'THORN;': '\xde', - 'thorn;': '\xfe', - 'Tilde;': '\u223c', - 'tilde;': '\u02dc', - 'TildeEqual;': '\u2243', - 'TildeFullEqual;': '\u2245', - 'TildeTilde;': '\u2248', - 'times': '\xd7', - 'times;': '\xd7', - 'timesb;': '\u22a0', - 'timesbar;': '\u2a31', - 'timesd;': '\u2a30', - 'tint;': '\u222d', - 'toea;': '\u2928', - 'top;': '\u22a4', - 'topbot;': '\u2336', - 'topcir;': '\u2af1', - 'Topf;': '\U0001d54b', - 'topf;': '\U0001d565', - 'topfork;': '\u2ada', - 'tosa;': '\u2929', - 'tprime;': '\u2034', - 'TRADE;': '\u2122', - 'trade;': '\u2122', - 'triangle;': '\u25b5', - 'triangledown;': '\u25bf', - 'triangleleft;': '\u25c3', - 'trianglelefteq;': '\u22b4', - 'triangleq;': '\u225c', - 'triangleright;': '\u25b9', - 'trianglerighteq;': '\u22b5', - 'tridot;': '\u25ec', - 'trie;': '\u225c', - 'triminus;': '\u2a3a', - 'TripleDot;': '\u20db', - 'triplus;': '\u2a39', - 'trisb;': '\u29cd', - 'tritime;': '\u2a3b', - 'trpezium;': '\u23e2', - 'Tscr;': '\U0001d4af', - 'tscr;': '\U0001d4c9', - 'TScy;': '\u0426', - 'tscy;': '\u0446', - 'TSHcy;': '\u040b', - 'tshcy;': '\u045b', - 'Tstrok;': '\u0166', - 'tstrok;': '\u0167', - 'twixt;': '\u226c', - 'twoheadleftarrow;': '\u219e', - 'twoheadrightarrow;': '\u21a0', - 'Uacute': '\xda', - 'uacute': '\xfa', - 'Uacute;': '\xda', - 'uacute;': '\xfa', - 'Uarr;': '\u219f', - 'uArr;': '\u21d1', - 'uarr;': '\u2191', - 'Uarrocir;': '\u2949', - 'Ubrcy;': '\u040e', - 'ubrcy;': '\u045e', - 'Ubreve;': '\u016c', - 'ubreve;': '\u016d', - 'Ucirc': '\xdb', - 'ucirc': '\xfb', - 'Ucirc;': '\xdb', - 'ucirc;': '\xfb', - 'Ucy;': '\u0423', - 'ucy;': '\u0443', - 'udarr;': '\u21c5', - 'Udblac;': '\u0170', - 'udblac;': '\u0171', - 'udhar;': '\u296e', - 'ufisht;': '\u297e', - 'Ufr;': '\U0001d518', - 'ufr;': '\U0001d532', - 'Ugrave': '\xd9', - 'ugrave': '\xf9', - 'Ugrave;': '\xd9', - 'ugrave;': '\xf9', - 'uHar;': '\u2963', - 'uharl;': '\u21bf', - 'uharr;': '\u21be', - 'uhblk;': '\u2580', - 'ulcorn;': '\u231c', - 'ulcorner;': '\u231c', - 'ulcrop;': '\u230f', - 'ultri;': '\u25f8', - 'Umacr;': '\u016a', - 'umacr;': '\u016b', - 'uml': '\xa8', - 'uml;': '\xa8', - 'UnderBar;': '_', - 'UnderBrace;': '\u23df', - 'UnderBracket;': '\u23b5', - 'UnderParenthesis;': '\u23dd', - 'Union;': '\u22c3', - 'UnionPlus;': '\u228e', - 'Uogon;': '\u0172', - 'uogon;': '\u0173', - 'Uopf;': '\U0001d54c', - 'uopf;': '\U0001d566', - 'UpArrow;': '\u2191', - 'Uparrow;': '\u21d1', - 'uparrow;': '\u2191', - 'UpArrowBar;': '\u2912', - 'UpArrowDownArrow;': '\u21c5', - 'UpDownArrow;': '\u2195', - 'Updownarrow;': '\u21d5', - 'updownarrow;': '\u2195', - 'UpEquilibrium;': '\u296e', - 'upharpoonleft;': '\u21bf', - 'upharpoonright;': '\u21be', - 'uplus;': '\u228e', - 'UpperLeftArrow;': '\u2196', - 'UpperRightArrow;': '\u2197', - 'Upsi;': '\u03d2', - 'upsi;': '\u03c5', - 'upsih;': '\u03d2', - 'Upsilon;': '\u03a5', - 'upsilon;': '\u03c5', - 'UpTee;': '\u22a5', - 'UpTeeArrow;': '\u21a5', - 'upuparrows;': '\u21c8', - 'urcorn;': '\u231d', - 'urcorner;': '\u231d', - 'urcrop;': '\u230e', - 'Uring;': '\u016e', - 'uring;': '\u016f', - 'urtri;': '\u25f9', - 'Uscr;': '\U0001d4b0', - 'uscr;': '\U0001d4ca', - 'utdot;': '\u22f0', - 'Utilde;': '\u0168', - 'utilde;': '\u0169', - 'utri;': '\u25b5', - 'utrif;': '\u25b4', - 'uuarr;': '\u21c8', - 'Uuml': '\xdc', - 'uuml': '\xfc', - 'Uuml;': '\xdc', - 'uuml;': '\xfc', - 'uwangle;': '\u29a7', - 'vangrt;': '\u299c', - 'varepsilon;': '\u03f5', - 'varkappa;': '\u03f0', - 'varnothing;': '\u2205', - 'varphi;': '\u03d5', - 'varpi;': '\u03d6', - 'varpropto;': '\u221d', - 'vArr;': '\u21d5', - 'varr;': '\u2195', - 'varrho;': '\u03f1', - 'varsigma;': '\u03c2', - 'varsubsetneq;': '\u228a\ufe00', - 'varsubsetneqq;': '\u2acb\ufe00', - 'varsupsetneq;': '\u228b\ufe00', - 'varsupsetneqq;': '\u2acc\ufe00', - 'vartheta;': '\u03d1', - 'vartriangleleft;': '\u22b2', - 'vartriangleright;': '\u22b3', - 'Vbar;': '\u2aeb', - 'vBar;': '\u2ae8', - 'vBarv;': '\u2ae9', - 'Vcy;': '\u0412', - 'vcy;': '\u0432', - 'VDash;': '\u22ab', - 'Vdash;': '\u22a9', - 'vDash;': '\u22a8', - 'vdash;': '\u22a2', - 'Vdashl;': '\u2ae6', - 'Vee;': '\u22c1', - 'vee;': '\u2228', - 'veebar;': '\u22bb', - 'veeeq;': '\u225a', - 'vellip;': '\u22ee', - 'Verbar;': '\u2016', - 'verbar;': '|', - 'Vert;': '\u2016', - 'vert;': '|', - 'VerticalBar;': '\u2223', - 'VerticalLine;': '|', - 'VerticalSeparator;': '\u2758', - 'VerticalTilde;': '\u2240', - 'VeryThinSpace;': '\u200a', - 'Vfr;': '\U0001d519', - 'vfr;': '\U0001d533', - 'vltri;': '\u22b2', - 'vnsub;': '\u2282\u20d2', - 'vnsup;': '\u2283\u20d2', - 'Vopf;': '\U0001d54d', - 'vopf;': '\U0001d567', - 'vprop;': '\u221d', - 'vrtri;': '\u22b3', - 'Vscr;': '\U0001d4b1', - 'vscr;': '\U0001d4cb', - 'vsubnE;': '\u2acb\ufe00', - 'vsubne;': '\u228a\ufe00', - 'vsupnE;': '\u2acc\ufe00', - 'vsupne;': '\u228b\ufe00', - 'Vvdash;': '\u22aa', - 'vzigzag;': '\u299a', - 'Wcirc;': '\u0174', - 'wcirc;': '\u0175', - 'wedbar;': '\u2a5f', - 'Wedge;': '\u22c0', - 'wedge;': '\u2227', - 'wedgeq;': '\u2259', - 'weierp;': '\u2118', - 'Wfr;': '\U0001d51a', - 'wfr;': '\U0001d534', - 'Wopf;': '\U0001d54e', - 'wopf;': '\U0001d568', - 'wp;': '\u2118', - 'wr;': '\u2240', - 'wreath;': '\u2240', - 'Wscr;': '\U0001d4b2', - 'wscr;': '\U0001d4cc', - 'xcap;': '\u22c2', - 'xcirc;': '\u25ef', - 'xcup;': '\u22c3', - 'xdtri;': '\u25bd', - 'Xfr;': '\U0001d51b', - 'xfr;': '\U0001d535', - 'xhArr;': '\u27fa', - 'xharr;': '\u27f7', - 'Xi;': '\u039e', - 'xi;': '\u03be', - 'xlArr;': '\u27f8', - 'xlarr;': '\u27f5', - 'xmap;': '\u27fc', - 'xnis;': '\u22fb', - 'xodot;': '\u2a00', - 'Xopf;': '\U0001d54f', - 'xopf;': '\U0001d569', - 'xoplus;': '\u2a01', - 'xotime;': '\u2a02', - 'xrArr;': '\u27f9', - 'xrarr;': '\u27f6', - 'Xscr;': '\U0001d4b3', - 'xscr;': '\U0001d4cd', - 'xsqcup;': '\u2a06', - 'xuplus;': '\u2a04', - 'xutri;': '\u25b3', - 'xvee;': '\u22c1', - 'xwedge;': '\u22c0', - 'Yacute': '\xdd', - 'yacute': '\xfd', - 'Yacute;': '\xdd', - 'yacute;': '\xfd', - 'YAcy;': '\u042f', - 'yacy;': '\u044f', - 'Ycirc;': '\u0176', - 'ycirc;': '\u0177', - 'Ycy;': '\u042b', - 'ycy;': '\u044b', - 'yen': '\xa5', - 'yen;': '\xa5', - 'Yfr;': '\U0001d51c', - 'yfr;': '\U0001d536', - 'YIcy;': '\u0407', - 'yicy;': '\u0457', - 'Yopf;': '\U0001d550', - 'yopf;': '\U0001d56a', - 'Yscr;': '\U0001d4b4', - 'yscr;': '\U0001d4ce', - 'YUcy;': '\u042e', - 'yucy;': '\u044e', - 'yuml': '\xff', - 'Yuml;': '\u0178', - 'yuml;': '\xff', - 'Zacute;': '\u0179', - 'zacute;': '\u017a', - 'Zcaron;': '\u017d', - 'zcaron;': '\u017e', - 'Zcy;': '\u0417', - 'zcy;': '\u0437', - 'Zdot;': '\u017b', - 'zdot;': '\u017c', - 'zeetrf;': '\u2128', - 'ZeroWidthSpace;': '\u200b', - 'Zeta;': '\u0396', - 'zeta;': '\u03b6', - 'Zfr;': '\u2128', - 'zfr;': '\U0001d537', - 'ZHcy;': '\u0416', - 'zhcy;': '\u0436', - 'zigrarr;': '\u21dd', - 'Zopf;': '\u2124', - 'zopf;': '\U0001d56b', - 'Zscr;': '\U0001d4b5', - 'zscr;': '\U0001d4cf', - 'zwj;': '\u200d', - 'zwnj;': '\u200c', -} - -# maps the Unicode code point to the HTML entity name +# maps the Unicode codepoint to the HTML entity name codepoint2name = {} # maps the HTML entity name to the character diff -r 6db40a9955dc -r 0d413f60cc23 Lib/html/parser.py --- a/Lib/html/parser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/html/parser.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,14 +8,8 @@ # and CDATA (character data -- only end tags are special). +import _markupbase import re -import warnings -import _markupbase - -from html import unescape - - -__all__ = ['HTMLParser'] # Regular expressions used for parsing @@ -28,18 +22,38 @@ starttagopen = re.compile('<[a-zA-Z]') piclose = re.compile('>') commentclose = re.compile(r'--\s*>') -# Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will -# explode, so don't do it. +tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile('([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') +tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*') +# Note: +# 1) the strict attrfind isn't really strict, but we can't make it +# correctly strict without breaking backward compatibility; +# 2) if you change attrfind remember to update locatestarttagend too; +# 3) if you change attrfind and/or locatestarttagend the parser will +# explode, so don't do it. +attrfind = re.compile( + r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' + r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'[\s/]*((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +locatestarttagend = re.compile(r""" + <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name + (?:\s+ # whitespace before attribute name + (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name + (?:\s*=\s* # value indicator + (?:'[^']*' # LITA-enclosed value + |\"[^\"]*\" # LIT-enclosed value + |[^'\">\s]+ # bare value + ) + )? + ) + )* + \s* # trailing whitespace +""", re.VERBOSE) locatestarttagend_tolerant = re.compile(r""" - <[a-zA-Z][^\t\n\r\f />\x00]* # tag name + <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:[\s/]* # optional whitespace before attribute name (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name (?:\s*=+\s* # value indicator @@ -59,6 +73,23 @@ endtagfind = re.compile('') +class HTMLParseError(Exception): + """Exception raised for all parse errors.""" + + def __init__(self, msg, position=(None, None)): + assert msg + self.msg = msg + self.lineno = position[0] + self.offset = position[1] + + def __str__(self): + result = self.msg + if self.lineno is not None: + result = result + ", at line %d" % self.lineno + if self.offset is not None: + result = result + ", column %d" % (self.offset + 1) + return result + class HTMLParser(_markupbase.ParserBase): """Find tags and other markup and call handler functions. @@ -73,24 +104,24 @@ self.handle_startendtag(); end tags by self.handle_endtag(). The data between tags is passed from the parser to the derived class by calling self.handle_data() with the data as argument (the data - may be split up in arbitrary chunks). If convert_charrefs is - True the character references are converted automatically to the - corresponding Unicode character (and self.handle_data() is no - longer split in chunks), otherwise they are passed by calling - self.handle_entityref() or self.handle_charref() with the string - containing respectively the named or numeric reference as the - argument. + may be split up in arbitrary chunks). Entity references are + passed by calling self.handle_entityref() with the entity + reference as the argument. Numeric character references are + passed to self.handle_charref() with the string containing the + reference as the argument. """ CDATA_CONTENT_ELEMENTS = ("script", "style") - def __init__(self, *, convert_charrefs=True): + def __init__(self, strict=True): """Initialize and reset this instance. - If convert_charrefs is True (the default), all character references - are automatically converted to the corresponding Unicode characters. + If strict is set to True (the default), errors are raised when invalid + HTML is encountered. If set to False, an attempt is instead made to + continue parsing, making "best guesses" about the intended meaning, in + a fashion similar to what browsers typically do. """ - self.convert_charrefs = convert_charrefs + self.strict = strict self.reset() def reset(self): @@ -114,6 +145,9 @@ """Handle any buffered data.""" self.goahead(1) + def error(self, message): + raise HTMLParseError(message, self.getpos()) + __starttag_text = None def get_starttag_text(self): @@ -136,33 +170,14 @@ i = 0 n = len(rawdata) while i < n: - if self.convert_charrefs and not self.cdata_elem: - j = rawdata.find('<', i) - if j < 0: - # if we can't find the next <, either we are at the end - # or there's more text incoming. If the latter is True, - # we can't pass the text to handle_data in case we have - # a charref cut in half at end. Try to determine if - # this is the case before proceding by looking for an - # & near the end and see if it's followed by a space or ;. - amppos = rawdata.rfind('&', max(i, n-34)) - if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): - break # wait till we get all the text - j = n + match = self.interesting.search(rawdata, i) # < or & + if match: + j = match.start() else: - match = self.interesting.search(rawdata, i) # < or & - if match: - j = match.start() - else: - if self.cdata_elem: - break - j = n - if i < j: - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:j])) - else: - self.handle_data(rawdata[i:j]) + if self.cdata_elem: + break + j = n + if i < j: self.handle_data(rawdata[i:j]) i = self.updatepos(i, j) if i == n: break startswith = rawdata.startswith @@ -176,7 +191,10 @@ elif startswith("', i + 1) if k < 0: k = rawdata.find('<', i + 1) @@ -192,10 +212,7 @@ k = i + 1 else: k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) - else: - self.handle_data(rawdata[i:k]) + self.handle_data(rawdata[i:k]) i = self.updatepos(i, k) elif startswith("&#", i): match = charref.match(rawdata, i) @@ -208,9 +225,9 @@ i = self.updatepos(i, k) continue else: - if ";" in rawdata[i:]: # bail by consuming &# - self.handle_data(rawdata[i:i+2]) - i = self.updatepos(i, i+2) + if ";" in rawdata[i:]: #bail by consuming &# + self.handle_data(rawdata[0:2]) + i = self.updatepos(i, 2) break elif startswith('&', i): match = entityref.match(rawdata, i) @@ -226,10 +243,12 @@ if match: # match.group() will contain at least 2 chars if end and match.group() == rawdata[i:]: - k = match.end() - if k <= i: - k = n - i = self.updatepos(i, i + 1) + if self.strict: + self.error("EOF in middle of entity or char ref") + else: + if k <= i: + k = n + i = self.updatepos(i, i + 1) # incomplete break elif (i + 1) < n: @@ -243,10 +262,7 @@ assert 0, "interesting.search() lied" # end while if end and i < n and not self.cdata_elem: - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:n])) - else: - self.handle_data(rawdata[i:n]) + self.handle_data(rawdata[i:n]) i = self.updatepos(i, n) self.rawdata = rawdata[i:] @@ -255,8 +271,8 @@ # See also parse_declaration in _markupbase def parse_html_declaration(self, i): rawdata = self.rawdata - assert rawdata[i:i+2] == '', i+2) if pos == -1: return -1 @@ -308,12 +324,15 @@ # Now parse the data between i+1 and j into a tag and attrs attrs = [] - match = tagfind_tolerant.match(rawdata, i+1) + match = tagfind.match(rawdata, i+1) assert match, 'unexpected call to parse_starttag()' k = match.end() - self.lasttag = tag = match.group(1).lower() + self.lasttag = tag = rawdata[i+1:k].lower() while k < endpos: - m = attrfind_tolerant.match(rawdata, k) + if self.strict: + m = attrfind.match(rawdata, k) + else: + m = attrfind_tolerant.match(rawdata, k) if not m: break attrname, rest, attrvalue = m.group(1, 2, 3) @@ -323,7 +342,7 @@ attrvalue[:1] == '"' == attrvalue[-1:]: attrvalue = attrvalue[1:-1] if attrvalue: - attrvalue = unescape(attrvalue) + attrvalue = self.unescape(attrvalue) attrs.append((attrname.lower(), attrvalue)) k = m.end() @@ -336,6 +355,9 @@ - self.__starttag_text.rfind("\n") else: offset = offset + len(self.__starttag_text) + if self.strict: + self.error("junk characters in start tag: %r" + % (rawdata[k:endpos][:20],)) self.handle_data(rawdata[i:endpos]) return endpos if end.endswith('/>'): @@ -351,7 +373,10 @@ # or -1 if incomplete. def check_for_whole_start_tag(self, i): rawdata = self.rawdata - m = locatestarttagend_tolerant.match(rawdata, i) + if self.strict: + m = locatestarttagend.match(rawdata, i) + else: + m = locatestarttagend_tolerant.match(rawdata, i) if m: j = m.end() next = rawdata[j:j+1] @@ -364,6 +389,9 @@ # buffer boundary return -1 # else bogus input + if self.strict: + self.updatepos(i, j + 1) + self.error("malformed empty start tag") if j > i: return j else: @@ -376,6 +404,9 @@ # end of input in or before attribute value, or we have the # '/' from a '/>' ending return -1 + if self.strict: + self.updatepos(i, j) + self.error("malformed start tag") if j > i: return j else: @@ -395,6 +426,8 @@ if self.cdata_elem is not None: self.handle_data(rawdata[i:gtpos]) return gtpos + if self.strict: + self.error("bad end tag: %r" % (rawdata[i:gtpos],)) # find the name: w3.org/TR/html5/tokenization.html#tag-name-state namematch = tagfind_tolerant.match(rawdata, i+2) if not namematch: @@ -403,7 +436,7 @@ return i+3 else: return self.parse_bogus_comment(i) - tagname = namematch.group(1).lower() + tagname = namematch.group().lower() # consume and ignore other stuff between the name and the > # Note: this is not 100% correct, since we might have things like # , but looking for > after tha name should cover @@ -460,11 +493,38 @@ pass def unknown_decl(self, data): - pass + if self.strict: + self.error("unknown declaration: %r" % (data,)) # Internal -- helper to remove special character quoting + entitydefs = None def unescape(self, s): - warnings.warn('The unescape method is deprecated and will be removed ' - 'in 3.5, use html.unescape() instead.', - DeprecationWarning, stacklevel=2) - return unescape(s) + if '&' not in s: + return s + def replaceEntities(s): + s = s.groups()[0] + try: + if s[0] == "#": + s = s[1:] + if s[0] in ['x','X']: + c = int(s[1:], 16) + else: + c = int(s) + return chr(c) + except ValueError: + return '&#'+ s +';' + else: + # Cannot use name2codepoint directly, because HTMLParser + # supports apos, which is not part of HTML 4 + import html.entities + if HTMLParser.entitydefs is None: + entitydefs = HTMLParser.entitydefs = {'apos':"'"} + for k, v in html.entities.name2codepoint.items(): + entitydefs[k] = chr(v) + try: + return self.entitydefs[s] + except KeyError: + return '&'+s+';' + + return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));", + replaceEntities, s, flags=re.ASCII) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/http/__init__.py --- a/Lib/http/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/http/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,134 +1,1 @@ -from enum import IntEnum - -__all__ = ['HTTPStatus'] - -class HTTPStatus(IntEnum): - """HTTP status codes and reason phrases - - Status codes from the following RFCs are all observed: - - * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616 - * RFC 6585: Additional HTTP Status Codes - * RFC 3229: Delta encoding in HTTP - * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518 - * RFC 5842: Binding Extensions to WebDAV - * RFC 7238: Permanent Redirect - * RFC 2295: Transparent Content Negotiation in HTTP - * RFC 2774: An HTTP Extension Framework - """ - def __new__(cls, value, phrase, description=''): - obj = int.__new__(cls, value) - obj._value_ = value - - obj.phrase = phrase - obj.description = description - return obj - - # informational - CONTINUE = 100, 'Continue', 'Request received, please continue' - SWITCHING_PROTOCOLS = (101, 'Switching Protocols', - 'Switching to new protocol; obey Upgrade header') - PROCESSING = 102, 'Processing' - - # success - OK = 200, 'OK', 'Request fulfilled, document follows' - CREATED = 201, 'Created', 'Document created, URL follows' - ACCEPTED = (202, 'Accepted', - 'Request accepted, processing continues off-line') - NON_AUTHORITATIVE_INFORMATION = (203, - 'Non-Authoritative Information', 'Request fulfilled from cache') - NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows' - RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input' - PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows' - MULTI_STATUS = 207, 'Multi-Status' - ALREADY_REPORTED = 208, 'Already Reported' - IM_USED = 226, 'IM Used' - - # redirection - MULTIPLE_CHOICES = (300, 'Multiple Choices', - 'Object has several resources -- see URI list') - MOVED_PERMANENTLY = (301, 'Moved Permanently', - 'Object moved permanently -- see URI list') - FOUND = 302, 'Found', 'Object moved temporarily -- see URI list' - SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list' - NOT_MODIFIED = (304, 'Not Modified', - 'Document has not changed since given time') - USE_PROXY = (305, 'Use Proxy', - 'You must use proxy specified in Location to access this resource') - TEMPORARY_REDIRECT = (307, 'Temporary Redirect', - 'Object moved temporarily -- see URI list') - PERMANENT_REDIRECT = (308, 'Permanent Redirect', - 'Object moved temporarily -- see URI list') - - # client error - BAD_REQUEST = (400, 'Bad Request', - 'Bad request syntax or unsupported method') - UNAUTHORIZED = (401, 'Unauthorized', - 'No permission -- see authorization schemes') - PAYMENT_REQUIRED = (402, 'Payment Required', - 'No payment -- see charging schemes') - FORBIDDEN = (403, 'Forbidden', - 'Request forbidden -- authorization will not help') - NOT_FOUND = (404, 'Not Found', - 'Nothing matches the given URI') - METHOD_NOT_ALLOWED = (405, 'Method Not Allowed', - 'Specified method is invalid for this resource') - NOT_ACCEPTABLE = (406, 'Not Acceptable', - 'URI not available in preferred format') - PROXY_AUTHENTICATION_REQUIRED = (407, - 'Proxy Authentication Required', - 'You must authenticate with this proxy before proceeding') - REQUEST_TIMEOUT = (408, 'Request Timeout', - 'Request timed out; try again later') - CONFLICT = 409, 'Conflict', 'Request conflict' - GONE = (410, 'Gone', - 'URI no longer exists and has been permanently removed') - LENGTH_REQUIRED = (411, 'Length Required', - 'Client must specify Content-Length') - PRECONDITION_FAILED = (412, 'Precondition Failed', - 'Precondition in headers is false') - REQUEST_ENTITY_TOO_LARGE = (413, 'Request Entity Too Large', - 'Entity is too large') - REQUEST_URI_TOO_LONG = (414, 'Request-URI Too Long', - 'URI is too long') - UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type', - 'Entity body in unsupported format') - REQUESTED_RANGE_NOT_SATISFIABLE = (416, - 'Requested Range Not Satisfiable', - 'Cannot satisfy request range') - EXPECTATION_FAILED = (417, 'Expectation Failed', - 'Expect condition could not be satisfied') - UNPROCESSABLE_ENTITY = 422, 'Unprocessable Entity' - LOCKED = 423, 'Locked' - FAILED_DEPENDENCY = 424, 'Failed Dependency' - UPGRADE_REQUIRED = 426, 'Upgrade Required' - PRECONDITION_REQUIRED = (428, 'Precondition Required', - 'The origin server requires the request to be conditional') - TOO_MANY_REQUESTS = (429, 'Too Many Requests', - 'The user has sent too many requests in ' - 'a given amount of time ("rate limiting")') - REQUEST_HEADER_FIELDS_TOO_LARGE = (431, - 'Request Header Fields Too Large', - 'The server is unwilling to process the request because its header ' - 'fields are too large') - - # server errors - INTERNAL_SERVER_ERROR = (500, 'Internal Server Error', - 'Server got itself in trouble') - NOT_IMPLEMENTED = (501, 'Not Implemented', - 'Server does not support this operation') - BAD_GATEWAY = (502, 'Bad Gateway', - 'Invalid responses from another server/proxy') - SERVICE_UNAVAILABLE = (503, 'Service Unavailable', - 'The server cannot process the request due to a high load') - GATEWAY_TIMEOUT = (504, 'Gateway Timeout', - 'The gateway server did not receive a timely response') - HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported', - 'Cannot fulfill request') - VARIANT_ALSO_NEGOTIATES = 506, 'Variant Also Negotiates' - INSUFFICIENT_STORAGE = 507, 'Insufficient Storage' - LOOP_DETECTED = 508, 'Loop Detected' - NOT_EXTENDED = 510, 'Not Extended' - NETWORK_AUTHENTICATION_REQUIRED = (511, - 'Network Authentication Required', - 'The client needs to authenticate to gain network access') +# This directory is a Python package. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/http/client.py --- a/Lib/http/client.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/http/client.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,12 +20,10 @@ | ( putheader() )* endheaders() v Request-sent - |\_____________________________ - | | getresponse() raises - | response = getresponse() | ConnectionError - v v - Unread-response Idle - [Response-headers-read] + | + | response = getresponse() + v + Unread-response [Response-headers-read] |\____________________ | | | response.read() | putrequest() @@ -70,23 +68,19 @@ import email.parser import email.message -import http import io import os -import re import socket import collections from urllib.parse import urlsplit +import warnings -# HTTPMessage, parse_headers(), and the HTTP status code constants are -# intentionally omitted for simplicity __all__ = ["HTTPResponse", "HTTPConnection", "HTTPException", "NotConnected", "UnknownProtocol", "UnknownTransferEncoding", "UnimplementedFileMode", "IncompleteRead", "InvalidURL", "ImproperConnectionState", "CannotSendRequest", "CannotSendHeader", "ResponseNotReady", - "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error", - "responses"] + "BadStatusLine", "error", "responses"] HTTP_PORT = 80 HTTPS_PORT = 443 @@ -98,53 +92,120 @@ _CS_REQ_STARTED = 'Request-started' _CS_REQ_SENT = 'Request-sent' +# status codes +# informational +CONTINUE = 100 +SWITCHING_PROTOCOLS = 101 +PROCESSING = 102 -# hack to maintain backwards compatibility -globals().update(http.HTTPStatus.__members__) +# successful +OK = 200 +CREATED = 201 +ACCEPTED = 202 +NON_AUTHORITATIVE_INFORMATION = 203 +NO_CONTENT = 204 +RESET_CONTENT = 205 +PARTIAL_CONTENT = 206 +MULTI_STATUS = 207 +IM_USED = 226 -# another hack to maintain backwards compatibility +# redirection +MULTIPLE_CHOICES = 300 +MOVED_PERMANENTLY = 301 +FOUND = 302 +SEE_OTHER = 303 +NOT_MODIFIED = 304 +USE_PROXY = 305 +TEMPORARY_REDIRECT = 307 + +# client error +BAD_REQUEST = 400 +UNAUTHORIZED = 401 +PAYMENT_REQUIRED = 402 +FORBIDDEN = 403 +NOT_FOUND = 404 +METHOD_NOT_ALLOWED = 405 +NOT_ACCEPTABLE = 406 +PROXY_AUTHENTICATION_REQUIRED = 407 +REQUEST_TIMEOUT = 408 +CONFLICT = 409 +GONE = 410 +LENGTH_REQUIRED = 411 +PRECONDITION_FAILED = 412 +REQUEST_ENTITY_TOO_LARGE = 413 +REQUEST_URI_TOO_LONG = 414 +UNSUPPORTED_MEDIA_TYPE = 415 +REQUESTED_RANGE_NOT_SATISFIABLE = 416 +EXPECTATION_FAILED = 417 +UNPROCESSABLE_ENTITY = 422 +LOCKED = 423 +FAILED_DEPENDENCY = 424 +UPGRADE_REQUIRED = 426 + +# server error +INTERNAL_SERVER_ERROR = 500 +NOT_IMPLEMENTED = 501 +BAD_GATEWAY = 502 +SERVICE_UNAVAILABLE = 503 +GATEWAY_TIMEOUT = 504 +HTTP_VERSION_NOT_SUPPORTED = 505 +INSUFFICIENT_STORAGE = 507 +NOT_EXTENDED = 510 + # Mapping status codes to official W3C names -responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()} +responses = { + 100: 'Continue', + 101: 'Switching Protocols', + + 200: 'OK', + 201: 'Created', + 202: 'Accepted', + 203: 'Non-Authoritative Information', + 204: 'No Content', + 205: 'Reset Content', + 206: 'Partial Content', + + 300: 'Multiple Choices', + 301: 'Moved Permanently', + 302: 'Found', + 303: 'See Other', + 304: 'Not Modified', + 305: 'Use Proxy', + 306: '(Unused)', + 307: 'Temporary Redirect', + + 400: 'Bad Request', + 401: 'Unauthorized', + 402: 'Payment Required', + 403: 'Forbidden', + 404: 'Not Found', + 405: 'Method Not Allowed', + 406: 'Not Acceptable', + 407: 'Proxy Authentication Required', + 408: 'Request Timeout', + 409: 'Conflict', + 410: 'Gone', + 411: 'Length Required', + 412: 'Precondition Failed', + 413: 'Request Entity Too Large', + 414: 'Request-URI Too Long', + 415: 'Unsupported Media Type', + 416: 'Requested Range Not Satisfiable', + 417: 'Expectation Failed', + + 500: 'Internal Server Error', + 501: 'Not Implemented', + 502: 'Bad Gateway', + 503: 'Service Unavailable', + 504: 'Gateway Timeout', + 505: 'HTTP Version Not Supported', +} # maximal amount of data to read at one time in _safe_read MAXAMOUNT = 1048576 # maximal line length when calling readline(). _MAXLINE = 65536 -_MAXHEADERS = 100 - -# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) -# -# VCHAR = %x21-7E -# obs-text = %x80-FF -# header-field = field-name ":" OWS field-value OWS -# field-name = token -# field-value = *( field-content / obs-fold ) -# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -# field-vchar = VCHAR / obs-text -# -# obs-fold = CRLF 1*( SP / HTAB ) -# ; obsolete line folding -# ; see Section 3.2.4 - -# token = 1*tchar -# -# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" -# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" -# / DIGIT / ALPHA -# ; any VCHAR, except delimiters -# -# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1 - -# the patterns for both name and value are more leniant than RFC -# definitions to allow for backwards compatibility -_is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch -_is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search - -# We always set the Content-Length header for these methods because some -# servers will otherwise respond with a 411 -_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} - class HTTPMessage(email.message.Message): # XXX The only usage of this method is in @@ -192,15 +253,15 @@ if len(line) > _MAXLINE: raise LineTooLong("header line") headers.append(line) - if len(headers) > _MAXHEADERS: - raise HTTPException("got more than %d headers" % _MAXHEADERS) if line in (b'\r\n', b'\n', b''): break hstring = b''.join(headers).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) -class HTTPResponse(io.BufferedIOBase): +_strict_sentinel = object() + +class HTTPResponse(io.RawIOBase): # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details. @@ -209,7 +270,7 @@ # text following RFC 2047. The basic status line parsing only # accepts iso-8859-1. - def __init__(self, sock, debuglevel=0, method=None, url=None): + def __init__(self, sock, debuglevel=0, strict=_strict_sentinel, method=None, url=None): # If the response includes a content-length header, we need to # make sure that the client doesn't read more than the # specified number of bytes. If it does, it will block until @@ -219,6 +280,10 @@ # clients unless they know what they are doing. self.fp = sock.makefile("rb") self.debuglevel = debuglevel + if strict is not _strict_sentinel: + warnings.warn("the 'strict' argument isn't supported anymore; " + "http.client now always assumes HTTP/1.x compliant servers.", + DeprecationWarning, 2) self._method = method # The HTTPResponse object is returned via urllib. The clients @@ -248,8 +313,7 @@ if not line: # Presumably, the server closed the connection before # sending a valid response. - raise RemoteDisconnected("Remote end closed connection without" - " response") + raise BadStatusLine(line) try: version, status, reason = line.split(None, 2) except ValueError: @@ -260,7 +324,7 @@ # empty version will cause next test to fail. version = "" if not version.startswith("HTTP/"): - self._close_conn() + self.close() raise BadStatusLine(line) # The status code is a three-digit number @@ -382,36 +446,29 @@ # otherwise, assume it will close return True - def _close_conn(self): - fp = self.fp - self.fp = None - fp.close() - def close(self): - try: - super().close() # set "closed" flag - finally: - if self.fp: - self._close_conn() + if self.fp: + self.fp.close() + self.fp = None # These implementations are for the benefit of io.BufferedReader. # XXX This class should probably be revised to act more like # the "raw stream" that BufferedReader expects. + @property + def closed(self): + return self.isclosed() + def flush(self): - super().flush() - if self.fp: - self.fp.flush() + self.fp.flush() def readable(self): - """Always returns True""" return True # End of "raw stream" methods def isclosed(self): - """True if the connection is closed.""" # NOTE: it is possible that we will not ever call self.close(). This # case occurs when will_close is TRUE, length is None, and we # read up to the last byte, but NOT past it. @@ -425,14 +482,13 @@ return b"" if self._method == "HEAD": - self._close_conn() + self.close() return b"" if amt is not None: - # Amount is given, implement using readinto - b = bytearray(amt) - n = self.readinto(b) - return memoryview(b)[:n].tobytes() + # Amount is given, so call base class version + # (which is implemented in terms of self.readinto) + return super(HTTPResponse, self).read(amt) else: # Amount is not given (unbounded read) so we must check self.length # and self.chunked @@ -443,25 +499,17 @@ if self.length is None: s = self.fp.read() else: - try: - s = self._safe_read(self.length) - except IncompleteRead: - self._close_conn() - raise + s = self._safe_read(self.length) self.length = 0 - self._close_conn() # we read everything + self.close() # we read everything return s def readinto(self, b): - """Read up to len(b) bytes into bytearray b and return the number - of bytes read. - """ - if self.fp is None: return 0 if self._method == "HEAD": - self._close_conn() + self.close() return 0 if self.chunked: @@ -476,14 +524,10 @@ # connection, and the user is reading more bytes than will be provided # (for example, reading in 1k chunks) n = self.fp.readinto(b) - if not n and b: - # Ideally, we would raise IncompleteRead if the content-length - # wasn't satisfied, but it might break compatibility. - self._close_conn() - elif self.length is not None: + if self.length is not None: self.length -= n if not self.length: - self._close_conn() + self.close() return n def _read_next_chunk_size(self): @@ -499,7 +543,7 @@ except ValueError: # close the connection as protocol synchronisation is # probably lost - self._close_conn() + self.close() raise def _read_and_discard_trailer(self): @@ -513,70 +557,74 @@ # a vanishingly small number of sites EOF without # sending the trailer break - if line in (b'\r\n', b'\n', b''): + if line == b"\r\n": break - def _get_chunk_left(self): - # return self.chunk_left, reading a new chunk if necessary. - # chunk_left == 0: at the end of the current chunk, need to close it - # chunk_left == None: No current chunk, should read next. - # This function returns non-zero or None if the last chunk has - # been read. - chunk_left = self.chunk_left - if not chunk_left: # Can be 0 or None - if chunk_left is not None: - # We are at the end of chunk. dicard chunk end - self._safe_read(2) # toss the CRLF at the end of the chunk - try: - chunk_left = self._read_next_chunk_size() - except ValueError: - raise IncompleteRead(b'') - if chunk_left == 0: - # last chunk: 1*("0") [ chunk-extension ] CRLF - self._read_and_discard_trailer() - # we read everything; close the "file" - self._close_conn() - chunk_left = None - self.chunk_left = chunk_left - return chunk_left - def _readall_chunked(self): assert self.chunked != _UNKNOWN + chunk_left = self.chunk_left value = [] - try: - while True: - chunk_left = self._get_chunk_left() - if chunk_left is None: - break - value.append(self._safe_read(chunk_left)) - self.chunk_left = 0 - return b''.join(value) - except IncompleteRead: - raise IncompleteRead(b''.join(value)) + while True: + if chunk_left is None: + try: + chunk_left = self._read_next_chunk_size() + if chunk_left == 0: + break + except ValueError: + raise IncompleteRead(b''.join(value)) + value.append(self._safe_read(chunk_left)) + + # we read the whole chunk, get another + self._safe_read(2) # toss the CRLF at the end of the chunk + chunk_left = None + + self._read_and_discard_trailer() + + # we read everything; close the "file" + self.close() + + return b''.join(value) def _readinto_chunked(self, b): assert self.chunked != _UNKNOWN + chunk_left = self.chunk_left + total_bytes = 0 mvb = memoryview(b) - try: - while True: - chunk_left = self._get_chunk_left() - if chunk_left is None: - return total_bytes + while True: + if chunk_left is None: + try: + chunk_left = self._read_next_chunk_size() + if chunk_left == 0: + break + except ValueError: + raise IncompleteRead(bytes(b[0:total_bytes])) - if len(mvb) <= chunk_left: - n = self._safe_readinto(mvb) - self.chunk_left = chunk_left - n - return total_bytes + n - - temp_mvb = mvb[:chunk_left] + if len(mvb) < chunk_left: + n = self._safe_readinto(mvb) + self.chunk_left = chunk_left - n + return total_bytes + n + elif len(mvb) == chunk_left: + n = self._safe_readinto(mvb) + self._safe_read(2) # toss the CRLF at the end of the chunk + self.chunk_left = None + return total_bytes + n + else: + temp_mvb = mvb[0:chunk_left] n = self._safe_readinto(temp_mvb) mvb = mvb[n:] total_bytes += n - self.chunk_left = 0 - except IncompleteRead: - raise IncompleteRead(bytes(b[0:total_bytes])) + # we read the whole chunk, get another + self._safe_read(2) # toss the CRLF at the end of the chunk + chunk_left = None + + self._read_and_discard_trailer() + + # we read everything; close the "file" + self.close() + + return total_bytes def _safe_read(self, amt): """Read the number of bytes requested, compensating for partial reads. @@ -617,88 +665,10 @@ total_bytes += n return total_bytes - def read1(self, n=-1): - """Read with at most one underlying system call. If at least one - byte is buffered, return that instead. - """ - if self.fp is None or self._method == "HEAD": - return b"" - if self.chunked: - return self._read1_chunked(n) - try: - result = self.fp.read1(n) - except ValueError: - if n >= 0: - raise - # some implementations, like BufferedReader, don't support -1 - # Read an arbitrarily selected largeish chunk. - result = self.fp.read1(16*1024) - if not result and n: - self._close_conn() - return result - - def peek(self, n=-1): - # Having this enables IOBase.readline() to read more than one - # byte at a time - if self.fp is None or self._method == "HEAD": - return b"" - if self.chunked: - return self._peek_chunked(n) - return self.fp.peek(n) - - def readline(self, limit=-1): - if self.fp is None or self._method == "HEAD": - return b"" - if self.chunked: - # Fallback to IOBase readline which uses peek() and read() - return super().readline(limit) - result = self.fp.readline(limit) - if not result and limit: - self._close_conn() - return result - - def _read1_chunked(self, n): - # Strictly speaking, _get_chunk_left() may cause more than one read, - # but that is ok, since that is to satisfy the chunked protocol. - chunk_left = self._get_chunk_left() - if chunk_left is None or n == 0: - return b'' - if not (0 <= n <= chunk_left): - n = chunk_left # if n is negative or larger than chunk_left - read = self.fp.read1(n) - self.chunk_left -= len(read) - if not read: - raise IncompleteRead(b"") - return read - - def _peek_chunked(self, n): - # Strictly speaking, _get_chunk_left() may cause more than one read, - # but that is ok, since that is to satisfy the chunked protocol. - try: - chunk_left = self._get_chunk_left() - except IncompleteRead: - return b'' # peek doesn't worry about protocol - if chunk_left is None: - return b'' # eof - # peek is allowed to return more than requested. Just request the - # entire chunk, and truncate what we get. - return self.fp.peek(chunk_left)[:chunk_left] - def fileno(self): return self.fp.fileno() def getheader(self, name, default=None): - '''Returns the value of the header matching *name*. - - If there are multiple matching headers, the values are - combined into a single string separated by commas and spaces. - - If no matching header is found, returns *default* or None if - the *default* is not specified. - - If the headers are unknown, raises http.client.ResponseNotReady. - - ''' if self.headers is None: raise ResponseNotReady() headers = self.headers.get_all(name) or default @@ -721,45 +691,12 @@ # For compatibility with old-style urllib responses. def info(self): - '''Returns an instance of the class mimetools.Message containing - meta-information associated with the URL. - - When the method is HTTP, these headers are those returned by - the server at the head of the retrieved HTML page (including - Content-Length and Content-Type). - - When the method is FTP, a Content-Length header will be - present if (as is now usual) the server passed back a file - length in response to the FTP retrieval request. A - Content-Type header will be present if the MIME type can be - guessed. - - When the method is local-file, returned headers will include - a Date representing the file's last-modified time, a - Content-Length giving file size, and a Content-Type - containing a guess at the file's type. See also the - description of the mimetools module. - - ''' return self.headers def geturl(self): - '''Return the real URL of the page. - - In some cases, the HTTP server redirects a client to another - URL. The urlopen() function handles this transparently, but in - some cases the caller needs to know which URL the client was - redirected to. The geturl() method can be used to get at this - redirected URL. - - ''' return self.url def getcode(self): - '''Return the HTTP status code that was sent with the response, - or None if the URL is not an HTTP URL. - - ''' return self.status class HTTPConnection: @@ -772,8 +709,12 @@ auto_open = 1 debuglevel = 0 - def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None): + def __init__(self, host, port=None, strict=_strict_sentinel, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): + if strict is not _strict_sentinel: + warnings.warn("the 'strict' argument isn't supported anymore; " + "http.client now always assumes HTTP/1.x compliant servers.", + DeprecationWarning, 2) self.timeout = timeout self.source_address = source_address self.sock = None @@ -785,37 +726,22 @@ self._tunnel_port = None self._tunnel_headers = {} - (self.host, self.port) = self._get_hostport(host, port) - - # This is stored as an instance variable to allow unit - # tests to replace it with a suitable mockup - self._create_connection = socket.create_connection + self._set_hostport(host, port) def set_tunnel(self, host, port=None, headers=None): - """Set up host and port for HTTP CONNECT tunnelling. + """ Sets up the host and the port for the HTTP CONNECT Tunnelling. - In a connection that uses HTTP CONNECT tunneling, the host passed to the - constructor is used as a proxy server that relays all communication to - the endpoint passed to `set_tunnel`. This done by sending an HTTP - CONNECT request to the proxy server when the connection is established. - - This method must be called before the HTML connection has been - established. - - The headers argument should be a mapping of extra HTTP headers to send - with the CONNECT request. + The headers argument should be a mapping of extra HTTP headers + to send with the CONNECT request. """ - - if self.sock: - raise RuntimeError("Can't set up tunnel for established connection") - - self._tunnel_host, self._tunnel_port = self._get_hostport(host, port) + self._tunnel_host = host + self._tunnel_port = port if headers: self._tunnel_headers = headers else: self._tunnel_headers.clear() - def _get_hostport(self, host, port): + def _set_hostport(self, host, port): if port is None: i = host.rfind(':') j = host.rfind(']') # ipv6 addresses have [...] @@ -832,15 +758,15 @@ port = self.default_port if host and host[0] == '[' and host[-1] == ']': host = host[1:-1] - - return (host, port) + self.host = host + self.port = port def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, - self._tunnel_port) + self._set_hostport(self._tunnel_host, self._tunnel_port) + connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port) connect_bytes = connect_str.encode("ascii") self.send(connect_bytes) for header, value in self._tunnel_headers.items(): @@ -852,45 +778,33 @@ response = self.response_class(self.sock, method=self._method) (version, code, message) = response._read_status() - if code != http.HTTPStatus.OK: + if code != 200: self.close() - raise OSError("Tunnel connection failed: %d %s" % (code, - message.strip())) + raise socket.error("Tunnel connection failed: %d %s" % (code, + message.strip())) while True: line = response.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer + if line == b'\r\n': break - if line in (b'\r\n', b'\n', b''): - break - - if self.debuglevel > 0: - print('header:', line.decode()) def connect(self): """Connect to the host and port specified in __init__.""" - self.sock = self._create_connection( - (self.host,self.port), self.timeout, self.source_address) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - + self.sock = socket.create_connection((self.host,self.port), + self.timeout, self.source_address) if self._tunnel_host: self._tunnel() def close(self): """Close the connection to the HTTP server.""" + if self.sock: + self.sock.close() # close it manually... there may be other refs + self.sock = None + if self.__response: + self.__response.close() + self.__response = None self.__state = _CS_IDLE - try: - sock = self.sock - if sock: - self.sock = None - sock.close() # close it manually... there may be other refs - finally: - response = self.__response - if response: - self.__response = None - response.close() def send(self, data): """Send `data' to the server. @@ -929,7 +843,7 @@ if encode: datablock = datablock.encode("iso-8859-1") self.sock.sendall(datablock) - return + try: self.sock.sendall(data) except TypeError: @@ -956,9 +870,16 @@ self._buffer.extend((b"", b"")) msg = b"\r\n".join(self._buffer) del self._buffer[:] - + # If msg and message_body are sent in a single send() call, + # it will avoid performance problems caused by the interaction + # between delayed ack and the Nagle algorithm. + if isinstance(message_body, bytes): + msg += message_body + message_body = None self.send(msg) if message_body is not None: + # message_body was not a string (i.e. it is a file), and + # we must run the risk of Nagle. self.send(message_body) def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0): @@ -1037,29 +958,22 @@ netloc_enc = netloc.encode("idna") self.putheader('Host', netloc_enc) else: - if self._tunnel_host: - host = self._tunnel_host - port = self._tunnel_port - else: - host = self.host - port = self.port - try: - host_enc = host.encode("ascii") + host_enc = self.host.encode("ascii") except UnicodeEncodeError: - host_enc = host.encode("idna") + host_enc = self.host.encode("idna") # As per RFC 273, IPv6 address should be wrapped with [] # when used as Host header - if host.find(':') >= 0: + if self.host.find(':') >= 0: host_enc = b'[' + host_enc + b']' - if port == self.default_port: + if self.port == self.default_port: self.putheader('Host', host_enc) else: host_enc = host_enc.decode("ascii") - self.putheader('Host', "%s:%s" % (host_enc, port)) + self.putheader('Host', "%s:%s" % (host_enc, self.port)) # note: we are assuming that clients will not attempt to set these # headers since *this* library must deal with the @@ -1094,20 +1008,12 @@ if hasattr(header, 'encode'): header = header.encode('ascii') - - if not _is_legal_header_name(header): - raise ValueError('Invalid header name %r' % (header,)) - values = list(values) for i, one_value in enumerate(values): if hasattr(one_value, 'encode'): values[i] = one_value.encode('latin-1') elif isinstance(one_value, int): values[i] = str(one_value).encode('ascii') - - if _is_illegal_header_value(values[i]): - raise ValueError('Invalid header value %r' % (values[i],)) - value = b'\r\n\t'.join(values) header = header + b': ' + value self._output(header) @@ -1131,26 +1037,19 @@ """Send a complete request to the server.""" self._send_request(method, url, body, headers) - def _set_content_length(self, body, method): - # Set the content-length based on the body. If the body is "empty", we - # set Content-Length: 0 for methods that expect a body (RFC 7230, - # Section 3.3.2). If the body is set for other methods, we set the - # header provided we can figure out what the length is. + def _set_content_length(self, body): + # Set the content-length based on the body. thelen = None - method_expects_body = method.upper() in _METHODS_EXPECTING_BODY - if body is None and method_expects_body: - thelen = '0' - elif body is not None: + try: + thelen = str(len(body)) + except TypeError as te: + # If this is a file-like object, try to + # fstat its file descriptor try: - thelen = str(len(body)) - except TypeError: - # If this is a file-like object, try to - # fstat its file descriptor - try: - thelen = str(os.fstat(body.fileno()).st_size) - except (AttributeError, OSError): - # Don't send a length if this failed - if self.debuglevel > 0: print("Cannot stat!!") + thelen = str(os.fstat(body.fileno()).st_size) + except (AttributeError, OSError): + # Don't send a length if this failed + if self.debuglevel > 0: print("Cannot stat!!") if thelen is not None: self.putheader('Content-Length', thelen) @@ -1166,8 +1065,8 @@ self.putrequest(method, url, **skips) - if 'content-length' not in header_names: - self._set_content_length(body, method) + if body and ('content-length' not in header_names): + self._set_content_length(body) for hdr, value in headers.items(): self.putheader(hdr, value) if isinstance(body, str): @@ -1218,26 +1117,18 @@ else: response = self.response_class(self.sock, method=self._method) - try: - try: - response.begin() - except ConnectionError: - self.close() - raise - assert response.will_close != _UNKNOWN - self.__state = _CS_IDLE + response.begin() + assert response.will_close != _UNKNOWN + self.__state = _CS_IDLE - if response.will_close: - # this effectively passes the connection to the response - self.close() - else: - # remember this, so we can tell when it is complete - self.__response = response + if response.will_close: + # this effectively passes the connection to the response + self.close() + else: + # remember this, so we can tell when it is complete + self.__response = response - return response - except: - response.close() - raise + return response try: import ssl @@ -1252,19 +1143,20 @@ # XXX Should key_file and cert_file be deprecated in favour of context? def __init__(self, host, port=None, key_file=None, cert_file=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, *, context=None, - check_hostname=None): - super(HTTPSConnection, self).__init__(host, port, timeout, + strict=_strict_sentinel, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, *, context=None, check_hostname=None): + super(HTTPSConnection, self).__init__(host, port, strict, timeout, source_address) self.key_file = key_file self.cert_file = cert_file if context is None: - context = ssl._create_default_https_context() + # Some reasonable defaults + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.options |= ssl.OP_NO_SSLv2 will_verify = context.verify_mode != ssl.CERT_NONE if check_hostname is None: - check_hostname = context.check_hostname - if check_hostname and not will_verify: + check_hostname = will_verify + elif check_hostname and not will_verify: raise ValueError("check_hostname needs a SSL context with " "either CERT_OPTIONAL or CERT_REQUIRED") if key_file or cert_file: @@ -1275,22 +1167,23 @@ def connect(self): "Connect to a host on a given (SSL) port." - super().connect() + sock = socket.create_connection((self.host, self.port), + self.timeout, self.source_address) if self._tunnel_host: - server_hostname = self._tunnel_host - else: - server_hostname = self.host + self.sock = sock + self._tunnel() - self.sock = self._context.wrap_socket(self.sock, + server_hostname = self.host if ssl.HAS_SNI else None + self.sock = self._context.wrap_socket(sock, server_hostname=server_hostname) - if not self._context.check_hostname and self._check_hostname: - try: - ssl.match_hostname(self.sock.getpeercert(), server_hostname) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise + try: + if self._check_hostname: + ssl.match_hostname(self.sock.getpeercert(), self.host) + except Exception: + self.sock.shutdown(socket.SHUT_RDWR) + self.sock.close() + raise __all__.append("HTTPSConnection") @@ -1326,8 +1219,7 @@ e = ', %i more expected' % self.expected else: e = '' - return '%s(%i bytes read%s)' % (self.__class__.__name__, - len(self.partial), e) + return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e) def __str__(self): return repr(self) @@ -1355,10 +1247,5 @@ HTTPException.__init__(self, "got more than %d bytes when reading %s" % (_MAXLINE, line_type)) -class RemoteDisconnected(ConnectionResetError, BadStatusLine): - def __init__(self, *pos, **kw): - BadStatusLine.__init__(self, "") - ConnectionResetError.__init__(self, *pos, **kw) - # for backwards compatibility error = HTTPException diff -r 6db40a9955dc -r 0d413f60cc23 Lib/http/cookiejar.py --- a/Lib/http/cookiejar.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/http/cookiejar.py Mon Jan 25 17:05:13 2016 +0100 @@ -423,10 +423,10 @@ Takes a list of lists of (key, value) pairs and produces a single header value. Attribute values are quoted if needed. - >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) - 'text/plain; charset="iso-8859-1"' - >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) - 'text/plain, charset="iso-8859-1"' + >>> join_header_words([[("text/plain", None), ("charset", "iso-8859/1")]]) + 'text/plain; charset="iso-8859/1"' + >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859/1")]]) + 'text/plain, charset="iso-8859/1"' """ headers = [] @@ -472,42 +472,26 @@ for ns_header in ns_headers: pairs = [] version_set = False - - # XXX: The following does not strictly adhere to RFCs in that empty - # names and values are legal (the former will only appear once and will - # be overwritten if multiple occurrences are present). This is - # mostly to deal with backwards compatibility. - for ii, param in enumerate(ns_header.split(';')): - param = param.strip() - - key, sep, val = param.partition('=') - key = key.strip() - - if not key: - if ii == 0: - break - else: - continue - - # allow for a distinction between present and empty and missing - # altogether - val = val.strip() if sep else None - + for ii, param in enumerate(re.split(r";\s*", ns_header)): + param = param.rstrip() + if param == "": continue + if "=" not in param: + k, v = param, None + else: + k, v = re.split(r"\s*=\s*", param, 1) + k = k.lstrip() if ii != 0: - lc = key.lower() + lc = k.lower() if lc in known_attrs: - key = lc - - if key == "version": + k = lc + if k == "version": # This is an RFC 2109 cookie. - if val is not None: - val = strip_quotes(val) + v = strip_quotes(v) version_set = True - elif key == "expires": + if k == "expires": # convert expires date to seconds since epoch - if val is not None: - val = http2time(strip_quotes(val)) # None if invalid - pairs.append((key, val)) + v = http2time(strip_quotes(v)) # None if invalid + pairs.append((k, v)) if pairs: if not version_set: @@ -641,7 +625,7 @@ return path def request_port(request): - host = request.host + host = request.get_host() i = host.find(':') if i >= 0: port = host[i+1:] @@ -720,7 +704,7 @@ """ req_host = request_host(request) - if not domain_match(req_host, reach(request.origin_req_host)): + if not domain_match(req_host, reach(request.get_origin_req_host())): return True else: return False @@ -758,7 +742,7 @@ ): if version is not None: version = int(version) - if expires is not None: expires = int(float(expires)) + if expires is not None: expires = int(expires) if port is None and port_specified is True: raise ValueError("if port is None, port_specified must be false") @@ -821,7 +805,7 @@ args.append("%s=%s" % (name, repr(attr))) args.append("rest=%s" % repr(self._rest)) args.append("rfc2109=%s" % repr(self.rfc2109)) - return "%s(%s)" % (self.__class__.__name__, ", ".join(args)) + return "Cookie(%s)" % ", ".join(args) class CookiePolicy: @@ -965,7 +949,7 @@ return True def set_ok_verifiability(self, cookie, request): - if request.unverifiable and is_third_party(request): + if request.is_unverifiable() and is_third_party(request): if cookie.version > 0 and self.strict_rfc2965_unverifiable: _debug(" third-party RFC 2965 cookie during " "unverifiable transaction") @@ -1104,7 +1088,7 @@ return True def return_ok_verifiability(self, cookie, request): - if request.unverifiable and is_third_party(request): + if request.is_unverifiable() and is_third_party(request): if cookie.version > 0 and self.strict_rfc2965_unverifiable: _debug(" third-party RFC 2965 cookie during unverifiable " "transaction") @@ -1116,7 +1100,7 @@ return True def return_ok_secure(self, cookie, request): - if cookie.secure and request.type != "https": + if cookie.secure and request.get_type() != "https": _debug(" secure cookie with non-secure request") return False return True @@ -1209,7 +1193,8 @@ pass else: mapping = True - yield from deepvalues(obj) + for subobj in deepvalues(obj): + yield subobj if not mapping: yield obj @@ -1437,7 +1422,7 @@ break # convert RFC 2965 Max-Age to seconds since epoch # XXX Strictly you're supposed to follow RFC 2616 - # age-calculation rules. Remember that zero Max-Age + # age-calculation rules. Remember that zero Max-Age is a # is a request to discard (old and new) cookie, though. k = "expires" v = self._now + v @@ -1738,16 +1723,16 @@ def __repr__(self): r = [] for cookie in self: r.append(repr(cookie)) - return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__, ", ".join(r)) def __str__(self): r = [] for cookie in self: r.append(str(cookie)) - return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) + return "<%s[%s]>" % (self.__class__, ", ".join(r)) -# derives from OSError for backwards-compatibility with Python 2.4.0 -class LoadError(OSError): pass +# derives from IOError for backwards-compatibility with Python 2.4.0 +class LoadError(IOError): pass class FileCookieJar(CookieJar): """CookieJar that can be loaded from and saved to a file.""" @@ -1777,14 +1762,17 @@ if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with open(filename) as f: + f = open(filename) + try: self._really_load(f, filename, ignore_discard, ignore_expires) + finally: + f.close() def revert(self, filename=None, ignore_discard=False, ignore_expires=False): """Clear all cookies and reload cookies from a saved file. - Raises LoadError (or OSError) if reversion is not successful; the + Raises LoadError (or IOError) if reversion is not successful; the object's state will not be altered if this happens. """ @@ -1799,7 +1787,7 @@ self._cookies = {} try: self.load(filename, ignore_discard, ignore_expires) - except OSError: + except (LoadError, IOError): self._cookies = old_state raise @@ -1808,7 +1796,7 @@ def lwp_cookie_str(cookie): - """Return string representation of Cookie in the LWP cookie file format. + """Return string representation of Cookie in an the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. @@ -1837,7 +1825,7 @@ class LWPCookieJar(FileCookieJar): """ - The LWPCookieJar saves a sequence of "Set-Cookie3" lines. + The LWPCookieJar saves a sequence of"Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl libary, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. @@ -1849,7 +1837,7 @@ """ def as_lwp_str(self, ignore_discard=True, ignore_expires=True): - """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers. + """Return cookies as a string of "\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save @@ -1869,12 +1857,15 @@ if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with open(filename, "w") as f: + f = open(filename, "w") + try: # There really isn't an LWP Cookies 2.0 format, but this indicates # that there is extra information in here (domain_dot and # port_spec) while still being compatible with libwww-perl, I hope. f.write("#LWP-Cookies-2.0\n") f.write(self.as_lwp_str(ignore_discard, ignore_expires)) + finally: + f.close() def _really_load(self, f, filename, ignore_discard, ignore_expires): magic = f.readline() @@ -1947,7 +1938,8 @@ if not ignore_expires and c.is_expired(now): continue self.set_cookie(c) - except OSError: + + except IOError: raise except Exception: _warn_unhandled_exception() @@ -1989,7 +1981,7 @@ magic_re = re.compile("#( Netscape)? HTTP Cookie File") header = """\ # Netscape HTTP Cookie File -# http://curl.haxx.se/rfc/cookie_spec.html +# http://www.netscape.com/newsref/std/cookie_spec.html # This is a generated file! Do not edit. """ @@ -1999,6 +1991,7 @@ magic = f.readline() if not self.magic_re.search(magic): + f.close() raise LoadError( "%r does not look like a Netscape format cookies file" % filename) @@ -2052,7 +2045,7 @@ continue self.set_cookie(c) - except OSError: + except IOError: raise except Exception: _warn_unhandled_exception() @@ -2064,7 +2057,8 @@ if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) - with open(filename, "w") as f: + f = open(filename, "w") + try: f.write(self.header) now = time.time() for cookie in self: @@ -2093,3 +2087,5 @@ "\t".join([cookie.domain, initial_dot, cookie.path, secure, expires, name, value])+ "\n") + finally: + f.close() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/http/cookies.py --- a/Lib/http/cookies.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/http/cookies.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# + #### # Copyright 2000 by Timothy O'Malley # @@ -138,12 +141,6 @@ _semispacejoin = '; '.join _spacejoin = ' '.join -def _warn_deprecated_setter(setter): - import warnings - msg = ('The .%s setter is deprecated. The attribute will be read-only in ' - 'future releases. Please use the set() method instead.' % setter) - warnings.warn(msg, DeprecationWarning, stacklevel=3) - # # Define an exception visible to External modules # @@ -157,36 +154,88 @@ # into a 4 character sequence: a forward-slash followed by the # three-digit octal equivalent of the character. Any '\' or '"' is # quoted with a preceeding '\' slash. -# Because of the way browsers really handle cookies (as opposed to what -# the RFC says) we also encode "," and ";". # # These are taken from RFC2068 and RFC2109. # _LegalChars is the list of chars which don't require "'s # _Translator hash-table for fast quoting # -_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~:" -_UnescapedChars = _LegalChars + ' ()/<=>?@[]{}' +_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~" +_Translator = { + '\000' : '\\000', '\001' : '\\001', '\002' : '\\002', + '\003' : '\\003', '\004' : '\\004', '\005' : '\\005', + '\006' : '\\006', '\007' : '\\007', '\010' : '\\010', + '\011' : '\\011', '\012' : '\\012', '\013' : '\\013', + '\014' : '\\014', '\015' : '\\015', '\016' : '\\016', + '\017' : '\\017', '\020' : '\\020', '\021' : '\\021', + '\022' : '\\022', '\023' : '\\023', '\024' : '\\024', + '\025' : '\\025', '\026' : '\\026', '\027' : '\\027', + '\030' : '\\030', '\031' : '\\031', '\032' : '\\032', + '\033' : '\\033', '\034' : '\\034', '\035' : '\\035', + '\036' : '\\036', '\037' : '\\037', -_Translator = {n: '\\%03o' % n - for n in set(range(256)) - set(map(ord, _UnescapedChars))} -_Translator.update({ - ord('"'): '\\"', - ord('\\'): '\\\\', -}) + # Because of the way browsers really handle cookies (as opposed + # to what the RFC says) we also encode , and ; -_is_legal_key = re.compile('[%s]+' % _LegalChars).fullmatch + ',' : '\\054', ';' : '\\073', -def _quote(str): + '"' : '\\"', '\\' : '\\\\', + + '\177' : '\\177', '\200' : '\\200', '\201' : '\\201', + '\202' : '\\202', '\203' : '\\203', '\204' : '\\204', + '\205' : '\\205', '\206' : '\\206', '\207' : '\\207', + '\210' : '\\210', '\211' : '\\211', '\212' : '\\212', + '\213' : '\\213', '\214' : '\\214', '\215' : '\\215', + '\216' : '\\216', '\217' : '\\217', '\220' : '\\220', + '\221' : '\\221', '\222' : '\\222', '\223' : '\\223', + '\224' : '\\224', '\225' : '\\225', '\226' : '\\226', + '\227' : '\\227', '\230' : '\\230', '\231' : '\\231', + '\232' : '\\232', '\233' : '\\233', '\234' : '\\234', + '\235' : '\\235', '\236' : '\\236', '\237' : '\\237', + '\240' : '\\240', '\241' : '\\241', '\242' : '\\242', + '\243' : '\\243', '\244' : '\\244', '\245' : '\\245', + '\246' : '\\246', '\247' : '\\247', '\250' : '\\250', + '\251' : '\\251', '\252' : '\\252', '\253' : '\\253', + '\254' : '\\254', '\255' : '\\255', '\256' : '\\256', + '\257' : '\\257', '\260' : '\\260', '\261' : '\\261', + '\262' : '\\262', '\263' : '\\263', '\264' : '\\264', + '\265' : '\\265', '\266' : '\\266', '\267' : '\\267', + '\270' : '\\270', '\271' : '\\271', '\272' : '\\272', + '\273' : '\\273', '\274' : '\\274', '\275' : '\\275', + '\276' : '\\276', '\277' : '\\277', '\300' : '\\300', + '\301' : '\\301', '\302' : '\\302', '\303' : '\\303', + '\304' : '\\304', '\305' : '\\305', '\306' : '\\306', + '\307' : '\\307', '\310' : '\\310', '\311' : '\\311', + '\312' : '\\312', '\313' : '\\313', '\314' : '\\314', + '\315' : '\\315', '\316' : '\\316', '\317' : '\\317', + '\320' : '\\320', '\321' : '\\321', '\322' : '\\322', + '\323' : '\\323', '\324' : '\\324', '\325' : '\\325', + '\326' : '\\326', '\327' : '\\327', '\330' : '\\330', + '\331' : '\\331', '\332' : '\\332', '\333' : '\\333', + '\334' : '\\334', '\335' : '\\335', '\336' : '\\336', + '\337' : '\\337', '\340' : '\\340', '\341' : '\\341', + '\342' : '\\342', '\343' : '\\343', '\344' : '\\344', + '\345' : '\\345', '\346' : '\\346', '\347' : '\\347', + '\350' : '\\350', '\351' : '\\351', '\352' : '\\352', + '\353' : '\\353', '\354' : '\\354', '\355' : '\\355', + '\356' : '\\356', '\357' : '\\357', '\360' : '\\360', + '\361' : '\\361', '\362' : '\\362', '\363' : '\\363', + '\364' : '\\364', '\365' : '\\365', '\366' : '\\366', + '\367' : '\\367', '\370' : '\\370', '\371' : '\\371', + '\372' : '\\372', '\373' : '\\373', '\374' : '\\374', + '\375' : '\\375', '\376' : '\\376', '\377' : '\\377' + } + +def _quote(str, LegalChars=_LegalChars): r"""Quote a string for use in a cookie header. If the string does not need to be double-quoted, then just return the string. Otherwise, surround the string in doublequotes and quote (with a \) special characters. """ - if str is None or _is_legal_key(str): + if all(c in LegalChars for c in str): return str else: - return '"' + str.translate(_Translator) + '"' + return '"' + _nulljoin(_Translator.get(s, s) for s in str) + '"' _OctalPatt = re.compile(r"\\[0-3][0-7][0-7]") @@ -195,7 +244,7 @@ def _unquote(str): # If there aren't any doublequotes, # then there can't be any special characters. See RFC 2109. - if str is None or len(str) < 2: + if len(str) < 2: return str if str[0] != '"' or str[-1] != '"': return str @@ -252,7 +301,7 @@ from time import gmtime, time now = time() year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future) - return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ + return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \ (weekdayname[wd], day, monthname[month], year, hh, mm, ss) @@ -284,117 +333,40 @@ "comment" : "Comment", "domain" : "Domain", "max-age" : "Max-Age", - "secure" : "Secure", - "httponly" : "HttpOnly", + "secure" : "secure", + "httponly" : "httponly", "version" : "Version", } - _flags = {'secure', 'httponly'} - def __init__(self): # Set defaults - self._key = self._value = self._coded_value = None + self.key = self.value = self.coded_value = None # Set default attributes for key in self._reserved: dict.__setitem__(self, key, "") - @property - def key(self): - return self._key - - @key.setter - def key(self, key): - _warn_deprecated_setter('key') - self._key = key - - @property - def value(self): - return self._value - - @value.setter - def value(self, value): - _warn_deprecated_setter('value') - self._value = value - - @property - def coded_value(self): - return self._coded_value - - @coded_value.setter - def coded_value(self, coded_value): - _warn_deprecated_setter('coded_value') - self._coded_value = coded_value - def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: - raise CookieError("Invalid attribute %r" % (K,)) + raise CookieError("Invalid Attribute %s" % K) dict.__setitem__(self, K, V) - def setdefault(self, key, val=None): - key = key.lower() - if key not in self._reserved: - raise CookieError("Invalid attribute %r" % (key,)) - return dict.setdefault(self, key, val) - - def __eq__(self, morsel): - if not isinstance(morsel, Morsel): - return NotImplemented - return (dict.__eq__(self, morsel) and - self._value == morsel._value and - self._key == morsel._key and - self._coded_value == morsel._coded_value) - - __ne__ = object.__ne__ - - def copy(self): - morsel = Morsel() - dict.update(morsel, self) - morsel.__dict__.update(self.__dict__) - return morsel - - def update(self, values): - data = {} - for key, val in dict(values).items(): - key = key.lower() - if key not in self._reserved: - raise CookieError("Invalid attribute %r" % (key,)) - data[key] = val - dict.update(self, data) - def isReservedKey(self, K): return K.lower() in self._reserved def set(self, key, val, coded_val, LegalChars=_LegalChars): - if LegalChars != _LegalChars: - import warnings - warnings.warn( - 'LegalChars parameter is deprecated, ignored and will ' - 'be removed in future versions.', DeprecationWarning, - stacklevel=2) - + # First we verify that the key isn't a reserved word + # Second we make sure it only contains legal characters if key.lower() in self._reserved: - raise CookieError('Attempt to set a reserved key %r' % (key,)) - if not _is_legal_key(key): - raise CookieError('Illegal key %r' % (key,)) + raise CookieError("Attempt to set a reserved key: %s" % key) + if any(c not in LegalChars for c in key): + raise CookieError("Illegal key value: %s" % key) # It's a good key, so save it. - self._key = key - self._value = val - self._coded_value = coded_val - - def __getstate__(self): - return { - 'key': self._key, - 'value': self._value, - 'coded_value': self._coded_value, - } - - def __setstate__(self, state): - self._key = state['key'] - self._value = state['value'] - self._coded_value = state['coded_value'] + self.key = key + self.value = val + self.coded_value = coded_val def output(self, attrs=None, header="Set-Cookie:"): return "%s %s" % (header, self.OutputString(attrs)) @@ -402,7 +374,8 @@ __str__ = output def __repr__(self): - return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) + return '<%s: %s=%s>' % (self.__class__.__name__, + self.key, repr(self.value)) def js_output(self, attrs=None): # Print javascript @@ -436,9 +409,10 @@ append("%s=%s" % (self._reserved[key], _getdate(value))) elif key == "max-age" and isinstance(value, int): append("%s=%d" % (self._reserved[key], value)) - elif key in self._flags: - if value: - append(str(self._reserved[key])) + elif key == "secure": + append(str(self._reserved[key])) + elif key == "httponly": + append(str(self._reserved[key])) else: append("%s=%s" % (self._reserved[key], value)) @@ -455,26 +429,21 @@ # result, the parsing rules here are less strict. # -_LegalKeyChars = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=" -_LegalValueChars = _LegalKeyChars + '\[\]' +_LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" _CookiePattern = re.compile(r""" (?x) # This is a verbose pattern - \s* # Optional whitespace at start of cookie (?P # Start of group 'key' - [""" + _LegalKeyChars + r"""]+? # Any word of at least one letter + """ + _LegalCharsPatt + r"""+? # Any word of at least one letter ) # End of group 'key' - ( # Optional group: there may not be a value. - \s*=\s* # Equal Sign - (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string - | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr - | # or - [""" + _LegalValueChars + r"""]* # Any word or empty string - ) # End of group 'val' - )? # End of optional value group - \s* # Any number of spaces. - (\s+|;|$) # Ending either at space, semicolon, or EOS. + \s*=\s* # Equal Sign + (?P # Start of group 'val' + "(?:[^\\"]|\\.)*" # Any doublequoted string + | # or + \w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + | # or + """ + _LegalCharsPatt + r"""* # Any word or empty string + ) # End of group 'val' + \s*;? # Probably ending in a semi-colon """, re.ASCII) # May be removed if safe. @@ -514,12 +483,8 @@ def __setitem__(self, key, value): """Dictionary style assignment.""" - if isinstance(value, Morsel): - # allow assignment of constructed Morsels (e.g. for pickling) - dict.__setitem__(self, key, value) - else: - rval, cval = self.value_encode(value) - self.__set(key, rval, cval) + rval, cval = self.value_encode(value) + self.__set(key, rval, cval) def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): """Return a string suitable for HTTP.""" @@ -561,20 +526,13 @@ return def __parse_string(self, str, patt=_CookiePattern): - i = 0 # Our starting point - n = len(str) # Length of string - parsed_items = [] # Parsed (type, key, value) triples - morsel_seen = False # A key=value pair was previously encountered + i = 0 # Our starting point + n = len(str) # Length of string + M = None # current morsel - TYPE_ATTRIBUTE = 1 - TYPE_KEYVALUE = 2 - - # We first parse the whole cookie string and reject it if it's - # syntactically invalid (this helps avoid some classes of injection - # attacks). while 0 <= i < n: # Start looking for a cookie - match = patt.match(str, i) + match = patt.search(str, i) if not match: # No more cookies break @@ -582,41 +540,18 @@ key, value = match.group("key"), match.group("val") i = match.end(0) + # Parse the key, value in case it's metainfo if key[0] == "$": - if not morsel_seen: - # We ignore attributes which pertain to the cookie - # mechanism as a whole, such as "$Version". - # See RFC 2965. (Does anyone care?) - continue - parsed_items.append((TYPE_ATTRIBUTE, key[1:], value)) + # We ignore attributes which pertain to the cookie + # mechanism as a whole. See RFC 2109. + # (Does anyone care?) + if M: + M[key[1:]] = value elif key.lower() in Morsel._reserved: - if not morsel_seen: - # Invalid cookie string - return - if value is None: - if key.lower() in Morsel._flags: - parsed_items.append((TYPE_ATTRIBUTE, key, True)) - else: - # Invalid cookie string - return - else: - parsed_items.append((TYPE_ATTRIBUTE, key, _unquote(value))) - elif value is not None: - parsed_items.append((TYPE_KEYVALUE, key, self.value_decode(value))) - morsel_seen = True + if M: + M[key] = _unquote(value) else: - # Invalid cookie string - return - - # The cookie string is valid, apply it. - M = None # current morsel - for tp, key, value in parsed_items: - if tp == TYPE_ATTRIBUTE: - assert M is not None - M[key] = value - else: - assert tp == TYPE_KEYVALUE - rval, cval = value + rval, cval = self.value_decode(value) self.__set(key, rval, cval) M = self[key] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/http/server.py --- a/Lib/http/server.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/http/server.py Mon Jan 25 17:05:13 2016 +0100 @@ -82,12 +82,11 @@ __version__ = "0.6" -__all__ = [ - "HTTPServer", "BaseHTTPRequestHandler", - "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", -] +__all__ = ["HTTPServer", "BaseHTTPRequestHandler"] import html +import email.message +import email.parser import http.client import io import mimetypes @@ -101,10 +100,6 @@ import time import urllib.parse import copy -import argparse - -from http import HTTPStatus - # Default error message template DEFAULT_ERROR_MESSAGE = """\ @@ -275,7 +270,7 @@ """ self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version - self.close_connection = True + self.close_connection = 1 requestline = str(self.raw_requestline, 'iso-8859-1') requestline = requestline.rstrip('\r\n') self.requestline = requestline @@ -283,9 +278,7 @@ if len(words) == 3: command, path, version = words if version[:5] != 'HTTP/': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) + self.send_error(400, "Bad request version (%r)" % version) return False try: base_version_number = version.split('/', 1)[1] @@ -300,31 +293,25 @@ raise ValueError version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) + self.send_error(400, "Bad request version (%r)" % version) return False if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": - self.close_connection = False + self.close_connection = 0 if version_number >= (2, 0): - self.send_error( - HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, - "Invalid HTTP Version (%s)" % base_version_number) + self.send_error(505, + "Invalid HTTP Version (%s)" % base_version_number) return False elif len(words) == 2: command, path = words - self.close_connection = True + self.close_connection = 1 if command != 'GET': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad HTTP/0.9 request type (%r)" % command) + self.send_error(400, + "Bad HTTP/0.9 request type (%r)" % command) return False elif not words: return False else: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request syntax (%r)" % requestline) + self.send_error(400, "Bad request syntax (%r)" % requestline) return False self.command, self.path, self.request_version = command, path, version @@ -333,17 +320,15 @@ self.headers = http.client.parse_headers(self.rfile, _class=self.MessageClass) except http.client.LineTooLong: - self.send_error( - HTTPStatus.BAD_REQUEST, - "Line too long") + self.send_error(400, "Line too long") return False conntype = self.headers.get('Connection', "") if conntype.lower() == 'close': - self.close_connection = True + self.close_connection = 1 elif (conntype.lower() == 'keep-alive' and self.protocol_version >= "HTTP/1.1"): - self.close_connection = False + self.close_connection = 0 # Examine the headers and look for an Expect directive expect = self.headers.get('Expect', "") if (expect.lower() == "100-continue" and @@ -367,8 +352,8 @@ False. """ - self.send_response_only(HTTPStatus.CONTINUE) - self.end_headers() + self.send_response_only(100) + self.flush_headers() return True def handle_one_request(self): @@ -385,19 +370,17 @@ self.requestline = '' self.request_version = '' self.command = '' - self.send_error(HTTPStatus.REQUEST_URI_TOO_LONG) + self.send_error(414) return if not self.raw_requestline: - self.close_connection = True + self.close_connection = 1 return if not self.parse_request(): # An error code has been sent, just exit return mname = 'do_' + self.command if not hasattr(self, mname): - self.send_error( - HTTPStatus.NOT_IMPLEMENTED, - "Unsupported method (%r)" % self.command) + self.send_error(501, "Unsupported method (%r)" % self.command) return method = getattr(self, mname) method() @@ -405,28 +388,23 @@ except socket.timeout as e: #a read or a write timed out. Discard this connection self.log_error("Request timed out: %r", e) - self.close_connection = True + self.close_connection = 1 return def handle(self): """Handle multiple requests if necessary.""" - self.close_connection = True + self.close_connection = 1 self.handle_one_request() while not self.close_connection: self.handle_one_request() - def send_error(self, code, message=None, explain=None): + def send_error(self, code, message=None): """Send and log an error reply. - Arguments are - * code: an HTTP error code - 3 digits - * message: a simple optional 1 line reason phrase. - *( HTAB / SP / VCHAR / %x80-FF ) - defaults to short entry matching the response code - * explain: a detailed message defaults to the long entry - matching the response code. + Arguments are the error code, and a detailed message. + The detailed message defaults to the short entry matching the + response code. This sends an error response (so it must be called before any output has been generated), logs the error, and finally sends @@ -440,24 +418,17 @@ shortmsg, longmsg = '???', '???' if message is None: message = shortmsg - if explain is None: - explain = longmsg + explain = longmsg self.log_error("code %d, message %s", code, message) # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': _quote_html(explain)}) - body = content.encode('UTF-8', 'replace') + {'code': code, 'message': _quote_html(message), 'explain': explain}) self.send_response(code, message) self.send_header("Content-Type", self.error_content_type) self.send_header('Connection', 'close') - self.send_header('Content-Length', int(len(body))) self.end_headers() - - if (self.command != 'HEAD' and - code >= 200 and - code not in ( - HTTPStatus.NO_CONTENT, HTTPStatus.NOT_MODIFIED)): - self.wfile.write(body) + if self.command != 'HEAD' and code >= 200 and code not in (204, 304): + self.wfile.write(content.encode('UTF-8', 'replace')) def send_response(self, code, message=None): """Add the response header to the headers buffer and log the @@ -496,9 +467,9 @@ if keyword.lower() == 'connection': if value.lower() == 'close': - self.close_connection = True + self.close_connection = 1 elif value.lower() == 'keep-alive': - self.close_connection = False + self.close_connection = 0 def end_headers(self): """Send the blank line ending the MIME headers.""" @@ -517,8 +488,7 @@ This is called by send_response(). """ - if isinstance(code, HTTPStatus): - code = code.value + self.log_message('"%s" %s %s', self.requestline, str(code), str(size)) @@ -548,7 +518,7 @@ specified as subsequent arguments (it's just like printf!). - The client ip and current date/time are prefixed to + The client host and current date/time are prefixed to every message. """ @@ -588,9 +558,15 @@ 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] def address_string(self): - """Return the client address.""" + """Return the client address formatted for logging. - return self.client_address[0] + This version looks up the full hostname using gethostbyaddr(), + and tries to find a name that contains at least one dot. + + """ + + host, port = self.client_address[:2] + return socket.getfqdn(host) # Essentially static class variables @@ -601,11 +577,74 @@ # MessageClass used to parse headers MessageClass = http.client.HTTPMessage - # hack to maintain backwards compatibility + # Table mapping response codes to messages; entries have the + # form {code: (shortmessage, longmessage)}. + # See RFC 2616. responses = { - v: (v.phrase, v.description) - for v in HTTPStatus.__members__.values() - } + 100: ('Continue', 'Request received, please continue'), + 101: ('Switching Protocols', + 'Switching to new protocol; obey Upgrade header'), + + 200: ('OK', 'Request fulfilled, document follows'), + 201: ('Created', 'Document created, URL follows'), + 202: ('Accepted', + 'Request accepted, processing continues off-line'), + 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), + 204: ('No Content', 'Request fulfilled, nothing follows'), + 205: ('Reset Content', 'Clear input form for further input.'), + 206: ('Partial Content', 'Partial content follows.'), + + 300: ('Multiple Choices', + 'Object has several resources -- see URI list'), + 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), + 302: ('Found', 'Object moved temporarily -- see URI list'), + 303: ('See Other', 'Object moved -- see Method and URL list'), + 304: ('Not Modified', + 'Document has not changed since given time'), + 305: ('Use Proxy', + 'You must use proxy specified in Location to access this ' + 'resource.'), + 307: ('Temporary Redirect', + 'Object moved temporarily -- see URI list'), + + 400: ('Bad Request', + 'Bad request syntax or unsupported method'), + 401: ('Unauthorized', + 'No permission -- see authorization schemes'), + 402: ('Payment Required', + 'No payment -- see charging schemes'), + 403: ('Forbidden', + 'Request forbidden -- authorization will not help'), + 404: ('Not Found', 'Nothing matches the given URI'), + 405: ('Method Not Allowed', + 'Specified method is invalid for this resource.'), + 406: ('Not Acceptable', 'URI not available in preferred format.'), + 407: ('Proxy Authentication Required', 'You must authenticate with ' + 'this proxy before proceeding.'), + 408: ('Request Timeout', 'Request timed out; try again later.'), + 409: ('Conflict', 'Request conflict.'), + 410: ('Gone', + 'URI no longer exists and has been permanently removed.'), + 411: ('Length Required', 'Client must specify Content-Length.'), + 412: ('Precondition Failed', 'Precondition in headers is false.'), + 413: ('Request Entity Too Large', 'Entity is too large.'), + 414: ('Request-URI Too Long', 'URI is too long.'), + 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), + 416: ('Requested Range Not Satisfiable', + 'Cannot satisfy request range.'), + 417: ('Expectation Failed', + 'Expect condition could not be satisfied.'), + + 500: ('Internal Server Error', 'Server got itself in trouble'), + 501: ('Not Implemented', + 'Server does not support this operation'), + 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), + 503: ('Service Unavailable', + 'The server cannot process the request due to a high load'), + 504: ('Gateway Timeout', + 'The gateway server did not receive a timely response'), + 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), + } class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): @@ -627,10 +666,8 @@ """Serve a GET request.""" f = self.send_head() if f: - try: - self.copyfile(f, self.wfile) - finally: - f.close() + self.copyfile(f, self.wfile) + f.close() def do_HEAD(self): """Serve a HEAD request.""" @@ -652,14 +689,10 @@ path = self.translate_path(self.path) f = None if os.path.isdir(path): - parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): + if not self.path.endswith('/'): # redirect browser - doing basically what apache does - self.send_response(HTTPStatus.MOVED_PERMANENTLY) - new_parts = (parts[0], parts[1], parts[2] + '/', - parts[3], parts[4]) - new_url = urllib.parse.urlunsplit(new_parts) - self.send_header("Location", new_url) + self.send_response(301) + self.send_header("Location", self.path + "/") self.end_headers() return None for index in "index.html", "index.htm": @@ -672,20 +705,16 @@ ctype = self.guess_type(path) try: f = open(path, 'rb') - except OSError: - self.send_error(HTTPStatus.NOT_FOUND, "File not found") + except IOError: + self.send_error(404, "File not found") return None - try: - self.send_response(HTTPStatus.OK) - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - self.end_headers() - return f - except: - f.close() - raise + self.send_response(200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + return f def list_directory(self, path): """Helper to produce a directory listing (absent index.html). @@ -697,19 +726,12 @@ """ try: list = os.listdir(path) - except OSError: - self.send_error( - HTTPStatus.NOT_FOUND, - "No permission to list directory") + except os.error: + self.send_error(404, "No permission to list directory") return None list.sort(key=lambda a: a.lower()) r = [] - try: - displaypath = urllib.parse.unquote(self.path, - errors='surrogatepass') - except UnicodeDecodeError: - displaypath = urllib.parse.unquote(path) - displaypath = html.escape(displaypath) + displaypath = html.escape(urllib.parse.unquote(self.path)) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('%s' - % (urllib.parse.quote(linkname, - errors='surrogatepass'), - html.escape(displayname))) + % (urllib.parse.quote(linkname), html.escape(displayname))) r.append('\n

    \n\n\n') - encoded = '\n'.join(r).encode(enc, 'surrogateescape') + encoded = '\n'.join(r).encode(enc) f = io.BytesIO() f.write(encoded) f.seek(0) - self.send_response(HTTPStatus.OK) + self.send_response(200) self.send_header("Content-type", "text/html; charset=%s" % enc) self.send_header("Content-Length", str(len(encoded))) self.end_headers() @@ -756,13 +776,7 @@ # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] - # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') - try: - path = urllib.parse.unquote(path, errors='surrogatepass') - except UnicodeDecodeError: - path = urllib.parse.unquote(path) - path = posixpath.normpath(path) + path = posixpath.normpath(urllib.parse.unquote(path)) words = path.split('/') words = filter(None, words) path = os.getcwd() @@ -771,8 +785,6 @@ head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) - if trailing_slash: - path += '/' return path def copyfile(self, source, outputfile): @@ -828,52 +840,51 @@ # Utilities for CGIHTTPRequestHandler -def _url_collapse_path(path): +# TODO(gregory.p.smith): Move this into an appropriate library. +def _url_collapse_path_split(path): """ Given a URL path, remove extra '/'s and '.' path elements and collapse - any '..' references and returns a collapsed path. + any '..' references. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. - The utility of this function is limited to is_cgi method and helps - preventing some security attacks. - Returns: The reconstituted URL, which will always start with a '/'. + Returns: A tuple of (head, tail) where tail is everything after the final / + and head is everything before it. Head will always start with a '/' and, + if it contains anything else, never have a trailing '/'. Raises: IndexError if too many '..' occur within the path. - """ - # Query component should not be involved. - path, _, query = path.partition('?') - path = urllib.parse.unquote(path) - # Similar to os.path.split(os.path.normpath(path)) but specific to URL # path semantics rather than local operating system semantics. - path_parts = path.split('/') - head_parts = [] - for part in path_parts[:-1]: - if part == '..': - head_parts.pop() # IndexError if more '..' than prior parts - elif part and part != '.': - head_parts.append( part ) + path_parts = [] + for part in path.split('/'): + if part == '.': + path_parts.append('') + else: + path_parts.append(part) + # Filter out blank non trailing parts before consuming the '..'. + path_parts = [part for part in path_parts[:-1] if part] + path_parts[-1:] if path_parts: - tail_part = path_parts.pop() - if tail_part: - if tail_part == '..': - head_parts.pop() - tail_part = '' - elif tail_part == '.': - tail_part = '' + # Special case for CGI's for PATH_INFO + if path.startswith('/cgi-bin') or path.startswith('/htbin'): + tail_part = [] + while path_parts[-1] not in ('cgi-bin','htbin'): + tail_part.insert(0,path_parts.pop()) + tail_part = "/".join(tail_part) + else: + tail_part = path_parts.pop() else: tail_part = '' - - if query: - tail_part = '?'.join((tail_part, query)) - - splitpath = ('/' + '/'.join(head_parts), tail_part) - collapsed_path = "/".join(splitpath) - - return collapsed_path - + head_parts = [] + for part in path_parts: + if part == '..': + head_parts.pop() + else: + head_parts.append(part) + if tail_part and tail_part == '..': + head_parts.pop() + tail_part = '' + return ('/' + '/'.join(head_parts), tail_part) nobody = None @@ -926,9 +937,7 @@ if self.is_cgi(): self.run_cgi() else: - self.send_error( - HTTPStatus.NOT_IMPLEMENTED, - "Can only POST to CGI scripts") + self.send_error(501, "Can only POST to CGI scripts") def send_head(self): """Version of send_head that support CGI scripts""" @@ -952,15 +961,13 @@ (and the next character is a '/' or the end of the string). """ - collapsed_path = _url_collapse_path(self.path) - dir_sep = collapsed_path.find('/', 1) - head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] - if head in self.cgi_directories: - self.cgi_info = head, tail + + splitpath = _url_collapse_path_split(self.path) + if splitpath[0] in self.cgi_directories: + self.cgi_info = splitpath return True return False - cgi_directories = ['/cgi-bin', '/htbin'] def is_executable(self, path): @@ -974,9 +981,10 @@ def run_cgi(self): """Execute a CGI script.""" + path = self.path dir, rest = self.cgi_info - path = dir + '/' + rest - i = path.find('/', len(dir)+1) + + i = path.find('/', len(dir) + 1) while i >= 0: nextdir = path[:i] nextrest = path[i+1:] @@ -984,12 +992,16 @@ scriptdir = self.translate_path(nextdir) if os.path.isdir(scriptdir): dir, rest = nextdir, nextrest - i = path.find('/', len(dir)+1) + i = path.find('/', len(dir) + 1) else: break # find an explicit query string, if present. - rest, _, query = rest.partition('?') + i = rest.rfind('?') + if i >= 0: + rest, query = rest[:i], rest[i+1:] + else: + query = '' # dissect the part after the directory name into a script name & # a possible additional path, to be stored in PATH_INFO. @@ -1002,21 +1014,17 @@ scriptname = dir + '/' + script scriptfile = self.translate_path(scriptname) if not os.path.exists(scriptfile): - self.send_error( - HTTPStatus.NOT_FOUND, - "No such CGI script (%r)" % scriptname) + self.send_error(404, "No such CGI script (%r)" % scriptname) return if not os.path.isfile(scriptfile): - self.send_error( - HTTPStatus.FORBIDDEN, - "CGI script is not a plain file (%r)" % scriptname) + self.send_error(403, "CGI script is not a plain file (%r)" % + scriptname) return ispy = self.is_python(scriptname) if self.have_fork or not ispy: if not self.is_executable(scriptfile): - self.send_error( - HTTPStatus.FORBIDDEN, - "CGI script is not executable (%r)" % scriptname) + self.send_error(403, "CGI script is not executable (%r)" % + scriptname) return # Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html @@ -1034,6 +1042,9 @@ env['SCRIPT_NAME'] = scriptname if query: env['QUERY_STRING'] = query + host = self.address_string() + if host != self.client_address[0]: + env['REMOTE_HOST'] = host env['REMOTE_ADDR'] = self.client_address[0] authorization = self.headers.get("authorization") if authorization: @@ -1084,7 +1095,7 @@ 'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'): env.setdefault(k, "") - self.send_response(HTTPStatus.OK, "Script output follows") + self.send_response(200, "Script output follows") self.flush_headers() decoded_query = query.replace('+', ' ') @@ -1111,7 +1122,7 @@ try: try: os.setuid(nobody) - except OSError: + except os.error: pass os.dup2(self.rfile.fileno(), 0) os.dup2(self.wfile.fileno(), 1) @@ -1164,14 +1175,20 @@ self.log_message("CGI script exited OK") -def test(HandlerClass=BaseHTTPRequestHandler, - ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""): +def test(HandlerClass = BaseHTTPRequestHandler, + ServerClass = HTTPServer, protocol="HTTP/1.0"): """Test the HTTP request handler class. - This runs an HTTP server on port 8000 (or the port argument). + This runs an HTTP server on port 8000 (or the first command line + argument). """ - server_address = (bind, port) + + if sys.argv[1:]: + port = int(sys.argv[1]) + else: + port = 8000 + server_address = ('', port) HandlerClass.protocol_version = protocol httpd = ServerClass(server_address, HandlerClass) @@ -1186,19 +1203,4 @@ sys.exit(0) if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--cgi', action='store_true', - help='Run as CGI Server') - parser.add_argument('--bind', '-b', default='', metavar='ADDRESS', - help='Specify alternate bind address ' - '[default: all interfaces]') - parser.add_argument('port', action='store', - default=8000, type=int, - nargs='?', - help='Specify alternate port [default: 8000]') - args = parser.parse_args() - if args.cgi: - handler_class = CGIHTTPRequestHandler - else: - handler_class = SimpleHTTPRequestHandler - test(HandlerClass=handler_class, port=args.port, bind=args.bind) + test(HandlerClass=SimpleHTTPRequestHandler) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/AutoComplete.py --- a/Lib/idlelib/AutoComplete.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/AutoComplete.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,6 +9,9 @@ from idlelib.configHandler import idleConf +# This string includes all chars that may be in a file name (without a path +# separator) +FILENAME_CHARS = string.ascii_letters + string.digits + os.curdir + "._~#$:-" # This string includes all chars that may be in an identifier ID_CHARS = string.ascii_letters + string.digits + "_" @@ -121,26 +124,19 @@ curline = self.text.get("insert linestart", "insert") i = j = len(curline) if hp.is_in_string() and (not mode or mode==COMPLETE_FILES): - # Find the beginning of the string - # fetch_completions will look at the file system to determine whether the - # string value constitutes an actual file name - # XXX could consider raw strings here and unescape the string value if it's - # not raw. self._remove_autocomplete_window() mode = COMPLETE_FILES - # Find last separator or string start - while i and curline[i-1] not in "'\"" + SEPS: + while i and curline[i-1] in FILENAME_CHARS: i -= 1 comp_start = curline[i:j] j = i - # Find string start - while i and curline[i-1] not in "'\"": + while i and curline[i-1] in FILENAME_CHARS + SEPS: i -= 1 comp_what = curline[i:j] elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): self._remove_autocomplete_window() mode = COMPLETE_ATTRIBUTES - while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): + while i and curline[i-1] in ID_CHARS: i -= 1 comp_start = curline[i:j] if i and curline[i-1] == '.': @@ -160,9 +156,12 @@ if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - return not self.autocompletewindow.show_window( - comp_lists, "insert-%dc" % len(comp_start), - complete, mode, userWantsWin) + self.autocompletewindow.show_window(comp_lists, + "insert-%dc" % len(comp_start), + complete, + mode, + userWantsWin) + return True def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list @@ -226,8 +225,3 @@ namespace = sys.modules.copy() namespace.update(__main__.__dict__) return eval(name, namespace) - - -if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_autocomplete', verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/AutoCompleteWindow.py --- a/Lib/idlelib/AutoCompleteWindow.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/AutoCompleteWindow.py Mon Jan 25 17:05:13 2016 +0100 @@ -157,14 +157,13 @@ self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) - start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return completed == start + return self.userwantswindow = userWantsWin self.lasttypedstart = self.start @@ -192,7 +191,6 @@ scrollbar.config(command=listbox.yview) scrollbar.pack(side=RIGHT, fill=Y) listbox.pack(side=LEFT, fill=BOTH, expand=True) - acw.lift() # work around bug in Tk 8.5.18+ (issue #24570) # Initialize the listbox selection self.listbox.select_set(self._binary_search(self.start)) @@ -356,15 +354,6 @@ # A modifier key, so ignore return - elif event.char and event.char >= ' ': - # Regular character with a non-length-1 keycode - self._change_start(self.start + event.char) - self.lasttypedstart = self.start - self.listbox.select_clear(0, int(self.listbox.curselection()[0])) - self.listbox.select_set(self._binary_search(self.start)) - self._selection_changed() - return "break" - else: # Unknown event, close the window and let it through. self.hide_window() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/AutoExpand.py --- a/Lib/idlelib/AutoExpand.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/AutoExpand.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,17 +1,3 @@ -'''Complete the current word before the cursor with words in the editor. - -Each menu selection or shortcut key selection replaces the word with a -different word with the same prefix. The search for matches begins -before the target and moves toward the top of the editor. It then starts -after the cursor and moves down. It then returns to the original word and -the cycle starts again. - -Changing the current text line or leaving the cursor in a different -place before requesting the next selection causes AutoExpand to reset -its state. - -This is an extension file and there is only one instance of AutoExpand. -''' import string import re @@ -34,7 +20,6 @@ self.state = None def expand_word_event(self, event): - "Replace the current word with the next expansion." curinsert = self.text.index("insert") curline = self.text.get("insert linestart", "insert lineend") if not self.state: @@ -61,7 +46,6 @@ return "break" def getwords(self): - "Return a list of words that match the prefix before the cursor." word = self.getprevword() if not word: return [] @@ -92,13 +76,8 @@ return words def getprevword(self): - "Return the word prefix before the cursor." line = self.text.get("insert linestart", "insert") i = len(line) while i > 0 and line[i-1] in self.wordchars: i = i-1 return line[i:] - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_autoexpand', verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Bindings.py --- a/Lib/idlelib/Bindings.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/Bindings.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,21 +8,14 @@ windows. """ -from importlib.util import find_spec - +import sys from idlelib.configHandler import idleConf - -# Warning: menudefs is altered in macosxSupport.overrideRootMenu() -# after it is determined that an OS X Aqua Tk is in use, -# which cannot be done until after Tk() is first called. -# Do not alter the 'file', 'options', or 'help' cascades here -# without altering overrideRootMenu() as well. -# TODO: Make this more robust +from idlelib import macosxSupport menudefs = [ # underscore prefixes character to underscore ('file', [ - ('_New File', '<>'), + ('_New Window', '<>'), ('_Open...', '<>'), ('Open _Module...', '<>'), ('Class _Browser', '<>'), @@ -77,7 +70,7 @@ ('!_Auto-open Stack Viewer', '<>'), ]), ('options', [ - ('Configure _IDLE', '<>'), + ('_Configure IDLE...', '<>'), None, ]), ('help', [ @@ -88,7 +81,23 @@ ]), ] -if find_spec('turtledemo'): - menudefs[-1][1].append(('Turtle Demo', '<>')) +if macosxSupport.runningAsOSXApp(): + # Running as a proper MacOS application bundle. This block restructures + # the menus a little to make them conform better to the HIG. + + quitItem = menudefs[0][1][-1] + closeItem = menudefs[0][1][-2] + + # Remove the last 3 items of the file menu: a separator, close window and + # quit. Close window will be reinserted just above the save item, where + # it should be according to the HIG. Quit is in the application menu. + del menudefs[0][1][-3:] + menudefs[0][1].insert(6, closeItem) + + # Remove the 'About' entry from the help menu, it is in the application + # menu + del menudefs[-1][1][0:2] default_keydefs = idleConf.GetCurrentKeySet() + +del sys diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/CREDITS.txt --- a/Lib/idlelib/CREDITS.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/CREDITS.txt Mon Jan 25 17:05:13 2016 +0100 @@ -24,7 +24,7 @@ integration, debugger integration and persistent breakpoints). Scott David Daniels, Tal Einat, Hernan Foffani, Christos Georgiou, -Jim Jewett, Martin v. Löwis, Jason Orendorff, Guilherme Polo, Josh Robb, +Jim Jewett, Martin v. Lwis, Jason Orendorff, Guilherme Polo, Josh Robb, Nigel Rowe, Bruce Sherwood, Jeff Shute, and Weeble have submitted useful patches. Thanks, guys! diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/CallTipWindow.py --- a/Lib/idlelib/CallTipWindow.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/CallTipWindow.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,8 +2,9 @@ After ToolTip.py, which uses ideas gleaned from PySol Used by the CallTips IDLE extension. + """ -from tkinter import Toplevel, Label, LEFT, SOLID, TclError +from tkinter import * HIDE_VIRTUAL_EVENT_NAME = "<>" HIDE_SEQUENCES = ("", "") @@ -47,7 +48,13 @@ def showtip(self, text, parenleft, parenright): """Show the calltip, bind events which will close it and reposition it. """ - # Only called in CallTips, where lines are truncated + # truncate overly long calltip + if len(text) >= 79: + textlines = text.splitlines() + for i, line in enumerate(textlines): + if len(line) > 79: + textlines[i] = line[:75] + ' ...' + text = '\n'.join(textlines) self.text = text if self.tipwindow or not self.text: return @@ -72,7 +79,6 @@ background="#ffffe0", relief=SOLID, borderwidth=1, font = self.widget['font']) self.label.pack() - tw.lift() # work around bug in Tk 8.5.18+ (issue #24570) self.checkhideid = self.widget.bind(CHECKHIDE_VIRTUAL_EVENT_NAME, self.checkhide_event) @@ -133,29 +139,37 @@ return bool(self.tipwindow) -def _calltip_window(parent): # htest # - from tkinter import Toplevel, Text, LEFT, BOTH - top = Toplevel(parent) - top.title("Test calltips") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - text = Text(top) - text.pack(side=LEFT, fill=BOTH, expand=1) - text.insert("insert", "string.split") - top.update() - calltip = CallTip(text) +############################### +# +# Test Code +# +class container: # Conceptually an editor_window + def __init__(self): + root = Tk() + text = self.text = Text(root) + text.pack(side=LEFT, fill=BOTH, expand=1) + text.insert("insert", "string.split") + root.update() + self.calltip = CallTip(text) - def calltip_show(event): - calltip.showtip("(s=Hello world)", "insert", "end") - def calltip_hide(event): - calltip.hidetip() - text.event_add("<>", "(") - text.event_add("<>", ")") - text.bind("<>", calltip_show) - text.bind("<>", calltip_hide) - text.focus_set() + text.event_add("<>", "(") + text.event_add("<>", ")") + text.bind("<>", self.calltip_show) + text.bind("<>", self.calltip_hide) + + text.focus_set() + root.mainloop() + + def calltip_show(self, event): + self.calltip.showtip("Hello world") + + def calltip_hide(self, event): + self.calltip.hidetip() + +def main(): + # Test code + c=container() if __name__=='__main__': - from idlelib.idle_test.htest import run - run(_calltip_window) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/CallTips.py --- a/Lib/idlelib/CallTips.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/CallTips.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,16 +5,16 @@ which disappear when you type a closing parenthesis. """ -import __main__ -import inspect import re import sys -import textwrap import types +import inspect from idlelib import CallTipWindow from idlelib.HyperParser import HyperParser +import __main__ + class CallTips: menudefs = [ @@ -67,18 +67,18 @@ if not sur_paren: return hp.set_index(sur_paren[0]) - expression = hp.get_expression() - if not expression: + name = hp.get_expression() + if not name: return - if not evalfuncs and (expression.find('(') != -1): + if not evalfuncs and (name.find('(') != -1): return - argspec = self.fetch_tip(expression) + argspec = self.fetch_tip(name) if not argspec: return self.active_calltip = self._calltip_window() self.active_calltip.showtip(argspec, sur_paren[0], sur_paren[1]) - def fetch_tip(self, expression): + def fetch_tip(self, name): """Return the argument list and docstring of a function or class. If there is a Python subprocess, get the calltip there. Otherwise, @@ -94,82 +94,110 @@ """ try: rpcclt = self.editwin.flist.pyshell.interp.rpcclt - except AttributeError: + except: rpcclt = None if rpcclt: return rpcclt.remotecall("exec", "get_the_calltip", - (expression,), {}) + (name,), {}) else: - return get_argspec(get_entity(expression)) + entity = self.get_entity(name) + return get_argspec(entity) -def get_entity(expression): - """Return the object corresponding to expression evaluated - in a namespace spanning sys.modules and __main.dict__. - """ - if expression: - namespace = sys.modules.copy() - namespace.update(__main__.__dict__) - try: - return eval(expression, namespace) - except BaseException: - # An uncaught exception closes idle, and eval can raise any - # exception, especially if user classes are involved. - return None + def get_entity(self, name): + "Lookup name in a namespace spanning sys.modules and __main.dict__." + if name: + namespace = sys.modules.copy() + namespace.update(__main__.__dict__) + try: + return eval(name, namespace) + except (NameError, AttributeError): + return None -# The following are used in get_argspec and some in tests -_MAX_COLS = 85 -_MAX_LINES = 5 # enough for bytes -_INDENT = ' '*4 # for wrapped signatures -_first_param = re.compile('(?<=\()\w*\,?\s*') -_default_callable_argspec = "See source or doc" - +def _find_constructor(class_ob): + "Find the nearest __init__() in the class tree." + try: + return class_ob.__init__.__func__ + except AttributeError: + for base in class_ob.__bases__: + init = _find_constructor(base) + if init: + return init + return None def get_argspec(ob): - '''Return a string describing the signature of a callable object, or ''. - - For Python-coded functions and methods, the first line is introspected. - Delete 'self' parameter for classes (.__init__) and bound methods. - The next lines are the first lines of the doc string up to the first - empty line or _MAX_LINES. For builtins, this typically includes - the arguments in addition to the return value. - ''' + """Get a string describing the arguments for the given object.""" argspec = "" - try: - ob_call = ob.__call__ - except BaseException: - return argspec - if isinstance(ob, type): - fob = ob.__init__ - elif isinstance(ob_call, types.MethodType): - fob = ob_call - else: - fob = ob - if isinstance(fob, (types.FunctionType, types.MethodType)): - argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) - if (isinstance(ob, (type, types.MethodType)) or - isinstance(ob_call, types.MethodType)): - argspec = _first_param.sub("", argspec) - - lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) - if len(argspec) > _MAX_COLS else [argspec] if argspec else []) - - if isinstance(ob_call, types.MethodType): - doc = ob_call.__doc__ - else: + if ob is not None: + if isinstance(ob, type): + fob = _find_constructor(ob) + if fob is None: + fob = lambda: None + elif isinstance(ob, types.MethodType): + fob = ob.__func__ + else: + fob = ob + if isinstance(fob, (types.FunctionType, types.LambdaType)): + argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) + pat = re.compile('self\,?\s*') + argspec = pat.sub("", argspec) doc = getattr(ob, "__doc__", "") - if doc: - for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: - line = line.strip() - if not line: - break - if len(line) > _MAX_COLS: - line = line[: _MAX_COLS - 3] + '...' - lines.append(line) - argspec = '\n'.join(lines) - if not argspec: - argspec = _default_callable_argspec + if doc: + doc = doc.lstrip() + pos = doc.find("\n") + if pos < 0 or pos > 70: + pos = 70 + if argspec: + argspec += "\n" + argspec += doc[:pos] return argspec +################################################# +# +# Test code +# +def main(): + def t1(): "()" + def t2(a, b=None): "(a, b=None)" + def t3(a, *args): "(a, *args)" + def t4(*args): "(*args)" + def t5(a, *args): "(a, *args)" + def t6(a, b=None, *args, **kw): "(a, b=None, *args, **kw)" + + class TC(object): + "(ai=None, *b)" + def __init__(self, ai=None, *b): "(ai=None, *b)" + def t1(self): "()" + def t2(self, ai, b=None): "(ai, b=None)" + def t3(self, ai, *args): "(ai, *args)" + def t4(self, *args): "(*args)" + def t5(self, ai, *args): "(ai, *args)" + def t6(self, ai, b=None, *args, **kw): "(ai, b=None, *args, **kw)" + + __main__.__dict__.update(locals()) + + def test(tests): + ct = CallTips() + failed=[] + for t in tests: + expected = t.__doc__ + "\n" + t.__doc__ + name = t.__name__ + # exercise fetch_tip(), not just get_argspec() + try: + qualified_name = "%s.%s" % (t.__self__.__class__.__name__, name) + except AttributeError: + qualified_name = name + argspec = ct.fetch_tip(qualified_name) + if argspec != expected: + failed.append(t) + fmt = "%s - expected %s, but got %s" + print(fmt % (t.__name__, expected, get_argspec(t))) + print("%d of %d tests failed" % (len(failed), len(tests))) + + tc = TC() + tests = (t1, t2, t3, t4, t5, t6, + TC, tc.t1, tc.t2, tc.t3, tc.t4, tc.t5, tc.t6) + + test(tests) + if __name__ == '__main__': - from unittest import main - main('idlelib.idle_test.test_calltips', verbosity=2) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ChangeLog --- a/Lib/idlelib/ChangeLog Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ChangeLog Mon Jan 25 17:05:13 2016 +0100 @@ -20,7 +20,7 @@ 2001-07-19 14:49 elguavas * ChangeLog, EditorWindow.py, INSTALLATION, NEWS.txt, README.txt, - TODO.txt, idlever.py: + TODO.txt, idlever.py: minor tidy-ups ready for 0.8.1 alpha tarball release 2001-07-17 15:12 kbk @@ -172,7 +172,7 @@ all this work w/ a future-stmt just looks harder and harder." --tim_one - (From Rel 1.8: "Hack to make this still work with Python 1.5.2. + (From Rel 1.8: "Hack to make this still work with Python 1.5.2. ;-( " --fdrake) 2001-07-14 14:51 kbk @@ -193,7 +193,7 @@ test() to _test()." --GvR This was an interesting merge. The join completely missed removing - goodname(), which was adjacent, but outside of, a small conflict. + goodname(), which was adjacent, but outside of, a small conflict. I only caught it by comparing the 1.1.3.2/1.1.3.3 diff. CVS ain't infallible. @@ -516,12 +516,12 @@ 2000-08-15 22:51 nowonder - * IDLEFORK.html: + * IDLEFORK.html: corrected email address 2000-08-15 22:47 nowonder - * IDLEFORK.html: + * IDLEFORK.html: added .html file for http://idlefork.sourceforge.net 2000-08-15 11:13 dscherer diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ClassBrowser.py --- a/Lib/idlelib/ClassBrowser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ClassBrowser.py Mon Jan 25 17:05:13 2016 +0100 @@ -19,23 +19,13 @@ from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.configHandler import idleConf -file_open = None # Method...Item and Class...Item use this. -# Normally PyShell.flist.open, but there is no PyShell.flist for htest. - class ClassBrowser: - def __init__(self, flist, name, path, _htest=False): + def __init__(self, flist, name, path): # XXX This API should change, if the file doesn't end in ".py" # XXX the code here is bogus! - """ - _htest - bool, change box when location running htest. - """ - global file_open - if not _htest: - file_open = PyShell.flist.open self.name = name self.file = os.path.join(path[0], self.name + ".py") - self._htest = _htest self.init(flist) def close(self, event=None): @@ -50,13 +40,10 @@ self.top = top = ListedToplevel(flist.root) top.protocol("WM_DELETE_WINDOW", self.close) top.bind("", self.close) - if self._htest: # place dialog below parent if running htest - top.geometry("+%d+%d" % - (flist.root.winfo_rootx(), flist.root.winfo_rooty() + 200)) self.settitle() top.focus_set() # create scrolled canvas - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') background = idleConf.GetHighlight(theme, 'normal')['background'] sc = ScrolledCanvas(top, bg=background, highlightthickness=0, takefocus=1) sc.frame.pack(expand=1, fill="both") @@ -107,7 +94,7 @@ return [] try: dict = pyclbr.readmodule_ex(name, [dir] + sys.path) - except ImportError: + except ImportError as msg: return [] items = [] self.classes = {} @@ -176,7 +163,7 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = file_open(self.file) + edit = PyShell.flist.open(self.file) if hasattr(self.cl, 'lineno'): lineno = self.cl.lineno edit.gotoline(lineno) @@ -212,10 +199,10 @@ def OnDoubleClick(self): if not os.path.exists(self.file): return - edit = file_open(self.file) + edit = PyShell.flist.open(self.file) edit.gotoline(self.cl.methods[self.name]) -def _class_browser(parent): #Wrapper for htest +def main(): try: file = __file__ except NameError: @@ -226,11 +213,9 @@ file = sys.argv[0] dir, file = os.path.split(file) name = os.path.splitext(file)[0] - flist = PyShell.PyShellFileList(parent) - global file_open - file_open = flist.open - ClassBrowser(flist, name, [dir], _htest=True) + ClassBrowser(PyShell.flist, name, [dir]) + if sys.stdin is sys.__stdin__: + mainloop() if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_class_browser) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/CodeContext.py --- a/Lib/idlelib/CodeContext.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/CodeContext.py Mon Jan 25 17:05:13 2016 +0100 @@ -15,8 +15,8 @@ from sys import maxsize as INFINITY from idlelib.configHandler import idleConf -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with"} +BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for", + "if", "try", "while", "with"]) UPDATEINTERVAL = 100 # millisec FONTUPDATEINTERVAL = 1000 # millisec @@ -57,18 +57,18 @@ # Calculate the border width and horizontal padding required to # align the context with the text in the main Text widget. # - # All values are passed through getint(), since some + # All values are passed through int(str()), since some # values may be pixel objects, which can't simply be added to ints. widgets = self.editwin.text, self.editwin.text_frame # Calculate the required vertical padding padx = 0 for widget in widgets: - padx += widget.tk.getint(widget.pack_info()['padx']) - padx += widget.tk.getint(widget.cget('padx')) + padx += int(str( widget.pack_info()['padx'] )) + padx += int(str( widget.cget('padx') )) # Calculate the required border width border = 0 for widget in widgets: - border += widget.tk.getint(widget.cget('border')) + border += int(str( widget.cget('border') )) self.label = tkinter.Label(self.editwin.top, text="\n" * (self.context_depth - 1), anchor=W, justify=LEFT, diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ColorDelegator.py --- a/Lib/idlelib/ColorDelegator.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ColorDelegator.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,6 +2,7 @@ import re import keyword import builtins +from tkinter import * from idlelib.Delegator import Delegator from idlelib.configHandler import idleConf @@ -20,17 +21,17 @@ # 1st 'file' colorized normal, 2nd as builtin, 3rd as string builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" comment = any("COMMENT", [r"#[^\n]*"]) - stringprefix = r"(\br|u|ur|R|U|UR|Ur|uR|b|B|br|Br|bR|BR|rb|rB|Rb|RB)?" - sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?" - dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?' - sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" - dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' + sqstring = r"(\b[rRbB])?'[^'\\\n]*(\\.[^'\\\n]*)*'?" + dqstring = r'(\b[rRbB])?"[^"\\\n]*(\\.[^"\\\n]*)*"?' + sq3string = r"(\b[rRbB])?'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" + dq3string = r'(\b[rRbB])?"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) return kw + "|" + builtin + "|" + comment + "|" + string +\ "|" + any("SYNC", [r"\n"]) prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) +asprog = re.compile(r".*?\b(as)\b") class ColorDelegator(Delegator): @@ -38,6 +39,7 @@ Delegator.__init__(self) self.prog = prog self.idprog = idprog + self.asprog = asprog self.LoadTagDefs() def setdelegate(self, delegate): @@ -48,10 +50,6 @@ self.config_colors() self.bind("<>", self.toggle_colorize_event) self.notify_range("1.0", "end") - else: - # No delegate - stop any colorizing - self.stop_colorizing = True - self.allow_colorizing = False def config_colors(self): for tag, cnf in self.tagdefs.items(): @@ -60,7 +58,7 @@ self.tag_raise('sel') def LoadTagDefs(self): - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') self.tagdefs = { "COMMENT": idleConf.GetHighlight(theme, "comment"), "KEYWORD": idleConf.GetHighlight(theme, "keyword"), @@ -69,6 +67,7 @@ "DEFINITION": idleConf.GetHighlight(theme, "definition"), "SYNC": {'background':None,'foreground':None}, "TODO": {'background':None,'foreground':None}, + "BREAK": idleConf.GetHighlight(theme, "break"), "ERROR": idleConf.GetHighlight(theme, "error"), # The following is used by ReplaceDialog: "hit": idleConf.GetHighlight(theme, "hit"), @@ -150,9 +149,9 @@ self.stop_colorizing = False self.colorizing = True if DEBUG: print("colorizing...") - t0 = time.perf_counter() + t0 = time.clock() self.recolorize_main() - t1 = time.perf_counter() + t1 = time.clock() if DEBUG: print("%.3f seconds" % (t1-t0)) finally: self.colorizing = False @@ -210,6 +209,22 @@ self.tag_add("DEFINITION", head + "+%dc" % a, head + "+%dc" % b) + elif value == "import": + # color all the "as" words on same line, except + # if in a comment; cheap approximation to the + # truth + if '#' in chars: + endpos = chars.index('#') + else: + endpos = len(chars) + while True: + m1 = self.asprog.match(chars, b, endpos) + if not m1: + break + a, b = m1.span(1) + self.tag_add("KEYWORD", + head + "+%dc" % a, + head + "+%dc" % b) m = self.prog.search(chars, m.end()) if "SYNC" in self.tag_names(next + "-1c"): head = next @@ -233,24 +248,17 @@ for tag in self.tagdefs: self.tag_remove(tag, "1.0", "end") -def _color_delegator(parent): # htest # - from tkinter import Toplevel, Text +def main(): from idlelib.Percolator import Percolator - - top = Toplevel(parent) - top.title("Test ColorDelegator") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - source = "if somename: x = 'abc' # comment\nprint\n" - text = Text(top, background="white") + root = Tk() + root.wm_protocol("WM_DELETE_WINDOW", root.quit) + text = Text(background="white") text.pack(expand=1, fill="both") - text.insert("insert", source) text.focus_set() - p = Percolator(text) d = ColorDelegator() p.insertfilter(d) + root.mainloop() if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_color_delegator) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Debugger.py --- a/Lib/idlelib/Debugger.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/Debugger.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,6 @@ import os import bdb +import types from tkinter import * from idlelib.WindowList import ListedToplevel from idlelib.ScrolledList import ScrolledList @@ -17,10 +18,7 @@ self.set_step() return message = self.__frame2message(frame) - try: - self.gui.interaction(message, frame) - except TclError: # When closing debugger window with [x] in 3.x - pass + self.gui.interaction(message, frame) def user_exception(self, frame, info): if self.in_rpc_code(frame): @@ -62,42 +60,8 @@ self.frame = None self.make_gui() self.interacting = 0 - self.nesting_level = 0 def run(self, *args): - # Deal with the scenario where we've already got a program running - # in the debugger and we want to start another. If that is the case, - # our second 'run' was invoked from an event dispatched not from - # the main event loop, but from the nested event loop in 'interaction' - # below. So our stack looks something like this: - # outer main event loop - # run() - # - # callback to debugger's interaction() - # nested event loop - # run() for second command - # - # This kind of nesting of event loops causes all kinds of problems - # (see e.g. issue #24455) especially when dealing with running as a - # subprocess, where there's all kinds of extra stuff happening in - # there - insert a traceback.print_stack() to check it out. - # - # By this point, we've already called restart_subprocess() in - # ScriptBinding. However, we also need to unwind the stack back to - # that outer event loop. To accomplish this, we: - # - return immediately from the nested run() - # - abort_loop ensures the nested event loop will terminate - # - the debugger's interaction routine completes normally - # - the restart_subprocess() will have taken care of stopping - # the running program, which will also let the outer run complete - # - # That leaves us back at the outer main event loop, at which point our - # after event can fire, and we'll come back to this routine with a - # clean stack. - if self.nesting_level > 0: - self.abort_loop() - self.root.after(100, lambda: self.run(*args)) - return try: self.interacting = 1 return self.idb.run(*args) @@ -105,10 +69,6 @@ self.interacting = 0 def close(self, event=None): - try: - self.quit() - except Exception: - pass if self.interacting: self.top.bell() return @@ -232,12 +192,7 @@ b.configure(state="normal") # self.top.wakeup() - # Nested main loop: Tkinter's main loop is not reentrant, so use - # Tcl's vwait facility, which reenters the event loop until an - # event handler sets the variable we're waiting on - self.nesting_level += 1 - self.root.tk.call('vwait', '::idledebugwait') - self.nesting_level -= 1 + self.root.mainloop() # for b in self.buttons: b.configure(state="disabled") @@ -261,26 +216,23 @@ def cont(self): self.idb.set_continue() - self.abort_loop() + self.root.quit() def step(self): self.idb.set_step() - self.abort_loop() + self.root.quit() def next(self): self.idb.set_next(self.frame) - self.abort_loop() + self.root.quit() def ret(self): self.idb.set_return(self.frame) - self.abort_loop() + self.root.quit() def quit(self): self.idb.set_quit() - self.abort_loop() - - def abort_loop(self): - self.root.tk.call('set', '::idledebugwait', '1') + self.root.quit() stackviewer = None @@ -302,7 +254,8 @@ self.sync_source_line() def show_frame(self, stackitem): - self.frame = stackitem[0] # lineno is stackitem[1] + frame, lineno = stackitem + self.frame = frame self.show_variables() localsviewer = None @@ -370,7 +323,7 @@ class StackViewer(ScrolledList): def __init__(self, master, flist, gui): - if macosxSupport.isAquaTk(): + if macosxSupport.runningAsOSXApp(): # At least on with the stock AquaTk version on OSX 10.4 you'll # get an shaking GUI that eventually kills IDLE if the width # argument is specified. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Delegator.py --- a/Lib/idlelib/Delegator.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/Delegator.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,12 +4,12 @@ def __init__(self, delegate=None): self.delegate = delegate - self.__cache = set() + self.__cache = {} def __getattr__(self, name): attr = getattr(self.delegate, name) # May raise AttributeError setattr(self, name, attr) - self.__cache.add(name) + self.__cache[name] = attr return attr def resetcache(self): @@ -20,6 +20,14 @@ pass self.__cache.clear() + def cachereport(self): + keys = list(self.__cache.keys()) + keys.sort() + print(keys) + def setdelegate(self, delegate): self.resetcache() self.delegate = delegate + + def getdelegate(self): + return self.delegate diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/EditorWindow.py --- a/Lib/idlelib/EditorWindow.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/EditorWindow.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,8 @@ -import importlib -import importlib.abc -import importlib.util +import sys import os -import platform import re import string -import sys +import imp from tkinter import * import tkinter.simpledialog as tkSimpleDialog import tkinter.messagebox as tkMessageBox @@ -13,6 +10,7 @@ import webbrowser from idlelib.MultiCall import MultiCallCreator +from idlelib import idlever from idlelib import WindowList from idlelib import SearchDialog from idlelib import GrepDialog @@ -21,24 +19,50 @@ from idlelib.configHandler import idleConf from idlelib import aboutDialog, textView, configDialog from idlelib import macosxSupport -from idlelib import help # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 -_py_version = ' (%s)' % platform.python_version() - def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info release = '%s%s' % (major, minor) - release += '%s' % (micro,) + if micro: + release += '%s' % (micro,) if level == 'candidate': release += 'rc%s' % (serial,) elif level != 'final': release += '%s%s' % (level[0], serial) return release +def _find_module(fullname, path=None): + """Version of imp.find_module() that handles hierarchical module names""" + + file = None + for tgt in fullname.split('.'): + if file is not None: + file.close() # close intermediate files + (file, filename, descr) = imp.find_module(tgt, path) + if descr[2] == imp.PY_SOURCE: + break # find but not load the source file + module = imp.load_module(tgt, file, filename, descr) + try: + path = module.__path__ + except AttributeError: + raise ImportError('No source for module ' + module.__name__) + if descr[2] != imp.PY_SOURCE: + # If all of the above fails and didn't raise an exception,fallback + # to a straight import which can find __init__.py in a package. + m = __import__(fullname) + try: + filename = m.__file__ + except AttributeError: + pass + else: + file = None + descr = os.path.splitext(filename)[1], None, imp.PY_SOURCE + return file, filename, descr + class HelpDialog(object): @@ -54,11 +78,6 @@ near - a Toplevel widget (e.g. EditorWindow or PyShell) to use as a reference for placing the help window """ - import warnings as w - w.warn("EditorWindow.HelpDialog is no longer used by Idle.\n" - "It will be removed in 3.6 or later.\n" - "It has been replaced by private help.HelpWindow\n", - DeprecationWarning, stacklevel=2) if self.dlg is None: self.show_dialog(parent) if near: @@ -85,7 +104,7 @@ self.dlg = None self.parent = None -helpDialog = HelpDialog() # singleton instance, no longer used +helpDialog = HelpDialog() # singleton instance class EditorWindow(object): @@ -101,7 +120,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): if EditorWindow.help_url is None: - dochome = os.path.join(sys.base_prefix, 'Doc', 'index.html') + dochome = os.path.join(sys.prefix, 'Doc', 'index.html') if sys.platform.count('linux'): # look for html docs in a couple of standard places pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3] @@ -112,13 +131,13 @@ dochome = os.path.join(basepath, pyver, 'Doc', 'index.html') elif sys.platform[:3] == 'win': - chmfile = os.path.join(sys.base_prefix, 'Doc', + chmfile = os.path.join(sys.prefix, 'Doc', 'Python%s.chm' % _sphinx_version()) if os.path.isfile(chmfile): dochome = chmfile - elif sys.platform == 'darwin': - # documentation may be stored inside a python framework - dochome = os.path.join(sys.base_prefix, + elif macosxSupport.runningAsOSXApp(): + # documentation is stored inside the python framework + dochome = os.path.join(sys.prefix, 'Resources/English.lproj/Documentation/index.html') dochome = os.path.normpath(dochome) if os.path.isfile(dochome): @@ -127,7 +146,8 @@ # Safari requires real file:-URLs EditorWindow.help_url = 'file://' + EditorWindow.help_url else: - EditorWindow.help_url = "https://docs.python.org/%d.%d/" % sys.version_info[:2] + EditorWindow.help_url = "http://docs.python.org/%d.%d" % sys.version_info[:2] + currentTheme=idleConf.CurrentTheme() self.flist = flist root = root or flist.root self.root = root @@ -150,16 +170,13 @@ 'recent-files.lst') self.text_frame = text_frame = Frame(top) self.vbar = vbar = Scrollbar(text_frame, name='vbar') - self.width = idleConf.GetOption('main', 'EditorWindow', - 'width', type='int') + self.width = idleConf.GetOption('main','EditorWindow','width') text_options = { 'name': 'text', 'padx': 5, 'wrap': 'none', - 'highlightthickness': 0, 'width': self.width, - 'height': idleConf.GetOption('main', 'EditorWindow', - 'height', type='int')} + 'height': idleConf.GetOption('main', 'EditorWindow', 'height')} if TkVersion >= 8.5: # Starting with tk 8.5 we have to set the new tabstyle option # to 'wordprocessor' to achieve the same display of tabs as in @@ -173,16 +190,16 @@ self.top.protocol("WM_DELETE_WINDOW", self.close) self.top.bind("<>", self.close_event) - if macosxSupport.isAquaTk(): + if macosxSupport.runningAsOSXApp(): # Command-W on editorwindows doesn't work without this. text.bind('<>', self.close_event) - # Some OS X systems have only one mouse button, so use - # control-click for popup context menus there. For two - # buttons, AquaTk defines <2> as the right button, not <3>. + # Some OS X systems have only one mouse button, + # so use control-click for pulldown menus there. + # (Note, AquaTk defines <2> as the right button if + # present and the Tk Text widget already binds <2>.) text.bind("",self.right_menu_event) - text.bind("<2>", self.right_menu_event) else: - # Elsewhere, use right-click for popup menus. + # Elsewhere, use right-click for pulldown menus. text.bind("<3>",self.right_menu_event) text.bind("<>", self.cut) text.bind("<>", self.copy) @@ -227,13 +244,17 @@ text.bind("<>", self.flist.close_all_callback) text.bind("<>", self.open_class_browser) text.bind("<>", self.open_path_browser) - text.bind("<>", self.open_turtle_demo) self.set_status_bar() vbar['command'] = text.yview vbar.pack(side=RIGHT, fill=Y) text['yscrollcommand'] = vbar.set - text['font'] = idleConf.GetFont(self.root, 'main', 'EditorWindow') + fontWeight = 'normal' + if idleConf.GetOption('main', 'EditorWindow', 'font-bold', type='bool'): + fontWeight='bold' + text.config(font=(idleConf.GetOption('main', 'EditorWindow', 'font'), + idleConf.GetOption('main', 'EditorWindow', 'font-size'), + fontWeight)) text_frame.pack(side=LEFT, fill=BOTH, expand=1) text.pack(side=TOP, fill=BOTH, expand=1) text.focus_set() @@ -247,8 +268,7 @@ # Although use-spaces=0 can be configured manually in config-main.def, # configuration of tabs v. spaces is not supported in the configuration # dialog. IDLE promotes the preferred Python indentation: use spaces! - usespaces = idleConf.GetOption('main', 'Indent', - 'use-spaces', type='bool') + usespaces = idleConf.GetOption('main', 'Indent', 'use-spaces', type='bool') self.usetabs = not usespaces # tabwidth is the display width of a literal tab character. @@ -292,10 +312,11 @@ self.good_load = True is_py_src = self.ispythonsource(filename) self.set_indentation_params(is_py_src) + if is_py_src: + self.color = color = self.ColorDelegator() + per.insertfilter(color) else: io.set_filename(filename) - self.good_load = True - self.ResetColorizer() self.saved_change_hook() self.update_recent_files_list() @@ -317,19 +338,19 @@ self.showerror = tkMessageBox.showerror def _filename_to_unicode(self, filename): - """Return filename as BMP unicode so diplayable in Tk.""" - # Decode bytes to unicode. - if isinstance(filename, bytes): + """convert filename to unicode in order to display it in Tk""" + if isinstance(filename, str) or not filename: + return filename + else: try: - filename = filename.decode(self.filesystemencoding) + return filename.decode(self.filesystemencoding) except UnicodeDecodeError: + # XXX try: - filename = filename.decode(self.encoding) + return filename.decode(self.encoding) except UnicodeDecodeError: # byte-to-byte conversion - filename = filename.decode('iso8859-1') - # Replace non-BMP char with diamond questionmark. - return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename) + return filename.decode('iso8859-1') def new_callback(self, event): dirname, basename = self.io.defaultfilename() @@ -361,11 +382,9 @@ self.text.tag_remove("sel", "1.0", "end") else: if not self.text.index("sel.first"): - # there was no previous selection - self.text.mark_set("my_anchor", "insert") + self.text.mark_set("my_anchor", "insert") # there was no previous selection else: - if self.text.compare(self.text.index("sel.first"), "<", - self.text.index("insert")): + if self.text.compare(self.text.index("sel.first"), "<", self.text.index("insert")): self.text.mark_set("my_anchor", "sel.first") # extend back else: self.text.mark_set("my_anchor", "sel.last") # extend forward @@ -381,15 +400,13 @@ def set_status_bar(self): self.status_bar = self.MultiStatusBar(self.top) - sep = Frame(self.top, height=1, borderwidth=1, background='grey75') - if sys.platform == "darwin": + if macosxSupport.runningAsOSXApp(): # Insert some padding to avoid obscuring some of the statusbar # by the resize widget. self.status_bar.set_label('_padding1', ' ', side=RIGHT) self.status_bar.set_label('column', 'Col: ?', side=RIGHT) self.status_bar.set_label('line', 'Ln: ?', side=RIGHT) self.status_bar.pack(side=BOTTOM, fill=X) - sep.pack(side=BOTTOM, fill=X) self.text.bind("<>", self.set_line_and_column) self.text.event_add("<>", "", "") @@ -406,25 +423,28 @@ ("format", "F_ormat"), ("run", "_Run"), ("options", "_Options"), - ("windows", "_Window"), + ("windows", "_Windows"), ("help", "_Help"), ] + if macosxSupport.runningAsOSXApp(): + del menu_specs[-3] + menu_specs[-2] = ("windows", "_Window") + def createmenubar(self): mbar = self.menubar self.menudict = menudict = {} for name, label in self.menu_specs: underline, label = prepstr(label) - menudict[name] = menu = Menu(mbar, name=name, tearoff=0) + menudict[name] = menu = Menu(mbar, name=name) mbar.add_cascade(label=label, menu=menu, underline=underline) - if macosxSupport.isCarbonTk(): + if macosxSupport.isCarbonAquaTk(self.root): # Insert the application menu - menudict['application'] = menu = Menu(mbar, name='apple', - tearoff=0) + menudict['application'] = menu = Menu(mbar, name='apple') mbar.add_cascade(label='IDLE', menu=menu) self.fill_menus() - self.recent_files_menu = Menu(self.menubar, tearoff=0) + self.recent_files_menu = Menu(self.menubar) self.menudict['file'].insert_cascade(3, label='Recent Files', underline=0, menu=self.recent_files_menu) @@ -444,6 +464,7 @@ rmenu = None def right_menu_event(self, event): + self.text.tag_remove("sel", "1.0", "end") self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) if not self.rmenu: self.make_rmenu() @@ -452,83 +473,41 @@ iswin = sys.platform[:3] == 'win' if iswin: self.text.config(cursor="arrow") - - for item in self.rmenu_specs: - try: - label, eventname, verify_state = item - except ValueError: # see issue1207589 - continue - - if verify_state is None: - continue - state = getattr(self, verify_state)() - rmenu.entryconfigure(label, state=state) - - rmenu.tk_popup(event.x_root, event.y_root) if iswin: self.text.config(cursor="ibeam") rmenu_specs = [ - # ("Label", "<>", "statefuncname"), ... - ("Close", "<>", None), # Example + # ("Label", "<>"), ... + ("Close", "<>"), # Example ] def make_rmenu(self): rmenu = Menu(self.text, tearoff=0) - for item in self.rmenu_specs: - label, eventname = item[0], item[1] - if label is not None: - def command(text=self.text, eventname=eventname): - text.event_generate(eventname) - rmenu.add_command(label=label, command=command) - else: - rmenu.add_separator() + for label, eventname in self.rmenu_specs: + def command(text=self.text, eventname=eventname): + text.event_generate(eventname) + rmenu.add_command(label=label, command=command) self.rmenu = rmenu - def rmenu_check_cut(self): - return self.rmenu_check_copy() - - def rmenu_check_copy(self): - try: - indx = self.text.index('sel.first') - except TclError: - return 'disabled' - else: - return 'normal' if indx else 'disabled' - - def rmenu_check_paste(self): - try: - self.text.tk.call('tk::GetSelection', self.text, 'CLIPBOARD') - except TclError: - return 'disabled' - else: - return 'normal' - def about_dialog(self, event=None): - "Handle Help 'About IDLE' event." - # Synchronize with macosxSupport.overrideRootMenu.about_dialog. aboutDialog.AboutDialog(self.top,'About IDLE') def config_dialog(self, event=None): - "Handle Options 'Configure IDLE' event." - # Synchronize with macosxSupport.overrideRootMenu.config_dialog. configDialog.ConfigDialog(self.top,'Settings') def help_dialog(self, event=None): - "Handle Help 'IDLE Help' event." - # Synchronize with macosxSupport.overrideRootMenu.help_dialog. if self.root: parent = self.root else: parent = self.top - help.show_idlehelp(parent) + helpDialog.display(parent, near=self.top) def python_docs(self, event=None): if sys.platform[:3] == 'win': try: os.startfile(self.help_url) - except OSError as why: + except WindowsError as why: tkMessageBox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: @@ -639,38 +618,30 @@ return # XXX Ought to insert current file's directory in front of path try: - spec = importlib.util.find_spec(name) - except (ValueError, ImportError) as msg: + (f, file, (suffix, mode, type)) = _find_module(name) + except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return - if spec is None: - tkMessageBox.showerror("Import error", "module not found", - parent=self.text) + if type != imp.PY_SOURCE: + tkMessageBox.showerror("Unsupported type", + "%s is not a source module" % name, parent=self.text) return - if not isinstance(spec.loader, importlib.abc.SourceLoader): - tkMessageBox.showerror("Import error", "not a source-based module", - parent=self.text) - return - try: - file_path = spec.loader.get_filename(name) - except AttributeError: - tkMessageBox.showerror("Import error", - "loader does not support get_filename", - parent=self.text) - return + if f: + f.close() if self.flist: - self.flist.open(file_path) + self.flist.open(file) else: - self.io.loadfile(file_path) - return file_path + self.io.loadfile(file) def open_class_browser(self, event=None): filename = self.io.filename - if not (self.__class__.__name__ == 'PyShellEditorWindow' - and filename): - filename = self.open_module() - if filename is None: - return + if not filename: + tkMessageBox.showerror( + "No filename", + "This buffer has no associated filename", + master=self.text) + self.text.focus_set() + return None head, tail = os.path.split(filename) base, ext = os.path.splitext(tail) from idlelib import ClassBrowser @@ -680,14 +651,6 @@ from idlelib import PathBrowser PathBrowser.PathBrowser(self.flist) - def open_turtle_demo(self, event = None): - import subprocess - - cmd = [sys.executable, - '-c', - 'from turtledemo.__main__ import main; main()'] - subprocess.Popen(cmd, shell=False) - def gotoline(self, lineno): if lineno is not None and lineno > 0: self.text.mark_set("insert", "%d.0" % lineno) @@ -738,11 +701,11 @@ self.color = None def ResetColorizer(self): - "Update the color theme" + "Update the colour theme" # Called from self.filename_change_hook and from configDialog.py self._rmcolorizer() self._addcolorizer() - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') normal_colors = idleConf.GetHighlight(theme, 'normal') cursor_color = idleConf.GetHighlight(theme, 'cursor', fgBg='fg') select_colors = idleConf.GetHighlight(theme, 'hilite') @@ -753,9 +716,6 @@ selectforeground=select_colors['foreground'], selectbackground=select_colors['background'], ) - if TkVersion >= 8.5: - self.text.config( - inactiveselectbackground=select_colors['background']) IDENTCHARS = string.ascii_letters + string.digits + "_" @@ -773,8 +733,12 @@ def ResetFont(self): "Update the text widgets' font if it is changed" # Called from configDialog.py - - self.text['font'] = idleConf.GetFont(self.root, 'main','EditorWindow') + fontWeight='normal' + if idleConf.GetOption('main','EditorWindow','font-bold',type='bool'): + fontWeight='bold' + self.text.config(font=(idleConf.GetOption('main','EditorWindow','font'), + idleConf.GetOption('main','EditorWindow','font-size'), + fontWeight)) def RemoveKeybindings(self): "Remove the keybindings before they are changed." @@ -806,11 +770,7 @@ menuEventDict[menu[0]][prepstr(item[0])[1]] = item[1] for menubarItem in self.menudict: menu = self.menudict[menubarItem] - end = menu.index(END) - if end is None: - # Skip empty menus - continue - end += 1 + end = menu.index(END) + 1 for index in range(0, end): if menu.type(index) == 'command': accel = menu.entrycget(index, 'accelerator') @@ -856,7 +816,7 @@ if sys.platform[:3] == 'win': try: os.startfile(helpfile) - except OSError as why: + except WindowsError as why: tkMessageBox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: @@ -867,9 +827,12 @@ "Load and update the recent files list and menus" rf_list = [] if os.path.exists(self.recent_files_path): - with open(self.recent_files_path, 'r', - encoding='utf_8', errors='replace') as rf_list_file: + rf_list_file = open(self.recent_files_path,'r', + encoding='utf_8', errors='replace') + try: rf_list = rf_list_file.readlines() + finally: + rf_list_file.close() if new_file: new_file = os.path.abspath(new_file) + '\n' if new_file in rf_list: @@ -887,19 +850,17 @@ with open(self.recent_files_path, 'w', encoding='utf_8', errors='replace') as rf_file: rf_file.writelines(rf_list) - except OSError as err: + except IOError as err: if not getattr(self.root, "recentfilelist_error_displayed", False): self.root.recentfilelist_error_displayed = True - tkMessageBox.showwarning(title='IDLE Warning', - message="Cannot update File menu Recent Files list. " - "Your operating system says:\n%s\n" - "Select OK and IDLE will continue without updating." - % self._filename_to_unicode(str(err)), + tkMessageBox.showerror(title='IDLE Error', + message='Unable to update Recent Files list:\n%s' + % str(err), parent=self.text) # for each edit window instance, construct the recent files menu for instance in self.top.instance_dict: menu = instance.recent_files_menu - menu.delete(0, END) # clear, and rebuild: + menu.delete(1, END) # clear, and rebuild: for i, file_name in enumerate(rf_list): file_name = file_name.rstrip() # zap \n # make unicode string to display non-ASCII chars correctly @@ -918,7 +879,7 @@ short = self.short_title() long = self.long_title() if short and long: - title = short + " - " + long + _py_version + title = short + " - " + long elif short: title = short elif long: @@ -945,8 +906,6 @@ filename = self.io.filename if filename: filename = os.path.basename(filename) - else: - filename = "Untitled" # return unicode string to display non-ASCII chars correctly return self._filename_to_unicode(filename) @@ -1046,10 +1005,7 @@ def load_extension(self, name): try: - try: - mod = importlib.import_module('.' + name, package=__package__) - except (ImportError, TypeError): - mod = importlib.import_module(name) + mod = __import__(name, globals(), locals(), []) except ImportError: print("\nFailed to import extension: ", name) raise @@ -1382,7 +1338,7 @@ text.see("insert") text.undo_block_stop() - # Our editwin provides an is_char_in_string function that works + # Our editwin provides a is_char_in_string function that works # with a Tk text index, but PyParse only knows about offsets into # a string. This builds a function for PyParse that accepts an # offset. @@ -1438,7 +1394,6 @@ def tabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() - if tabwidth is None: return for pos in range(len(lines)): line = lines[pos] if line: @@ -1450,7 +1405,6 @@ def untabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() - if tabwidth is None: return for pos in range(len(lines)): lines[pos] = lines[pos].expandtabs(tabwidth) self.set_region(head, tail, chars, lines) @@ -1544,7 +1498,7 @@ parent=self.text, initialvalue=self.indentwidth, minvalue=2, - maxvalue=16) + maxvalue=16) or self.tabwidth # Guess indentwidth from text content. # Return guessed indentwidth. This should not be believed unless @@ -1628,7 +1582,7 @@ tokens = _tokenize.generate_tokens(self.readline) for token in tokens: self.tokeneater(*token) - except (_tokenize.TokenError, SyntaxError): + except _tokenize.TokenError: # since we cut off the tokenizer early, we can trigger # spurious errors pass @@ -1657,7 +1611,7 @@ keylist = keydefs.get(eventname) # issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5 # if not keylist: - if (not keylist) or (macosxSupport.isCocoaTk() and eventname in { + if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in { "<>", "<>", "<>"}): @@ -1684,20 +1638,19 @@ tk.call('set', 'tcl_nonwordchars', '[^a-zA-Z0-9_]') -def _editor_window(parent): # htest # - # error if close master window first - timer event, after script - root = parent +def test(): + root = Tk() fixwordbreaks(root) + root.withdraw() if sys.argv[1:]: filename = sys.argv[1] else: filename = None - macosxSupport.setupApp(root, None) edit = EditorWindow(root=root, filename=filename) + edit.set_close_hook(root.quit) edit.text.bind("<>", edit.close_event) - # Does not stop error, neither does following - # edit.text.bind("<>", edit.close_event) + root.mainloop() + root.destroy() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_editor_window) + test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/FileList.py --- a/Lib/idlelib/FileList.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/FileList.py Mon Jan 25 17:05:13 2016 +0100 @@ -103,7 +103,7 @@ if not os.path.isabs(filename): try: pwd = os.getcwd() - except OSError: + except os.error: pass else: filename = os.path.join(pwd, filename) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/FormatParagraph.py --- a/Lib/idlelib/FormatParagraph.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/FormatParagraph.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,19 +1,18 @@ -"""Extension to format a paragraph or selection to a max width. +# Extension to format a paragraph -Does basic, standard text formatting, and also understands Python -comment blocks. Thus, for editing Python source code, this -extension is really only suitable for reformatting these comment -blocks or triple-quoted strings. +# Does basic, standard text formatting, and also understands Python +# comment blocks. Thus, for editing Python source code, this +# extension is really only suitable for reformatting these comment +# blocks or triple-quoted strings. -Known problems with comment reformatting: -* If there is a selection marked, and the first line of the - selection is not complete, the block will probably not be detected - as comments, and will have the normal "text formatting" rules - applied. -* If a comment block has leading whitespace that mixes tabs and - spaces, they will not be considered part of the same block. -* Fancy comments, like this bulleted list, aren't handled :-) -""" +# Known problems with comment reformatting: +# * If there is a selection marked, and the first line of the +# selection is not complete, the block will probably not be detected +# as comments, and will have the normal "text formatting" rules +# applied. +# * If a comment block has leading whitespace that mixes tabs and +# spaces, they will not be considered part of the same block. +# * Fancy comments, like this bulleted list, arent handled :-) import re from idlelib.configHandler import idleConf @@ -32,37 +31,41 @@ def close(self): self.editwin = None - def format_paragraph_event(self, event, limit=None): - """Formats paragraph to a max width specified in idleConf. - - If text is selected, format_paragraph_event will start breaking lines - at the max width, starting from the beginning selection. - - If no text is selected, format_paragraph_event uses the current - cursor location to determine the paragraph (lines of text surrounded - by blank lines) and formats it. - - The length limit parameter is for testing with a known value. - """ - if limit is None: - # The default length limit is that defined by pep8 - limit = idleConf.GetOption( - 'extensions', 'FormatParagraph', 'max-width', - type='int', default=72) + def format_paragraph_event(self, event): + maxformatwidth = int(idleConf.GetOption('main','FormatParagraph','paragraph')) text = self.editwin.text first, last = self.editwin.get_selection_indices() if first and last: data = text.get(first, last) - comment_header = get_comment_header(data) + comment_header = '' else: first, last, comment_header, data = \ find_paragraph(text, text.index("insert")) if comment_header: - newdata = reformat_comment(data, limit, comment_header) + # Reformat the comment lines - convert to text sans header. + lines = data.split("\n") + lines = map(lambda st, l=len(comment_header): st[l:], lines) + data = "\n".join(lines) + # Reformat to maxformatwidth chars or a 20 char width, whichever is greater. + format_width = max(maxformatwidth - len(comment_header), 20) + newdata = reformat_paragraph(data, format_width) + # re-split and re-insert the comment header. + newdata = newdata.split("\n") + # If the block ends in a \n, we dont want the comment + # prefix inserted after it. (Im not sure it makes sense to + # reformat a comment block that isnt made of complete + # lines, but whatever!) Can't think of a clean solution, + # so we hack away + block_suffix = "" + if not newdata[-1]: + block_suffix = "\n" + newdata = newdata[:-1] + builder = lambda item, prefix=comment_header: prefix+item + newdata = '\n'.join(map(builder, newdata)) + block_suffix else: - newdata = reformat_paragraph(data, limit) + # Just a normal text format + newdata = reformat_paragraph(data, maxformatwidth) text.tag_remove("sel", "1.0", "end") - if newdata != data: text.mark_set("insert", first) text.undo_block_start() @@ -75,44 +78,31 @@ return "break" def find_paragraph(text, mark): - """Returns the start/stop indices enclosing the paragraph that mark is in. - - Also returns the comment format string, if any, and paragraph of text - between the start/stop indices. - """ lineno, col = map(int, mark.split(".")) - line = text.get("%d.0" % lineno, "%d.end" % lineno) - - # Look for start of next paragraph if the index passed in is a blank line + line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) + line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) first_lineno = lineno comment_header = get_comment_header(line) comment_header_len = len(comment_header) - - # Once start line found, search for end of paragraph (a blank line) while get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) + line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) last = "%d.0" % lineno - - # Search back to beginning of paragraph (first blank line before) + # Search back to beginning of paragraph lineno = first_lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) + line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) while lineno > 0 and \ get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) + line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) first = "%d.0" % (lineno+1) - return first, last, comment_header, text.get(first, last) -# This should perhaps be replaced with textwrap.wrap def reformat_paragraph(data, limit): - """Return data reformatted to specified width (limit).""" lines = data.split("\n") i = 0 n = len(lines) @@ -135,7 +125,7 @@ if not word: continue # Can happen when line ends in whitespace if len((partial + word).expandtabs()) > limit and \ - partial != indent1: + partial != indent1: new.append(partial.rstrip()) partial = indent2 partial = partial + word + " " @@ -147,49 +137,13 @@ new.extend(lines[i:]) return "\n".join(new) -def reformat_comment(data, limit, comment_header): - """Return data reformatted to specified width with comment header.""" - - # Remove header from the comment lines - lc = len(comment_header) - data = "\n".join(line[lc:] for line in data.split("\n")) - # Reformat to maxformatwidth chars or a 20 char width, - # whichever is greater. - format_width = max(limit - len(comment_header), 20) - newdata = reformat_paragraph(data, format_width) - # re-split and re-insert the comment header. - newdata = newdata.split("\n") - # If the block ends in a \n, we dont want the comment prefix - # inserted after it. (Im not sure it makes sense to reformat a - # comment block that is not made of complete lines, but whatever!) - # Can't think of a clean solution, so we hack away - block_suffix = "" - if not newdata[-1]: - block_suffix = "\n" - newdata = newdata[:-1] - return '\n'.join(comment_header+line for line in newdata) + block_suffix - def is_all_white(line): - """Return True if line is empty or all whitespace.""" - return re.match(r"^\s*$", line) is not None def get_indent(line): - """Return the initial space or tab indent of line.""" - return re.match(r"^([ \t]*)", line).group() + return re.match(r"^(\s*)", line).group() def get_comment_header(line): - """Return string with leading whitespace and '#' from line or ''. - - A null return indicates that the line is not a comment line. A non- - null return, such as ' #', will be used to find the other lines of - a comment block with the same indent. - """ - m = re.match(r"^([ \t]*#*)", line) + m = re.match(r"^(\s*#*)", line) if m is None: return "" return m.group(1) - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_formatparagraph', - verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/GrepDialog.py --- a/Lib/idlelib/GrepDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/GrepDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,13 +1,9 @@ import os import fnmatch -import re # for htest import sys -from tkinter import StringVar, BooleanVar, Checkbutton # for GrepDialog -from tkinter import Tk, Text, Button, SEL, END # for htest +from tkinter import * from idlelib import SearchEngine from idlelib.SearchDialogBase import SearchDialogBase -# Importing OutputWindow fails due to import loop -# EditorWindow -> GrepDialop -> OutputWindow -> EditorWindow def grep(text, io=None, flist=None): root = text._root() @@ -44,10 +40,10 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.globent = self.make_entry("In files:", self.globvar)[0] + self.globent = self.make_entry("In files:", self.globvar) def create_other_buttons(self): - f = self.make_frame()[0] + f = self.make_frame() btn = Checkbutton(f, anchor="w", variable=self.recvar, @@ -67,7 +63,7 @@ if not path: self.top.bell() return - from idlelib.OutputWindow import OutputWindow # leave here! + from idlelib.OutputWindow import OutputWindow save = sys.stdout try: sys.stdout = OutputWindow(self.flist) @@ -83,31 +79,38 @@ pat = self.engine.getpat() print("Searching %r in %s ..." % (pat, path)) hits = 0 - try: - for fn in list: - try: - with open(fn, errors='replace') as f: - for lineno, line in enumerate(f, 1): - if line[-1:] == '\n': - line = line[:-1] - if prog.search(line): - sys.stdout.write("%s: %s: %s\n" % - (fn, lineno, line)) - hits += 1 - except OSError as msg: - print(msg) - print(("Hits found: %s\n" - "(Hint: right-click to open locations.)" - % hits) if hits else "No hits.") - except AttributeError: - # Tk window has been closed, OutputWindow.text = None, - # so in OW.write, OW.text.insert fails. - pass + for fn in list: + try: + f = open(fn) + except IOError as msg: + print(msg) + continue + lineno = 0 + while 1: + block = f.readlines(100000) + if not block: + break + for line in block: + lineno = lineno + 1 + if line[-1:] == '\n': + line = line[:-1] + if prog.search(line): + sys.stdout.write("%s: %s: %s\n" % (fn, lineno, line)) + hits = hits + 1 + if hits: + if hits == 1: + s = "" + else: + s = "s" + print("Found", hits, "hit%s." % s) + print("(Hint: right-click to open locations.)") + else: + print("No hits.") def findfiles(self, dir, base, rec): try: names = os.listdir(dir or os.curdir) - except OSError as msg: + except os.error as msg: print(msg) return [] list = [] @@ -128,31 +131,3 @@ if self.top: self.top.grab_release() self.top.withdraw() - - -def _grep_dialog(parent): # htest # - from idlelib.PyShell import PyShellFileList - root = Tk() - root.title("Test GrepDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - - flist = PyShellFileList(root) - text = Text(root, height=5) - text.pack() - - def show_grep_dialog(): - text.tag_add(SEL, "1.0", END) - grep(text, flist=flist) - text.tag_remove(SEL, "1.0", END) - - button = Button(root, text="Show GrepDialog", command=show_grep_dialog) - button.pack() - root.mainloop() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(_grep_dialog) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/HISTORY.txt --- a/Lib/idlelib/HISTORY.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/HISTORY.txt Mon Jan 25 17:05:13 2016 +0100 @@ -11,7 +11,7 @@ *Release date: 22-Jul-2001* - New tarball released as a result of the 'revitalisation' of the IDLEfork - project. + project. - This release requires python 2.1 or better. Compatibility with earlier versions of python (especially ancient ones like 1.5x) is no longer a @@ -26,8 +26,8 @@ not working, but I believe this was the case with the previous IDLE fork release (0.7.1) as well. -- This release is being made now to mark the point at which IDLEfork is - launching into a new stage of development. +- This release is being made now to mark the point at which IDLEfork is + launching into a new stage of development. - IDLEfork CVS will now be branched to enable further development and exploration of the two "execution in a remote process" patches submitted by @@ -96,7 +96,7 @@ instead of the IDLE help; shift-TAB is now a synonym for unindent. - New modules: - + ExecBinding.py Executes program through loader loader.py Bootstraps user program protocol.py RPC protocol diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/HyperParser.py --- a/Lib/idlelib/HyperParser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/HyperParser.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,31 +1,23 @@ -"""Provide advanced parsing abilities for ParenMatch and other extensions. - -HyperParser uses PyParser. PyParser mostly gives information on the -proper indentation of code. HyperParser gives additional information on -the structure of code. +""" +HyperParser +=========== +This module defines the HyperParser class, which provides advanced parsing +abilities for the ParenMatch and other extensions. +The HyperParser uses PyParser. PyParser is intended mostly to give information +on the proper indentation of code. HyperParser gives some information on the +structure of code, used by extensions to help the user. """ import string -from keyword import iskeyword +import keyword from idlelib import PyParse +class HyperParser: -# all ASCII chars that may be in an identifier -_ASCII_ID_CHARS = frozenset(string.ascii_letters + string.digits + "_") -# all ASCII chars that may be the first char of an identifier -_ASCII_ID_FIRST_CHARS = frozenset(string.ascii_letters + "_") - -# lookup table for whether 7-bit ASCII chars are valid in a Python identifier -_IS_ASCII_ID_CHAR = [(chr(x) in _ASCII_ID_CHARS) for x in range(128)] -# lookup table for whether 7-bit ASCII chars are valid as the first -# char in a Python identifier -_IS_ASCII_ID_FIRST_CHAR = \ - [(chr(x) in _ASCII_ID_FIRST_CHARS) for x in range(128)] - - -class HyperParser: def __init__(self, editwin, index): - "To initialize, analyze the surroundings of the given index." + """Initialize the HyperParser to analyze the surroundings of the given + index. + """ self.editwin = editwin self.text = text = editwin.text @@ -41,10 +33,9 @@ startat = max(lno - context, 1) startatindex = repr(startat) + ".0" stopatindex = "%d.end" % lno - # We add the newline because PyParse requires a newline - # at end. We add a space so that index won't be at end - # of line, so that its status will be the same as the - # char before it, if should. + # We add the newline because PyParse requires a newline at end. + # We add a space so that index won't be at end of line, so that + # its status will be the same as the char before it, if should. parser.set_str(text.get(startatindex, stopatindex)+' \n') bod = parser.find_good_parse_start( editwin._build_char_in_string_func(startatindex)) @@ -58,175 +49,122 @@ else: startatindex = "1.0" stopatindex = "%d.end" % lno - # We add the newline because PyParse requires it. We add a - # space so that index won't be at end of line, so that its - # status will be the same as the char before it, if should. + # We add the newline because PyParse requires a newline at end. + # We add a space so that index won't be at end of line, so that + # its status will be the same as the char before it, if should. parser.set_str(text.get(startatindex, stopatindex)+' \n') parser.set_lo(0) - # We want what the parser has, minus the last newline and space. + # We want what the parser has, except for the last newline and space. self.rawtext = parser.str[:-2] - # Parser.str apparently preserves the statement we are in, so - # that stopatindex can be used to synchronize the string with - # the text box indices. + # As far as I can see, parser.str preserves the statement we are in, + # so that stopatindex can be used to synchronize the string with the + # text box indices. self.stopatindex = stopatindex self.bracketing = parser.get_last_stmt_bracketing() - # find which pairs of bracketing are openers. These always - # correspond to a character of rawtext. - self.isopener = [i>0 and self.bracketing[i][1] > - self.bracketing[i-1][1] + # find which pairs of bracketing are openers. These always correspond + # to a character of rawtext. + self.isopener = [i>0 and self.bracketing[i][1] > self.bracketing[i-1][1] for i in range(len(self.bracketing))] self.set_index(index) def set_index(self, index): - """Set the index to which the functions relate. - - The index must be in the same statement. + """Set the index to which the functions relate. Note that it must be + in the same statement. """ - indexinrawtext = (len(self.rawtext) - - len(self.text.get(index, self.stopatindex))) + indexinrawtext = \ + len(self.rawtext) - len(self.text.get(index, self.stopatindex)) if indexinrawtext < 0: - raise ValueError("Index %s precedes the analyzed statement" - % index) + raise ValueError("The index given is before the analyzed statement") self.indexinrawtext = indexinrawtext # find the rightmost bracket to which index belongs self.indexbracket = 0 - while (self.indexbracket < len(self.bracketing)-1 and - self.bracketing[self.indexbracket+1][0] < self.indexinrawtext): + while self.indexbracket < len(self.bracketing)-1 and \ + self.bracketing[self.indexbracket+1][0] < self.indexinrawtext: self.indexbracket += 1 - if (self.indexbracket < len(self.bracketing)-1 and - self.bracketing[self.indexbracket+1][0] == self.indexinrawtext and - not self.isopener[self.indexbracket+1]): + if self.indexbracket < len(self.bracketing)-1 and \ + self.bracketing[self.indexbracket+1][0] == self.indexinrawtext and \ + not self.isopener[self.indexbracket+1]: self.indexbracket += 1 def is_in_string(self): - """Is the index given to the HyperParser in a string?""" + """Is the index given to the HyperParser is in a string?""" # The bracket to which we belong should be an opener. # If it's an opener, it has to have a character. - return (self.isopener[self.indexbracket] and - self.rawtext[self.bracketing[self.indexbracket][0]] - in ('"', "'")) + return self.isopener[self.indexbracket] and \ + self.rawtext[self.bracketing[self.indexbracket][0]] in ('"', "'") def is_in_code(self): - """Is the index given to the HyperParser in normal code?""" - return (not self.isopener[self.indexbracket] or - self.rawtext[self.bracketing[self.indexbracket][0]] - not in ('#', '"', "'")) + """Is the index given to the HyperParser is in a normal code?""" + return not self.isopener[self.indexbracket] or \ + self.rawtext[self.bracketing[self.indexbracket][0]] not in \ + ('#', '"', "'") def get_surrounding_brackets(self, openers='([{', mustclose=False): - """Return bracket indexes or None. - - If the index given to the HyperParser is surrounded by a - bracket defined in openers (or at least has one before it), - return the indices of the opening bracket and the closing - bracket (or the end of line, whichever comes first). - - If it is not surrounded by brackets, or the end of line comes - before the closing bracket and mustclose is True, returns None. + """If the index given to the HyperParser is surrounded by a bracket + defined in openers (or at least has one before it), return the + indices of the opening bracket and the closing bracket (or the + end of line, whichever comes first). + If it is not surrounded by brackets, or the end of line comes before + the closing bracket and mustclose is True, returns None. """ - bracketinglevel = self.bracketing[self.indexbracket][1] before = self.indexbracket - while (not self.isopener[before] or - self.rawtext[self.bracketing[before][0]] not in openers or - self.bracketing[before][1] > bracketinglevel): + while not self.isopener[before] or \ + self.rawtext[self.bracketing[before][0]] not in openers or \ + self.bracketing[before][1] > bracketinglevel: before -= 1 if before < 0: return None bracketinglevel = min(bracketinglevel, self.bracketing[before][1]) after = self.indexbracket + 1 - while (after < len(self.bracketing) and - self.bracketing[after][1] >= bracketinglevel): + while after < len(self.bracketing) and \ + self.bracketing[after][1] >= bracketinglevel: after += 1 beforeindex = self.text.index("%s-%dc" % (self.stopatindex, len(self.rawtext)-self.bracketing[before][0])) - if (after >= len(self.bracketing) or - self.bracketing[after][0] > len(self.rawtext)): + if after >= len(self.bracketing) or \ + self.bracketing[after][0] > len(self.rawtext): if mustclose: return None afterindex = self.stopatindex else: - # We are after a real char, so it is a ')' and we give the - # index before it. - afterindex = self.text.index( - "%s-%dc" % (self.stopatindex, + # We are after a real char, so it is a ')' and we give the index + # before it. + afterindex = self.text.index("%s-%dc" % + (self.stopatindex, len(self.rawtext)-(self.bracketing[after][0]-1))) return beforeindex, afterindex - # the set of built-in identifiers which are also keywords, - # i.e. keyword.iskeyword() returns True for them - _ID_KEYWORDS = frozenset({"True", "False", "None"}) + # This string includes all chars that may be in a white space + _whitespace_chars = " \t\n\\" + # This string includes all chars that may be in an identifier + _id_chars = string.ascii_letters + string.digits + "_" + # This string includes all chars that may be the first char of an identifier + _id_first_chars = string.ascii_letters + "_" - @classmethod - def _eat_identifier(cls, str, limit, pos): - """Given a string and pos, return the number of chars in the - identifier which ends at pos, or 0 if there is no such one. - - This ignores non-identifier eywords are not identifiers. - """ - is_ascii_id_char = _IS_ASCII_ID_CHAR - - # Start at the end (pos) and work backwards. + # Given a string and pos, return the number of chars in the identifier + # which ends at pos, or 0 if there is no such one. Saved words are not + # identifiers. + def _eat_identifier(self, str, limit, pos): i = pos - - # Go backwards as long as the characters are valid ASCII - # identifier characters. This is an optimization, since it - # is faster in the common case where most of the characters - # are ASCII. - while i > limit and ( - ord(str[i - 1]) < 128 and - is_ascii_id_char[ord(str[i - 1])] - ): + while i > limit and str[i-1] in self._id_chars: i -= 1 - - # If the above loop ended due to reaching a non-ASCII - # character, continue going backwards using the most generic - # test for whether a string contains only valid identifier - # characters. - if i > limit and ord(str[i - 1]) >= 128: - while i - 4 >= limit and ('a' + str[i - 4:pos]).isidentifier(): - i -= 4 - if i - 2 >= limit and ('a' + str[i - 2:pos]).isidentifier(): - i -= 2 - if i - 1 >= limit and ('a' + str[i - 1:pos]).isidentifier(): - i -= 1 - - # The identifier candidate starts here. If it isn't a valid - # identifier, don't eat anything. At this point that is only - # possible if the first character isn't a valid first - # character for an identifier. - if not str[i:pos].isidentifier(): - return 0 - elif i < pos: - # All characters in str[i:pos] are valid ASCII identifier - # characters, so it is enough to check that the first is - # valid as the first character of an identifier. - if not _IS_ASCII_ID_FIRST_CHAR[ord(str[i])]: - return 0 - - # All keywords are valid identifiers, but should not be - # considered identifiers here, except for True, False and None. - if i < pos and ( - iskeyword(str[i:pos]) and - str[i:pos] not in cls._ID_KEYWORDS - ): - return 0 - + if i < pos and (str[i] not in self._id_first_chars or \ + keyword.iskeyword(str[i:pos])): + i = pos return pos - i - # This string includes all chars that may be in a white space - _whitespace_chars = " \t\n\\" - def get_expression(self): - """Return a string with the Python expression which ends at the - given index, which is empty if there is no real one. + """Return a string with the Python expression which ends at the given + index, which is empty if there is no real one. """ if not self.is_in_code(): - raise ValueError("get_expression should only be called" - "if index is inside a code.") + raise ValueError("get_expression should only be called if index "\ + "is inside a code.") rawtext = self.rawtext bracketing = self.bracketing @@ -239,20 +177,20 @@ postdot_phase = True while 1: - # Eat whitespaces, comments, and if postdot_phase is False - a dot + # Eat whitespaces, comments, and if postdot_phase is False - one dot while 1: if pos>brck_limit and rawtext[pos-1] in self._whitespace_chars: # Eat a whitespace pos -= 1 - elif (not postdot_phase and - pos > brck_limit and rawtext[pos-1] == '.'): + elif not postdot_phase and \ + pos > brck_limit and rawtext[pos-1] == '.': # Eat a dot pos -= 1 postdot_phase = True - # The next line will fail if we are *inside* a comment, - # but we shouldn't be. - elif (pos == brck_limit and brck_index > 0 and - rawtext[bracketing[brck_index-1][0]] == '#'): + # The next line will fail if we are *inside* a comment, but we + # shouldn't be. + elif pos == brck_limit and brck_index > 0 and \ + rawtext[bracketing[brck_index-1][0]] == '#': # Eat a comment brck_index -= 2 brck_limit = bracketing[brck_index][0] @@ -262,8 +200,8 @@ break if not postdot_phase: - # We didn't find a dot, so the expression end at the - # last identifier pos. + # We didn't find a dot, so the expression end at the last + # identifier pos. break ret = self._eat_identifier(rawtext, brck_limit, pos) @@ -271,13 +209,13 @@ # There is an identifier to eat pos = pos - ret last_identifier_pos = pos - # Now, to continue the search, we must find a dot. + # Now, in order to continue the search, we must find a dot. postdot_phase = False # (the loop continues now) elif pos == brck_limit: - # We are at a bracketing limit. If it is a closing - # bracket, eat the bracket, otherwise, stop the search. + # We are at a bracketing limit. If it is a closing bracket, + # eat the bracket, otherwise, stop the search. level = bracketing[brck_index][1] while brck_index > 0 and bracketing[brck_index-1][1] > level: brck_index -= 1 @@ -294,11 +232,6 @@ pass else: # We can't continue after other types of brackets - if rawtext[pos] in "'\"": - # Scan a string prefix - while pos > 0 and rawtext[pos - 1] in "rRbBuU": - pos -= 1 - last_identifier_pos = pos break else: @@ -306,8 +239,3 @@ break return rawtext[last_identifier_pos:self.indexinrawtext] - - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_hyperparser', verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/IOBinding.py --- a/Lib/idlelib/IOBinding.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/IOBinding.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,16 +1,17 @@ +import os +import types +import sys import codecs -from codecs import BOM_UTF8 -import os -import re -import shlex -import sys import tempfile - import tkinter.filedialog as tkFileDialog import tkinter.messagebox as tkMessageBox +import re +from tkinter import * from tkinter.simpledialog import askstring +from idlelib.configHandler import idleConf +from codecs import BOM_UTF8 # Try setting the locale, so that we can find out # what encoding to use @@ -61,8 +62,7 @@ encoding = locale_encoding ### KBK 07Sep07 This is used all over IDLE, check! ### 'encoding' is used below in encode(), check! -coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) +coding_re = re.compile("coding[:=]\s*([-\w_.]+)") def coding_spec(data): """Return the encoding declaration according to PEP 263. @@ -83,18 +83,14 @@ lines = data # consider only the first two lines if '\n' in lines: - lst = lines.split('\n', 2)[:2] + lst = lines.split('\n')[:2] elif '\r' in lines: - lst = lines.split('\r', 2)[:2] + lst = lines.split('\r')[:2] else: - lst = [lines] - for line in lst: - match = coding_re.match(line) - if match is not None: - break - if not blank_re.match(line): - return None - else: + lst = list(lines) + str = '\n'.join(lst) + match = coding_re.search(str) + if not match: return None name = match.group(1) try: @@ -160,33 +156,29 @@ self.filename_change_hook() def open(self, event=None, editFile=None): - flist = self.editwin.flist - # Save in case parent window is closed (ie, during askopenfile()). - if flist: + if self.editwin.flist: if not editFile: filename = self.askopenfile() else: filename=editFile if filename: - # If editFile is valid and already open, flist.open will - # shift focus to its existing window. - # If the current window exists and is a fresh unnamed, - # unmodified editor window (not an interpreter shell), - # pass self.loadfile to flist.open so it will load the file - # in the current window (if the file is not already open) - # instead of a new window. - if (self.editwin and - not getattr(self.editwin, 'interp', None) and - not self.filename and - self.get_saved()): - flist.open(filename, self.loadfile) + # If the current window has no filename and hasn't been + # modified, we replace its contents (no loss). Otherwise + # we open a new window. But we won't replace the + # shell window (which has an interp(reter) attribute), which + # gets set to "not modified" at every new prompt. + try: + interp = self.editwin.interp + except AttributeError: + interp = None + if not self.filename and self.get_saved() and not interp: + self.editwin.flist.open(filename, self.loadfile) else: - flist.open(filename) + self.editwin.flist.open(filename) else: - if self.text: - self.text.focus_set() + self.text.focus_set() return "break" - + # # Code for use outside IDLE: if self.get_saved(): reply = self.maybesave() @@ -211,12 +203,13 @@ try: # open the file in binary mode so that we can handle # end-of-line convention ourselves. - with open(filename, 'rb') as f: - two_lines = f.readline() + f.readline() - f.seek(0) - bytes = f.read() - except OSError as msg: - tkMessageBox.showerror("I/O Error", str(msg), parent=self.text) + f = open(filename,'rb') + two_lines = f.readline() + f.readline() + f.seek(0) + bytes = f.read() + f.close() + except IOError as msg: + tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False chars, converted = self._decode(two_lines, bytes) if chars is None: @@ -265,7 +258,7 @@ title="Error loading the file", message="The encoding '%s' is not known to this Python "\ "installation. The file may not display correctly" % name, - parent = self.text) + master = self.text) enc = None except UnicodeDecodeError: return None, False @@ -320,7 +313,7 @@ title="Save On Close", message=message, default=tkMessageBox.YES, - parent=self.text) + master=self.text) if confirm: reply = "yes" self.save(None) @@ -375,12 +368,14 @@ text = text.replace("\n", self.eol_convention) chars = self.encode(text) try: - with open(filename, "wb") as f: - f.write(chars) + f = open(filename, "wb") + f.write(chars) + f.flush() + f.close() return True - except OSError as msg: + except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), - parent=self.text) + master=self.text) return False def encode(self, chars): @@ -417,7 +412,7 @@ tkMessageBox.showerror( "I/O Error", "%s.\nSaving as UTF-8" % failed, - parent = self.text) + master = self.text) # Fallback: save as UTF-8, with BOM - ignoring the incorrect # declared encoding return BOM_UTF8 + chars.encode("utf-8") @@ -432,7 +427,7 @@ title="Print", message="Print to Default Printer", default=tkMessageBox.OK, - parent=self.text) + master=self.text) if not confirm: self.text.focus_set() return "break" @@ -459,7 +454,7 @@ else: #no printing for this platform printPlatform = False if printPlatform: #we can try to print for this platform - command = command % shlex.quote(filename) + command = command % filename pipe = os.popen(command, "r") # things can get ugly on NT if there is no printer available. output = pipe.read().strip() @@ -469,10 +464,10 @@ status + output if output: output = "Printing command: %s\n" % repr(command) + output - tkMessageBox.showerror("Print status", output, parent=self.text) + tkMessageBox.showerror("Print status", output, master=self.text) else: #no printing for this platform message = "Printing is not enabled for this platform: %s" % platform - tkMessageBox.showinfo("Print status", message, parent=self.text) + tkMessageBox.showinfo("Print status", message, master=self.text) if tempfilename: os.unlink(tempfilename) return "break" @@ -486,12 +481,10 @@ ("All files", "*"), ] - defaultextension = '.py' if sys.platform == 'darwin' else '' - def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: - self.opendialog = tkFileDialog.Open(parent=self.text, + self.opendialog = tkFileDialog.Open(master=self.text, filetypes=self.filetypes) filename = self.opendialog.show(initialdir=dir, initialfile=base) return filename @@ -504,17 +497,15 @@ else: try: pwd = os.getcwd() - except OSError: + except os.error: pwd = "" return pwd, "" def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs( - parent=self.text, - filetypes=self.filetypes, - defaultextension=self.defaultextension) + self.savedialog = tkFileDialog.SaveAs(master=self.text, + filetypes=self.filetypes) filename = self.savedialog.show(initialdir=dir, initialfile=base) return filename @@ -523,20 +514,16 @@ if self.editwin.flist: self.editwin.update_recent_files_list(filename) -def _io_binding(parent): # htest # - from tkinter import Toplevel, Text - from idlelib.configHandler import idleConf - - root = Toplevel(parent) - root.title("Test IOBinding") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +def test(): + root = Tk() class MyEditWin: def __init__(self, text): self.text = text self.flist = None self.text.bind("", self.open) self.text.bind("", self.save) + self.text.bind("", self.save_as) + self.text.bind("", self.save_a_copy) def get_saved(self): return 0 def set_saved(self, flag): pass def reset_undo(self): pass @@ -544,13 +531,16 @@ self.text.event_generate("<>") def save(self, event): self.text.event_generate("<>") - + def save_as(self, event): + self.text.event_generate("<>") + def save_a_copy(self, event): + self.text.event_generate("<>") text = Text(root) text.pack() text.focus_set() editwin = MyEditWin(text) - IOBinding(editwin) + io = IOBinding(editwin) + root.mainloop() if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_io_binding) + test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle.ico Binary file Lib/idlelib/Icons/idle.ico has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_16.gif Binary file Lib/idlelib/Icons/idle_16.gif has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_16.png Binary file Lib/idlelib/Icons/idle_16.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_32.gif Binary file Lib/idlelib/Icons/idle_32.gif has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_32.png Binary file Lib/idlelib/Icons/idle_32.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_48.gif Binary file Lib/idlelib/Icons/idle_48.gif has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/idle_48.png Binary file Lib/idlelib/Icons/idle_48.png has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Icons/python.gif Binary file Lib/idlelib/Icons/python.gif has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/IdleHistory.py --- a/Lib/idlelib/IdleHistory.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/IdleHistory.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,93 +1,81 @@ -"Implement Idle Shell history mechanism with History class" - from idlelib.configHandler import idleConf class History: - ''' Implement Idle Shell history mechanism. - store - Store source statement (called from PyShell.resetoutput). - fetch - Fetch stored statement matching prefix already entered. - history_next - Bound to <> event (default Alt-N). - history_prev - Bound to <> event (default Alt-P). - ''' - def __init__(self, text): - '''Initialize data attributes and bind event methods. - - .text - Idle wrapper of tk Text widget, with .bell(). - .history - source statements, possibly with multiple lines. - .prefix - source already entered at prompt; filters history list. - .pointer - index into history. - .cyclic - wrap around history list (or not). - ''' + def __init__(self, text, output_sep = "\n"): self.text = text self.history = [] - self.prefix = None - self.pointer = None + self.history_prefix = None + self.history_pointer = None + self.output_sep = output_sep self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") text.bind("<>", self.history_prev) text.bind("<>", self.history_next) def history_next(self, event): - "Fetch later statement; start with ealiest if cyclic." - self.fetch(reverse=False) + self.history_do(0) return "break" def history_prev(self, event): - "Fetch earlier statement; start with most recent." - self.fetch(reverse=True) + self.history_do(1) return "break" - def fetch(self, reverse): - '''Fetch statememt and replace current line in text widget. + def _get_source(self, start, end): + # Get source code from start index to end index. Lines in the + # text control may be separated by sys.ps2 . + lines = self.text.get(start, end).split(self.output_sep) + return "\n".join(lines) - Set prefix and pointer as needed for successive fetches. - Reset them to None, None when returning to the start line. - Sound bell when return to start line or cannot leave a line - because cyclic is False. - ''' + def _put_source(self, where, source): + output = self.output_sep.join(source.split("\n")) + self.text.insert(where, output) + + def history_do(self, reverse): nhist = len(self.history) - pointer = self.pointer - prefix = self.prefix + pointer = self.history_pointer + prefix = self.history_prefix if pointer is not None and prefix is not None: if self.text.compare("insert", "!=", "end-1c") or \ - self.text.get("iomark", "end-1c") != self.history[pointer]: + self._get_source("iomark", "end-1c") != self.history[pointer]: pointer = prefix = None - self.text.mark_set("insert", "end-1c") # != after cursor move if pointer is None or prefix is None: - prefix = self.text.get("iomark", "end-1c") + prefix = self._get_source("iomark", "end-1c") if reverse: - pointer = nhist # will be decremented + pointer = nhist else: if self.cyclic: - pointer = -1 # will be incremented - else: # abort history_next + pointer = -1 + else: self.text.bell() return nprefix = len(prefix) while 1: - pointer += -1 if reverse else 1 + if reverse: + pointer = pointer - 1 + else: + pointer = pointer + 1 if pointer < 0 or pointer >= nhist: self.text.bell() - if not self.cyclic and pointer < 0: # abort history_prev + if not self.cyclic and pointer < 0: return else: - if self.text.get("iomark", "end-1c") != prefix: + if self._get_source("iomark", "end-1c") != prefix: self.text.delete("iomark", "end-1c") - self.text.insert("iomark", prefix) + self._put_source("iomark", prefix) pointer = prefix = None break item = self.history[pointer] if item[:nprefix] == prefix and len(item) > nprefix: self.text.delete("iomark", "end-1c") - self.text.insert("iomark", item) + self._put_source("iomark", item) break + self.text.mark_set("insert", "end-1c") self.text.see("insert") self.text.tag_remove("sel", "1.0", "end") - self.pointer = pointer - self.prefix = prefix + self.history_pointer = pointer + self.history_prefix = prefix - def store(self, source): - "Store Shell input statement into history list." + def history_store(self, source): source = source.strip() if len(source) > 2: # avoid duplicates @@ -96,9 +84,5 @@ except ValueError: pass self.history.append(source) - self.pointer = None - self.prefix = None - -if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) + self.history_pointer = None + self.history_prefix = None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/MultiCall.py --- a/Lib/idlelib/MultiCall.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/MultiCall.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,6 +32,7 @@ import sys import re import tkinter +from idlelib import macosxSupport # the event type constants, which define the meaning of mc_type MC_KEYPRESS=0; MC_KEYRELEASE=1; MC_BUTTONPRESS=2; MC_BUTTONRELEASE=3; @@ -44,7 +45,7 @@ MC_OPTION = 1<<6; MC_COMMAND = 1<<7 # define the list of modifiers, to be used in complex event types. -if sys.platform == "darwin": +if macosxSupport.runningAsOSXApp(): _modifiers = (("Shift",), ("Control",), ("Option",), ("Command",)) _modifier_masks = (MC_SHIFT, MC_CONTROL, MC_OPTION, MC_COMMAND) else: @@ -56,12 +57,6 @@ for number in range(len(_modifiers)) for name in _modifiers[number]]) -# In 3.4, if no shell window is ever open, the underlying Tk widget is -# destroyed before .__del__ methods here are called. The following -# is used to selectively ignore shutdown exceptions to avoid -# 'Exception ignored' messages. See http://bugs.python.org/issue20167 -APPLICATION_GONE = "application has been destroyed" - # A binder is a class which binds functions to one type of event. It has two # methods: bind and unbind, which get a function and a parsed sequence, as # returned by _parse_sequence(). There are two types of binders: @@ -103,12 +98,7 @@ def __del__(self): if self.handlerid: - try: - self.widget.unbind(self.widgetinst, self.sequence, - self.handlerid) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise + self.widget.unbind(self.widgetinst, self.sequence, self.handlerid) # An int in range(1 << len(_modifiers)) represents a combination of modifiers # (if the least significent bit is on, _modifiers[0] is on, and so on). @@ -180,9 +170,8 @@ break ishandlerrunning[:] = [] # Call all functions in doafterhandler and remove them from list - for f in doafterhandler: - f() - doafterhandler[:] = [] + while doafterhandler: + doafterhandler.pop()() if r: return r return handler @@ -237,11 +226,7 @@ def __del__(self): for seq, id in self.handlerids: - try: - self.widget.unbind(self.widgetinst, seq, id) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise + self.widget.unbind(self.widgetinst, seq, id) # define the list of event types to be handled by MultiEvent. the order is # compatible with the definition of event type constants. @@ -404,21 +389,15 @@ func, triplets = self.__eventinfo[virtual] if func: for triplet in triplets: - try: - self.__binders[triplet[1]].unbind(triplet, func) - except tkinter.TclError as e: - if not APPLICATION_GONE in e.args[0]: - raise + self.__binders[triplet[1]].unbind(triplet, func) + _multicall_dict[widget] = MultiCall return MultiCall - -def _multi_call(parent): +if __name__ == "__main__": + # Test root = tkinter.Tk() - root.title("Test MultiCall") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) text = MultiCallCreator(tkinter.Text)(root) text.pack() def bindseq(seq, n=[0]): @@ -434,13 +413,8 @@ bindseq("") bindseq("") bindseq("") - bindseq("") bindseq("") bindseq("") bindseq("") bindseq("") root.mainloop() - -if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_multi_call) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/MultiStatusBar.py --- a/Lib/idlelib/MultiStatusBar.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/MultiStatusBar.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,40 +8,25 @@ Frame.__init__(self, master, **kw) self.labels = {} - def set_label(self, name, text='', side=LEFT, width=0): + def set_label(self, name, text='', side=LEFT): if name not in self.labels: - label = Label(self, borderwidth=0, anchor=W) - label.pack(side=side, pady=0, padx=4) + label = Label(self, bd=1, relief=SUNKEN, anchor=W) + label.pack(side=side) self.labels[name] = label else: label = self.labels[name] - if width != 0: - label.config(width=width) label.config(text=text) -def _multistatus_bar(parent): - root = Tk() - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d" %(x, y + 150)) - root.title("Test multistatus bar") - frame = Frame(root) - text = Text(frame) - text.pack() - msb = MultiStatusBar(frame) - msb.set_label("one", "hello") - msb.set_label("two", "world") - msb.pack(side=BOTTOM, fill=X) - - def change(): - msb.set_label("one", "foo") - msb.set_label("two", "bar") - - button = Button(root, text="Update status", command=change) - button.pack(side=BOTTOM) - frame.pack() - frame.mainloop() - root.mainloop() +def _test(): + b = Frame() + c = Text(b) + c.pack(side=TOP) + a = MultiStatusBar(b) + a.set_label("one", "hello") + a.set_label("two", "world") + a.pack(side=BOTTOM, fill=X) + b.pack() + b.mainloop() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_multistatus_bar) + _test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/NEWS.txt --- a/Lib/idlelib/NEWS.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/NEWS.txt Mon Jan 25 17:05:13 2016 +0100 @@ -1,262 +1,13 @@ -What's New in IDLE 3.6.0a1? -=========================== -*Release date: 2017?* - -- Issue 15348: Stop the debugger engine (normally in a user process) - before closing the debugger window (running in the IDLE process). - This prevents the RuntimeErrors that were being caught and ignored. - -- Issue #24455: Prevent IDLE from hanging when a) closing the shell while the - debugger is active (15347); b) closing the debugger with the [X] button - (15348); and c) activating the debugger when already active (24455). - The patch by Mark Roseman does this by making two changes. - 1. Suspend and resume the gui.interaction method with the tcl vwait - mechanism intended for this purpose (instead of root.mainloop & .quit). - 2. In gui.run, allow any existing interaction to terminate first. - -- Change 'The program' to 'Your program' in an IDLE 'kill program?' message - to make it clearer that the program referred to is the currently running - user program, not IDLE itself. - -- Issue #24750: Improve the appearance of the IDLE editor window status bar. - Patch by Mark Roseman. - -- Issue #25313: Change the handling of new built-in text color themes to better - address the compatibility problem introduced by the addition of IDLE Dark. - Consistently use the revised idleConf.CurrentTheme everywhere in idlelib. - -- Issue #24782: Extension configuration is now a tab in the IDLE Preferences - dialog rather than a separate dialog. The former tabs are now a sorted - list. Patch by Mark Roseman. - -- Issue #22726: Re-activate the config dialog help button with some content - about the other buttons and the new IDLE Dark theme. - -- Issue #24820: IDLE now has an 'IDLE Dark' built-in text color theme. - It is more or less IDLE Classic inverted, with a cobalt blue background. - Strings, comments, keywords, ... are still green, red, orange, ... . - To use it with IDLEs released before November 2015, hit the - 'Save as New Custom Theme' button and enter a new name, - such as 'Custom Dark'. The custom theme will work with any IDLE - release, and can be modified. - -- Issue #25224: README.txt is now an idlelib index for IDLE developers and - curious users. The previous user content is now in the IDLE doc chapter. - 'IDLE' now means 'Integrated Development and Learning Environment'. - -- Issue #24820: Users can now set breakpoint colors in - Settings -> Custom Highlighting. Original patch by Mark Roseman. - -- Issue #24972: Inactive selection background now matches active selection - background, as configured by users, on all systems. Found items are now - always highlighted on Windows. Initial patch by Mark Roseman. - -- Issue #24570: Idle: make calltip and completion boxes appear on Macs - affected by a tk regression. Initial patch by Mark Roseman. - -- Issue #24988: Idle ScrolledList context menus (used in debugger) - now work on Mac Aqua. Patch by Mark Roseman. - -- Issue #24801: Make right-click for context menu work on Mac Aqua. - Patch by Mark Roseman. - -- Issue #25173: Associate tkinter messageboxes with a specific widget. - For Mac OSX, make them a 'sheet'. Patch by Mark Roseman. - -- Issue #25198: Enhance the initial html viewer now used for Idle Help. - * Properly indent fixed-pitch text (patch by Mark Roseman). - * Give code snippet a very Sphinx-like light blueish-gray background. - * Re-use initial width and height set by users for shell and editor. - * When the Table of Contents (TOC) menu is used, put the section header - at the top of the screen. - -- Issue #25225: Condense and rewrite Idle doc section on text colors. - -- Issue #21995: Explain some differences between IDLE and console Python. - -- Issue #22820: Explain need for *print* when running file from Idle editor. - -- Issue #25224: Doc: augment Idle feature list and no-subprocess section. - -- Issue #25219: Update doc for Idle command line options. - Some were missing and notes were not correct. - -- Issue #24861: Most of idlelib is private and subject to change. - Use idleib.idle.* to start Idle. See idlelib.__init__.__doc__. - -- Issue #25199: Idle: add synchronization comments for future maintainers. - -- Issue #16893: Replace help.txt with help.html for Idle doc display. - The new idlelib/help.html is rstripped Doc/build/html/library/idle.html. - It looks better than help.txt and will better document Idle as released. - The tkinter html viewer that works for this file was written by Mark Roseman. - The now unused EditorWindow.HelpDialog class and helt.txt file are deprecated. - -- Issue #24199: Deprecate unused idlelib.idlever with possible removal in 3.6. - -- Issue #24790: Remove extraneous code (which also create 2 & 3 conflicts). - - -What's New in IDLE 3.5.0? +What's New in IDLE 3.2.3? ========================= -*Release date: 2015-09-13* - -- Issue #23672: Allow Idle to edit and run files with astral chars in name. - Patch by Mohd Sanad Zaki Rizvi. - -- Issue 24745: Idle editor default font. Switch from Courier to - platform-sensitive TkFixedFont. This should not affect current customized - font selections. If there is a problem, edit $HOME/.idlerc/config-main.cfg - and remove 'fontxxx' entries from [Editor Window]. Patch by Mark Roseman. - -- Issue #21192: Idle editor. When a file is run, put its name in the restart bar. - Do not print false prompts. Original patch by Adnan Umer. - -- Issue #13884: Idle menus. Remove tearoff lines. Patch by Roger Serwy. - -- Issue #23184: remove unused names and imports in idlelib. - Initial patch by Al Sweigart. - -- Issue #20577: Configuration of the max line length for the FormatParagraph - extension has been moved from the General tab of the Idle preferences dialog - to the FormatParagraph tab of the Config Extensions dialog. - Patch by Tal Einat. - -- Issue #16893: Update Idle doc chapter to match current Idle and add new - information. - -- Issue #3068: Add Idle extension configuration dialog to Options menu. - Changes are written to HOME/.idlerc/config-extensions.cfg. - Original patch by Tal Einat. - -- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a - editor window with a filename. When Class Browser is requested otherwise, - from a shell, output window, or 'Untitled' editor, Idle no longer displays - an error box. It now pops up an Open Module box (Alt+M). If a valid name - is entered and a module is opened, a corresponding browser is also opened. - -- Issue #4832: Save As to type Python files automatically adds .py to the - name you enter (even if your system does not display it). Some systems - automatically add .txt when type is Text files. - -- Issue #21986: Code objects are not normally pickled by the pickle module. - To match this, they are no longer pickled when running under Idle. - -- Issue #23180: Rename IDLE "Windows" menu item to "Window". - Patch by Al Sweigart. - -- Issue #17390: Adjust Editor window title; remove 'Python', - move version to end. - -- Issue #14105: Idle debugger breakpoints no longer disappear - when inseting or deleting lines. - -- Issue #17172: Turtledemo can now be run from Idle. - Currently, the entry is on the Help menu, but it may move to Run. - Patch by Ramchandra Apt and Lita Cho. - -- Issue #21765: Add support for non-ascii identifiers to HyperParser. - -- Issue #21940: Add unittest for WidgetRedirector. Initial patch by Saimadhav - Heblikar. - -- Issue #18592: Add unittest for SearchDialogBase. Patch by Phil Webster. - -- Issue #21694: Add unittest for ParenMatch. Patch by Saimadhav Heblikar. - -- Issue #21686: add unittest for HyperParser. Original patch by Saimadhav - Heblikar. - -- Issue #12387: Add missing upper(lower)case versions of default Windows key - bindings for Idle so Caps Lock does not disable them. Patch by Roger Serwy. - -- Issue #21695: Closing a Find-in-files output window while the search is - still in progress no longer closes Idle. - -- Issue #18910: Add unittest for textView. Patch by Phil Webster. - -- Issue #18292: Add unittest for AutoExpand. Patch by Saihadhav Heblikar. - -- Issue #18409: Add unittest for AutoComplete. Patch by Phil Webster. - -- Issue #21477: htest.py - Improve framework, complete set of tests. - Patches by Saimadhav Heblikar - -- Issue #18104: Add idlelib/idle_test/htest.py with a few sample tests to begin - consolidating and improving human-validated tests of Idle. Change other files - as needed to work with htest. Running the module as __main__ runs all tests. - -- Issue #21139: Change default paragraph width to 72, the PEP 8 recommendation. - -- Issue #21284: Paragraph reformat test passes after user changes reformat width. - -- Issue #17654: Ensure IDLE menus are customized properly on OS X for - non-framework builds and for all variants of Tk. - - -What's New in IDLE 3.4.0? -========================= -*Release date: 2014-03-16* - -- Issue #17390: Display Python version on Idle title bar. - Initial patch by Edmond Burnett. - -- Issue #5066: Update IDLE docs. Patch by Todd Rovito. - -- Issue #17625: Close the replace dialog after it is used. - -- Issue #16226: Fix IDLE Path Browser crash. - (Patch by Roger Serwy) - -- Issue #15853: Prevent IDLE crash on OS X when opening Preferences menu - with certain versions of Tk 8.5. Initial patch by Kevin Walzer. - - -What's New in IDLE 3.3.0? -========================= -*Release date: 2012-09-29* - -- Issue #17625: Close the replace dialog after it is used. - -- Issue #7163: Propagate return value of sys.stdout.write. - -- Issue #15318: Prevent writing to sys.stdin. - -- Issue #4832: Modify IDLE to save files with .py extension by - default on Windows and OS X (Tk 8.5) as it already does with X11 Tk. - -- Issue #13532, #15319: Check that arguments to sys.stdout.write are strings. - -- Issue # 12510: Attempt to get certain tool tips no longer crashes IDLE. - Erroneous tool tips have been corrected. Default added for callables. - -- Issue10365: File open dialog now works instead of crashing even when - parent window is closed while dialog is open. - -- Issue 14876: use user-selected font for highlight configuration. - -- Issue #14937: Perform auto-completion of filenames in strings even for - non-ASCII filenames. Likewise for identifiers. - -- Issue #8515: Set __file__ when run file in IDLE. - Initial patch by Bruce Frederiksen. - -- IDLE can be launched as `python -m idlelib` - -- Issue #14409: IDLE now properly executes commands in the Shell window - when it cannot read the normal config files on startup and - has to use the built-in default key bindings. - There was previously a bug in one of the defaults. - Issue #3573: IDLE hangs when passing invalid command line args (directory(ies) instead of file(s)). -- Issue #14018: Update checks for unstable system Tcl/Tk versions on OS X - to include versions shipped with OS X 10.7 and 10.8 in addition to 10.6. - What's New in IDLE 3.2.1? ========================= + *Release date: 15-May-11* - Issue #6378: Further adjust idle.bat to start associated Python @@ -274,6 +25,7 @@ What's New in IDLE 3.1b1? ========================= + *Release date: 06-May-09* - Use of 'filter' in keybindingDialog.py was causing custom key assignment to @@ -282,6 +34,7 @@ What's New in IDLE 3.1a1? ========================= + *Release date: 07-Mar-09* - Issue #4815: Offer conversion to UTF-8 if source files have @@ -297,9 +50,9 @@ - Issue #2665: On Windows, an IDLE installation upgraded from an old version would not start if a custom theme was defined. - What's New in IDLE 2.7? (UNRELEASED, but merged into 3.1 releases above.) ======================= + *Release date: XX-XXX-2010* - idle.py modified and simplified to better support developing experimental @@ -323,9 +76,9 @@ - Issue #3549: On MacOS the preferences menu was not present +What's New in IDLE 3.0 final? +============================= -What's New in IDLE 3.0? -======================= *Release date: 03-Dec-2008* - IDLE would print a "Unhandled server exception!" message when internal @@ -337,6 +90,12 @@ - Issue #4383: When IDLE cannot make the connection to its subprocess, it would fail to properly display the error message. + +What's New in IDLE 3.0a3? +========================= + +*Release date: 29-Feb-2008* + - help() was not paging to the shell. Issue1650. - CodeContext was not importing. @@ -348,9 +107,21 @@ - Issue #1585: IDLE uses non-existent xrange() function. + +What's New in IDLE 3.0a2? +========================= + +*Release date: 06-Dec-2007* + - Windows EOL sequence not converted correctly, encoding error. Caused file save to fail. Bug 1130. + +What's New in IDLE 3.0a1? +========================= + +*Release date: 31-Aug-2007* + - IDLE converted to Python 3000 syntax. - Strings became Unicode. @@ -364,8 +135,9 @@ be cleared before IDLE exits. -What's New in IDLE 2.6 -====================== +What's New in IDLE 2.6 final? +============================= + *Release date: 01-Oct-2008*, merged into 3.0 releases detailed above (3.0rc2) - Issue #2665: On Windows, an IDLE installation upgraded from an old version @@ -450,8 +222,15 @@ What's New in IDLE 1.2? ======================= + *Release date: 19-SEP-2006* + +What's New in IDLE 1.2c1? +========================= + +*Release date: 17-AUG-2006* + - File menu hotkeys: there were three 'p' assignments. Reassign the 'Save Copy As' and 'Print' hotkeys to 'y' and 't'. Change the Shell hotkey from 's' to 'l'. @@ -472,6 +251,11 @@ - When used w/o subprocess, all exceptions were preceded by an error message claiming they were IDLE internal errors (since 1.2a1). +What's New in IDLE 1.2b3? +========================= + +*Release date: 03-AUG-2006* + - Bug #1525817: Don't truncate short lines in IDLE's tool tips. - Bug #1517990: IDLE keybindings on MacOS X now work correctly @@ -495,6 +279,26 @@ 'as' keyword in comment directly following import command. Closes 1325071. Patch 1479219 Tal Einat +What's New in IDLE 1.2b2? +========================= + +*Release date: 11-JUL-2006* + +What's New in IDLE 1.2b1? +========================= + +*Release date: 20-JUN-2006* + +What's New in IDLE 1.2a2? +========================= + +*Release date: 27-APR-2006* + +What's New in IDLE 1.2a1? +========================= + +*Release date: 05-APR-2006* + - Patch #1162825: Support non-ASCII characters in IDLE window titles. - Source file f.flush() after writing; trying to avoid lossage if user @@ -574,14 +378,19 @@ - The remote procedure call module rpc.py can now access data attributes of remote registered objects. Changes to these attributes are local, however. - What's New in IDLE 1.1? ======================= + *Release date: 30-NOV-2004* - On OpenBSD, terminating IDLE with ctrl-c from the command line caused a stuck subprocess MainThread because only the SocketThread was exiting. +What's New in IDLE 1.1b3/rc1? +============================= + +*Release date: 18-NOV-2004* + - Saving a Keyset w/o making changes (by using the "Save as New Custom Key Set" button) caused IDLE to fail on restart (no new keyset was created in config-keys.cfg). Also true for Theme/highlights. Python Bug 1064535. @@ -589,12 +398,28 @@ - A change to the linecache.py API caused IDLE to exit when an exception was raised while running without the subprocess (-n switch). Python Bug 1063840. +What's New in IDLE 1.1b2? +========================= + +*Release date: 03-NOV-2004* + - When paragraph reformat width was made configurable, a bug was introduced that caused reformatting of comment blocks to ignore how far the block was indented, effectively adding the indentation width to the reformat width. This has been repaired, and the reformat width is again a bound on the total width of reformatted lines. +What's New in IDLE 1.1b1? +========================= + +*Release date: 15-OCT-2004* + + +What's New in IDLE 1.1a3? +========================= + +*Release date: 02-SEP-2004* + - Improve keyboard focus binding, especially in Windows menu. Improve window raising, especially in the Windows menu and in the debugger. IDLEfork 763524. @@ -602,12 +427,24 @@ - If user passes a non-existent filename on the commandline, just open a new file, don't raise a dialog. IDLEfork 854928. + +What's New in IDLE 1.1a2? +========================= + +*Release date: 05-AUG-2004* + - EditorWindow.py was not finding the .chm help file on Windows. Typo at Rev 1.54. Python Bug 990954 - checking sys.platform for substring 'win' was breaking IDLE docs on Mac (darwin). Also, Mac Safari browser requires full file:// URIs. SF 900580. + +What's New in IDLE 1.1a1? +========================= + +*Release date: 08-JUL-2004* + - Redirect the warning stream to the shell during the ScriptBinding check of user code and format the warning similarly to an exception for both that check and for runtime warnings raised in the subprocess. @@ -670,13 +507,26 @@ What's New in IDLE 1.0? ======================= + *Release date: 29-Jul-2003* - Added a banner to the shell discussing warnings possibly raised by personal firewall software. Added same comment to README.txt. + +What's New in IDLE 1.0 release candidate 2? +=========================================== + +*Release date: 24-Jul-2003* + - Calltip error when docstring was None Python Bug 775541 + +What's New in IDLE 1.0 release candidate 1? +=========================================== + +*Release date: 18-Jul-2003* + - Updated extend.txt, help.txt, and config-extensions.def to correctly reflect the current status of the configuration system. Python Bug 768469 @@ -692,6 +542,12 @@ sys.std{in|out|err}.encoding, for both the local and the subprocess case. SF IDLEfork patch 682347. + +What's New in IDLE 1.0b2? +========================= + +*Release date: 29-Jun-2003* + - Extend AboutDialog.ViewFile() to support file encodings. Make the CREDITS file Latin-1. @@ -730,6 +586,7 @@ What's New in IDLEfork 0.9b1? ============================= + *Release date: 02-Jun-2003* - The current working directory of the execution environment (and shell @@ -831,8 +688,10 @@ exception formatting to the subprocess. + What's New in IDLEfork 0.9 Alpha 2? =================================== + *Release date: 27-Jan-2003* - Updated INSTALL.txt to claify use of the python2 rpm. @@ -936,6 +795,7 @@ What's New in IDLEfork 0.9 Alpha 1? =================================== + *Release date: 31-Dec-2002* - First release of major new functionality. For further details refer to @@ -963,3 +823,8 @@ -------------------------------------------------------------------- Refer to HISTORY.txt for additional information on earlier releases. -------------------------------------------------------------------- + + + + + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ObjectBrowser.py --- a/Lib/idlelib/ObjectBrowser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ObjectBrowser.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,8 +9,6 @@ # XXX TO DO: # - for classes/modules, add "open source" to object browser -import re - from idlelib.TreeWidget import TreeItem, TreeNode, ScrolledCanvas from reprlib import Repr @@ -121,14 +119,12 @@ c = ObjectTreeItem return c(labeltext, object, setfunction) +# Test script -def _object_browser(parent): +def _test(): import sys from tkinter import Tk root = Tk() - root.title("Test ObjectBrowser") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) root.configure(bd=0, bg="yellow") root.focus_set() sc = ScrolledCanvas(root, bg="white", highlightthickness=0, takefocus=1) @@ -139,5 +135,4 @@ root.mainloop() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_object_browser) + _test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/OutputWindow.py --- a/Lib/idlelib/OutputWindow.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/OutputWindow.py Mon Jan 25 17:05:13 2016 +0100 @@ -40,7 +40,6 @@ self.text.insert(mark, s, tags) self.text.see(mark) self.text.update() - return len(s) def writelines(self, lines): for line in lines: @@ -52,11 +51,7 @@ # Our own right-button menu rmenu_specs = [ - ("Cut", "<>", "rmenu_check_cut"), - ("Copy", "<>", "rmenu_check_copy"), - ("Paste", "<>", "rmenu_check_paste"), - (None, None, None), - ("Go to file/line", "<>", None), + ("Go to file/line", "<>"), ] file_line_pats = [ @@ -91,7 +86,7 @@ "No special line", "The line you point at doesn't look like " "a valid file name followed by a line number.", - parent=self.text) + master=self.text) return filename, lineno = result edit = self.flist.open(filename) @@ -106,7 +101,7 @@ f = open(filename, "r") f.close() break - except OSError: + except IOError: continue else: return None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ParenMatch.py --- a/Lib/idlelib/ParenMatch.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ParenMatch.py Mon Jan 25 17:05:13 2016 +0100 @@ -90,8 +90,7 @@ self.set_timeout = self.set_timeout_none def flash_paren_event(self, event): - indices = (HyperParser(self.editwin, "insert") - .get_surrounding_brackets()) + indices = HyperParser(self.editwin, "insert").get_surrounding_brackets() if indices is None: self.warn_mismatched() return @@ -168,11 +167,6 @@ # associate a counter with an event; only disable the "paren" # tag if the event is for the most recent timer. self.counter += 1 - self.editwin.text_frame.after( - self.FLASH_DELAY, - lambda self=self, c=self.counter: self.handle_restore_timer(c)) - - -if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_parenmatch', verbosity=2) + self.editwin.text_frame.after(self.FLASH_DELAY, + lambda self=self, c=self.counter: \ + self.handle_restore_timer(c)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/PathBrowser.py --- a/Lib/idlelib/PathBrowser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/PathBrowser.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,23 +1,16 @@ import os import sys -import importlib.machinery +import imp from idlelib.TreeWidget import TreeItem from idlelib.ClassBrowser import ClassBrowser, ModuleBrowserTreeItem -from idlelib.PyShell import PyShellFileList - class PathBrowser(ClassBrowser): - def __init__(self, flist, _htest=False): - """ - _htest - bool, change box location when running htest - """ - self._htest = _htest + def __init__(self, flist): self.init(flist) def settitle(self): - "Set window titles." self.top.wm_title("Path Browser") self.top.wm_iconname("Path Browser") @@ -51,7 +44,7 @@ def GetSubList(self): try: names = os.listdir(self.dir or os.curdir) - except OSError: + except os.error: return [] packages = [] for name in names: @@ -70,19 +63,16 @@ return sublist def ispackagedir(self, file): - " Return true for directories that are packages." if not os.path.isdir(file): - return False + return 0 init = os.path.join(file, "__init__.py") return os.path.exists(init) def listmodules(self, allnames): modules = {} - suffixes = importlib.machinery.EXTENSION_SUFFIXES[:] - suffixes += importlib.machinery.SOURCE_SUFFIXES - suffixes += importlib.machinery.BYTECODE_SUFFIXES + suffixes = imp.get_suffixes() sorted = [] - for suff in suffixes: + for suff, mode, flag in suffixes: i = -len(suff) for name in allnames[:]: normed_name = os.path.normcase(name) @@ -95,14 +85,11 @@ sorted.sort() return sorted -def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) - parent.mainloop() +def main(): + from idlelib import PyShell + PathBrowser(PyShell.flist) + if sys.stdin is sys.__stdin__: + mainloop() if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(_path_browser) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/Percolator.py --- a/Lib/idlelib/Percolator.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/Percolator.py Mon Jan 25 17:05:13 2016 +0100 @@ -51,9 +51,8 @@ f.setdelegate(filter.delegate) filter.setdelegate(None) -def _percolator(parent): - import tkinter as tk - import re +def main(): + import tkinter as Tk class Tracer(Delegator): def __init__(self, name): self.name = name @@ -64,41 +63,22 @@ def delete(self, *args): print(self.name, ": delete", args) self.delegate.delete(*args) - root = tk.Tk() - root.title("Test Percolator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - text = tk.Text(root) + root = Tk.Tk() + root.wm_protocol("WM_DELETE_WINDOW", root.quit) + text = Tk.Text() + text.pack() + text.focus_set() p = Percolator(text) t1 = Tracer("t1") t2 = Tracer("t2") - - def toggle1(): - if var1.get() == 0: - var1.set(1) - p.insertfilter(t1) - elif var1.get() == 1: - var1.set(0) - p.removefilter(t1) - - def toggle2(): - if var2.get() == 0: - var2.set(1) - p.insertfilter(t2) - elif var2.get() == 1: - var2.set(0) - p.removefilter(t2) - - text.pack() - var1 = tk.IntVar() - cb1 = tk.Checkbutton(root, text="Tracer1", command=toggle1, variable=var1) - cb1.pack() - var2 = tk.IntVar() - cb2 = tk.Checkbutton(root, text="Tracer2", command=toggle2, variable=var2) - cb2.pack() - + p.insertfilter(t1) + p.insertfilter(t2) + root.mainloop() # click close widget to continue... + p.removefilter(t2) + root.mainloop() + p.insertfilter(t2) + p.removefilter(t1) root.mainloop() if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_percolator) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/PyParse.py --- a/Lib/idlelib/PyParse.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/PyParse.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,5 @@ import re import sys -from collections import Mapping # Reason last stmt is continued (or C_NONE if it's not). (C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE, @@ -92,48 +91,19 @@ [^[\](){}#'"\\]+ """, re.VERBOSE).match +# Build translation table to map uninteresting chars to "x", open +# brackets to "(", and close brackets to ")". -class StringTranslatePseudoMapping(Mapping): - r"""Utility class to be used with str.translate() - - This Mapping class wraps a given dict. When a value for a key is - requested via __getitem__() or get(), the key is looked up in the - given dict. If found there, the value from the dict is returned. - Otherwise, the default value given upon initialization is returned. - - This allows using str.translate() to make some replacements, and to - replace all characters for which no replacement was specified with - a given character instead of leaving them as-is. - - For example, to replace everything except whitespace with 'x': - - >>> whitespace_chars = ' \t\n\r' - >>> preserve_dict = {ord(c): ord(c) for c in whitespace_chars} - >>> mapping = StringTranslatePseudoMapping(preserve_dict, ord('x')) - >>> text = "a + b\tc\nd" - >>> text.translate(mapping) - 'x x x\tx\nx' - """ - def __init__(self, non_defaults, default_value): - self._non_defaults = non_defaults - self._default_value = default_value - - def _get(key, _get=non_defaults.get, _default=default_value): - return _get(key, _default) - self._get = _get - - def __getitem__(self, item): - return self._get(item) - - def __len__(self): - return len(self._non_defaults) - - def __iter__(self): - return iter(self._non_defaults) - - def get(self, key, default=None): - return self._get(key) - +_tran = {} +for i in range(256): + _tran[i] = 'x' +for ch in "({[": + _tran[ord(ch)] = '(' +for ch in ")}]": + _tran[ord(ch)] = ')' +for ch in "\"'\\\n#": + _tran[ord(ch)] = ch +del i, ch class Parser: @@ -143,6 +113,19 @@ def set_str(self, s): assert len(s) == 0 or s[-1] == '\n' + if isinstance(s, str): + # The parse functions have no idea what to do with Unicode, so + # replace all Unicode characters with "x". This is "safe" + # so long as the only characters germane to parsing the structure + # of Python are 7-bit ASCII. It's *necessary* because Unicode + # strings don't have a .translate() method that supports + # deletechars. + uniphooey = s + s = [] + push = s.append + for raw in map(ord, uniphooey): + push(raw < 127 and chr(raw) or "x") + s = "".join(s) self.str = s self.study_level = 0 @@ -214,16 +197,6 @@ if lo > 0: self.str = self.str[lo:] - # Build a translation table to map uninteresting chars to 'x', open - # brackets to '(', close brackets to ')' while preserving quotes, - # backslashes, newlines and hashes. This is to be passed to - # str.translate() in _study1(). - _tran = {} - _tran.update((ord(c), ord('(')) for c in "({[") - _tran.update((ord(c), ord(')')) for c in ")}]") - _tran.update((ord(c), ord(c)) for c in "\"'\\\n#") - _tran = StringTranslatePseudoMapping(_tran, default_value=ord('x')) - # As quickly as humanly possible , find the line numbers (0- # based) of the non-continuation lines. # Creates self.{goodlines, continuation}. @@ -238,7 +211,7 @@ # uninteresting characters. This can cut the number of chars # by a factor of 10-40, and so greatly speed the following loop. str = self.str - str = str.translate(self._tran) + str = str.translate(_tran) str = str.replace('xxxxxxxx', 'x') str = str.replace('xxxx', 'x') str = str.replace('xx', 'x') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/PyShell.py --- a/Lib/idlelib/PyShell.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/PyShell.py Mon Jan 25 17:05:13 2016 +0100 @@ -10,16 +10,16 @@ import threading import time import tokenize -import io +import traceback +import types import linecache from code import InteractiveInterpreter -from platform import python_version, system try: from tkinter import * except ImportError: - print("** IDLE can't import Tkinter.\n" + print("** IDLE can't import Tkinter. " \ "Your Python may not be configured for Tk. **", file=sys.__stderr__) sys.exit(1) import tkinter.messagebox as tkMessageBox @@ -30,6 +30,7 @@ from idlelib.UndoDelegator import UndoDelegator from idlelib.OutputWindow import OutputWindow from idlelib.configHandler import idleConf +from idlelib import idlever from idlelib import rpc from idlelib import Debugger from idlelib import RemoteDebugger @@ -42,55 +43,35 @@ # internal warnings to the console. ScriptBinding.check_syntax() will # temporarily redirect the stream to the shell window to display warnings when # checking user's code. -warning_stream = sys.__stderr__ # None, at least on Windows, if no console. -import warnings - -def idle_formatwarning(message, category, filename, lineno, line=None): - """Format warnings the IDLE way.""" - - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n" % (category.__name__, message) - return s - -def idle_showwarning( - message, category, filename, lineno, file=None, line=None): - """Show Idle-format warning (after replacing warnings.showwarning). - - The differences are the formatter called, the file=None replacement, - which can be None, the capture of the consequence AttributeError, - and the output of a hard-coded prompt. - """ - if file is None: - file = warning_stream - try: - file.write(idle_formatwarning( - message, category, filename, lineno, line=line)) - file.write(">>> ") - except (AttributeError, OSError): - pass # if file (probably __stderr__) is invalid, skip warning. - -_warnings_showwarning = None - -def capture_warnings(capture): - "Replace warning.showwarning with idle_showwarning, or reverse." - - global _warnings_showwarning - if capture: - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning - warnings.showwarning = idle_showwarning - else: - if _warnings_showwarning is not None: - warnings.showwarning = _warnings_showwarning - _warnings_showwarning = None - -capture_warnings(True) +global warning_stream +warning_stream = sys.__stderr__ +try: + import warnings +except ImportError: + pass +else: + def idle_showwarning(message, category, filename, lineno, + file=None, line=None): + if file is None: + file = warning_stream + try: + file.write(warnings.formatwarning(message, category, filename, + lineno, line=line)) + except IOError: + pass ## file (probably __stderr__) is invalid, warning dropped. + warnings.showwarning = idle_showwarning + def idle_formatwarning(message, category, filename, lineno, line=None): + """Format warnings the IDLE way""" + s = "\nWarning (from warnings module):\n" + s += ' File \"%s\", line %s\n' % (filename, lineno) + if line is None: + line = linecache.getline(filename, lineno) + line = line.strip() + if line: + s += " %s\n" % line + s += "%s: %s\n>>> " % (category.__name__, message) + return s + warnings.formatwarning = idle_formatwarning def extended_linecache_checkcache(filename=None, orig_checkcache=linecache.checkcache): @@ -128,42 +109,22 @@ self.breakpointPath = os.path.join(idleConf.GetUserCfgDir(), 'breakpoints.lst') # whenever a file is changed, restore breakpoints + if self.io.filename: self.restore_file_breaks() def filename_changed_hook(old_hook=self.io.filename_change_hook, self=self): self.restore_file_breaks() old_hook() self.io.set_filename_change_hook(filename_changed_hook) - if self.io.filename: - self.restore_file_breaks() - self.color_breakpoint_text() - rmenu_specs = [ - ("Cut", "<>", "rmenu_check_cut"), - ("Copy", "<>", "rmenu_check_copy"), - ("Paste", "<>", "rmenu_check_paste"), - (None, None, None), - ("Set Breakpoint", "<>", None), - ("Clear Breakpoint", "<>", None) - ] - - def color_breakpoint_text(self, color=True): - "Turn colorizing of breakpoint text on or off" - if self.io is None: - # possible due to update in restore_file_breaks - return - if color: - theme = idleConf.CurrentTheme() - cfg = idleConf.GetHighlight(theme, "break") - else: - cfg = {'foreground': '', 'background': ''} - self.text.tag_config('BREAK', cfg) + rmenu_specs = [("Set Breakpoint", "<>"), + ("Clear Breakpoint", "<>")] def set_breakpoint(self, lineno): text = self.text filename = self.io.filename text.tag_add("BREAK", "%d.0" % lineno, "%d.0" % (lineno+1)) try: - self.breakpoints.index(lineno) + i = self.breakpoints.index(lineno) except ValueError: # only add if missing, i.e. do once self.breakpoints.append(lineno) try: # update the subprocess debugger @@ -227,8 +188,13 @@ # This is necessary to keep the saved breaks synched with the # saved file. # - # Breakpoints are set as tagged ranges in the text. - # Since a modified file has to be saved before it is + # Breakpoints are set as tagged ranges in the text. Certain + # kinds of edits cause these ranges to be deleted: Inserting + # or deleting a line just before a breakpoint, and certain + # deletions prior to a breakpoint. These issues need to be + # investigated and understood. It's not clear if they are + # Tk issues or IDLE issues, or whether they can actually + # be fixed. Since a modified file has to be saved before it is # run, and since self.breakpoints (from which the subprocess # debugger is loaded) is updated during the save, the visible # breaks stay synched with the subprocess even if one of these @@ -238,7 +204,7 @@ try: with open(self.breakpointPath, "r") as fp: lines = fp.readlines() - except OSError: + except IOError: lines = [] try: with open(self.breakpointPath, "w") as new_file: @@ -249,7 +215,7 @@ breaks = self.breakpoints if breaks: new_file.write(filename + '=' + str(breaks) + '\n') - except OSError as err: + except IOError as err: if not getattr(self.root, "breakpoint_error_displayed", False): self.root.breakpoint_error_displayed = True tkMessageBox.showerror(title='IDLE Error', @@ -259,9 +225,6 @@ def restore_file_breaks(self): self.text.update() # this enables setting "BREAK" tags to be visible - if self.io is None: - # can happen if IDLE closes due to the .update() call - return filename = self.io.filename if filename is None: return @@ -284,8 +247,8 @@ def ranges_to_linenumbers(self, ranges): lines = [] for index in range(0, len(ranges), 2): - lineno = int(float(ranges[index].string)) - end = int(float(ranges[index+1].string)) + lineno = int(float(ranges[index])) + end = int(float(ranges[index+1])) while lineno < end: lines.append(lineno) lineno += 1 @@ -338,7 +301,7 @@ def LoadTagDefs(self): ColorDelegator.LoadTagDefs(self) - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') self.tagdefs.update({ "stdin": {'background':None,'foreground':None}, "stdout": idleConf.GetHighlight(theme, "stdout"), @@ -346,11 +309,6 @@ "console": idleConf.GetHighlight(theme, "console"), }) - def removecolors(self): - # Don't remove shell color tags before "iomark" - for tag in self.tagdefs: - self.tag_remove(tag, "iomark", "end") - class ModifiedUndoDelegator(UndoDelegator): "Extend base class: forbid insert/delete before the I/O mark" @@ -392,7 +350,6 @@ self.port = PORT self.original_compiler_flags = self.compile.compiler.flags - _afterid = None rpcclt = None rpcsubproc = None @@ -424,7 +381,7 @@ try: self.rpcclt = MyRPCClient(addr) break - except OSError: + except socket.error as err: pass else: self.display_port_binding_error() @@ -445,11 +402,10 @@ self.rpcclt.listening_sock.settimeout(10) try: self.rpcclt.accept() - except socket.timeout: + except socket.timeout as err: self.display_no_subprocess_error() return None - self.rpcclt.register("console", self.tkconsole) - self.rpcclt.register("stdin", self.tkconsole.stdin) + self.rpcclt.register("stdin", self.tkconsole) self.rpcclt.register("stdout", self.tkconsole.stdout) self.rpcclt.register("stderr", self.tkconsole.stderr) self.rpcclt.register("flist", self.tkconsole.flist) @@ -459,7 +415,7 @@ self.poll_subprocess() return self.rpcclt - def restart_subprocess(self, with_cwd=False, filename=''): + def restart_subprocess(self, with_cwd=False): if self.restarting: return self.rpcclt self.restarting = True @@ -480,24 +436,24 @@ self.spawn_subprocess() try: self.rpcclt.accept() - except socket.timeout: + except socket.timeout as err: self.display_no_subprocess_error() return None self.transfer_path(with_cwd=with_cwd) - console.stop_readline() # annotate restart in shell window and mark it console.text.delete("iomark", "end-1c") - tag = 'RESTART: ' + (filename if filename else 'Shell') - halfbar = ((int(console.width) -len(tag) - 4) // 2) * '=' - console.write("\n{0} {1} {0}".format(halfbar, tag)) + if was_executing: + console.write('\n') + console.showprompt() + halfbar = ((int(console.width) - 16) // 2) * '=' + console.write(halfbar + ' RESTART ' + halfbar) console.text.mark_set("restart", "end-1c") console.text.mark_gravity("restart", "left") - if not filename: - console.showprompt() + console.showprompt() # restart subprocess debugger if debug: # Restarted debugger connects to current instance of debug GUI - RemoteDebugger.restart_subprocess_debugger(self.rpcclt) + gui = RemoteDebugger.restart_subprocess_debugger(self.rpcclt) # reload remote debugger breakpoints for all PyShellEditWindows debug.load_breakpoints() self.compile.compiler.flags = self.original_compiler_flags @@ -511,8 +467,6 @@ threading.Thread(target=self.__request_interrupt).start() def kill_subprocess(self): - if self._afterid is not None: - self.tkconsole.text.after_cancel(self._afterid) try: self.rpcclt.listening_sock.close() except AttributeError: # no socket @@ -559,7 +513,7 @@ return try: response = clt.pollresponse(self.active_seq, wait=0.05) - except (EOFError, OSError, KeyboardInterrupt): + except (EOFError, IOError, KeyboardInterrupt): # lost connection or subprocess terminated itself, restart # [the KBI is from rpc.SocketIO.handle_EOF()] if self.tkconsole.closing: @@ -588,8 +542,8 @@ pass # Reschedule myself if not self.tkconsole.closing: - self._afterid = self.tkconsole.text.after( - self.tkconsole.pollinterval, self.poll_subprocess) + self.tkconsole.text.after(self.tkconsole.pollinterval, + self.poll_subprocess) debugger = None @@ -621,7 +575,7 @@ item = RemoteObjectBrowser.StubObjectTreeItem(self.rpcclt, oid) from idlelib.TreeWidget import ScrolledCanvas, TreeNode top = Toplevel(self.tkconsole.root) - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') background = idleConf.GetHighlight(theme, 'normal')['background'] sc = ScrolledCanvas(top, bg=background, highlightthickness=0) sc.frame.pack(expand=1, fill="both") @@ -645,9 +599,9 @@ code = compile(source, filename, "exec") except (OverflowError, SyntaxError): self.tkconsole.resetoutput() - print('*** Error in script or command!\n' - 'Traceback (most recent call last):', - file=self.tkconsole.stderr) + tkerr = self.tkconsole.stderr + print('*** Error in script or command!\n', file=tkerr) + print('Traceback (most recent call last):', file=tkerr) InteractiveInterpreter.showsyntaxerror(self, filename) self.tkconsole.showprompt() else: @@ -774,7 +728,7 @@ "Exit?", "Do you want to exit altogether?", default="yes", - parent=self.tkconsole.text): + master=self.tkconsole.text): raise else: self.showtraceback() @@ -801,7 +755,7 @@ def write(self, s): "Override base class method" - return self.tkconsole.stderr.write(s) + self.tkconsole.stderr.write(s) def display_port_binding_error(self): tkMessageBox.showerror( @@ -812,7 +766,7 @@ "Run IDLE with the -n command line switch to start without a " "subprocess and refer to Help/IDLE Help 'Running without a " "subprocess' for further details.", - parent=self.tkconsole.text) + master=self.tkconsole.text) def display_no_subprocess_error(self): tkMessageBox.showerror( @@ -820,19 +774,19 @@ "IDLE's subprocess didn't make connection. Either IDLE can't " "start a subprocess or personal firewall software is blocking " "the connection.", - parent=self.tkconsole.text) + master=self.tkconsole.text) def display_executing_dialog(self): tkMessageBox.showerror( "Already executing", "The Python Shell window is already executing a command; " "please wait until it is finished.", - parent=self.tkconsole.text) + master=self.tkconsole.text) class PyShell(OutputWindow): - shell_title = "Python " + python_version() + " Shell" + shell_title = "Python Shell" # Override classes ColorDelegator = ModifiedColorDelegator @@ -844,10 +798,14 @@ ("edit", "_Edit"), ("debug", "_Debug"), ("options", "_Options"), - ("windows", "_Window"), + ("windows", "_Windows"), ("help", "_Help"), ] + if macosxSupport.runningAsOSXApp(): + del menu_specs[-3] + menu_specs[-2] = ("windows", "_Window") + # New classes from idlelib.IdleHistory import History @@ -881,6 +839,8 @@ text.bind("<>", self.open_stack_viewer) text.bind("<>", self.toggle_debugger) text.bind("<>", self.toggle_jit_stack_viewer) + self.color = color = self.ColorDelegator() + self.per.insertfilter(color) if use_subprocess: text.bind("<>", self.view_restart_mark) text.bind("<>", self.restart_shell) @@ -889,14 +849,13 @@ self.save_stderr = sys.stderr self.save_stdin = sys.stdin from idlelib import IOBinding - self.stdin = PseudoInputFile(self, "stdin", IOBinding.encoding) - self.stdout = PseudoOutputFile(self, "stdout", IOBinding.encoding) - self.stderr = PseudoOutputFile(self, "stderr", IOBinding.encoding) - self.console = PseudoOutputFile(self, "console", IOBinding.encoding) + self.stdout = PseudoFile(self, "stdout", IOBinding.encoding) + self.stderr = PseudoFile(self, "stderr", IOBinding.encoding) + self.console = PseudoFile(self, "console", IOBinding.encoding) if not use_subprocess: sys.stdout = self.stdout sys.stderr = self.stderr - sys.stdin = self.stdin + sys.stdin = self try: # page help() text to shell. import pydoc # import must be done here to capture i/o rebinding. @@ -918,7 +877,6 @@ canceled = False endoffile = False closing = False - _stop_readline_flag = False def set_warning_stream(self, stream): global warning_stream @@ -931,7 +889,7 @@ if self.executing: tkMessageBox.showerror("Don't debug now", "You can only toggle the debugger when idle", - parent=self.text) + master=self.text) self.set_debugger_indicator() return "break" else: @@ -989,14 +947,19 @@ if self.executing: response = tkMessageBox.askokcancel( "Kill?", - "Your program is still running!\n Do you want to kill it?", + "The program is still running!\n Do you want to kill it?", default="ok", parent=self.text) if response is False: return "cancel" - self.stop_readline() + if self.reading: + self.top.quit() self.canceled = True self.closing = True + # Wait for poll_subprocess() rescheduling to stop + self.text.after(2 * self.pollinterval, self.close2) + + def close2(self): return EditorWindow.close(self) def _close(self): @@ -1035,26 +998,16 @@ self.close() return False else: - nosub = ("==== No Subprocess ====\n\n" + - "WARNING: Running IDLE without a Subprocess is deprecated\n" + - "and will be removed in a later version. See Help/IDLE Help\n" + - "for details.\n\n") + nosub = "==== No Subprocess ====" sys.displayhook = rpc.displayhook self.write("Python %s on %s\n%s\n%s" % (sys.version, sys.platform, self.COPYRIGHT, nosub)) - self.text.focus_force() self.showprompt() import tkinter tkinter._default_root = None # 03Jan04 KBK What's this? return True - def stop_readline(self): - if not self.reading: # no nested mainloop to exit. - return - self._stop_readline_flag = True - self.top.quit() - def readline(self): save = self.reading try: @@ -1062,9 +1015,6 @@ self.top.mainloop() # nested mainloop() finally: self.reading = save - if self._stop_readline_flag: - self._stop_readline_flag = False - return "" line = self.text.get("iomark", "end-1c") if len(line) == 0: # may be EOF if we quit our mainloop with Ctrl-C line = "\n" @@ -1228,7 +1178,7 @@ while i > 0 and line[i-1] in " \t": i = i-1 line = line[:i] - self.interp.runsource(line) + more = self.interp.runsource(line) def open_stack_viewer(self, event=None): if self.interp.rpcclt: @@ -1239,10 +1189,10 @@ tkMessageBox.showerror("No stack trace", "There is no stack trace yet.\n" "(sys.last_traceback is not defined)", - parent=self.text) + master=self.text) return from idlelib.StackViewer import StackBrowser - StackBrowser(self.root, self.flist) + sv = StackBrowser(self.root, self.flist) def view_restart_mark(self, event=None): self.text.see("iomark") @@ -1266,7 +1216,7 @@ def resetoutput(self): source = self.text.get("iomark", "end-1c") if self.history: - self.history.store(source) + self.history.history_store(source) if self.text.get("end-2c") != "\n": self.text.insert("end-1c", "\n") self.text.mark_set("iomark", "end-1c") @@ -1285,7 +1235,7 @@ 'Non-BMP character not supported in Tk') try: self.text.mark_gravity("iomark", "right") - count = OutputWindow.write(self, s, tags, "iomark") + OutputWindow.write(self, s, tags, "iomark") self.text.mark_gravity("iomark", "left") except: raise ###pass # ### 11Aug07 KBK if we are expecting exceptions @@ -1294,108 +1244,28 @@ self.canceled = 0 if not use_subprocess: raise KeyboardInterrupt - return count - def rmenu_check_cut(self): - try: - if self.text.compare('sel.first', '<', 'iomark'): - return 'disabled' - except TclError: # no selection, so the index 'sel.first' doesn't exist - return 'disabled' - return super().rmenu_check_cut() - - def rmenu_check_paste(self): - if self.text.compare('insert','<','iomark'): - return 'disabled' - return super().rmenu_check_paste() - -class PseudoFile(io.TextIOBase): +class PseudoFile(object): def __init__(self, shell, tags, encoding=None): self.shell = shell self.tags = tags - self._encoding = encoding + self.encoding = encoding - @property - def encoding(self): - return self._encoding + def write(self, s): + self.shell.write(s, self.tags) - @property - def name(self): - return '<%s>' % self.tags + def writelines(self, lines): + for line in lines: + self.write(line) + + def flush(self): + pass def isatty(self): return True -class PseudoOutputFile(PseudoFile): - - def writable(self): - return True - - def write(self, s): - if self.closed: - raise ValueError("write to closed file") - if type(s) is not str: - if not isinstance(s, str): - raise TypeError('must be str, not ' + type(s).__name__) - # See issue #19481 - s = str.__str__(s) - return self.shell.write(s, self.tags) - - -class PseudoInputFile(PseudoFile): - - def __init__(self, shell, tags, encoding=None): - PseudoFile.__init__(self, shell, tags, encoding) - self._line_buffer = '' - - def readable(self): - return True - - def read(self, size=-1): - if self.closed: - raise ValueError("read from closed file") - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError('must be int, not ' + type(size).__name__) - result = self._line_buffer - self._line_buffer = '' - if size < 0: - while True: - line = self.shell.readline() - if not line: break - result += line - else: - while len(result) < size: - line = self.shell.readline() - if not line: break - result += line - self._line_buffer = result[size:] - result = result[:size] - return result - - def readline(self, size=-1): - if self.closed: - raise ValueError("read from closed file") - if size is None: - size = -1 - elif not isinstance(size, int): - raise TypeError('must be int, not ' + type(size).__name__) - line = self._line_buffer or self.shell.readline() - if size < 0: - size = len(line) - eol = line.find('\n', 0, size) - if eol >= 0: - size = eol + 1 - self._line_buffer = line[size:] - return line[:size] - - def close(self): - self.shell.close() - - usage_msg = """\ USAGE: idle [-deins] [-t title] [file]* @@ -1403,8 +1273,7 @@ idle [-dns] [-t title] - [arg]* -h print this help message and exit - -n run IDLE without a subprocess (DEPRECATED, - see Help/IDLE Help for details) + -n run IDLE without a subprocess (see Help/IDLE Help for details) The following options will override the IDLE 'settings' configuration: @@ -1453,9 +1322,8 @@ def main(): global flist, root, use_subprocess - capture_warnings(True) use_subprocess = True - enable_shell = False + enable_shell = True enable_edit = False debug = False cmd = None @@ -1464,7 +1332,8 @@ try: opts, args = getopt.getopt(sys.argv[1:], "c:deihnr:st:") except getopt.error as msg: - print("Error: %s\n%s" % (msg, usage_msg), file=sys.stderr) + sys.stderr.write("Error: %s\n" % str(msg)) + sys.stderr.write(usage_msg) sys.exit(2) for o, a in opts: if o == '-c': @@ -1475,14 +1344,13 @@ enable_shell = True if o == '-e': enable_edit = True + enable_shell = False if o == '-h': sys.stdout.write(usage_msg) sys.exit() if o == '-i': enable_shell = True if o == '-n': - print(" Warning: running IDLE without a subprocess is deprecated.", - file=sys.stderr) use_subprocess = False if o == '-r': script = a @@ -1527,35 +1395,14 @@ edit_start = idleConf.GetOption('main', 'General', 'editor-on-startup', type='bool') enable_edit = enable_edit or edit_start - enable_shell = enable_shell or not enable_edit # start editor and/or shell windows: root = Tk(className="Idle") - # set application icon - icondir = os.path.join(os.path.dirname(__file__), 'Icons') - if system() == 'Windows': - iconfile = os.path.join(icondir, 'idle.ico') - root.wm_iconbitmap(default=iconfile) - elif TkVersion >= 8.5: - ext = '.png' if TkVersion >= 8.6 else '.gif' - iconfiles = [os.path.join(icondir, 'idle_%d%s' % (size, ext)) - for size in (16, 32, 48)] - icons = [PhotoImage(file=iconfile) for iconfile in iconfiles] - root.wm_iconphoto(True, *icons) - fixwordbreaks(root) root.withdraw() flist = PyShellFileList(root) macosxSupport.setupApp(root, flist) - if macosxSupport.isAquaTk(): - # There are some screwed up <2> class bindings for text - # widgets defined in Tk which we need to do away with. - # See issue #24801. - root.unbind_class('Text', '') - root.unbind_class('Text', '') - root.unbind_class('Text', '<>') - if enable_edit: if not (cmd or script): for filename in args[:]: @@ -1564,22 +1411,20 @@ args.remove(filename) if not args: flist.new() - if enable_shell: shell = flist.open_shell() if not shell: return # couldn't open shell - if macosxSupport.isAquaTk() and flist.dict: + + if macosxSupport.runningAsOSXApp() and flist.dict: # On OSX: when the user has double-clicked on a file that causes # IDLE to be launched the shell window will open just in front of # the file she wants to see. Lower the interpreter window when # there are open files. shell.top.lower() - else: - shell = flist.pyshell - # Handle remaining options. If any of these are set, enable_shell - # was set also, so shell must be true to reach here. + shell = flist.pyshell + # handle remaining options: if debug: shell.open_debugger() if startup: @@ -1587,7 +1432,7 @@ os.environ.get("PYTHONSTARTUP") if filename and os.path.isfile(filename): shell.interp.execfile(filename) - if cmd or script: + if shell and cmd or script: shell.interp.runcommand("""if 1: import sys as _sys _sys.argv = %r @@ -1598,22 +1443,17 @@ elif script: shell.interp.prepend_syspath(script) shell.interp.execfile(script) - elif shell: - # If there is a shell window and no cmd or script in progress, - # check for problematic OS X Tk versions and print a warning - # message in the IDLE shell window; this is less intrusive - # than always opening a separate window. - tkversionwarning = macosxSupport.tkVersionWarning(root) - if tkversionwarning: - shell.interp.runcommand("print('%s')" % tkversionwarning) - while flist.inversedict: # keep IDLE running while files are open. - root.mainloop() + # Check for problematic OS X Tk versions and print a warning message + # in the IDLE shell window; this is less intrusive than always opening + # a separate window. + tkversionwarning = macosxSupport.tkVersionWarning(root) + if tkversionwarning: + shell.interp.runcommand(''.join(("print('", tkversionwarning, "')"))) + + root.mainloop() root.destroy() - capture_warnings(False) if __name__ == "__main__": sys.modules['PyShell'] = sys.modules['__main__'] main() - -capture_warnings(False) # Make sure turned off; see issue 18081 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/README.txt --- a/Lib/idlelib/README.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/README.txt Mon Jan 25 17:05:13 2016 +0100 @@ -1,229 +1,60 @@ -README.txt: an index to idlelib files and the IDLE menu. +IDLE is Python's Tkinter-based Integrated DeveLopment Environment. -IDLE is Python's Integrated Development and Learning -Environment. The user documentation is part of the Library Reference and -is available in IDLE by selecting Help => IDLE Help. This README documents -idlelib for IDLE developers and curious users. +IDLE emphasizes a lightweight, clean design with a simple user interface. +Although it is suitable for beginners, even advanced users will find that +IDLE has everything they really need to develop pure Python code. -IDLELIB FILES lists files alphabetically by category, -with a short description of each. +IDLE features a multi-window text editor with multiple undo, Python colorizing, +and many other capabilities, e.g. smart indent, call tips, and autocompletion. -IDLE MENU show the menu tree, annotated with the module -or module object that implements the corresponding function. +The editor has comprehensive search functions, including searching through +multiple files. Class browsers and path browsers provide fast access to +code objects from a top level viewpoint without dealing with code folding. -This file is descriptive, not prescriptive, and may have errors -and omissions and lag behind changes in idlelib. +There is a Python Shell window which features colorizing and command recall. +IDLE executes Python code in a separate process, which is restarted for each +Run (F5) initiated from an editor window. The environment can also be +restarted from the Shell window without restarting IDLE. -IDLELIB FILES -Implemetation files not in IDLE MENU are marked (nim). -Deprecated files and objects are listed separately as the end. +This enhancement has often been requested, and is now finally available. The +magic "reload/import *" incantations are no longer required when editing and +testing a module two or three steps down the import chain. -Startup -------- -__init__.py # import, does nothing -__main__.py # -m, starts IDLE -idle.bat -idle.py -idle.pyw +(Personal firewall software may warn about the connection IDLE makes to its +subprocess using this computer's internal loopback interface. This connection +is not visible on any external interface and no data is sent to or received +from the Internet.) -Implementation --------------- -AutoComplete.py # Complete attribute names or filenames. -AutoCompleteWindow.py # Display completions. -AutoExpand.py # Expand word with previous word in file. -Bindings.py # Define most of IDLE menu. -CallTipWindow.py # Display calltip. -CallTips.py # Create calltip text. -ClassBrowser.py # Create module browser window. -CodeContext.py # Show compound statement headers otherwise not visible. -ColorDelegator.py # Colorize text (nim). -Debugger.py # Debug code run from editor; show window. -Delegator.py # Define base class for delegators (nim). -EditorWindow.py # Define most of editor and utility functions. -FileList.py # Open files and manage list of open windows (nim). -FormatParagraph.py# Re-wrap multiline strings and comments. -GrepDialog.py # Find all occurrences of pattern in multiple files. -HyperParser.py # Parse code around a given index. -IOBinding.py # Open, read, and write files -IdleHistory.py # Get previous or next user input in shell (nim) -MultiCall.py # Wrap tk widget to allow multiple calls per event (nim). -MultiStatusBar.py # Define status bar for windows (nim). -ObjectBrowser.py # Define class used in StackViewer (nim). -OutputWindow.py # Create window for grep output. -ParenMatch.py # Match fenceposts: (), [], and {}. -PathBrowser.py # Create path browser window. -Percolator.py # Manage delegator stack (nim). -PyParse.py # Give information on code indentation -PyShell.py # Start IDLE, manage shell, complete editor window -RemoteDebugger.py # Debug code run in remote process. -RemoteObjectBrowser.py # Communicate objects between processes with rpc (nim). -ReplaceDialog.py # Search and replace pattern in text. -RstripExtension.py# Strip trailing whitespace -ScriptBinding.py # Check and run user code. -ScrolledList.py # Define ScrolledList widget for IDLE (nim). -SearchDialog.py # Search for pattern in text. -SearchDialogBase.py # Define base for search, replace, and grep dialogs. -SearchEngine.py # Define engine for all 3 search dialogs. -StackViewer.py # View stack after exception. -TreeWidget.py # Define tree widger, used in browsers (nim). -UndoDelegator.py # Manage undo stack. -WidgetRedirector.py # Intercept widget subcommands (for percolator) (nim). -WindowList.py # Manage window list and define listed top level. -ZoomHeight.py # Zoom window to full height of screen. -aboutDialog.py # Display About IDLE dialog. -configDialog.py # Display user configuration dialogs. -configHandler.py # Load, fetch, and save configuration (nim). -configHelpSourceEdit.py # Specify help source. -configSectionNameDialog.py # Spefify user config section name -dynOptionMenuWidget.py # define mutable OptionMenu widget (nim). -help.py # Display IDLE's html doc. -keybindingDialog.py # Change keybindings. -macosxSupport.py # Help IDLE run on Macs (nim). -rpc.py # Commuicate between idle and user processes (nim). -run.py # Manage user code execution subprocess. -tabbedpages.py # Define tabbed pages widget (nim). -textView.py # Define read-only text widget (nim). +It is possible to interrupt tightly looping user code, even on Windows. -Configuration -------------- -config-extensions.def # Defaults for extensions -config-highlight.def # Defaults for colorizing -config-keys.def # Defaults for key bindings -config-main.def # Defai;ts fpr font and geneal +Applications which cannot support subprocesses and/or sockets can still run +IDLE in a single process. -Text ----- -CREDITS.txt # not maintained, displayed by About IDLE -HISTORY.txt # NEWS up to July 2001 -NEWS.txt # commits, displayed by About IDLE -README.txt # this file, displeyed by About IDLE -TODO.txt # needs review -extend.txt # about writing extensions -help.html # copy of idle.html in docs, displayed by IDLE Help +IDLE has an integrated debugger with stepping, persistent breakpoints, and call +stack visibility. -Subdirectories --------------- -Icons # small image files -idle_test # files for human test and automated unit tests +There is a GUI configuration manager which makes it easy to select fonts, +colors, keybindings, and startup options. This facility includes a feature +which allows the user to specify additional help sources, either locally or on +the web. -Unused and Deprecated files and objects (nim) ---------------------------------------------- -EditorWindow.py: Helpdialog and helpDialog -ToolTip.py: unused. -help.txt -idlever.py +IDLE is coded in 100% pure Python, using the Tkinter GUI toolkit (Tk/Tcl) +and is cross-platform, working on Unix, Mac, and Windows. +IDLE accepts command line arguments. Try idle -h to see the options. -IDLE MENUS -Top level items and most submenu items are defined in Bindings. -Extenstions add submenu items when active. The names given are -found, quoted, in one of these modules, paired with a '<>'. -Each pseudoevent is bound to an event handler. Some event handlers -call another function that does the actual work. The annotations below -are intended to at least give the module where the actual work is done. -File # IOBindig except as noted - New File - Open... # IOBinding.open - Open Module - Recent Files - Class Browser # Class Browser - Path Browser # Path Browser - --- - Save # IDBinding.save - Save As... # IOBinding.save_as - Save Copy As... # IOBindling.save_a_copy - --- - Print Window # IOBinding.print_window - --- - Close - Exit +If you find bugs or have suggestions or patches, let us know about +them by using the Python issue tracker: -Edit - Undo # undoDelegator - Redo # undoDelegator - --- - Cut - Copy - Paste - Select All - --- # Next 5 items use SearchEngine; dialogs use SearchDialogBase - Find # Search Dialog - Find Again - Find Selection - Find in Files... # GrepDialog - Replace... # ReplaceDialog - Go to Line - Show Completions # AutoComplete extension and AutoCompleteWidow (&HP) - Expand Word # AutoExpand extension - Show call tip # Calltips extension and CalltipWindow (& Hyperparser) - Show surrounding parens # ParenMatch (& Hyperparser) +http://bugs.python.org -Shell # PyShell - View Last Restart # PyShell.? - Restart Shell # PyShell.? +For further details and links, read the Help files and check the IDLE home +page at -Debug (Shell only) - Go to File/Line - Debugger # Debugger, RemoteDebugger - Stack Viewer # StackViewer - Auto-open Stack Viewer # StackViewer +http://www.python.org/idle/ -Format (Editor only) - Indent Region - Dedent Region - Comment Out Region - Uncomment Region - Tabify Region - Untabify Region - Toggle Tabs - New Indent Width - Format Paragraph # FormatParagraph extension - --- - Strip tailing whitespace # RstripExtension extension +There is a mail list for IDLE: idle-dev@python.org. You can join at -Run (Editor only) - Python Shell # PyShell - --- - Check Module # ScriptBinding - Run Module # ScriptBinding - -Options - Configure IDLE # configDialog - (tabs in the dialog) - Font tab # onfig-main.def - Highlight tab # configSectionNameDialog, config-highlight.def - Keys tab # keybindingDialog, configSectionNameDialog, onfig-keus.def - General tab # configHelpSourceEdit, config-main.def - Configure Extensions # configDialog - Xyz tab # xyz.py, config-extensions.def - --- - Code Context (editor only) # CodeContext extension - -Window - Zoomheight # ZoomHeight extension - --- - # WindowList - -Help - About IDLE # aboutDialog - --- - IDLE Help # help - Python Doc - Turtle Demo - --- - - - (right click) -Defined in EditorWindow, PyShell, Output - Cut - Copy - Paste - --- - Go to file/line (shell and output only) - Set Breakpoint (editor only) - Clear Breakpoint (editor only) - Defined in Debugger - Go to source line - Show stack frame +http://mail.python.org/mailman/listinfo/idle-dev diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/RemoteDebugger.py --- a/Lib/idlelib/RemoteDebugger.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/RemoteDebugger.py Mon Jan 25 17:05:13 2016 +0100 @@ -21,6 +21,7 @@ """ import types +from idlelib import rpc from idlelib import Debugger debugging = 0 @@ -98,7 +99,7 @@ else: tb = tracebacktable[tbid] stack, i = self.idb.get_stack(frame, tb) - stack = [(wrap_frame(frame2), k) for frame2, k in stack] + stack = [(wrap_frame(frame), k) for frame, k in stack] return stack, i def run(self, cmd): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ReplaceDialog.py --- a/Lib/idlelib/ReplaceDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ReplaceDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,8 +2,6 @@ from idlelib import SearchEngine from idlelib.SearchDialogBase import SearchDialogBase -import re - def replace(text): root = text._root() @@ -13,7 +11,6 @@ dialog = engine._replacedialog dialog.open(text) - class ReplaceDialog(SearchDialogBase): title = "Replace Dialog" @@ -40,7 +37,7 @@ def create_entries(self): SearchDialogBase.create_entries(self) - self.replent = self.make_entry("Replace with:", self.replvar)[0] + self.replent = self.make_entry("Replace with:", self.replvar) def create_command_buttons(self): SearchDialogBase.create_command_buttons(self) @@ -58,23 +55,8 @@ def default_command(self, event=None): if self.do_find(self.ok): - if self.do_replace(): # Only find next match if replace succeeded. - # A bad re can cause it to fail. - self.do_find(0) - - def _replace_expand(self, m, repl): - """ Helper function for expanding a regular expression - in the replace field, if needed. """ - if self.engine.isre(): - try: - new = m.expand(repl) - except re.error: - self.engine.report_error(repl, 'Invalid Replace Expression') - new = None - else: - new = repl - - return new + self.do_replace() + self.do_find(0) def replace_all(self, event=None): prog = self.engine.getprog() @@ -104,9 +86,7 @@ line, m = res chars = text.get("%d.0" % line, "%d.0" % (line+1)) orig = m.group() - new = self._replace_expand(m, repl) - if new is None: - break + new = m.expand(repl) i, j = m.span() first = "%d.%d" % (line, i) last = "%d.%d" % (line, j) @@ -158,9 +138,7 @@ m = prog.match(chars, col) if not prog: return False - new = self._replace_expand(m, self.replvar.get()) - if new is None: - return False + new = m.expand(self.replvar.get()) text.mark_set("insert", first) text.undo_block_start() if m.group(): @@ -188,34 +166,3 @@ def close(self, event=None): SearchDialogBase.close(self, event) self.text.tag_remove("hit", "1.0", "end") - -def _replace_dialog(parent): - root = Tk() - root.title("Test ReplaceDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - - # mock undo delegator methods - def undo_block_start(): - pass - - def undo_block_stop(): - pass - - text = Text(root) - text.undo_block_start = undo_block_start - text.undo_block_stop = undo_block_stop - text.pack() - text.insert("insert","This is a sample string.\n"*10) - - def show_replace(): - text.tag_add(SEL, "1.0", END) - replace(text) - text.tag_remove(SEL, "1.0", END) - - button = Button(root, text="Replace", command=show_replace) - button.pack() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_replace_dialog) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/RstripExtension.py --- a/Lib/idlelib/RstripExtension.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/RstripExtension.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,13 @@ 'Provides "Strip trailing whitespace" under the "Format" menu.' +__author__ = "Roger D. Serwy " + class RstripExtension: menudefs = [ - ('format', [None, ('Strip trailing whitespace', '<>'), ] ), ] + ('format', [None, + ('Strip trailing whitespace', '<>'), + ]),] def __init__(self, editwin): self.editwin = editwin @@ -16,18 +20,10 @@ undo.undo_block_start() - end_line = int(float(text.index('end'))) + end_line = int(float(text.index('end'))) + 1 for cur in range(1, end_line): - txt = text.get('%i.0' % cur, '%i.end' % cur) - raw = len(txt) + txt = text.get('%i.0' % cur, '%i.0 lineend' % cur) cut = len(txt.rstrip()) - # Since text.delete() marks file as changed, even if not, - # only call it when needed to actually delete something. - if cut < raw: - text.delete('%i.%i' % (cur, cut), '%i.end' % cur) + text.delete('%i.%i' % (cur, cut), '%i.0 lineend' % cur) undo.undo_block_stop() - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_rstrip', verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ScriptBinding.py --- a/Lib/idlelib/ScriptBinding.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ScriptBinding.py Mon Jan 25 17:05:13 2016 +0100 @@ -18,10 +18,13 @@ """ import os +import re +import string import tabnanny import tokenize import tkinter.messagebox as tkMessageBox -from idlelib import PyShell +from idlelib.EditorWindow import EditorWindow +from idlelib import PyShell, IOBinding from idlelib.configHandler import idleConf from idlelib import macosxSupport @@ -36,7 +39,6 @@ by Format->Untabify Region and specify the number of columns used by each tab. """ - class ScriptBinding: menudefs = [ @@ -51,7 +53,7 @@ self.flist = self.editwin.flist self.root = self.editwin.root - if macosxSupport.isCocoaTk(): + if macosxSupport.runningAsOSXApp(): self.editwin.text_frame.bind('<>', self._run_module_event) def check_module_event(self, event): @@ -69,7 +71,7 @@ try: tabnanny.process_tokens(tokenize.generate_tokens(f.readline)) except tokenize.TokenError as msg: - msgtxt, (lineno, start) = msg.args + msgtxt, (lineno, start) = msg self.editwin.gotoline(lineno) self.errorbox("Tabnanny Tokenizing Error", "Token Error: %s" % msgtxt) @@ -85,8 +87,9 @@ self.shell = shell = self.flist.open_shell() saved_stream = shell.get_warning_stream() shell.set_warning_stream(shell.stderr) - with open(filename, 'rb') as f: - source = f.read() + f = open(filename, 'rb') + source = f.read() + f.close() if b'\r' in source: source = source.replace(b'\r\n', b'\n') source = source.replace(b'\r', b'\n') @@ -112,7 +115,7 @@ shell.set_warning_stream(saved_stream) def run_module_event(self, event): - if macosxSupport.isCocoaTk(): + if macosxSupport.runningAsOSXApp(): # Tk-Cocoa in MacOSX is broken until at least # Tk 8.5.9, and without this rather # crude workaround IDLE would hang when a user @@ -143,21 +146,20 @@ return 'break' interp = self.shell.interp if PyShell.use_subprocess: - interp.restart_subprocess(with_cwd=False, filename= - self.editwin._filename_to_unicode(filename)) + interp.restart_subprocess(with_cwd=False) dirname = os.path.dirname(filename) # XXX Too often this discards arguments the user just set... interp.runcommand("""if 1: - __file__ = {filename!r} + _filename = %r import sys as _sys from os.path import basename as _basename if (not _sys.argv or - _basename(_sys.argv[0]) != _basename(__file__)): - _sys.argv = [__file__] + _basename(_sys.argv[0]) != _basename(_filename)): + _sys.argv = [_filename] import os as _os - _os.chdir({dirname!r}) - del _sys, _basename, _os - \n""".format(filename=filename, dirname=dirname)) + _os.chdir(%r) + del _filename, _sys, _basename, _os + \n""" % (filename, dirname)) interp.prepend_syspath(filename) # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still # go to __stderr__. With subprocess, they go to the shell. @@ -197,10 +199,10 @@ confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", message=msg, default=tkMessageBox.OK, - parent=self.editwin.text) + master=self.editwin.text) return confirm def errorbox(self, title, message): # XXX This should really be a function of EditorWindow... - tkMessageBox.showerror(title, message, parent=self.editwin.text) + tkMessageBox.showerror(title, message, master=self.editwin.text) self.editwin.text.focus_set() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ScrolledList.py --- a/Lib/idlelib/ScrolledList.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ScrolledList.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,5 +1,4 @@ from tkinter import * -from idlelib import macosxSupport class ScrolledList: @@ -23,11 +22,7 @@ # Bind events to the list box listbox.bind("", self.click_event) listbox.bind("", self.double_click_event) - if macosxSupport.isAquaTk(): - listbox.bind("", self.popup_event) - listbox.bind("", self.popup_event) - else: - listbox.bind("", self.popup_event) + listbox.bind("", self.popup_event) listbox.bind("", self.up_event) listbox.bind("", self.down_event) # Mark as empty @@ -124,22 +119,21 @@ pass -def _scrolled_list(parent): +def test(): root = Tk() - root.title("Test ScrolledList") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) + root.protocol("WM_DELETE_WINDOW", root.destroy) class MyScrolledList(ScrolledList): - def fill_menu(self): self.menu.add_command(label="right click") + def fill_menu(self): self.menu.add_command(label="pass") def on_select(self, index): print("select", self.get(index)) def on_double(self, index): print("double", self.get(index)) + s = MyScrolledList(root) + for i in range(30): + s.append("item %02d" % i) + return root - scrolled_list = MyScrolledList(root) - for i in range(30): - scrolled_list.append("Item %02d" % i) - +def main(): + root = test() root.mainloop() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_scrolled_list) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/SearchDialog.py --- a/Lib/idlelib/SearchDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/SearchDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -23,13 +23,14 @@ class SearchDialog(SearchDialogBase): def create_widgets(self): - SearchDialogBase.create_widgets(self) - self.make_button("Find Next", self.default_command, 1) + f = SearchDialogBase.create_widgets(self) + self.make_button("Find", self.default_command, 1) def default_command(self, event=None): if not self.engine.getprog(): return - self.find_again(self.text) + if self.find_again(self.text): + self.close() def find_again(self, text): if not self.engine.getpat(): @@ -65,25 +66,3 @@ if pat: self.engine.setcookedpat(pat) return self.find_again(text) - -def _search_dialog(parent): - root = Tk() - root.title("Test SearchDialog") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - text = Text(root) - text.pack() - text.insert("insert","This is a sample string.\n"*10) - - def show_find(): - text.tag_add(SEL, "1.0", END) - s = _setup(text) - s.open(text) - text.tag_remove(SEL, "1.0", END) - - button = Button(root, text="Search", command=show_find) - button.pack() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_search_dialog) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/SearchDialogBase.py --- a/Lib/idlelib/SearchDialogBase.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/SearchDialogBase.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,51 +1,17 @@ -'''Define SearchDialogBase used by Search, Replace, and Grep dialogs.''' - -from tkinter import (Toplevel, Frame, Entry, Label, Button, - Checkbutton, Radiobutton) +from tkinter import * class SearchDialogBase: - '''Create most of a 3 or 4 row, 3 column search dialog. - The left and wide middle column contain: - 1 or 2 labeled text entry lines (make_entry, create_entries); - a row of standard Checkbuttons (make_frame, create_option_buttons), - each of which corresponds to a search engine Variable; - a row of dialog-specific Check/Radiobuttons (create_other_buttons). - - The narrow right column contains command buttons - (make_button, create_command_buttons). - These are bound to functions that execute the command. - - Except for command buttons, this base class is not limited to items - common to all three subclasses. Rather, it is the Find dialog minus - the "Find Next" command, its execution function, and the - default_command attribute needed in create_widgets. The other - dialogs override attributes and methods, the latter to replace and - add widgets. - ''' - - title = "Search Dialog" # replace in subclasses + title = "Search Dialog" icon = "Search" - needwrapbutton = 1 # not in Find in Files + needwrapbutton = 1 def __init__(self, root, engine): - '''Initialize root, engine, and top attributes. - - top (level widget): set in create_widgets() called from open(). - text (Text searched): set in open(), only used in subclasses(). - ent (ry): created in make_entry() called from create_entry(). - row (of grid): 0 in create_widgets(), +1 in make_entry/frame(). - default_command: set in subclasses, used in create_widgers(). - - title (of dialog): class attribute, override in subclasses. - icon (of dialog): ditto, use unclear if cannot minimize dialog. - ''' self.root = root self.engine = engine self.top = None def open(self, text, searchphrase=None): - "Make dialog visible on top of others and ready to use." self.text = text if not self.top: self.create_widgets() @@ -61,17 +27,11 @@ self.top.grab_set() def close(self, event=None): - "Put dialog away for later use." if self.top: self.top.grab_release() self.top.withdraw() def create_widgets(self): - '''Create basic 3 row x 3 col search (find) dialog. - - Other dialogs override subsidiary create_x methods as needed. - Replace and Find-in-Files add another entry row. - ''' top = Toplevel(self.root) top.bind("", self.default_command) top.bind("", self.close) @@ -84,84 +44,29 @@ self.top.grid_columnconfigure(0, pad=2, weight=0) self.top.grid_columnconfigure(1, pad=2, minsize=100, weight=100) - self.create_entries() # row 0 (and maybe 1), cols 0, 1 - self.create_option_buttons() # next row, cols 0, 1 - self.create_other_buttons() # next row, cols 0, 1 - self.create_command_buttons() # col 2, all rows + self.create_entries() + self.create_option_buttons() + self.create_other_buttons() + return self.create_command_buttons() - def make_entry(self, label_text, var): - '''Return (entry, label), . - - entry - gridded labeled Entry for text entry. - label - Label widget, returned for testing. - ''' - label = Label(self.top, text=label_text) - label.grid(row=self.row, column=0, sticky="nw") - entry = Entry(self.top, textvariable=var, exportselection=0) - entry.grid(row=self.row, column=1, sticky="nwe") + def make_entry(self, label, var): + l = Label(self.top, text=label) + l.grid(row=self.row, column=0, sticky="nw") + e = Entry(self.top, textvariable=var, exportselection=0) + e.grid(row=self.row, column=1, sticky="nwe") self.row = self.row + 1 - return entry, label - - def create_entries(self): - "Create one or more entry lines with make_entry." - self.ent = self.make_entry("Find:", self.engine.patvar)[0] + return e def make_frame(self,labeltext=None): - '''Return (frame, label). - - frame - gridded labeled Frame for option or other buttons. - label - Label widget, returned for testing. - ''' if labeltext: - label = Label(self.top, text=labeltext) - label.grid(row=self.row, column=0, sticky="nw") - else: - label = '' - frame = Frame(self.top) - frame.grid(row=self.row, column=1, columnspan=1, sticky="nwe") + l = Label(self.top, text=labeltext) + l.grid(row=self.row, column=0, sticky="nw") + f = Frame(self.top) + f.grid(row=self.row, column=1, columnspan=1, sticky="nwe") self.row = self.row + 1 - return frame, label - - def create_option_buttons(self): - '''Return (filled frame, options) for testing. - - Options is a list of SearchEngine booleanvar, label pairs. - A gridded frame from make_frame is filled with a Checkbutton - for each pair, bound to the var, with the corresponding label. - ''' - frame = self.make_frame("Options")[0] - engine = self.engine - options = [(engine.revar, "Regular expression"), - (engine.casevar, "Match case"), - (engine.wordvar, "Whole word")] - if self.needwrapbutton: - options.append((engine.wrapvar, "Wrap around")) - for var, label in options: - btn = Checkbutton(frame, anchor="w", variable=var, text=label) - btn.pack(side="left", fill="both") - if var.get(): - btn.select() - return frame, options - - def create_other_buttons(self): - '''Return (frame, others) for testing. - - Others is a list of value, label pairs. - A gridded frame from make_frame is filled with radio buttons. - ''' - frame = self.make_frame("Direction")[0] - var = self.engine.backvar - others = [(1, 'Up'), (0, 'Down')] - for val, label in others: - btn = Radiobutton(frame, anchor="w", - variable=var, value=val, text=label) - btn.pack(side="left", fill="both") - if var.get() == val: - btn.select() - return frame, others + return f def make_button(self, label, command, isdef=0): - "Return command button gridded in command frame." b = Button(self.buttonframe, text=label, command=command, default=isdef and "active" or "normal") @@ -170,15 +75,66 @@ self.buttonframe.grid(rowspan=rows+1) return b + def create_entries(self): + self.ent = self.make_entry("Find:", self.engine.patvar) + + def create_option_buttons(self): + f = self.make_frame("Options") + + btn = Checkbutton(f, anchor="w", + variable=self.engine.revar, + text="Regular expression") + btn.pack(side="left", fill="both") + if self.engine.isre(): + btn.select() + + btn = Checkbutton(f, anchor="w", + variable=self.engine.casevar, + text="Match case") + btn.pack(side="left", fill="both") + if self.engine.iscase(): + btn.select() + + btn = Checkbutton(f, anchor="w", + variable=self.engine.wordvar, + text="Whole word") + btn.pack(side="left", fill="both") + if self.engine.isword(): + btn.select() + + if self.needwrapbutton: + btn = Checkbutton(f, anchor="w", + variable=self.engine.wrapvar, + text="Wrap around") + btn.pack(side="left", fill="both") + if self.engine.iswrap(): + btn.select() + + def create_other_buttons(self): + f = self.make_frame("Direction") + + #lbl = Label(f, text="Direction: ") + #lbl.pack(side="left") + + btn = Radiobutton(f, anchor="w", + variable=self.engine.backvar, value=1, + text="Up") + btn.pack(side="left", fill="both") + if self.engine.isback(): + btn.select() + + btn = Radiobutton(f, anchor="w", + variable=self.engine.backvar, value=0, + text="Down") + btn.pack(side="left", fill="both") + if not self.engine.isback(): + btn.select() + def create_command_buttons(self): - "Place buttons in vertical command frame gridded on right." + # + # place button frame on the right f = self.buttonframe = Frame(self.top) f.grid(row=0,column=2,padx=2,pady=2,ipadx=2,ipady=2) b = self.make_button("close", self.close) b.lower() - -if __name__ == '__main__': - import unittest - unittest.main( - 'idlelib.idle_test.test_searchdialogbase', verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/SearchEngine.py --- a/Lib/idlelib/SearchEngine.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/SearchEngine.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,34 +1,26 @@ -'''Define SearchEngine for search dialogs.''' import re -from tkinter import StringVar, BooleanVar, TclError +from tkinter import * import tkinter.messagebox as tkMessageBox def get(root): - '''Return the singleton SearchEngine instance for the process. - - The single SearchEngine saves settings between dialog instances. - If there is not a SearchEngine already, make one. - ''' if not hasattr(root, "_searchengine"): root._searchengine = SearchEngine(root) - # This creates a cycle that persists until root is deleted. + # XXX This will never garbage-collect -- who cares return root._searchengine class SearchEngine: - """Handles searching a text widget for Find, Replace, and Grep.""" def __init__(self, root): - '''Initialize Variables that save search state. - - The dialogs bind these to the UI elements present in the dialogs. - ''' - self.root = root # need for report_error() - self.patvar = StringVar(root, '') # search pattern - self.revar = BooleanVar(root, False) # regular expression? - self.casevar = BooleanVar(root, False) # match case? - self.wordvar = BooleanVar(root, False) # match whole word? - self.wrapvar = BooleanVar(root, True) # wrap around buffer? - self.backvar = BooleanVar(root, False) # search backwards? + self.root = root + # State shared by search, replace, and grep; + # the search dialogs bind these to UI elements. + self.patvar = StringVar(root) # search pattern + self.revar = BooleanVar(root) # regular expression? + self.casevar = BooleanVar(root) # match case? + self.wordvar = BooleanVar(root) # match whole word? + self.wrapvar = BooleanVar(root) # wrap around buffer? + self.wrapvar.set(1) # (on by default) + self.backvar = BooleanVar(root) # search backwards? # Access methods @@ -55,23 +47,15 @@ # Higher level access methods - def setcookedpat(self, pat): - "Set pattern after escaping if re." - # called only in SearchDialog.py: 66 - if self.isre(): - pat = re.escape(pat) - self.setpat(pat) - def getcookedpat(self): pat = self.getpat() - if not self.isre(): # if True, see setcookedpat + if not self.isre(): pat = re.escape(pat) if self.isword(): pat = r"\b%s\b" % pat return pat def getprog(self): - "Return compiled cooked search pattern." pat = self.getpat() if not pat: self.report_error(pat, "Empty regular expression") @@ -83,41 +67,50 @@ try: prog = re.compile(pat, flags) except re.error as what: - args = what.args - msg = args[0] - col = args[1] if len(args) >= 2 else -1 + try: + msg, col = what + except: + msg = str(what) + col = -1 self.report_error(pat, msg, col) return None return prog def report_error(self, pat, msg, col=-1): - # Derived class could override this with something fancier + # Derived class could overrid this with something fancier msg = "Error: " + str(msg) if pat: - msg = msg + "\nPattern: " + str(pat) + msg = msg + "\np\Pattern: " + str(pat) if col >= 0: msg = msg + "\nOffset: " + str(col) tkMessageBox.showerror("Regular expression error", msg, master=self.root) + def setcookedpat(self, pat): + if self.isre(): + pat = re.escape(pat) + self.setpat(pat) + def search_text(self, text, prog=None, ok=0): - '''Return (lineno, matchobj) or None for forward/backward search. + """Search a text widget for the pattern. - This function calls the right function with the right arguments. - It directly return the result of that call. + If prog is given, it should be the precompiled pattern. + Return a tuple (lineno, matchobj); None if not found. - Text is a text widget. Prog is a precompiled pattern. - The ok parameter is a bit complicated as it has two effects. + This obeys the wrap and direction (back) settings. - If there is a selection, the search begin at either end, - depending on the direction setting and ok, with ok meaning that - the search starts with the selection. Otherwise, search begins - at the insert mark. + The search starts at the selection (if there is one) or + at the insert mark (otherwise). If the search is forward, + it starts at the right of the selection; for a backward + search, it starts at the left end. An empty match exactly + at either end of the selection (or at the insert mark if + there is no selection) is ignored unless the ok flag is true + -- this is done to guarantee progress. - To aid progress, the search functions do not return an empty - match at the starting position unless ok is True. - ''' + If the search is allowed to wrap around, it will return the + original selection if (and only if) it is the only match. + """ if not prog: prog = self.getprog() if not prog: @@ -186,19 +179,15 @@ col = len(chars) - 1 return None +# Helper to search backwards in a string. +# (Optimized for the case where the pattern isn't found.) + def search_reverse(prog, chars, col): - '''Search backwards and return an re match object or None. - - This is done by searching forwards until there is no match. - Prog: compiled re object with a search method returning a match. - Chars: line of text, without \\n. - Col: stop index for the search; the limit for match.end(). - ''' m = prog.search(chars) if not m: return None found = None - i, j = m.span() # m.start(), m.end() == match slice indexes + i, j = m.span() while i < col and j <= col: found = m if i == j: @@ -209,9 +198,10 @@ i, j = m.span() return found +# Helper to get selection end points, defaulting to insert mark. +# Return a tuple of indices ("line.col" strings). + def get_selection(text): - '''Return tuple of 'line.col' indexes from selection or insert mark. - ''' try: first = text.index("sel.first") last = text.index("sel.last") @@ -223,11 +213,8 @@ last = first return first, last +# Helper to parse a text index into a (line, col) tuple. + def get_line_col(index): - '''Return (line, col) tuple of ints from 'line.col' string.''' line, col = map(int, index.split(".")) # Fails on invalid index return line, col - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/StackViewer.py --- a/Lib/idlelib/StackViewer.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/StackViewer.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,16 +1,14 @@ import os import sys import linecache -import re -import tkinter as tk from idlelib.TreeWidget import TreeNode, TreeItem, ScrolledCanvas from idlelib.ObjectBrowser import ObjectTreeItem, make_objecttreeitem -from idlelib.PyShell import PyShellFileList def StackBrowser(root, flist=None, tb=None, top=None): if top is None: - top = tk.Toplevel(root) + from tkinter import Toplevel + top = Toplevel(root) sc = ScrolledCanvas(top, bg="white", highlightthickness=0) sc.frame.pack(expand=1, fill="both") item = StackTreeItem(flist, tb) @@ -107,9 +105,12 @@ def IsExpandable(self): return len(self.object) > 0 + def keys(self): + return list(self.object.keys()) + def GetSubList(self): sublist = [] - for key in self.object.keys(): + for key in self.keys(): try: value = self.object[key] except KeyError: @@ -119,33 +120,3 @@ item = make_objecttreeitem(key + " =", value, setfunction) sublist.append(item) return sublist - - def keys(self): # unused, left for possible 3rd party use - return list(self.object.keys()) - -def _stack_viewer(parent): - root = tk.Tk() - root.title("Test StackViewer") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - flist = PyShellFileList(root) - try: # to obtain a traceback object - intentional_name_error - except NameError: - exc_type, exc_value, exc_tb = sys.exc_info() - - # inject stack trace to sys - sys.last_type = exc_type - sys.last_value = exc_value - sys.last_traceback = exc_tb - - StackBrowser(root, flist=flist, top=root, tb=exc_tb) - - # restore sys to original state - del sys.last_type - del sys.last_value - del sys.last_traceback - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_stack_viewer) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ToolTip.py --- a/Lib/idlelib/ToolTip.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ToolTip.py Mon Jan 25 17:05:13 2016 +0100 @@ -76,22 +76,14 @@ for item in self.items: listbox.insert(END, item) -def _tooltip(parent): +def main(): + # Test code root = Tk() - root.title("Test tooltip") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - label = Label(root, text="Place your mouse over buttons") - label.pack() - button1 = Button(root, text="Button 1") - button2 = Button(root, text="Button 2") - button1.pack() - button2.pack() - ToolTip(button1, "This is tooltip text for button1.") - ListboxToolTip(button2, ["This is","multiple line", - "tooltip text","for button2"]) + b = Button(root, text="Hello", command=root.destroy) + b.pack() + root.update() + tip = ListboxToolTip(b, ["Hello", "world"]) root.mainloop() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tooltip) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/TreeWidget.py --- a/Lib/idlelib/TreeWidget.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/TreeWidget.py Mon Jan 25 17:05:13 2016 +0100 @@ -16,6 +16,7 @@ import os from tkinter import * +import imp from idlelib import ZoomHeight from idlelib.configHandler import idleConf @@ -173,12 +174,11 @@ def draw(self, x, y): # XXX This hard-codes too many geometry constants! - dy = 20 self.x, self.y = x, y self.drawicon() self.drawtext() if self.state != 'expanded': - return y + dy + return y+17 # draw children if not self.children: sublist = self.item._GetSubList() @@ -189,7 +189,7 @@ child = self.__class__(self.canvas, self, item) self.children.append(child) cx = x+20 - cy = y + dy + cy = y+17 cylast = 0 for child in self.children: cylast = cy @@ -228,7 +228,7 @@ def drawtext(self): textx = self.x+20-1 - texty = self.y-4 + texty = self.y-1 labeltext = self.item.GetLabelText() if labeltext: id = self.canvas.create_text(textx, texty, anchor="nw", @@ -245,11 +245,11 @@ else: self.edit_finish() try: - self.label + label = self.label except AttributeError: # padding carefully selected (on Windows) to match Entry widget: self.label = Label(self.canvas, text=text, bd=0, padx=2, pady=2) - theme = idleConf.CurrentTheme() + theme = idleConf.GetOption('main','Theme','name') if self.selected: self.label.configure(idleConf.GetHighlight(theme, 'hilite')) else: @@ -382,7 +382,7 @@ try: os.rename(self.path, newpath) self.path = newpath - except OSError: + except os.error: pass def GetIconName(self): @@ -395,7 +395,7 @@ def GetSubList(self): try: names = os.listdir(self.path) - except OSError: + except os.error: return [] names.sort(key = os.path.normcase) sublist = [] @@ -449,18 +449,29 @@ return "break" -def _tree_widget(parent): - root = Tk() - root.title("Test TreeWidget") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) +# Testing functions + +def test(): + from idlelib import PyShell + root = Toplevel(PyShell.root) + root.configure(bd=0, bg="yellow") + root.focus_set() sc = ScrolledCanvas(root, bg="white", highlightthickness=0, takefocus=1) - sc.frame.pack(expand=1, fill="both", side=LEFT) - item = FileTreeItem(os.getcwd()) + sc.frame.pack(expand=1, fill="both") + item = FileTreeItem("C:/windows/desktop") node = TreeNode(sc.canvas, None, item) node.expand() - root.mainloop() + +def test2(): + # test w/o scrolling canvas + root = Tk() + root.configure(bd=0) + canvas = Canvas(root, bg="white", highlightthickness=0) + canvas.pack(expand=1, fill="both") + item = FileTreeItem(os.curdir) + node = TreeNode(canvas, None, item) + node.update() + canvas.focus_set() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tree_widget) + test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/UndoDelegator.py --- a/Lib/idlelib/UndoDelegator.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/UndoDelegator.py Mon Jan 25 17:05:13 2016 +0100 @@ -336,30 +336,17 @@ self.depth = self.depth + incr return self.depth -def _undo_delegator(parent): +def main(): from idlelib.Percolator import Percolator root = Tk() - root.title("Test UndoDelegator") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - - text = Text(root) - text.config(height=10) + root.wm_protocol("WM_DELETE_WINDOW", root.quit) + text = Text() text.pack() text.focus_set() p = Percolator(text) d = UndoDelegator() p.insertfilter(d) - - undo = Button(root, text="Undo", command=lambda:d.undo_event(None)) - undo.pack(side='left') - redo = Button(root, text="Redo", command=lambda:d.redo_event(None)) - redo.pack(side='left') - dump = Button(root, text="Dump", command=lambda:d.dump_event(None)) - dump.pack(side='left') - root.mainloop() if __name__ == "__main__": - from idlelib.idle_test.htest import run - run(_undo_delegator) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/WidgetRedirector.py --- a/Lib/idlelib/WidgetRedirector.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/WidgetRedirector.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,40 +1,29 @@ -from tkinter import TclError +from tkinter import * class WidgetRedirector: + """Support for redirecting arbitrary widget subcommands. - Some Tk operations don't normally pass through tkinter. For example, if a + Some Tk operations don't normally pass through Tkinter. For example, if a character is inserted into a Text widget by pressing a key, a default Tk binding to the widget's 'insert' operation is activated, and the Tk library - processes the insert without calling back into tkinter. + processes the insert without calling back into Tkinter. - Although a binding to could be made via tkinter, what we really want - to do is to hook the Tk 'insert' operation itself. For one thing, we want - a text.insert call in idle code to have the same effect as a key press. + Although a binding to could be made via Tkinter, what we really want + to do is to hook the Tk 'insert' operation itself. When a widget is instantiated, a Tcl command is created whose name is the same as the pathname widget._w. This command is used to invoke the various widget operations, e.g. insert (for a Text widget). We are going to hook this command and provide a facility ('register') to intercept the widget - operation. We will also intercept method calls on the tkinter class - instance that represents the tk widget. + operation. - In IDLE, WidgetRedirector is used in Percolator to intercept Text - commands. The function being registered provides access to the top - of a Percolator chain. At the bottom of the chain is a call to the - original Tk widget operation. + In IDLE, the function being registered provides access to the top of a + Percolator chain. At the bottom of the chain is a call to the original + Tk widget operation. + """ def __init__(self, widget): - '''Initialize attributes and setup redirection. - - _operations: dict mapping operation name to new function. - widget: the widget whose tcl command is to be intercepted. - tk: widget.tk, a convenience attribute, probably not needed. - orig: new name of the original tcl command. - - Since renaming to orig fails with TclError when orig already - exists, only one WidgetDirector can exist for a given widget. - ''' self._operations = {} self.widget = widget # widget instance self.tk = tk = widget.tk # widget's root @@ -47,50 +36,31 @@ tk.createcommand(w, self.dispatch) def __repr__(self): - return "%s(%s<%s>)" % (self.__class__.__name__, - self.widget.__class__.__name__, - self.widget._w) + return "WidgetRedirector(%s<%s>)" % (self.widget.__class__.__name__, + self.widget._w) def close(self): - "Unregister operations and revert redirection created by .__init__." for operation in list(self._operations): self.unregister(operation) - widget = self.widget + widget = self.widget; del self.widget + orig = self.orig; del self.orig tk = widget.tk w = widget._w - # Restore the original widget Tcl command. tk.deletecommand(w) - tk.call("rename", self.orig, w) - del self.widget, self.tk # Should not be needed - # if instance is deleted after close, as in Percolator. + # restore the original widget Tcl command: + tk.call("rename", orig, w) def register(self, operation, function): - '''Return OriginalCommand(operation) after registering function. - - Registration adds an operation: function pair to ._operations. - It also adds an widget function attribute that masks the tkinter - class instance method. Method masking operates independently - from command dispatch. - - If a second function is registered for the same operation, the - first function is replaced in both places. - ''' self._operations[operation] = function setattr(self.widget, operation, function) return OriginalCommand(self, operation) def unregister(self, operation): - '''Return the function for the operation, or None. - - Deleting the instance attribute unmasks the class attribute. - ''' if operation in self._operations: function = self._operations[operation] del self._operations[operation] - try: + if hasattr(self.widget, operation): delattr(self.widget, operation) - except AttributeError: - pass return function else: return None @@ -118,59 +88,39 @@ class OriginalCommand: - '''Callable for original tk command that has been redirected. - - Returned by .register; can be used in the function registered. - redir = WidgetRedirector(text) - def my_insert(*args): - print("insert", args) - original_insert(*args) - original_insert = redir.register("insert", my_insert) - ''' def __init__(self, redir, operation): - '''Create .tk_call and .orig_and_operation for .__call__ method. - - .redir and .operation store the input args for __repr__. - .tk and .orig copy attributes of .redir (probably not needed). - ''' self.redir = redir self.operation = operation - self.tk = redir.tk # redundant with self.redir - self.orig = redir.orig # redundant with self.redir - # These two could be deleted after checking recipient code. - self.tk_call = redir.tk.call - self.orig_and_operation = (redir.orig, operation) + self.tk = redir.tk + self.orig = redir.orig + self.tk_call = self.tk.call + self.orig_and_operation = (self.orig, self.operation) def __repr__(self): - return "%s(%r, %r)" % (self.__class__.__name__, - self.redir, self.operation) + return "OriginalCommand(%r, %r)" % (self.redir, self.operation) def __call__(self, *args): return self.tk_call(self.orig_and_operation + args) -def _widget_redirector(parent): # htest # - from tkinter import Tk, Text - import re - +def main(): root = Tk() - root.title("Test WidgetRedirector") - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 150)) - text = Text(root) + root.wm_protocol("WM_DELETE_WINDOW", root.quit) + text = Text() text.pack() text.focus_set() redir = WidgetRedirector(text) + global previous_tcl_fcn def my_insert(*args): print("insert", args) - original_insert(*args) - original_insert = redir.register("insert", my_insert) + previous_tcl_fcn(*args) + previous_tcl_fcn = redir.register("insert", my_insert) root.mainloop() + redir.unregister("insert") # runs after first 'close window' + redir.close() + root.mainloop() + root.destroy() if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_widgetredir', - verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(_widget_redirector) + main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/ZoomHeight.py --- a/Lib/idlelib/ZoomHeight.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/ZoomHeight.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,7 +32,7 @@ newy = 0 newheight = newheight - 72 - elif macosxSupport.isAquaTk(): + elif macosxSupport.runningAsOSXApp(): # The '88' below is a magic number that avoids placing the bottom # of the window below the panel on my machine. I don't know how # to calculate the correct value for this with tkinter. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/__init__.py --- a/Lib/idlelib/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,8 +1,1 @@ -"""The idlelib package implements the Idle application. - -Idle includes an interactive shell and editor. -Use the files named idle.* to start Idle. - -The other files are private implementations. Their details are subject to -change. See PEP 434 for more. Import them at your own risk. -""" +# Dummy file to make this a package. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/__main__.py --- a/Lib/idlelib/__main__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,8 +0,0 @@ -""" -IDLE main entry point - -Run IDLE as python -m idlelib -""" -import idlelib.PyShell -idlelib.PyShell.main() -# This file does not work for 2.7; See issue 24212. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/aboutDialog.py --- a/Lib/idlelib/aboutDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/aboutDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,25 +2,21 @@ """ +from tkinter import * import os -from sys import version -from tkinter import * + from idlelib import textView +from idlelib import idlever class AboutDialog(Toplevel): """Modal about dialog for idle """ - def __init__(self, parent, title, _htest=False): - """ - _htest - bool, change box location when running htest - """ + def __init__(self,parent,title): Toplevel.__init__(self, parent) self.configure(borderwidth=5) - # place dialog below parent if running htest - self.geometry("+%d+%d" % ( - parent.winfo_rootx()+30, - parent.winfo_rooty()+(30 if not _htest else 100))) + self.geometry("+%d+%d" % (parent.winfo_rootx()+30, + parent.winfo_rooty()+30)) self.bg = "#707070" self.fg = "#ffffff" self.CreateWidgets() @@ -36,7 +32,6 @@ self.wait_window() def CreateWidgets(self): - release = version[:version.index(' ')] frameMain = Frame(self, borderwidth=2, relief=SUNKEN) frameButtons = Frame(self) frameButtons.pack(side=BOTTOM, fill=X) @@ -62,17 +57,21 @@ justify=LEFT, fg=self.fg, bg=self.bg) labelEmail.grid(row=6, column=0, columnspan=2, sticky=W, padx=10, pady=0) - labelWWW = Label(frameBg, text='https://docs.python.org/' + - version[:3] + '/library/idle.html', + labelWWW = Label(frameBg, text='www: http://www.python.org/idle/', justify=LEFT, fg=self.fg, bg=self.bg) labelWWW.grid(row=7, column=0, columnspan=2, sticky=W, padx=10, pady=0) Frame(frameBg, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=8, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - labelPythonVer = Label(frameBg, text='Python version: ' + - release, fg=self.fg, bg=self.bg) + labelPythonVer = Label(frameBg, text='Python version: ' + \ + sys.version.split()[0], fg=self.fg, bg=self.bg) labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - tkVer = self.tk.call('info', 'patchlevel') + # handle weird tk version num in windoze python >= 1.6 (?!?) + tkVer = repr(TkVersion).split('.') + tkVer[len(tkVer)-1] = str('%.3g' % (float('.'+tkVer[len(tkVer)-1])))[2:] + if tkVer[len(tkVer)-1] == '': + tkVer[len(tkVer)-1] = '0' + tkVer = '.'.join(tkVer) labelTkVer = Label(frameBg, text='Tk version: '+ tkVer, fg=self.fg, bg=self.bg) labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) @@ -93,7 +92,7 @@ Frame(frameBg, borderwidth=1, relief=SUNKEN, height=2, bg=self.bg).grid(row=11, column=0, sticky=EW, columnspan=3, padx=5, pady=5) - idle_v = Label(frameBg, text='IDLE version: ' + release, + idle_v = Label(frameBg, text='IDLE version: ' + idlever.IDLE_VERSION, fg=self.fg, bg=self.bg) idle_v.grid(row=12, column=0, sticky=W, padx=10, pady=0) idle_button_f = Frame(frameBg, bg=self.bg) @@ -111,7 +110,6 @@ command=self.ShowIDLECredits) idle_credits_b.pack(side=LEFT, padx=10, pady=10) - # License, et all, are of type _sitebuiltins._Printer def ShowLicense(self): self.display_printer_text('About - License', license) @@ -121,16 +119,14 @@ def ShowPythonCredits(self): self.display_printer_text('About - Python Credits', credits) - # Encode CREDITS.txt to utf-8 for proper version of Loewis. - # Specify others as ascii until need utf-8, so catch errors. def ShowIDLECredits(self): - self.display_file_text('About - Credits', 'CREDITS.txt', 'utf-8') + self.display_file_text('About - Credits', 'CREDITS.txt', 'iso-8859-1') def ShowIDLEAbout(self): - self.display_file_text('About - Readme', 'README.txt', 'ascii') + self.display_file_text('About - Readme', 'README.txt') def ShowIDLENEWS(self): - self.display_file_text('About - NEWS', 'NEWS.txt', 'ascii') + self.display_file_text('About - NEWS', 'NEWS.txt') def display_printer_text(self, title, printer): printer._Printer__setup() @@ -145,5 +141,10 @@ self.destroy() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(AboutDialog) + # test the dialog + root = Tk() + def run(): + from idlelib import aboutDialog + aboutDialog.AboutDialog(root, 'About') + Button(root, text='Dialog', command=run).pack() + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/config-extensions.def --- a/Lib/idlelib/config-extensions.def Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/config-extensions.def Mon Jan 25 17:05:13 2016 +0100 @@ -3,37 +3,73 @@ # IDLE reads several config files to determine user preferences. This # file is the default configuration file for IDLE extensions settings. # -# Each extension must have at least one section, named after the -# extension module. This section must contain an 'enable' item (=True to -# enable the extension, =False to disable it), it may contain -# 'enable_editor' or 'enable_shell' items, to apply it only to editor ir -# shell windows, and may also contain any other general configuration -# items for the extension. Other True/False values will also be -# recognized as boolean by the Extension Configuration dialog. +# Each extension must have at least one section, named after the extension +# module. This section must contain an 'enable' item (=1 to enable the +# extension, =0 to disable it), it may contain 'enable_editor' or 'enable_shell' +# items, to apply it only to editor/shell windows, and may also contain any +# other general configuration items for the extension. # -# Each extension must define at least one section named -# ExtensionName_bindings or ExtensionName_cfgBindings. If present, -# ExtensionName_bindings defines virtual event bindings for the -# extension that are not user re-configurable. If present, -# ExtensionName_cfgBindings defines virtual event bindings for the +# Each extension must define at least one section named ExtensionName_bindings +# or ExtensionName_cfgBindings. If present, ExtensionName_bindings defines +# virtual event bindings for the extension that are not user re-configurable. +# If present, ExtensionName_cfgBindings defines virtual event bindings for the # extension that may be sensibly re-configured. # -# If there are no keybindings for a menus' virtual events, include lines -# like <>= (See [CodeContext], below.) +# If there are no keybindings for a menus' virtual events, include lines like +# <>= (See [CodeContext], below.) # -# Currently it is necessary to manually modify this file to change -# extension key bindings and default values. To customize, create +# Currently it is necessary to manually modify this file to change extension +# key bindings and default values. To customize, create # ~/.idlerc/config-extensions.cfg and append the appropriate customized # section(s). Those sections will override the defaults in this file. # -# Note: If a keybinding is already in use when the extension is loaded, -# the extension's virtual event's keybinding will be set to ''. +# Note: If a keybinding is already in use when the extension is +# loaded, the extension's virtual event's keybinding will be set to ''. # # See config-keys.def for notes on specifying keys and extend.txt for # information on creating IDLE extensions. +[FormatParagraph] +enable=1 +[FormatParagraph_cfgBindings] +format-paragraph= + +[AutoExpand] +enable=1 +[AutoExpand_cfgBindings] +expand-word= + +[ZoomHeight] +enable=1 +[ZoomHeight_cfgBindings] +zoom-height= + +[ScriptBinding] +enable=1 +[ScriptBinding_cfgBindings] +run-module= +check-module= + +[CallTips] +enable=1 +[CallTips_cfgBindings] +force-open-calltip= +[CallTips_bindings] +try-open-calltip= +refresh-calltip= + +[ParenMatch] +enable=1 +style= expression +flash-delay= 500 +bell= 1 +[ParenMatch_cfgBindings] +flash-paren= +[ParenMatch_bindings] +paren-closed= + [AutoComplete] -enable=True +enable=1 popupwait=2000 [AutoComplete_cfgBindings] force-open-completions= @@ -41,59 +77,18 @@ autocomplete= try-open-completions= -[AutoExpand] -enable=True -[AutoExpand_cfgBindings] -expand-word= - -[CallTips] -enable=True -[CallTips_cfgBindings] -force-open-calltip= -[CallTips_bindings] -try-open-calltip= -refresh-calltip= - [CodeContext] -enable=True -enable_shell=False +enable=1 +enable_shell=0 numlines=3 -visible=False +visible=0 bgcolor=LightGray fgcolor=Black [CodeContext_bindings] toggle-code-context= -[FormatParagraph] -enable=True -max-width=72 -[FormatParagraph_cfgBindings] -format-paragraph= +[RstripExtension] +enable=1 +enable_shell=0 +enable_editor=1 -[ParenMatch] -enable=True -style= expression -flash-delay= 500 -bell=True -[ParenMatch_cfgBindings] -flash-paren= -[ParenMatch_bindings] -paren-closed= - -[RstripExtension] -enable=True -enable_shell=False -enable_editor=True - -[ScriptBinding] -enable=True -enable_shell=False -enable_editor=True -[ScriptBinding_cfgBindings] -run-module= -check-module= - -[ZoomHeight] -enable=True -[ZoomHeight_cfgBindings] -zoom-height= diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/config-highlight.def --- a/Lib/idlelib/config-highlight.def Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/config-highlight.def Mon Jan 25 17:05:13 2016 +0100 @@ -62,32 +62,3 @@ stderr-background= #ffffff console-foreground= #770000 console-background= #ffffff - -[IDLE Dark] -comment-foreground = #dd0000 -console-foreground = #ff4d4d -error-foreground = #FFFFFF -hilite-background = #7e7e7e -string-foreground = #02ff02 -stderr-background = #002240 -stderr-foreground = #ffb3b3 -console-background = #002240 -hit-background = #fbfbfb -string-background = #002240 -normal-background = #002240 -hilite-foreground = #FFFFFF -keyword-foreground = #ff8000 -error-background = #c86464 -keyword-background = #002240 -builtin-background = #002240 -break-background = #808000 -builtin-foreground = #ff00ff -definition-foreground = #5e5eff -stdout-foreground = #c2d1fa -definition-background = #002240 -normal-foreground = #FFFFFF -cursor-foreground = #ffffff -stdout-background = #002240 -hit-foreground = #002240 -comment-background = #002240 -break-foreground = #FFFFFF diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/config-keys.def --- a/Lib/idlelib/config-keys.def Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/config-keys.def Mon Jan 25 17:05:13 2016 +0100 @@ -13,37 +13,37 @@ paste= beginning-of-line= center-insert= -close-all-windows= +close-all-windows= close-window= do-nothing= end-of-file= python-docs= python-context-help= -history-next= -history-previous= +history-next= +history-previous= interrupt-execution= view-restart= restart-shell= -open-class-browser= -open-module= +open-class-browser= +open-module= open-new-window= open-window-from-file= plain-newline-and-indent= print-window= -redo= +redo= remove-selection= -save-copy-of-window-as-file= -save-window-as-file= -save-window= -select-all= +save-copy-of-window-as-file= +save-window-as-file= +save-window= +select-all= toggle-auto-coloring= undo= find= -find-again= +find-again= find-in-files= find-selection= replace= -goto-line= +goto-line= smart-backspace= newline-and-indent= smart-indent= @@ -53,8 +53,8 @@ uncomment-region= tabify-region= untabify-region= -toggle-tabs= -change-indentwidth= +toggle-tabs= +change-indentwidth= del-word-left= del-word-right= diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/config-main.def --- a/Lib/idlelib/config-main.def Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/config-main.def Mon Jan 25 17:05:13 2016 +0100 @@ -53,11 +53,14 @@ [EditorWindow] width= 80 height= 40 -font= TkFixedFont +font= courier font-size= 10 font-bold= 0 encoding= none +[FormatParagraph] +paragraph=70 + [Indent] use-spaces= 1 num-spaces= 4 @@ -65,8 +68,6 @@ [Theme] default= 1 name= IDLE Classic -name2= -# name2 set in user config-main.cfg for themes added after 2015 Oct 1 [Keys] default= 1 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/configDialog.py --- a/Lib/idlelib/configDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/configDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -13,576 +13,533 @@ import tkinter.messagebox as tkMessageBox import tkinter.colorchooser as tkColorChooser import tkinter.font as tkFont +import copy from idlelib.configHandler import idleConf from idlelib.dynOptionMenuWidget import DynOptionMenu +from idlelib.tabbedpages import TabbedPageSet from idlelib.keybindingDialog import GetKeysDialog from idlelib.configSectionNameDialog import GetCfgSectionNameDialog from idlelib.configHelpSourceEdit import GetHelpSourceDialog -from idlelib.tabbedpages import TabbedPageSet -from idlelib.textView import view_text from idlelib import macosxSupport class ConfigDialog(Toplevel): - def __init__(self, parent, title='', _htest=False, _utest=False): - """ - _htest - bool, change box location when running htest - _utest - bool, don't wait_window when running unittest - """ + def __init__(self,parent,title): Toplevel.__init__(self, parent) - self.parent = parent - if _htest: - parent.instance_dict = {} self.wm_withdraw() self.configure(borderwidth=5) - self.title(title or 'IDLE Preferences') - self.geometry( - "+%d+%d" % (parent.winfo_rootx() + 20, - parent.winfo_rooty() + (30 if not _htest else 150))) + self.title('IDLE Preferences') + self.geometry("+%d+%d" % (parent.winfo_rootx()+20, + parent.winfo_rooty()+30)) #Theme Elements. Each theme element key is its display name. #The first value of the tuple is the sample area tag name. #The second value is the display name list sort index. - self.themeElements={ - 'Normal Text': ('normal', '00'), - 'Python Keywords': ('keyword', '01'), - 'Python Definitions': ('definition', '02'), - 'Python Builtins': ('builtin', '03'), - 'Python Comments': ('comment', '04'), - 'Python Strings': ('string', '05'), - 'Selected Text': ('hilite', '06'), - 'Found Text': ('hit', '07'), - 'Cursor': ('cursor', '08'), - 'Editor Breakpoint': ('break', '09'), - 'Shell Normal Text': ('console', '10'), - 'Shell Error Text': ('error', '11'), - 'Shell Stdout Text': ('stdout', '12'), - 'Shell Stderr Text': ('stderr', '13'), + self.themeElements={'Normal Text':('normal','00'), + 'Python Keywords':('keyword','01'), + 'Python Definitions':('definition','02'), + 'Python Builtins':('builtin', '03'), + 'Python Comments':('comment','04'), + 'Python Strings':('string','05'), + 'Selected Text':('hilite','06'), + 'Found Text':('hit','07'), + 'Cursor':('cursor','08'), + 'Error Text':('error','09'), + 'Shell Normal Text':('console','10'), + 'Shell Stdout Text':('stdout','11'), + 'Shell Stderr Text':('stderr','12'), } self.ResetChangedItems() #load initial values in changed items dict self.CreateWidgets() - self.resizable(height=FALSE, width=FALSE) + self.resizable(height=FALSE,width=FALSE) self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.Cancel) + self.parent = parent self.tabPages.focus_set() #key bindings for this dialog - #self.bind('', self.Cancel) #dismiss dialog, no save - #self.bind('', self.Apply) #apply changes, save - #self.bind('', self.Help) #context help + #self.bind('',self.Cancel) #dismiss dialog, no save + #self.bind('',self.Apply) #apply changes, save + #self.bind('',self.Help) #context help self.LoadConfigs() self.AttachVarCallbacks() #avoid callbacks during LoadConfigs - if not _utest: - self.wm_deiconify() - self.wait_window() + self.wm_deiconify() + self.wait_window() def CreateWidgets(self): self.tabPages = TabbedPageSet(self, - page_names=['Fonts/Tabs', 'Highlighting', 'Keys', 'General', - 'Extensions']) - self.tabPages.pack(side=TOP, expand=TRUE, fill=BOTH) + page_names=['Fonts/Tabs','Highlighting','Keys','General']) + frameActionButtons = Frame(self,pady=2) + #action buttons + + if macosxSupport.runningAsOSXApp(): + # Surpress the padx and pady arguments when + # running as IDLE.app, otherwise the text + # on these buttons will not be readable. + extraKwds={} + else: + extraKwds=dict(padx=6, pady=3) + + self.buttonHelp = Button(frameActionButtons,text='Help', + command=self.Help,takefocus=FALSE, + **extraKwds) + self.buttonOk = Button(frameActionButtons,text='Ok', + command=self.Ok,takefocus=FALSE, + **extraKwds) + self.buttonApply = Button(frameActionButtons,text='Apply', + command=self.Apply,takefocus=FALSE, + **extraKwds) + self.buttonCancel = Button(frameActionButtons,text='Cancel', + command=self.Cancel,takefocus=FALSE, + **extraKwds) self.CreatePageFontTab() self.CreatePageHighlight() self.CreatePageKeys() self.CreatePageGeneral() - self.CreatePageExtensions() - self.create_action_buttons().pack(side=BOTTOM) - - def create_action_buttons(self): - if macosxSupport.isAquaTk(): - # Changing the default padding on OSX results in unreadable - # text in the buttons - paddingArgs = {} - else: - paddingArgs = {'padx':6, 'pady':3} - outer = Frame(self, pady=2) - buttons = Frame(outer, pady=2) - for txt, cmd in ( - ('Ok', self.Ok), - ('Apply', self.Apply), - ('Cancel', self.Cancel), - ('Help', self.Help)): - Button(buttons, text=txt, command=cmd, takefocus=FALSE, - **paddingArgs).pack(side=LEFT, padx=5) - # add space above buttons - Frame(outer, height=2, borderwidth=0).pack(side=TOP) - buttons.pack(side=BOTTOM) - return outer + self.buttonHelp.pack(side=RIGHT,padx=5) + self.buttonOk.pack(side=LEFT,padx=5) + self.buttonApply.pack(side=LEFT,padx=5) + self.buttonCancel.pack(side=LEFT,padx=5) + frameActionButtons.pack(side=BOTTOM) + Frame(self, height=2, borderwidth=0).pack(side=BOTTOM) + self.tabPages.pack(side=TOP,expand=TRUE,fill=BOTH) def CreatePageFontTab(self): - parent = self.parent - self.fontSize = StringVar(parent) - self.fontBold = BooleanVar(parent) - self.fontName = StringVar(parent) - self.spaceNum = IntVar(parent) - self.editFont = tkFont.Font(parent, ('courier', 10, 'normal')) - + #tkVars + self.fontSize=StringVar(self) + self.fontBold=BooleanVar(self) + self.fontName=StringVar(self) + self.spaceNum=IntVar(self) + self.editFont=tkFont.Font(self,('courier',10,'normal')) ##widget creation #body frame - frame = self.tabPages.pages['Fonts/Tabs'].frame + frame=self.tabPages.pages['Fonts/Tabs'].frame #body section frames - frameFont = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Base Editor Font ') - frameIndent = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Indentation Width ') + frameFont=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Base Editor Font ') + frameIndent=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Indentation Width ') #frameFont - frameFontName = Frame(frameFont) - frameFontParam = Frame(frameFont) - labelFontNameTitle = Label( - frameFontName, justify=LEFT, text='Font Face :') - self.listFontName = Listbox( - frameFontName, height=5, takefocus=FALSE, exportselection=FALSE) - self.listFontName.bind( - '', self.OnListFontButtonRelease) - scrollFont = Scrollbar(frameFontName) + frameFontName=Frame(frameFont) + frameFontParam=Frame(frameFont) + labelFontNameTitle=Label(frameFontName,justify=LEFT, + text='Font Face :') + self.listFontName=Listbox(frameFontName,height=5,takefocus=FALSE, + exportselection=FALSE) + self.listFontName.bind('',self.OnListFontButtonRelease) + scrollFont=Scrollbar(frameFontName) scrollFont.config(command=self.listFontName.yview) self.listFontName.config(yscrollcommand=scrollFont.set) - labelFontSizeTitle = Label(frameFontParam, text='Size :') - self.optMenuFontSize = DynOptionMenu( - frameFontParam, self.fontSize, None, command=self.SetFontSample) - checkFontBold = Checkbutton( - frameFontParam, variable=self.fontBold, onvalue=1, - offvalue=0, text='Bold', command=self.SetFontSample) - frameFontSample = Frame(frameFont, relief=SOLID, borderwidth=1) - self.labelFontSample = Label( - frameFontSample, justify=LEFT, font=self.editFont, - text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]') + labelFontSizeTitle=Label(frameFontParam,text='Size :') + self.optMenuFontSize=DynOptionMenu(frameFontParam,self.fontSize,None, + command=self.SetFontSample) + checkFontBold=Checkbutton(frameFontParam,variable=self.fontBold, + onvalue=1,offvalue=0,text='Bold',command=self.SetFontSample) + frameFontSample=Frame(frameFont,relief=SOLID,borderwidth=1) + self.labelFontSample=Label(frameFontSample, + text='AaBbCcDdEe\nFfGgHhIiJjK\n1234567890\n#:+=(){}[]', + justify=LEFT,font=self.editFont) #frameIndent - frameIndentSize = Frame(frameIndent) - labelSpaceNumTitle = Label( - frameIndentSize, justify=LEFT, - text='Python Standard: 4 Spaces!') - self.scaleSpaceNum = Scale( - frameIndentSize, variable=self.spaceNum, - orient='horizontal', tickinterval=2, from_=2, to=16) - + frameIndentSize=Frame(frameIndent) + labelSpaceNumTitle=Label(frameIndentSize, justify=LEFT, + text='Python Standard: 4 Spaces!') + self.scaleSpaceNum=Scale(frameIndentSize, variable=self.spaceNum, + orient='horizontal', + tickinterval=2, from_=2, to=16) #widget packing #body - frameFont.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameIndent.pack(side=LEFT, padx=5, pady=5, fill=Y) + frameFont.pack(side=LEFT,padx=5,pady=5,expand=TRUE,fill=BOTH) + frameIndent.pack(side=LEFT,padx=5,pady=5,fill=Y) #frameFont - frameFontName.pack(side=TOP, padx=5, pady=5, fill=X) - frameFontParam.pack(side=TOP, padx=5, pady=5, fill=X) - labelFontNameTitle.pack(side=TOP, anchor=W) - self.listFontName.pack(side=LEFT, expand=TRUE, fill=X) - scrollFont.pack(side=LEFT, fill=Y) - labelFontSizeTitle.pack(side=LEFT, anchor=W) - self.optMenuFontSize.pack(side=LEFT, anchor=W) - checkFontBold.pack(side=LEFT, anchor=W, padx=20) - frameFontSample.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - self.labelFontSample.pack(expand=TRUE, fill=BOTH) + frameFontName.pack(side=TOP,padx=5,pady=5,fill=X) + frameFontParam.pack(side=TOP,padx=5,pady=5,fill=X) + labelFontNameTitle.pack(side=TOP,anchor=W) + self.listFontName.pack(side=LEFT,expand=TRUE,fill=X) + scrollFont.pack(side=LEFT,fill=Y) + labelFontSizeTitle.pack(side=LEFT,anchor=W) + self.optMenuFontSize.pack(side=LEFT,anchor=W) + checkFontBold.pack(side=LEFT,anchor=W,padx=20) + frameFontSample.pack(side=TOP,padx=5,pady=5,expand=TRUE,fill=BOTH) + self.labelFontSample.pack(expand=TRUE,fill=BOTH) #frameIndent - frameIndentSize.pack(side=TOP, fill=X) - labelSpaceNumTitle.pack(side=TOP, anchor=W, padx=5) - self.scaleSpaceNum.pack(side=TOP, padx=5, fill=X) + frameIndentSize.pack(side=TOP,fill=X) + labelSpaceNumTitle.pack(side=TOP,anchor=W,padx=5) + self.scaleSpaceNum.pack(side=TOP,padx=5,fill=X) return frame def CreatePageHighlight(self): - parent = self.parent - self.builtinTheme = StringVar(parent) - self.customTheme = StringVar(parent) - self.fgHilite = BooleanVar(parent) - self.colour = StringVar(parent) - self.fontName = StringVar(parent) - self.themeIsBuiltin = BooleanVar(parent) - self.highlightTarget = StringVar(parent) - + self.builtinTheme=StringVar(self) + self.customTheme=StringVar(self) + self.fgHilite=BooleanVar(self) + self.colour=StringVar(self) + self.fontName=StringVar(self) + self.themeIsBuiltin=BooleanVar(self) + self.highlightTarget=StringVar(self) ##widget creation #body frame - frame = self.tabPages.pages['Highlighting'].frame + frame=self.tabPages.pages['Highlighting'].frame #body section frames - frameCustom = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Custom Highlighting ') - frameTheme = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Highlighting Theme ') + frameCustom=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Custom Highlighting ') + frameTheme=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Highlighting Theme ') #frameCustom - self.textHighlightSample=Text( - frameCustom, relief=SOLID, borderwidth=1, - font=('courier', 12, ''), cursor='hand2', width=21, height=11, - takefocus=FALSE, highlightthickness=0, wrap=NONE) + self.textHighlightSample=Text(frameCustom,relief=SOLID,borderwidth=1, + font=('courier',12,''),cursor='hand2',width=21,height=10, + takefocus=FALSE,highlightthickness=0,wrap=NONE) text=self.textHighlightSample - text.bind('', lambda e: 'break') - text.bind('', lambda e: 'break') - textAndTags=( - ('#you can click here', 'comment'), ('\n', 'normal'), - ('#to choose items', 'comment'), ('\n', 'normal'), - ('def', 'keyword'), (' ', 'normal'), - ('func', 'definition'), ('(param):\n ', 'normal'), - ('"""string"""', 'string'), ('\n var0 = ', 'normal'), - ("'string'", 'string'), ('\n var1 = ', 'normal'), - ("'selected'", 'hilite'), ('\n var2 = ', 'normal'), - ("'found'", 'hit'), ('\n var3 = ', 'normal'), - ('list', 'builtin'), ('(', 'normal'), - ('None', 'keyword'), (')\n', 'normal'), - (' breakpoint("line")', 'break'), ('\n\n', 'normal'), - (' error ', 'error'), (' ', 'normal'), - ('cursor |', 'cursor'), ('\n ', 'normal'), - ('shell', 'console'), (' ', 'normal'), - ('stdout', 'stdout'), (' ', 'normal'), - ('stderr', 'stderr'), ('\n', 'normal')) + text.bind('',lambda e: 'break') + text.bind('',lambda e: 'break') + textAndTags=(('#you can click here','comment'),('\n','normal'), + ('#to choose items','comment'),('\n','normal'),('def','keyword'), + (' ','normal'),('func','definition'),('(param):','normal'), + ('\n ','normal'),('"""string"""','string'),('\n var0 = ','normal'), + ("'string'",'string'),('\n var1 = ','normal'),("'selected'",'hilite'), + ('\n var2 = ','normal'),("'found'",'hit'), + ('\n var3 = ','normal'),('list', 'builtin'), ('(','normal'), + ('None', 'keyword'),(')\n\n','normal'), + (' error ','error'),(' ','normal'),('cursor |','cursor'), + ('\n ','normal'),('shell','console'),(' ','normal'),('stdout','stdout'), + (' ','normal'),('stderr','stderr'),('\n','normal')) for txTa in textAndTags: - text.insert(END, txTa[0], txTa[1]) + text.insert(END,txTa[0],txTa[1]) for element in self.themeElements: - def tem(event, elem=element): - event.widget.winfo_toplevel().highlightTarget.set(elem) - text.tag_bind( - self.themeElements[element][0], '', tem) + text.tag_bind(self.themeElements[element][0],'', + lambda event,elem=element: event.widget.winfo_toplevel() + .highlightTarget.set(elem)) text.config(state=DISABLED) - self.frameColourSet = Frame(frameCustom, relief=SOLID, borderwidth=1) - frameFgBg = Frame(frameCustom) - buttonSetColour = Button( - self.frameColourSet, text='Choose Colour for :', - command=self.GetColour, highlightthickness=0) - self.optMenuHighlightTarget = DynOptionMenu( - self.frameColourSet, self.highlightTarget, None, - highlightthickness=0) #, command=self.SetHighlightTargetBinding - self.radioFg = Radiobutton( - frameFgBg, variable=self.fgHilite, value=1, - text='Foreground', command=self.SetColourSampleBinding) - self.radioBg=Radiobutton( - frameFgBg, variable=self.fgHilite, value=0, - text='Background', command=self.SetColourSampleBinding) + self.frameColourSet=Frame(frameCustom,relief=SOLID,borderwidth=1) + frameFgBg=Frame(frameCustom) + buttonSetColour=Button(self.frameColourSet,text='Choose Colour for :', + command=self.GetColour,highlightthickness=0) + self.optMenuHighlightTarget=DynOptionMenu(self.frameColourSet, + self.highlightTarget,None,highlightthickness=0)#,command=self.SetHighlightTargetBinding + self.radioFg=Radiobutton(frameFgBg,variable=self.fgHilite, + value=1,text='Foreground',command=self.SetColourSampleBinding) + self.radioBg=Radiobutton(frameFgBg,variable=self.fgHilite, + value=0,text='Background',command=self.SetColourSampleBinding) self.fgHilite.set(1) - buttonSaveCustomTheme = Button( - frameCustom, text='Save as New Custom Theme', - command=self.SaveAsNewTheme) + buttonSaveCustomTheme=Button(frameCustom, + text='Save as New Custom Theme',command=self.SaveAsNewTheme) #frameTheme - labelTypeTitle = Label(frameTheme, text='Select : ') - self.radioThemeBuiltin = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=1, - command=self.SetThemeType, text='a Built-in Theme') - self.radioThemeCustom = Radiobutton( - frameTheme, variable=self.themeIsBuiltin, value=0, - command=self.SetThemeType, text='a Custom Theme') - self.optMenuThemeBuiltin = DynOptionMenu( - frameTheme, self.builtinTheme, None, command=None) - self.optMenuThemeCustom=DynOptionMenu( - frameTheme, self.customTheme, None, command=None) - self.buttonDeleteCustomTheme=Button( - frameTheme, text='Delete Custom Theme', + labelTypeTitle=Label(frameTheme,text='Select : ') + self.radioThemeBuiltin=Radiobutton(frameTheme,variable=self.themeIsBuiltin, + value=1,command=self.SetThemeType,text='a Built-in Theme') + self.radioThemeCustom=Radiobutton(frameTheme,variable=self.themeIsBuiltin, + value=0,command=self.SetThemeType,text='a Custom Theme') + self.optMenuThemeBuiltin=DynOptionMenu(frameTheme, + self.builtinTheme,None,command=None) + self.optMenuThemeCustom=DynOptionMenu(frameTheme, + self.customTheme,None,command=None) + self.buttonDeleteCustomTheme=Button(frameTheme,text='Delete Custom Theme', command=self.DeleteCustomTheme) - self.new_custom_theme = Label(frameTheme, bd=2) - ##widget packing #body - frameCustom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameTheme.pack(side=LEFT, padx=5, pady=5, fill=Y) + frameCustom.pack(side=LEFT,padx=5,pady=5,expand=TRUE,fill=BOTH) + frameTheme.pack(side=LEFT,padx=5,pady=5,fill=Y) #frameCustom - self.frameColourSet.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) - frameFgBg.pack(side=TOP, padx=5, pady=0) - self.textHighlightSample.pack( - side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - buttonSetColour.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) - self.optMenuHighlightTarget.pack( - side=TOP, expand=TRUE, fill=X, padx=8, pady=3) - self.radioFg.pack(side=LEFT, anchor=E) - self.radioBg.pack(side=RIGHT, anchor=W) - buttonSaveCustomTheme.pack(side=BOTTOM, fill=X, padx=5, pady=5) + self.frameColourSet.pack(side=TOP,padx=5,pady=5,expand=TRUE,fill=X) + frameFgBg.pack(side=TOP,padx=5,pady=0) + self.textHighlightSample.pack(side=TOP,padx=5,pady=5,expand=TRUE, + fill=BOTH) + buttonSetColour.pack(side=TOP,expand=TRUE,fill=X,padx=8,pady=4) + self.optMenuHighlightTarget.pack(side=TOP,expand=TRUE,fill=X,padx=8,pady=3) + self.radioFg.pack(side=LEFT,anchor=E) + self.radioBg.pack(side=RIGHT,anchor=W) + buttonSaveCustomTheme.pack(side=BOTTOM,fill=X,padx=5,pady=5) #frameTheme - labelTypeTitle.pack(side=TOP, anchor=W, padx=5, pady=5) - self.radioThemeBuiltin.pack(side=TOP, anchor=W, padx=5) - self.radioThemeCustom.pack(side=TOP, anchor=W, padx=5, pady=2) - self.optMenuThemeBuiltin.pack(side=TOP, fill=X, padx=5, pady=5) - self.optMenuThemeCustom.pack(side=TOP, fill=X, anchor=W, padx=5, pady=5) - self.buttonDeleteCustomTheme.pack(side=TOP, fill=X, padx=5, pady=5) - self.new_custom_theme.pack(side=TOP, fill=X, pady=5) + labelTypeTitle.pack(side=TOP,anchor=W,padx=5,pady=5) + self.radioThemeBuiltin.pack(side=TOP,anchor=W,padx=5) + self.radioThemeCustom.pack(side=TOP,anchor=W,padx=5,pady=2) + self.optMenuThemeBuiltin.pack(side=TOP,fill=X,padx=5,pady=5) + self.optMenuThemeCustom.pack(side=TOP,fill=X,anchor=W,padx=5,pady=5) + self.buttonDeleteCustomTheme.pack(side=TOP,fill=X,padx=5,pady=5) return frame def CreatePageKeys(self): - parent = self.parent - self.bindingTarget = StringVar(parent) - self.builtinKeys = StringVar(parent) - self.customKeys = StringVar(parent) - self.keysAreBuiltin = BooleanVar(parent) - self.keyBinding = StringVar(parent) - + #tkVars + self.bindingTarget=StringVar(self) + self.builtinKeys=StringVar(self) + self.customKeys=StringVar(self) + self.keysAreBuiltin=BooleanVar(self) + self.keyBinding=StringVar(self) ##widget creation #body frame - frame = self.tabPages.pages['Keys'].frame + frame=self.tabPages.pages['Keys'].frame #body section frames - frameCustom = LabelFrame( - frame, borderwidth=2, relief=GROOVE, - text=' Custom Key Bindings ') - frameKeySets = LabelFrame( - frame, borderwidth=2, relief=GROOVE, text=' Key Set ') + frameCustom=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Custom Key Bindings ') + frameKeySets=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Key Set ') #frameCustom - frameTarget = Frame(frameCustom) - labelTargetTitle = Label(frameTarget, text='Action - Key(s)') - scrollTargetY = Scrollbar(frameTarget) - scrollTargetX = Scrollbar(frameTarget, orient=HORIZONTAL) - self.listBindings = Listbox( - frameTarget, takefocus=FALSE, exportselection=FALSE) - self.listBindings.bind('', self.KeyBindingSelected) + frameTarget=Frame(frameCustom) + labelTargetTitle=Label(frameTarget,text='Action - Key(s)') + scrollTargetY=Scrollbar(frameTarget) + scrollTargetX=Scrollbar(frameTarget,orient=HORIZONTAL) + self.listBindings=Listbox(frameTarget,takefocus=FALSE, + exportselection=FALSE) + self.listBindings.bind('',self.KeyBindingSelected) scrollTargetY.config(command=self.listBindings.yview) scrollTargetX.config(command=self.listBindings.xview) self.listBindings.config(yscrollcommand=scrollTargetY.set) self.listBindings.config(xscrollcommand=scrollTargetX.set) - self.buttonNewKeys = Button( - frameCustom, text='Get New Keys for Selection', - command=self.GetNewKeys, state=DISABLED) + self.buttonNewKeys=Button(frameCustom,text='Get New Keys for Selection', + command=self.GetNewKeys,state=DISABLED) #frameKeySets frames = [Frame(frameKeySets, padx=2, pady=2, borderwidth=0) for i in range(2)] - self.radioKeysBuiltin = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=1, - command=self.SetKeysType, text='Use a Built-in Key Set') - self.radioKeysCustom = Radiobutton( - frames[0], variable=self.keysAreBuiltin, value=0, - command=self.SetKeysType, text='Use a Custom Key Set') - self.optMenuKeysBuiltin = DynOptionMenu( - frames[0], self.builtinKeys, None, command=None) - self.optMenuKeysCustom = DynOptionMenu( - frames[0], self.customKeys, None, command=None) - self.buttonDeleteCustomKeys = Button( - frames[1], text='Delete Custom Key Set', + self.radioKeysBuiltin=Radiobutton(frames[0],variable=self.keysAreBuiltin, + value=1,command=self.SetKeysType,text='Use a Built-in Key Set') + self.radioKeysCustom=Radiobutton(frames[0],variable=self.keysAreBuiltin, + value=0,command=self.SetKeysType,text='Use a Custom Key Set') + self.optMenuKeysBuiltin=DynOptionMenu(frames[0], + self.builtinKeys,None,command=None) + self.optMenuKeysCustom=DynOptionMenu(frames[0], + self.customKeys,None,command=None) + self.buttonDeleteCustomKeys=Button(frames[1],text='Delete Custom Key Set', command=self.DeleteCustomKeys) - buttonSaveCustomKeys = Button( - frames[1], text='Save as New Custom Key Set', - command=self.SaveAsNewKeySet) - + buttonSaveCustomKeys=Button(frames[1], + text='Save as New Custom Key Set',command=self.SaveAsNewKeySet) ##widget packing #body - frameCustom.pack(side=BOTTOM, padx=5, pady=5, expand=TRUE, fill=BOTH) - frameKeySets.pack(side=BOTTOM, padx=5, pady=5, fill=BOTH) + frameCustom.pack(side=BOTTOM,padx=5,pady=5,expand=TRUE,fill=BOTH) + frameKeySets.pack(side=BOTTOM,padx=5,pady=5,fill=BOTH) #frameCustom - self.buttonNewKeys.pack(side=BOTTOM, fill=X, padx=5, pady=5) - frameTarget.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.buttonNewKeys.pack(side=BOTTOM,fill=X,padx=5,pady=5) + frameTarget.pack(side=LEFT,padx=5,pady=5,expand=TRUE,fill=BOTH) #frame target - frameTarget.columnconfigure(0, weight=1) - frameTarget.rowconfigure(1, weight=1) - labelTargetTitle.grid(row=0, column=0, columnspan=2, sticky=W) - self.listBindings.grid(row=1, column=0, sticky=NSEW) - scrollTargetY.grid(row=1, column=1, sticky=NS) - scrollTargetX.grid(row=2, column=0, sticky=EW) + frameTarget.columnconfigure(0,weight=1) + frameTarget.rowconfigure(1,weight=1) + labelTargetTitle.grid(row=0,column=0,columnspan=2,sticky=W) + self.listBindings.grid(row=1,column=0,sticky=NSEW) + scrollTargetY.grid(row=1,column=1,sticky=NS) + scrollTargetX.grid(row=2,column=0,sticky=EW) #frameKeySets self.radioKeysBuiltin.grid(row=0, column=0, sticky=W+NS) self.radioKeysCustom.grid(row=1, column=0, sticky=W+NS) self.optMenuKeysBuiltin.grid(row=0, column=1, sticky=NSEW) self.optMenuKeysCustom.grid(row=1, column=1, sticky=NSEW) - self.buttonDeleteCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) - buttonSaveCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) + self.buttonDeleteCustomKeys.pack(side=LEFT,fill=X,expand=True,padx=2) + buttonSaveCustomKeys.pack(side=LEFT,fill=X,expand=True,padx=2) frames[0].pack(side=TOP, fill=BOTH, expand=True) frames[1].pack(side=TOP, fill=X, expand=True, pady=2) return frame def CreatePageGeneral(self): - parent = self.parent - self.winWidth = StringVar(parent) - self.winHeight = StringVar(parent) - self.startupEdit = IntVar(parent) - self.autoSave = IntVar(parent) - self.encoding = StringVar(parent) - self.userHelpBrowser = BooleanVar(parent) - self.helpBrowser = StringVar(parent) - + #tkVars + self.winWidth=StringVar(self) + self.winHeight=StringVar(self) + self.paraWidth=StringVar(self) + self.startupEdit=IntVar(self) + self.autoSave=IntVar(self) + self.encoding=StringVar(self) + self.userHelpBrowser=BooleanVar(self) + self.helpBrowser=StringVar(self) #widget creation #body - frame = self.tabPages.pages['General'].frame + frame=self.tabPages.pages['General'].frame #body section frames - frameRun = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Autosave Preferences ') - frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE) - frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') + frameRun=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Startup Preferences ') + frameSave=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Autosave Preferences ') + frameWinSize=Frame(frame,borderwidth=2,relief=GROOVE) + frameParaSize=Frame(frame,borderwidth=2,relief=GROOVE) + frameHelp=LabelFrame(frame,borderwidth=2,relief=GROOVE, + text=' Additional Help Sources ') #frameRun - labelRunChoiceTitle = Label(frameRun, text='At Startup') - radioStartupEdit = Radiobutton( - frameRun, variable=self.startupEdit, value=1, - command=self.SetKeysType, text="Open Edit Window") - radioStartupShell = Radiobutton( - frameRun, variable=self.startupEdit, value=0, - command=self.SetKeysType, text='Open Shell Window') + labelRunChoiceTitle=Label(frameRun,text='At Startup') + radioStartupEdit=Radiobutton(frameRun,variable=self.startupEdit, + value=1,command=self.SetKeysType,text="Open Edit Window") + radioStartupShell=Radiobutton(frameRun,variable=self.startupEdit, + value=0,command=self.SetKeysType,text='Open Shell Window') #frameSave - labelRunSaveTitle = Label(frameSave, text='At Start of Run (F5) ') - radioSaveAsk = Radiobutton( - frameSave, variable=self.autoSave, value=0, - command=self.SetKeysType, text="Prompt to Save") - radioSaveAuto = Radiobutton( - frameSave, variable=self.autoSave, value=1, - command=self.SetKeysType, text='No Prompt') + labelRunSaveTitle=Label(frameSave,text='At Start of Run (F5) ') + radioSaveAsk=Radiobutton(frameSave,variable=self.autoSave, + value=0,command=self.SetKeysType,text="Prompt to Save") + radioSaveAuto=Radiobutton(frameSave,variable=self.autoSave, + value=1,command=self.SetKeysType,text='No Prompt') #frameWinSize - labelWinSizeTitle = Label( - frameWinSize, text='Initial Window Size (in characters)') - labelWinWidthTitle = Label(frameWinSize, text='Width') - entryWinWidth = Entry( - frameWinSize, textvariable=self.winWidth, width=3) - labelWinHeightTitle = Label(frameWinSize, text='Height') - entryWinHeight = Entry( - frameWinSize, textvariable=self.winHeight, width=3) + labelWinSizeTitle=Label(frameWinSize,text='Initial Window Size'+ + ' (in characters)') + labelWinWidthTitle=Label(frameWinSize,text='Width') + entryWinWidth=Entry(frameWinSize,textvariable=self.winWidth, + width=3) + labelWinHeightTitle=Label(frameWinSize,text='Height') + entryWinHeight=Entry(frameWinSize,textvariable=self.winHeight, + width=3) + #paragraphFormatWidth + labelParaWidthTitle=Label(frameParaSize,text='Paragraph reformat'+ + ' width (in characters)') + entryParaWidth=Entry(frameParaSize,textvariable=self.paraWidth, + width=3) #frameHelp - frameHelpList = Frame(frameHelp) - frameHelpListButtons = Frame(frameHelpList) - scrollHelpList = Scrollbar(frameHelpList) - self.listHelp = Listbox( - frameHelpList, height=5, takefocus=FALSE, + frameHelpList=Frame(frameHelp) + frameHelpListButtons=Frame(frameHelpList) + scrollHelpList=Scrollbar(frameHelpList) + self.listHelp=Listbox(frameHelpList,height=5,takefocus=FALSE, exportselection=FALSE) scrollHelpList.config(command=self.listHelp.yview) self.listHelp.config(yscrollcommand=scrollHelpList.set) - self.listHelp.bind('', self.HelpSourceSelected) - self.buttonHelpListEdit = Button( - frameHelpListButtons, text='Edit', state=DISABLED, - width=8, command=self.HelpListItemEdit) - self.buttonHelpListAdd = Button( - frameHelpListButtons, text='Add', - width=8, command=self.HelpListItemAdd) - self.buttonHelpListRemove = Button( - frameHelpListButtons, text='Remove', state=DISABLED, - width=8, command=self.HelpListItemRemove) - + self.listHelp.bind('',self.HelpSourceSelected) + self.buttonHelpListEdit=Button(frameHelpListButtons,text='Edit', + state=DISABLED,width=8,command=self.HelpListItemEdit) + self.buttonHelpListAdd=Button(frameHelpListButtons,text='Add', + width=8,command=self.HelpListItemAdd) + self.buttonHelpListRemove=Button(frameHelpListButtons,text='Remove', + state=DISABLED,width=8,command=self.HelpListItemRemove) #widget packing #body - frameRun.pack(side=TOP, padx=5, pady=5, fill=X) - frameSave.pack(side=TOP, padx=5, pady=5, fill=X) - frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X) - frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + frameRun.pack(side=TOP,padx=5,pady=5,fill=X) + frameSave.pack(side=TOP,padx=5,pady=5,fill=X) + frameWinSize.pack(side=TOP,padx=5,pady=5,fill=X) + frameParaSize.pack(side=TOP,padx=5,pady=5,fill=X) + frameHelp.pack(side=TOP,padx=5,pady=5,expand=TRUE,fill=BOTH) #frameRun - labelRunChoiceTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - radioStartupShell.pack(side=RIGHT, anchor=W, padx=5, pady=5) - radioStartupEdit.pack(side=RIGHT, anchor=W, padx=5, pady=5) + labelRunChoiceTitle.pack(side=LEFT,anchor=W,padx=5,pady=5) + radioStartupShell.pack(side=RIGHT,anchor=W,padx=5,pady=5) + radioStartupEdit.pack(side=RIGHT,anchor=W,padx=5,pady=5) #frameSave - labelRunSaveTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - radioSaveAuto.pack(side=RIGHT, anchor=W, padx=5, pady=5) - radioSaveAsk.pack(side=RIGHT, anchor=W, padx=5, pady=5) + labelRunSaveTitle.pack(side=LEFT,anchor=W,padx=5,pady=5) + radioSaveAuto.pack(side=RIGHT,anchor=W,padx=5,pady=5) + radioSaveAsk.pack(side=RIGHT,anchor=W,padx=5,pady=5) #frameWinSize - labelWinSizeTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - entryWinHeight.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5) - entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) - labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5) + labelWinSizeTitle.pack(side=LEFT,anchor=W,padx=5,pady=5) + entryWinHeight.pack(side=RIGHT,anchor=E,padx=10,pady=5) + labelWinHeightTitle.pack(side=RIGHT,anchor=E,pady=5) + entryWinWidth.pack(side=RIGHT,anchor=E,padx=10,pady=5) + labelWinWidthTitle.pack(side=RIGHT,anchor=E,pady=5) + #paragraphFormatWidth + labelParaWidthTitle.pack(side=LEFT,anchor=W,padx=5,pady=5) + entryParaWidth.pack(side=RIGHT,anchor=E,padx=10,pady=5) #frameHelp - frameHelpListButtons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frameHelpList.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scrollHelpList.pack(side=RIGHT, anchor=W, fill=Y) - self.listHelp.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.buttonHelpListEdit.pack(side=TOP, anchor=W, pady=5) - self.buttonHelpListAdd.pack(side=TOP, anchor=W) - self.buttonHelpListRemove.pack(side=TOP, anchor=W, pady=5) + frameHelpListButtons.pack(side=RIGHT,padx=5,pady=5,fill=Y) + frameHelpList.pack(side=TOP,padx=5,pady=5,expand=TRUE,fill=BOTH) + scrollHelpList.pack(side=RIGHT,anchor=W,fill=Y) + self.listHelp.pack(side=LEFT,anchor=E,expand=TRUE,fill=BOTH) + self.buttonHelpListEdit.pack(side=TOP,anchor=W,pady=5) + self.buttonHelpListAdd.pack(side=TOP,anchor=W) + self.buttonHelpListRemove.pack(side=TOP,anchor=W,pady=5) return frame def AttachVarCallbacks(self): - self.fontSize.trace_variable('w', self.VarChanged_font) - self.fontName.trace_variable('w', self.VarChanged_font) - self.fontBold.trace_variable('w', self.VarChanged_font) - self.spaceNum.trace_variable('w', self.VarChanged_spaceNum) - self.colour.trace_variable('w', self.VarChanged_colour) - self.builtinTheme.trace_variable('w', self.VarChanged_builtinTheme) - self.customTheme.trace_variable('w', self.VarChanged_customTheme) - self.themeIsBuiltin.trace_variable('w', self.VarChanged_themeIsBuiltin) - self.highlightTarget.trace_variable('w', self.VarChanged_highlightTarget) - self.keyBinding.trace_variable('w', self.VarChanged_keyBinding) - self.builtinKeys.trace_variable('w', self.VarChanged_builtinKeys) - self.customKeys.trace_variable('w', self.VarChanged_customKeys) - self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin) - self.winWidth.trace_variable('w', self.VarChanged_winWidth) - self.winHeight.trace_variable('w', self.VarChanged_winHeight) - self.startupEdit.trace_variable('w', self.VarChanged_startupEdit) - self.autoSave.trace_variable('w', self.VarChanged_autoSave) - self.encoding.trace_variable('w', self.VarChanged_encoding) + self.fontSize.trace_variable('w',self.VarChanged_fontSize) + self.fontName.trace_variable('w',self.VarChanged_fontName) + self.fontBold.trace_variable('w',self.VarChanged_fontBold) + self.spaceNum.trace_variable('w',self.VarChanged_spaceNum) + self.colour.trace_variable('w',self.VarChanged_colour) + self.builtinTheme.trace_variable('w',self.VarChanged_builtinTheme) + self.customTheme.trace_variable('w',self.VarChanged_customTheme) + self.themeIsBuiltin.trace_variable('w',self.VarChanged_themeIsBuiltin) + self.highlightTarget.trace_variable('w',self.VarChanged_highlightTarget) + self.keyBinding.trace_variable('w',self.VarChanged_keyBinding) + self.builtinKeys.trace_variable('w',self.VarChanged_builtinKeys) + self.customKeys.trace_variable('w',self.VarChanged_customKeys) + self.keysAreBuiltin.trace_variable('w',self.VarChanged_keysAreBuiltin) + self.winWidth.trace_variable('w',self.VarChanged_winWidth) + self.winHeight.trace_variable('w',self.VarChanged_winHeight) + self.paraWidth.trace_variable('w',self.VarChanged_paraWidth) + self.startupEdit.trace_variable('w',self.VarChanged_startupEdit) + self.autoSave.trace_variable('w',self.VarChanged_autoSave) + self.encoding.trace_variable('w',self.VarChanged_encoding) - def VarChanged_font(self, *params): - '''When one font attribute changes, save them all, as they are - not independent from each other. In particular, when we are - overriding the default font, we need to write out everything. - ''' - value = self.fontName.get() - self.AddChangedItem('main', 'EditorWindow', 'font', value) - value = self.fontSize.get() - self.AddChangedItem('main', 'EditorWindow', 'font-size', value) - value = self.fontBold.get() - self.AddChangedItem('main', 'EditorWindow', 'font-bold', value) + def VarChanged_fontSize(self,*params): + value=self.fontSize.get() + self.AddChangedItem('main','EditorWindow','font-size',value) - def VarChanged_spaceNum(self, *params): - value = self.spaceNum.get() - self.AddChangedItem('main', 'Indent', 'num-spaces', value) + def VarChanged_fontName(self,*params): + value=self.fontName.get() + self.AddChangedItem('main','EditorWindow','font',value) - def VarChanged_colour(self, *params): + def VarChanged_fontBold(self,*params): + value=self.fontBold.get() + self.AddChangedItem('main','EditorWindow','font-bold',value) + + def VarChanged_spaceNum(self,*params): + value=self.spaceNum.get() + self.AddChangedItem('main','Indent','num-spaces',value) + + def VarChanged_colour(self,*params): self.OnNewColourSet() - def VarChanged_builtinTheme(self, *params): - value = self.builtinTheme.get() - if value == 'IDLE Dark': - if idleConf.GetOption('main', 'Theme', 'name') != 'IDLE New': - self.AddChangedItem('main', 'Theme', 'name', 'IDLE Classic') - self.AddChangedItem('main', 'Theme', 'name2', value) - self.new_custom_theme.config(text='New theme, see Help', - fg='#500000') - else: - self.AddChangedItem('main', 'Theme', 'name', value) - self.AddChangedItem('main', 'Theme', 'name2', '') - self.new_custom_theme.config(text='', fg='black') + def VarChanged_builtinTheme(self,*params): + value=self.builtinTheme.get() + self.AddChangedItem('main','Theme','name',value) self.PaintThemeSample() - def VarChanged_customTheme(self, *params): - value = self.customTheme.get() + def VarChanged_customTheme(self,*params): + value=self.customTheme.get() if value != '- no custom themes -': - self.AddChangedItem('main', 'Theme', 'name', value) + self.AddChangedItem('main','Theme','name',value) self.PaintThemeSample() - def VarChanged_themeIsBuiltin(self, *params): - value = self.themeIsBuiltin.get() - self.AddChangedItem('main', 'Theme', 'default', value) + def VarChanged_themeIsBuiltin(self,*params): + value=self.themeIsBuiltin.get() + self.AddChangedItem('main','Theme','default',value) if value: self.VarChanged_builtinTheme() else: self.VarChanged_customTheme() - def VarChanged_highlightTarget(self, *params): + def VarChanged_highlightTarget(self,*params): self.SetHighlightTarget() - def VarChanged_keyBinding(self, *params): - value = self.keyBinding.get() - keySet = self.customKeys.get() - event = self.listBindings.get(ANCHOR).split()[0] + def VarChanged_keyBinding(self,*params): + value=self.keyBinding.get() + keySet=self.customKeys.get() + event=self.listBindings.get(ANCHOR).split()[0] if idleConf.IsCoreBinding(event): #this is a core keybinding - self.AddChangedItem('keys', keySet, event, value) + self.AddChangedItem('keys',keySet,event,value) else: #this is an extension key binding - extName = idleConf.GetExtnNameForEvent(event) - extKeybindSection = extName + '_cfgBindings' - self.AddChangedItem('extensions', extKeybindSection, event, value) + extName=idleConf.GetExtnNameForEvent(event) + extKeybindSection=extName+'_cfgBindings' + self.AddChangedItem('extensions',extKeybindSection,event,value) - def VarChanged_builtinKeys(self, *params): - value = self.builtinKeys.get() - self.AddChangedItem('main', 'Keys', 'name', value) + def VarChanged_builtinKeys(self,*params): + value=self.builtinKeys.get() + self.AddChangedItem('main','Keys','name',value) self.LoadKeysList(value) - def VarChanged_customKeys(self, *params): - value = self.customKeys.get() + def VarChanged_customKeys(self,*params): + value=self.customKeys.get() if value != '- no custom keys -': - self.AddChangedItem('main', 'Keys', 'name', value) + self.AddChangedItem('main','Keys','name',value) self.LoadKeysList(value) - def VarChanged_keysAreBuiltin(self, *params): - value = self.keysAreBuiltin.get() - self.AddChangedItem('main', 'Keys', 'default', value) + def VarChanged_keysAreBuiltin(self,*params): + value=self.keysAreBuiltin.get() + self.AddChangedItem('main','Keys','default',value) if value: self.VarChanged_builtinKeys() else: self.VarChanged_customKeys() - def VarChanged_winWidth(self, *params): - value = self.winWidth.get() - self.AddChangedItem('main', 'EditorWindow', 'width', value) + def VarChanged_winWidth(self,*params): + value=self.winWidth.get() + self.AddChangedItem('main','EditorWindow','width',value) - def VarChanged_winHeight(self, *params): - value = self.winHeight.get() - self.AddChangedItem('main', 'EditorWindow', 'height', value) + def VarChanged_winHeight(self,*params): + value=self.winHeight.get() + self.AddChangedItem('main','EditorWindow','height',value) - def VarChanged_startupEdit(self, *params): - value = self.startupEdit.get() - self.AddChangedItem('main', 'General', 'editor-on-startup', value) + def VarChanged_paraWidth(self,*params): + value=self.paraWidth.get() + self.AddChangedItem('main','FormatParagraph','paragraph',value) - def VarChanged_autoSave(self, *params): - value = self.autoSave.get() - self.AddChangedItem('main', 'General', 'autosave', value) + def VarChanged_startupEdit(self,*params): + value=self.startupEdit.get() + self.AddChangedItem('main','General','editor-on-startup',value) - def VarChanged_encoding(self, *params): - value = self.encoding.get() - self.AddChangedItem('main', 'EditorWindow', 'encoding', value) + def VarChanged_autoSave(self,*params): + value=self.autoSave.get() + self.AddChangedItem('main','General','autosave',value) + + def VarChanged_encoding(self,*params): + value=self.encoding.get() + self.AddChangedItem('main','EditorWindow','encoding',value) def ResetChangedItems(self): #When any config item is changed in this dialog, an entry @@ -590,25 +547,24 @@ #dictionary. The key should be the config file section name and the #value a dictionary, whose key:value pairs are item=value pairs for #that config file section. - self.changedItems = {'main':{}, 'highlight':{}, 'keys':{}, - 'extensions':{}} + self.changedItems={'main':{},'highlight':{},'keys':{},'extensions':{}} - def AddChangedItem(self, typ, section, item, value): - value = str(value) #make sure we use a string - if section not in self.changedItems[typ]: - self.changedItems[typ][section] = {} - self.changedItems[typ][section][item] = value + def AddChangedItem(self,type,section,item,value): + value=str(value) #make sure we use a string + if section not in self.changedItems[type]: + self.changedItems[type][section]={} + self.changedItems[type][section][item]=value def GetDefaultItems(self): - dItems={'main':{}, 'highlight':{}, 'keys':{}, 'extensions':{}} + dItems={'main':{},'highlight':{},'keys':{},'extensions':{}} for configType in dItems: - sections = idleConf.GetSectionList('default', configType) + sections=idleConf.GetSectionList('default',configType) for section in sections: - dItems[configType][section] = {} - options = idleConf.defaultCfg[configType].GetOptionList(section) + dItems[configType][section]={} + options=idleConf.defaultCfg[configType].GetOptionList(section) for option in options: - dItems[configType][section][option] = ( - idleConf.defaultCfg[configType].Get(section, option)) + dItems[configType][section][option]=( + idleConf.defaultCfg[configType].Get(section,option)) return dItems def SetThemeType(self): @@ -634,26 +590,26 @@ self.buttonDeleteCustomKeys.config(state=NORMAL) def GetNewKeys(self): - listIndex = self.listBindings.index(ANCHOR) - binding = self.listBindings.get(listIndex) - bindName = binding.split()[0] #first part, up to first space + listIndex=self.listBindings.index(ANCHOR) + binding=self.listBindings.get(listIndex) + bindName=binding.split()[0] #first part, up to first space if self.keysAreBuiltin.get(): - currentKeySetName = self.builtinKeys.get() + currentKeySetName=self.builtinKeys.get() else: - currentKeySetName = self.customKeys.get() - currentBindings = idleConf.GetCurrentKeySet() + currentKeySetName=self.customKeys.get() + currentBindings=idleConf.GetCurrentKeySet() if currentKeySetName in self.changedItems['keys']: #unsaved changes - keySetChanges = self.changedItems['keys'][currentKeySetName] + keySetChanges=self.changedItems['keys'][currentKeySetName] for event in keySetChanges: - currentBindings[event] = keySetChanges[event].split() + currentBindings[event]=keySetChanges[event].split() currentKeySequences = list(currentBindings.values()) - newKeys = GetKeysDialog(self, 'Get New Keys', bindName, + newKeys=GetKeysDialog(self,'Get New Keys',bindName, currentKeySequences).result if newKeys: #new keys were specified if self.keysAreBuiltin.get(): #current key set is a built-in - message = ('Your changes will be saved as a new Custom Key Set.' - ' Enter a name for your new Custom Key Set below.') - newKeySet = self.GetNewKeysName(message) + message=('Your changes will be saved as a new Custom Key Set. '+ + 'Enter a name for your new Custom Key Set below.') + newKeySet=self.GetNewKeysName(message) if not newKeySet: #user cancelled custom key set creation self.listBindings.select_set(listIndex) self.listBindings.select_anchor(listIndex) @@ -661,7 +617,7 @@ else: #create new custom key set based on previously active key set self.CreateNewKeySet(newKeySet) self.listBindings.delete(listIndex) - self.listBindings.insert(listIndex, bindName+' - '+newKeys) + self.listBindings.insert(listIndex,bindName+' - '+newKeys) self.listBindings.select_set(listIndex) self.listBindings.select_anchor(listIndex) self.keyBinding.set(newKeys) @@ -669,65 +625,65 @@ self.listBindings.select_set(listIndex) self.listBindings.select_anchor(listIndex) - def GetNewKeysName(self, message): - usedNames = (idleConf.GetSectionList('user', 'keys') + - idleConf.GetSectionList('default', 'keys')) - newKeySet = GetCfgSectionNameDialog( - self, 'New Custom Key Set', message, usedNames).result + def GetNewKeysName(self,message): + usedNames=(idleConf.GetSectionList('user','keys')+ + idleConf.GetSectionList('default','keys')) + newKeySet=GetCfgSectionNameDialog(self,'New Custom Key Set', + message,usedNames).result return newKeySet def SaveAsNewKeySet(self): - newKeysName = self.GetNewKeysName('New Key Set Name:') + newKeysName=self.GetNewKeysName('New Key Set Name:') if newKeysName: self.CreateNewKeySet(newKeysName) - def KeyBindingSelected(self, event): + def KeyBindingSelected(self,event): self.buttonNewKeys.config(state=NORMAL) - def CreateNewKeySet(self, newKeySetName): + def CreateNewKeySet(self,newKeySetName): #creates new custom key set based on the previously active key set, #and makes the new key set active if self.keysAreBuiltin.get(): - prevKeySetName = self.builtinKeys.get() + prevKeySetName=self.builtinKeys.get() else: - prevKeySetName = self.customKeys.get() - prevKeys = idleConf.GetCoreKeys(prevKeySetName) - newKeys = {} + prevKeySetName=self.customKeys.get() + prevKeys=idleConf.GetCoreKeys(prevKeySetName) + newKeys={} for event in prevKeys: #add key set to changed items - eventName = event[2:-2] #trim off the angle brackets - binding = ' '.join(prevKeys[event]) - newKeys[eventName] = binding + eventName=event[2:-2] #trim off the angle brackets + binding=' '.join(prevKeys[event]) + newKeys[eventName]=binding #handle any unsaved changes to prev key set if prevKeySetName in self.changedItems['keys']: - keySetChanges = self.changedItems['keys'][prevKeySetName] + keySetChanges=self.changedItems['keys'][prevKeySetName] for event in keySetChanges: - newKeys[event] = keySetChanges[event] + newKeys[event]=keySetChanges[event] #save the new theme - self.SaveNewKeySet(newKeySetName, newKeys) + self.SaveNewKeySet(newKeySetName,newKeys) #change gui over to the new key set - customKeyList = idleConf.GetSectionList('user', 'keys') + customKeyList=idleConf.GetSectionList('user','keys') customKeyList.sort() - self.optMenuKeysCustom.SetMenu(customKeyList, newKeySetName) + self.optMenuKeysCustom.SetMenu(customKeyList,newKeySetName) self.keysAreBuiltin.set(0) self.SetKeysType() - def LoadKeysList(self, keySetName): - reselect = 0 - newKeySet = 0 + def LoadKeysList(self,keySetName): + reselect=0 + newKeySet=0 if self.listBindings.curselection(): - reselect = 1 - listIndex = self.listBindings.index(ANCHOR) - keySet = idleConf.GetKeySet(keySetName) + reselect=1 + listIndex=self.listBindings.index(ANCHOR) + keySet=idleConf.GetKeySet(keySetName) bindNames = list(keySet.keys()) bindNames.sort() - self.listBindings.delete(0, END) + self.listBindings.delete(0,END) for bindName in bindNames: - key = ' '.join(keySet[bindName]) #make key(s) into a string - bindName = bindName[2:-2] #trim off the angle brackets + key=' '.join(keySet[bindName]) #make key(s) into a string + bindName=bindName[2:-2] #trim off the angle brackets if keySetName in self.changedItems['keys']: #handle any unsaved changes to this key set if bindName in self.changedItems['keys'][keySetName]: - key = self.changedItems['keys'][keySetName][bindName] + key=self.changedItems['keys'][keySetName][bindName] self.listBindings.insert(END, bindName+' - '+key) if reselect: self.listBindings.see(listIndex) @@ -736,9 +692,9 @@ def DeleteCustomKeys(self): keySetName=self.customKeys.get() - delmsg = 'Are you sure you wish to delete the key set %r ?' - if not tkMessageBox.askyesno( - 'Delete Key Set', delmsg % keySetName, parent=self): + if not tkMessageBox.askyesno('Delete Key Set','Are you sure you wish '+ + 'to delete the key set %r ?' % (keySetName), + parent=self): return #remove key set from config idleConf.userCfg['keys'].remove_section(keySetName) @@ -747,25 +703,25 @@ #write changes idleConf.userCfg['keys'].Save() #reload user key set list - itemList = idleConf.GetSectionList('user', 'keys') + itemList=idleConf.GetSectionList('user','keys') itemList.sort() if not itemList: self.radioKeysCustom.config(state=DISABLED) - self.optMenuKeysCustom.SetMenu(itemList, '- no custom keys -') + self.optMenuKeysCustom.SetMenu(itemList,'- no custom keys -') else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + self.optMenuKeysCustom.SetMenu(itemList,itemList[0]) #revert to default key set - self.keysAreBuiltin.set(idleConf.defaultCfg['main'].Get('Keys', 'default')) - self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name')) + self.keysAreBuiltin.set(idleConf.defaultCfg['main'].Get('Keys','default')) + self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys','name')) #user can't back out of these changes, they must be applied now self.Apply() self.SetKeysType() def DeleteCustomTheme(self): - themeName = self.customTheme.get() - delmsg = 'Are you sure you wish to delete the theme %r ?' - if not tkMessageBox.askyesno( - 'Delete Theme', delmsg % themeName, parent=self): + themeName=self.customTheme.get() + if not tkMessageBox.askyesno('Delete Theme','Are you sure you wish '+ + 'to delete the theme %r ?' % (themeName,), + parent=self): return #remove theme from config idleConf.userCfg['highlight'].remove_section(themeName) @@ -774,149 +730,152 @@ #write changes idleConf.userCfg['highlight'].Save() #reload user theme list - itemList = idleConf.GetSectionList('user', 'highlight') + itemList=idleConf.GetSectionList('user','highlight') itemList.sort() if not itemList: self.radioThemeCustom.config(state=DISABLED) - self.optMenuThemeCustom.SetMenu(itemList, '- no custom themes -') + self.optMenuThemeCustom.SetMenu(itemList,'- no custom themes -') else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + self.optMenuThemeCustom.SetMenu(itemList,itemList[0]) #revert to default theme - self.themeIsBuiltin.set(idleConf.defaultCfg['main'].Get('Theme', 'default')) - self.builtinTheme.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) + self.themeIsBuiltin.set(idleConf.defaultCfg['main'].Get('Theme','default')) + self.builtinTheme.set(idleConf.defaultCfg['main'].Get('Theme','name')) #user can't back out of these changes, they must be applied now self.Apply() self.SetThemeType() def GetColour(self): - target = self.highlightTarget.get() - prevColour = self.frameColourSet.cget('bg') - rgbTuplet, colourString = tkColorChooser.askcolor( - parent=self, title='Pick new colour for : '+target, - initialcolor=prevColour) - if colourString and (colourString != prevColour): + target=self.highlightTarget.get() + prevColour=self.frameColourSet.cget('bg') + rgbTuplet, colourString = tkColorChooser.askcolor(parent=self, + title='Pick new colour for : '+target,initialcolor=prevColour) + if colourString and (colourString!=prevColour): #user didn't cancel, and they chose a new colour - if self.themeIsBuiltin.get(): #current theme is a built-in - message = ('Your changes will be saved as a new Custom Theme. ' - 'Enter a name for your new Custom Theme below.') - newTheme = self.GetNewThemeName(message) - if not newTheme: #user cancelled custom theme creation + if self.themeIsBuiltin.get(): #current theme is a built-in + message=('Your changes will be saved as a new Custom Theme. '+ + 'Enter a name for your new Custom Theme below.') + newTheme=self.GetNewThemeName(message) + if not newTheme: #user cancelled custom theme creation return - else: #create new custom theme based on previously active theme + else: #create new custom theme based on previously active theme self.CreateNewTheme(newTheme) self.colour.set(colourString) - else: #current theme is user defined + else: #current theme is user defined self.colour.set(colourString) def OnNewColourSet(self): newColour=self.colour.get() - self.frameColourSet.config(bg=newColour) #set sample - plane ='foreground' if self.fgHilite.get() else 'background' - sampleElement = self.themeElements[self.highlightTarget.get()][0] + self.frameColourSet.config(bg=newColour)#set sample + if self.fgHilite.get(): plane='foreground' + else: plane='background' + sampleElement=self.themeElements[self.highlightTarget.get()][0] self.textHighlightSample.tag_config(sampleElement, **{plane:newColour}) - theme = self.customTheme.get() - themeElement = sampleElement + '-' + plane - self.AddChangedItem('highlight', theme, themeElement, newColour) + theme=self.customTheme.get() + themeElement=sampleElement+'-'+plane + self.AddChangedItem('highlight',theme,themeElement,newColour) - def GetNewThemeName(self, message): - usedNames = (idleConf.GetSectionList('user', 'highlight') + - idleConf.GetSectionList('default', 'highlight')) - newTheme = GetCfgSectionNameDialog( - self, 'New Custom Theme', message, usedNames).result + def GetNewThemeName(self,message): + usedNames=(idleConf.GetSectionList('user','highlight')+ + idleConf.GetSectionList('default','highlight')) + newTheme=GetCfgSectionNameDialog(self,'New Custom Theme', + message,usedNames).result return newTheme def SaveAsNewTheme(self): - newThemeName = self.GetNewThemeName('New Theme Name:') + newThemeName=self.GetNewThemeName('New Theme Name:') if newThemeName: self.CreateNewTheme(newThemeName) - def CreateNewTheme(self, newThemeName): + def CreateNewTheme(self,newThemeName): #creates new custom theme based on the previously active theme, #and makes the new theme active if self.themeIsBuiltin.get(): - themeType = 'default' - themeName = self.builtinTheme.get() + themeType='default' + themeName=self.builtinTheme.get() else: - themeType = 'user' - themeName = self.customTheme.get() - newTheme = idleConf.GetThemeDict(themeType, themeName) + themeType='user' + themeName=self.customTheme.get() + newTheme=idleConf.GetThemeDict(themeType,themeName) #apply any of the old theme's unsaved changes to the new theme if themeName in self.changedItems['highlight']: - themeChanges = self.changedItems['highlight'][themeName] + themeChanges=self.changedItems['highlight'][themeName] for element in themeChanges: - newTheme[element] = themeChanges[element] + newTheme[element]=themeChanges[element] #save the new theme - self.SaveNewTheme(newThemeName, newTheme) + self.SaveNewTheme(newThemeName,newTheme) #change gui over to the new theme - customThemeList = idleConf.GetSectionList('user', 'highlight') + customThemeList=idleConf.GetSectionList('user','highlight') customThemeList.sort() - self.optMenuThemeCustom.SetMenu(customThemeList, newThemeName) + self.optMenuThemeCustom.SetMenu(customThemeList,newThemeName) self.themeIsBuiltin.set(0) self.SetThemeType() - def OnListFontButtonRelease(self, event): + def OnListFontButtonRelease(self,event): font = self.listFontName.get(ANCHOR) self.fontName.set(font.lower()) self.SetFontSample() - def SetFontSample(self, event=None): - fontName = self.fontName.get() - fontWeight = tkFont.BOLD if self.fontBold.get() else tkFont.NORMAL - newFont = (fontName, self.fontSize.get(), fontWeight) - self.labelFontSample.config(font=newFont) - self.textHighlightSample.configure(font=newFont) + def SetFontSample(self,event=None): + fontName=self.fontName.get() + if self.fontBold.get(): + fontWeight=tkFont.BOLD + else: + fontWeight=tkFont.NORMAL + self.editFont.config(size=self.fontSize.get(), + weight=fontWeight,family=fontName) def SetHighlightTarget(self): - if self.highlightTarget.get() == 'Cursor': #bg not possible + if self.highlightTarget.get()=='Cursor': #bg not possible self.radioFg.config(state=DISABLED) self.radioBg.config(state=DISABLED) self.fgHilite.set(1) - else: #both fg and bg can be set + else: #both fg and bg can be set self.radioFg.config(state=NORMAL) self.radioBg.config(state=NORMAL) self.fgHilite.set(1) self.SetColourSample() - def SetColourSampleBinding(self, *args): + def SetColourSampleBinding(self,*args): self.SetColourSample() def SetColourSample(self): #set the colour smaple area - tag = self.themeElements[self.highlightTarget.get()][0] - plane = 'foreground' if self.fgHilite.get() else 'background' - colour = self.textHighlightSample.tag_cget(tag, plane) + tag=self.themeElements[self.highlightTarget.get()][0] + if self.fgHilite.get(): plane='foreground' + else: plane='background' + colour=self.textHighlightSample.tag_cget(tag,plane) self.frameColourSet.config(bg=colour) def PaintThemeSample(self): - if self.themeIsBuiltin.get(): #a default theme - theme = self.builtinTheme.get() - else: #a user theme - theme = self.customTheme.get() + if self.themeIsBuiltin.get(): #a default theme + theme=self.builtinTheme.get() + else: #a user theme + theme=self.customTheme.get() for elementTitle in self.themeElements: - element = self.themeElements[elementTitle][0] - colours = idleConf.GetHighlight(theme, element) - if element == 'cursor': #cursor sample needs special painting - colours['background'] = idleConf.GetHighlight( - theme, 'normal', fgBg='bg') + element=self.themeElements[elementTitle][0] + colours=idleConf.GetHighlight(theme,element) + if element=='cursor': #cursor sample needs special painting + colours['background']=idleConf.GetHighlight(theme, + 'normal', fgBg='bg') #handle any unsaved changes to this theme if theme in self.changedItems['highlight']: - themeDict = self.changedItems['highlight'][theme] - if element + '-foreground' in themeDict: - colours['foreground'] = themeDict[element + '-foreground'] - if element + '-background' in themeDict: - colours['background'] = themeDict[element + '-background'] + themeDict=self.changedItems['highlight'][theme] + if element+'-foreground' in themeDict: + colours['foreground']=themeDict[element+'-foreground'] + if element+'-background' in themeDict: + colours['background']=themeDict[element+'-background'] self.textHighlightSample.tag_config(element, **colours) self.SetColourSample() - def HelpSourceSelected(self, event): + def HelpSourceSelected(self,event): self.SetHelpListButtonStates() def SetHelpListButtonStates(self): - if self.listHelp.size() < 1: #no entries in list + if self.listHelp.size()<1: #no entries in list self.buttonHelpListEdit.config(state=DISABLED) self.buttonHelpListRemove.config(state=DISABLED) else: #there are some entries - if self.listHelp.curselection(): #there currently is a selection + if self.listHelp.curselection(): #there currently is a selection self.buttonHelpListEdit.config(state=NORMAL) self.buttonHelpListRemove.config(state=NORMAL) else: #there currently is not a selection @@ -924,29 +883,28 @@ self.buttonHelpListRemove.config(state=DISABLED) def HelpListItemAdd(self): - helpSource = GetHelpSourceDialog(self, 'New Help Source').result + helpSource=GetHelpSourceDialog(self,'New Help Source').result if helpSource: - self.userHelpList.append((helpSource[0], helpSource[1])) - self.listHelp.insert(END, helpSource[0]) + self.userHelpList.append( (helpSource[0],helpSource[1]) ) + self.listHelp.insert(END,helpSource[0]) self.UpdateUserHelpChangedItems() self.SetHelpListButtonStates() def HelpListItemEdit(self): - itemIndex = self.listHelp.index(ANCHOR) - helpSource = self.userHelpList[itemIndex] - newHelpSource = GetHelpSourceDialog( - self, 'Edit Help Source', menuItem=helpSource[0], - filePath=helpSource[1]).result - if (not newHelpSource) or (newHelpSource == helpSource): + itemIndex=self.listHelp.index(ANCHOR) + helpSource=self.userHelpList[itemIndex] + newHelpSource=GetHelpSourceDialog(self,'Edit Help Source', + menuItem=helpSource[0],filePath=helpSource[1]).result + if (not newHelpSource) or (newHelpSource==helpSource): return #no changes - self.userHelpList[itemIndex] = newHelpSource + self.userHelpList[itemIndex]=newHelpSource self.listHelp.delete(itemIndex) - self.listHelp.insert(itemIndex, newHelpSource[0]) + self.listHelp.insert(itemIndex,newHelpSource[0]) self.UpdateUserHelpChangedItems() self.SetHelpListButtonStates() def HelpListItemRemove(self): - itemIndex = self.listHelp.index(ANCHOR) + itemIndex=self.listHelp.index(ANCHOR) del(self.userHelpList[itemIndex]) self.listHelp.delete(itemIndex) self.UpdateUserHelpChangedItems() @@ -955,126 +913,125 @@ def UpdateUserHelpChangedItems(self): "Clear and rebuild the HelpFiles section in self.changedItems" self.changedItems['main']['HelpFiles'] = {} - for num in range(1, len(self.userHelpList) + 1): - self.AddChangedItem( - 'main', 'HelpFiles', str(num), + for num in range(1,len(self.userHelpList)+1): + self.AddChangedItem('main','HelpFiles',str(num), ';'.join(self.userHelpList[num-1][:2])) def LoadFontCfg(self): ##base editor font selection list - fonts = list(tkFont.families(self)) + fonts=list(tkFont.families(self)) fonts.sort() for font in fonts: - self.listFontName.insert(END, font) - configuredFont = idleConf.GetFont(self, 'main', 'EditorWindow') - fontName = configuredFont[0].lower() - fontSize = configuredFont[1] - fontBold = configuredFont[2]=='bold' - self.fontName.set(fontName) + self.listFontName.insert(END,font) + configuredFont=idleConf.GetOption('main','EditorWindow','font', + default='courier') + lc_configuredFont = configuredFont.lower() + self.fontName.set(lc_configuredFont) lc_fonts = [s.lower() for s in fonts] - try: - currentFontIndex = lc_fonts.index(fontName) + if lc_configuredFont in lc_fonts: + currentFontIndex = lc_fonts.index(lc_configuredFont) self.listFontName.see(currentFontIndex) self.listFontName.select_set(currentFontIndex) self.listFontName.select_anchor(currentFontIndex) - except ValueError: - pass ##font size dropdown - self.optMenuFontSize.SetMenu(('7', '8', '9', '10', '11', '12', '13', - '14', '16', '18', '20', '22'), fontSize ) + fontSize=idleConf.GetOption('main','EditorWindow','font-size', + default='10') + self.optMenuFontSize.SetMenu(('7','8','9','10','11','12','13','14', + '16','18','20','22'),fontSize ) ##fontWeight - self.fontBold.set(fontBold) + self.fontBold.set(idleConf.GetOption('main','EditorWindow', + 'font-bold',default=0,type='bool')) ##font sample self.SetFontSample() def LoadTabCfg(self): ##indent sizes - spaceNum = idleConf.GetOption( - 'main', 'Indent', 'num-spaces', default=4, type='int') + spaceNum=idleConf.GetOption('main','Indent','num-spaces', + default=4,type='int') self.spaceNum.set(spaceNum) def LoadThemeCfg(self): ##current theme type radiobutton - self.themeIsBuiltin.set(idleConf.GetOption( - 'main', 'Theme', 'default', type='bool', default=1)) + self.themeIsBuiltin.set(idleConf.GetOption('main','Theme','default', + type='bool',default=1)) ##currently set theme - currentOption = idleConf.CurrentTheme() + currentOption=idleConf.CurrentTheme() ##load available theme option menus if self.themeIsBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'highlight') + itemList=idleConf.GetSectionList('default','highlight') itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'highlight') + self.optMenuThemeBuiltin.SetMenu(itemList,currentOption) + itemList=idleConf.GetSectionList('user','highlight') itemList.sort() if not itemList: self.radioThemeCustom.config(state=DISABLED) self.customTheme.set('- no custom themes -') else: - self.optMenuThemeCustom.SetMenu(itemList, itemList[0]) + self.optMenuThemeCustom.SetMenu(itemList,itemList[0]) else: #user theme selected - itemList = idleConf.GetSectionList('user', 'highlight') + itemList=idleConf.GetSectionList('user','highlight') itemList.sort() - self.optMenuThemeCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'highlight') + self.optMenuThemeCustom.SetMenu(itemList,currentOption) + itemList=idleConf.GetSectionList('default','highlight') itemList.sort() - self.optMenuThemeBuiltin.SetMenu(itemList, itemList[0]) + self.optMenuThemeBuiltin.SetMenu(itemList,itemList[0]) self.SetThemeType() ##load theme element option menu themeNames = list(self.themeElements.keys()) themeNames.sort(key=lambda x: self.themeElements[x][1]) - self.optMenuHighlightTarget.SetMenu(themeNames, themeNames[0]) + self.optMenuHighlightTarget.SetMenu(themeNames,themeNames[0]) self.PaintThemeSample() self.SetHighlightTarget() def LoadKeyCfg(self): ##current keys type radiobutton - self.keysAreBuiltin.set(idleConf.GetOption( - 'main', 'Keys', 'default', type='bool', default=1)) + self.keysAreBuiltin.set(idleConf.GetOption('main','Keys','default', + type='bool',default=1)) ##currently set keys - currentOption = idleConf.CurrentKeys() + currentOption=idleConf.CurrentKeys() ##load available keyset option menus if self.keysAreBuiltin.get(): #default theme selected - itemList = idleConf.GetSectionList('default', 'keys') + itemList=idleConf.GetSectionList('default','keys') itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('user', 'keys') + self.optMenuKeysBuiltin.SetMenu(itemList,currentOption) + itemList=idleConf.GetSectionList('user','keys') itemList.sort() if not itemList: self.radioKeysCustom.config(state=DISABLED) self.customKeys.set('- no custom keys -') else: - self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) + self.optMenuKeysCustom.SetMenu(itemList,itemList[0]) else: #user key set selected - itemList = idleConf.GetSectionList('user', 'keys') + itemList=idleConf.GetSectionList('user','keys') itemList.sort() - self.optMenuKeysCustom.SetMenu(itemList, currentOption) - itemList = idleConf.GetSectionList('default', 'keys') + self.optMenuKeysCustom.SetMenu(itemList,currentOption) + itemList=idleConf.GetSectionList('default','keys') itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, itemList[0]) + self.optMenuKeysBuiltin.SetMenu(itemList,itemList[0]) self.SetKeysType() ##load keyset element list - keySetName = idleConf.CurrentKeys() + keySetName=idleConf.CurrentKeys() self.LoadKeysList(keySetName) def LoadGeneralCfg(self): #startup state - self.startupEdit.set(idleConf.GetOption( - 'main', 'General', 'editor-on-startup', default=1, type='bool')) + self.startupEdit.set(idleConf.GetOption('main','General', + 'editor-on-startup',default=1,type='bool')) #autosave state - self.autoSave.set(idleConf.GetOption( - 'main', 'General', 'autosave', default=0, type='bool')) + self.autoSave.set(idleConf.GetOption('main', 'General', 'autosave', + default=0, type='bool')) #initial window size - self.winWidth.set(idleConf.GetOption( - 'main', 'EditorWindow', 'width', type='int')) - self.winHeight.set(idleConf.GetOption( - 'main', 'EditorWindow', 'height', type='int')) + self.winWidth.set(idleConf.GetOption('main','EditorWindow','width')) + self.winHeight.set(idleConf.GetOption('main','EditorWindow','height')) + #initial paragraph reformat size + self.paraWidth.set(idleConf.GetOption('main','FormatParagraph','paragraph')) # default source encoding - self.encoding.set(idleConf.GetOption( - 'main', 'EditorWindow', 'encoding', default='none')) + self.encoding.set(idleConf.GetOption('main', 'EditorWindow', + 'encoding', default='none')) # additional help sources self.userHelpList = idleConf.GetAllExtraHelpSourcesList() for helpItem in self.userHelpList: - self.listHelp.insert(END, helpItem[0]) + self.listHelp.insert(END,helpItem[0]) self.SetHelpListButtonStates() def LoadConfigs(self): @@ -1091,9 +1048,8 @@ self.LoadKeyCfg() ### general page self.LoadGeneralCfg() - # note: extension page handled separately - def SaveNewKeySet(self, keySetName, keySet): + def SaveNewKeySet(self,keySetName,keySet): """ save a newly created core key set. keySetName - string, the name of the new key set @@ -1102,10 +1058,10 @@ if not idleConf.userCfg['keys'].has_section(keySetName): idleConf.userCfg['keys'].add_section(keySetName) for event in keySet: - value = keySet[event] - idleConf.userCfg['keys'].SetOption(keySetName, event, value) + value=keySet[event] + idleConf.userCfg['keys'].SetOption(keySetName,event,value) - def SaveNewTheme(self, themeName, theme): + def SaveNewTheme(self,themeName,theme): """ save a newly created theme. themeName - string, the name of the new theme @@ -1114,16 +1070,16 @@ if not idleConf.userCfg['highlight'].has_section(themeName): idleConf.userCfg['highlight'].add_section(themeName) for element in theme: - value = theme[element] - idleConf.userCfg['highlight'].SetOption(themeName, element, value) + value=theme[element] + idleConf.userCfg['highlight'].SetOption(themeName,element,value) - def SetUserValue(self, configType, section, item, value): - if idleConf.defaultCfg[configType].has_option(section, item): - if idleConf.defaultCfg[configType].Get(section, item) == value: + def SetUserValue(self,configType,section,item,value): + if idleConf.defaultCfg[configType].has_option(section,item): + if idleConf.defaultCfg[configType].Get(section,item)==value: #the setting equals a default setting, remove it from user cfg - return idleConf.userCfg[configType].RemoveOption(section, item) + return idleConf.userCfg[configType].RemoveOption(section,item) #if we got here set the option - return idleConf.userCfg[configType].SetOption(section, item, value) + return idleConf.userCfg[configType].SetOption(section,item,value) def SaveAllChangedConfigs(self): "Save configuration changes to the user config file." @@ -1137,7 +1093,7 @@ cfgTypeHasChanges = True for item in self.changedItems[configType][section]: value = self.changedItems[configType][section][item] - if self.SetUserValue(configType, section, item, value): + if self.SetUserValue(configType,section,item,value): cfgTypeHasChanges = True if cfgTypeHasChanges: idleConf.userCfg[configType].Save() @@ -1145,7 +1101,6 @@ # save these even if unchanged! idleConf.userCfg[configType].Save() self.ResetChangedItems() #clear the changed items dict - self.save_all_changed_extensions() # uses a different mechanism def DeactivateCurrentConfig(self): #Before a config is saved, some cleanup of current @@ -1177,247 +1132,12 @@ self.ActivateConfigChanges() def Help(self): - page = self.tabPages._current_page - view_text(self, title='Help for IDLE preferences', - text=help_common+help_pages.get(page, '')) - - def CreatePageExtensions(self): - """Part of the config dialog used for configuring IDLE extensions. - - This code is generic - it works for any and all IDLE extensions. - - IDLE extensions save their configuration options using idleConf. - This code reads the current configuration using idleConf, supplies a - GUI interface to change the configuration values, and saves the - changes using idleConf. - - Not all changes take effect immediately - some may require restarting IDLE. - This depends on each extension's implementation. - - All values are treated as text, and it is up to the user to supply - reasonable values. The only exception to this are the 'enable*' options, - which are boolean, and can be toggled with an True/False button. - """ - parent = self.parent - frame = self.tabPages.pages['Extensions'].frame - self.ext_defaultCfg = idleConf.defaultCfg['extensions'] - self.ext_userCfg = idleConf.userCfg['extensions'] - self.is_int = self.register(is_int) - self.load_extensions() - # create widgets - a listbox shows all available extensions, with the - # controls for the extension selected in the listbox to the right - self.extension_names = StringVar(self) - frame.rowconfigure(0, weight=1) - frame.columnconfigure(2, weight=1) - self.extension_list = Listbox(frame, listvariable=self.extension_names, - selectmode='browse') - self.extension_list.bind('<>', self.extension_selected) - scroll = Scrollbar(frame, command=self.extension_list.yview) - self.extension_list.yscrollcommand=scroll.set - self.details_frame = LabelFrame(frame, width=250, height=250) - self.extension_list.grid(column=0, row=0, sticky='nws') - scroll.grid(column=1, row=0, sticky='ns') - self.details_frame.grid(column=2, row=0, sticky='nsew', padx=[10, 0]) - frame.configure(padx=10, pady=10) - self.config_frame = {} - self.current_extension = None - - self.outerframe = self # TEMPORARY - self.tabbed_page_set = self.extension_list # TEMPORARY - - # create the frame holding controls for each extension - ext_names = '' - for ext_name in sorted(self.extensions): - self.create_extension_frame(ext_name) - ext_names = ext_names + '{' + ext_name + '} ' - self.extension_names.set(ext_names) - self.extension_list.selection_set(0) - self.extension_selected(None) - - def load_extensions(self): - "Fill self.extensions with data from the default and user configs." - self.extensions = {} - for ext_name in idleConf.GetExtensions(active_only=False): - self.extensions[ext_name] = [] - - for ext_name in self.extensions: - opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) - - # bring 'enable' options to the beginning of the list - enables = [opt_name for opt_name in opt_list - if opt_name.startswith('enable')] - for opt_name in enables: - opt_list.remove(opt_name) - opt_list = enables + opt_list - - for opt_name in opt_list: - def_str = self.ext_defaultCfg.Get( - ext_name, opt_name, raw=True) - try: - def_obj = {'True':True, 'False':False}[def_str] - opt_type = 'bool' - except KeyError: - try: - def_obj = int(def_str) - opt_type = 'int' - except ValueError: - def_obj = def_str - opt_type = None - try: - value = self.ext_userCfg.Get( - ext_name, opt_name, type=opt_type, raw=True, - default=def_obj) - except ValueError: # Need this until .Get fixed - value = def_obj # bad values overwritten by entry - var = StringVar(self) - var.set(str(value)) - - self.extensions[ext_name].append({'name': opt_name, - 'type': opt_type, - 'default': def_str, - 'value': value, - 'var': var, - }) - - def extension_selected(self, event): - newsel = self.extension_list.curselection() - if newsel: - newsel = self.extension_list.get(newsel) - if newsel is None or newsel != self.current_extension: - if self.current_extension: - self.details_frame.config(text='') - self.config_frame[self.current_extension].grid_forget() - self.current_extension = None - if newsel: - self.details_frame.config(text=newsel) - self.config_frame[newsel].grid(column=0, row=0, sticky='nsew') - self.current_extension = newsel - - def create_extension_frame(self, ext_name): - """Create a frame holding the widgets to configure one extension""" - f = VerticalScrolledFrame(self.details_frame, height=250, width=250) - self.config_frame[ext_name] = f - entry_area = f.interior - # create an entry for each configuration option - for row, opt in enumerate(self.extensions[ext_name]): - # create a row with a label and entry/checkbutton - label = Label(entry_area, text=opt['name']) - label.grid(row=row, column=0, sticky=NW) - var = opt['var'] - if opt['type'] == 'bool': - Checkbutton(entry_area, textvariable=var, variable=var, - onvalue='True', offvalue='False', - indicatoron=FALSE, selectcolor='', width=8 - ).grid(row=row, column=1, sticky=W, padx=7) - elif opt['type'] == 'int': - Entry(entry_area, textvariable=var, validate='key', - validatecommand=(self.is_int, '%P') - ).grid(row=row, column=1, sticky=NSEW, padx=7) - - else: - Entry(entry_area, textvariable=var - ).grid(row=row, column=1, sticky=NSEW, padx=7) - return - - def set_extension_value(self, section, opt): - name = opt['name'] - default = opt['default'] - value = opt['var'].get().strip() or default - opt['var'].set(value) - # if self.defaultCfg.has_section(section): - # Currently, always true; if not, indent to return - if (value == default): - return self.ext_userCfg.RemoveOption(section, name) - # set the option - return self.ext_userCfg.SetOption(section, name, value) - - def save_all_changed_extensions(self): - """Save configuration changes to the user config file.""" - has_changes = False - for ext_name in self.extensions: - options = self.extensions[ext_name] - for opt in options: - if self.set_extension_value(ext_name, opt): - has_changes = True - if has_changes: - self.ext_userCfg.Save() - - -help_common = '''\ -When you click either the Apply or Ok buttons, settings in this -dialog that are different from IDLE's default are saved in -a .idlerc directory in your home directory. Except as noted, -these changes apply to all versions of IDLE installed on this -machine. Some do not take affect until IDLE is restarted. -[Cancel] only cancels changes made since the last save. -''' -help_pages = { - 'Highlighting':''' -Highlighting: -The IDLE Dark color theme is new in October 2015. It can only -be used with older IDLE releases if it is saved as a custom -theme, with a different name. -''' -} - - -def is_int(s): - "Return 's is blank or represents an int'" - if not s: - return True - try: - int(s) - return True - except ValueError: - return False - - -class VerticalScrolledFrame(Frame): - """A pure Tkinter vertically scrollable frame. - - * Use the 'interior' attribute to place widgets inside the scrollable frame - * Construct and pack/place/grid normally - * This frame only allows vertical scrolling - """ - def __init__(self, parent, *args, **kw): - Frame.__init__(self, parent, *args, **kw) - - # create a canvas object and a vertical scrollbar for scrolling it - vscrollbar = Scrollbar(self, orient=VERTICAL) - vscrollbar.pack(fill=Y, side=RIGHT, expand=FALSE) - canvas = Canvas(self, bd=0, highlightthickness=0, - yscrollcommand=vscrollbar.set, width=240) - canvas.pack(side=LEFT, fill=BOTH, expand=TRUE) - vscrollbar.config(command=canvas.yview) - - # reset the view - canvas.xview_moveto(0) - canvas.yview_moveto(0) - - # create a frame inside the canvas which will be scrolled with it - self.interior = interior = Frame(canvas) - interior_id = canvas.create_window(0, 0, window=interior, anchor=NW) - - # track changes to the canvas and frame width and sync them, - # also updating the scrollbar - def _configure_interior(event): - # update the scrollbars to match the size of the inner frame - size = (interior.winfo_reqwidth(), interior.winfo_reqheight()) - canvas.config(scrollregion="0 0 %s %s" % size) - interior.bind('', _configure_interior) - - def _configure_canvas(event): - if interior.winfo_reqwidth() != canvas.winfo_width(): - # update the inner frame's width to fill the canvas - canvas.itemconfigure(interior_id, width=canvas.winfo_width()) - canvas.bind('', _configure_canvas) - - return - + pass if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_configdialog', - verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(ConfigDialog) + #test the dialog + root=Tk() + Button(root,text='Dialog', + command=lambda:ConfigDialog(root,'Settings')).pack() + root.instance_dict={} + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/configHandler.py --- a/Lib/idlelib/configHandler.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/configHandler.py Mon Jan 25 17:05:13 2016 +0100 @@ -15,15 +15,13 @@ the retrieval of config information. When a default is returned instead of a requested config value, a message is printed to stderr to aid in configuration problem notification and resolution. + """ -# TODOs added Oct 2014, tjr - import os import sys -from configparser import ConfigParser -from tkinter import TkVersion -from tkinter.font import Font, nametofont +from idlelib import macosxSupport +from configparser import ConfigParser, NoOptionError, NoSectionError class InvalidConfigType(Exception): pass class InvalidConfigSet(Exception): pass @@ -38,35 +36,36 @@ """ cfgFile - string, fully specified configuration file name """ - self.file = cfgFile - ConfigParser.__init__(self, defaults=cfgDefaults, strict=False) + self.file=cfgFile + ConfigParser.__init__(self,defaults=cfgDefaults) def Get(self, section, option, type=None, default=None, raw=False): """ Get an option value for given section/option or return default. If type is specified, return as type. """ - # TODO Use default as fallback, at least if not None - # Should also print Warning(file, section, option). - # Currently may raise ValueError if not self.has_option(section, option): return default - if type == 'bool': + if type=='bool': return self.getboolean(section, option) - elif type == 'int': + elif type=='int': return self.getint(section, option) else: return self.get(section, option, raw=raw) - def GetOptionList(self, section): - "Return a list of options for given section, else []." + def GetOptionList(self,section): + """ + Get an option list for given section + """ if self.has_section(section): return self.options(section) else: #return a default value return [] def Load(self): - "Load the configuration file from disk." + """ + Load the configuration file from disk + """ self.read(self.file) class IdleUserConfParser(IdleConfParser): @@ -74,50 +73,61 @@ IdleConfigParser specialised for user configuration handling. """ - def AddSection(self, section): - "If section doesn't exist, add it." + def AddSection(self,section): + """ + if section doesn't exist, add it + """ if not self.has_section(section): self.add_section(section) def RemoveEmptySections(self): - "Remove any sections that have no options." + """ + remove any sections that have no options + """ for section in self.sections(): if not self.GetOptionList(section): self.remove_section(section) def IsEmpty(self): - "Return True if no sections after removing empty sections." + """ + Remove empty sections and then return 1 if parser has no sections + left, else return 0. + """ self.RemoveEmptySections() - return not self.sections() + if self.sections(): + return 0 + else: + return 1 - def RemoveOption(self, section, option): - """Return True if option is removed from section, else False. - - False if either section does not exist or did not have option. + def RemoveOption(self,section,option): + """ + If section/option exists, remove it. + Returns 1 if option was removed, 0 otherwise. """ if self.has_section(section): - return self.remove_option(section, option) - return False + return self.remove_option(section,option) - def SetOption(self, section, option, value): - """Return True if option is added or changed to value, else False. - - Add section if required. False means option already had value. + def SetOption(self,section,option,value): """ - if self.has_option(section, option): - if self.get(section, option) == value: - return False + Sets option to value, adding section if required. + Returns 1 if option was added or changed, otherwise 0. + """ + if self.has_option(section,option): + if self.get(section,option)==value: + return 0 else: - self.set(section, option, value) - return True + self.set(section,option,value) + return 1 else: if not self.has_section(section): self.add_section(section) - self.set(section, option, value) - return True + self.set(section,option,value) + return 1 def RemoveFile(self): - "Remove user config file self.file from disk if it exists." + """ + Removes the user config file from disk if it exists. + """ if os.path.exists(self.file): os.remove(self.file) @@ -132,7 +142,7 @@ fname = self.file try: cfgFile = open(fname, 'w') - except OSError: + except IOError: os.unlink(fname) cfgFile = open(fname, 'w') with cfgFile: @@ -141,60 +151,63 @@ self.RemoveFile() class IdleConf: - """Hold config parsers for all idle config files in singleton instance. - - Default config files, self.defaultCfg -- - for config_type in self.config_types: - (idle install dir)/config-{config-type}.def - - User config files, self.userCfg -- - for config_type in self.config_types: - (user home dir)/.idlerc/config-{config-type}.cfg + """ + holds config parsers for all idle config files: + default config files + (idle install dir)/config-main.def + (idle install dir)/config-extensions.def + (idle install dir)/config-highlight.def + (idle install dir)/config-keys.def + user config files + (user home dir)/.idlerc/config-main.cfg + (user home dir)/.idlerc/config-extensions.cfg + (user home dir)/.idlerc/config-highlight.cfg + (user home dir)/.idlerc/config-keys.cfg """ def __init__(self): - self.config_types = ('main', 'extensions', 'highlight', 'keys') - self.defaultCfg = {} - self.userCfg = {} - self.cfg = {} # TODO use to select userCfg vs defaultCfg + self.defaultCfg={} + self.userCfg={} + self.cfg={} self.CreateConfigHandlers() self.LoadCfgFiles() - + #self.LoadCfg() def CreateConfigHandlers(self): - "Populate default and user config parser dictionaries." + """ + set up a dictionary of config parsers for default and user + configurations respectively + """ #build idle install path if __name__ != '__main__': # we were imported idleDir=os.path.dirname(__file__) else: # we were exec'ed (for testing only) idleDir=os.path.abspath(sys.path[0]) userDir=self.GetUserCfgDir() - - defCfgFiles = {} - usrCfgFiles = {} - # TODO eliminate these temporaries by combining loops - for cfgType in self.config_types: #build config file names - defCfgFiles[cfgType] = os.path.join( - idleDir, 'config-' + cfgType + '.def') - usrCfgFiles[cfgType] = os.path.join( - userDir, 'config-' + cfgType + '.cfg') - for cfgType in self.config_types: #create config parsers - self.defaultCfg[cfgType] = IdleConfParser(defCfgFiles[cfgType]) - self.userCfg[cfgType] = IdleUserConfParser(usrCfgFiles[cfgType]) + configTypes=('main','extensions','highlight','keys') + defCfgFiles={} + usrCfgFiles={} + for cfgType in configTypes: #build config file names + defCfgFiles[cfgType]=os.path.join(idleDir,'config-'+cfgType+'.def') + usrCfgFiles[cfgType]=os.path.join(userDir,'config-'+cfgType+'.cfg') + for cfgType in configTypes: #create config parsers + self.defaultCfg[cfgType]=IdleConfParser(defCfgFiles[cfgType]) + self.userCfg[cfgType]=IdleUserConfParser(usrCfgFiles[cfgType]) def GetUserCfgDir(self): - """Return a filesystem directory for storing user config files. + """ + Creates (if required) and returns a filesystem directory for storing + user config files. - Creates it if required. """ cfgDir = '.idlerc' userDir = os.path.expanduser('~') if userDir != '~': # expanduser() found user home dir if not os.path.exists(userDir): - warn = ('\n Warning: os.path.expanduser("~") points to\n ' + - userDir + ',\n but the path does not exist.') + warn = ('\n Warning: os.path.expanduser("~") points to\n '+ + userDir+',\n but the path does not exist.\n') try: - print(warn, file=sys.stderr) - except OSError: + sys.stderr.write(warn) + except IOError: pass userDir = '~' if userDir == "~": # still no path to home! @@ -204,74 +217,63 @@ if not os.path.exists(userDir): try: os.mkdir(userDir) - except OSError: - warn = ('\n Warning: unable to create user config directory\n' + - userDir + '\n Check path and permissions.\n Exiting!\n') - print(warn, file=sys.stderr) + except (OSError, IOError): + warn = ('\n Warning: unable to create user config directory\n'+ + userDir+'\n Check path and permissions.\n Exiting!\n\n') + sys.stderr.write(warn) raise SystemExit - # TODO continue without userDIr instead of exit return userDir def GetOption(self, configType, section, option, default=None, type=None, warn_on_default=True, raw=False): - """Return a value for configType section option, or default. + """ + Get an option value for given config type and given general + configuration section/option or return a default. If type is specified, + return as type. Firstly the user configuration is checked, with a + fallback to the default configuration, and a final 'catch all' + fallback to a useable passed-in default if the option isn't present in + either the user or the default configuration. + configType must be one of ('main','extensions','highlight','keys') + If a default is returned, and warn_on_default is True, a warning is + printed to stderr. - If type is not None, return a value of that type. Also pass raw - to the config parser. First try to return a valid value - (including type) from a user configuration. If that fails, try - the default configuration. If that fails, return default, with a - default of None. - - Warn if either user or default configurations have an invalid value. - Warn if default is returned and warn_on_default is True. """ - try: - if self.userCfg[configType].has_option(section, option): - return self.userCfg[configType].Get(section, option, - type=type, raw=raw) - except ValueError: - warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' - ' invalid %r value for configuration option %r\n' - ' from section %r: %r' % - (type, option, section, - self.userCfg[configType].Get(section, option, raw=raw))) - try: - print(warning, file=sys.stderr) - except OSError: - pass - try: - if self.defaultCfg[configType].has_option(section,option): - return self.defaultCfg[configType].Get( - section, option, type=type, raw=raw) - except ValueError: - pass - #returning default, print warning - if warn_on_default: - warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' - ' problem retrieving configuration option %r\n' - ' from section %r.\n' - ' returning default value: %r' % - (option, section, default)) - try: - print(warning, file=sys.stderr) - except OSError: - pass - return default + if self.userCfg[configType].has_option(section,option): + return self.userCfg[configType].Get(section, option, + type=type, raw=raw) + elif self.defaultCfg[configType].has_option(section,option): + return self.defaultCfg[configType].Get(section, option, + type=type, raw=raw) + else: #returning default, print warning + if warn_on_default: + warning = ('\n Warning: configHandler.py - IdleConf.GetOption -\n' + ' problem retrieving configuration option %r\n' + ' from section %r.\n' + ' returning default value: %r\n' % + (option, section, default)) + try: + sys.stderr.write(warning) + except IOError: + pass + return default def SetOption(self, configType, section, option, value): - """Set section option to value in user config file.""" + """In user's config file, set section's option to value. + + """ self.userCfg[configType].SetOption(section, option, value) def GetSectionList(self, configSet, configType): - """Return sections for configSet configType configuration. - + """ + Get a list of sections from either the user or default config for + the given config type. configSet must be either 'user' or 'default' - configType must be in self.config_types. + configType must be one of ('main','extensions','highlight','keys') """ - if not (configType in self.config_types): + if not (configType in ('main','extensions','highlight','keys')): raise InvalidConfigType('Invalid configType specified') if configSet == 'user': - cfgParser = self.userCfg[configType] + cfgParser=self.userCfg[configType] elif configSet == 'default': cfgParser=self.defaultCfg[configType] else: @@ -279,27 +281,25 @@ return cfgParser.sections() def GetHighlight(self, theme, element, fgBg=None): - """Return individual theme element highlight color(s). - - fgBg - string ('fg' or 'bg') or None. - If None, return a dictionary containing fg and bg colors with - keys 'foreground' and 'background'. Otherwise, only return - fg or bg color, as specified. Colors are intended to be - appropriate for passing to Tkinter in, e.g., a tag_config call). + """ + return individual highlighting theme elements. + fgBg - string ('fg'or'bg') or None, if None return a dictionary + containing fg and bg colours (appropriate for passing to Tkinter in, + e.g., a tag_config call), otherwise fg or bg colour only as specified. """ if self.defaultCfg['highlight'].has_section(theme): - themeDict = self.GetThemeDict('default', theme) + themeDict=self.GetThemeDict('default',theme) else: - themeDict = self.GetThemeDict('user', theme) - fore = themeDict[element + '-foreground'] - if element == 'cursor': # There is no config value for cursor bg - back = themeDict['normal-background'] + themeDict=self.GetThemeDict('user',theme) + fore=themeDict[element+'-foreground'] + if element=='cursor': #there is no config value for cursor bg + back=themeDict['normal-background'] else: - back = themeDict[element + '-background'] - highlight = {"foreground": fore, "background": back} - if not fgBg: # Return dict of both colors + back=themeDict[element+'-background'] + highlight={"foreground": fore,"background": back} + if not fgBg: #return dict of both colours return highlight - else: # Return specified color only + else: #return specified colour only if fgBg == 'fg': return highlight["foreground"] if fgBg == 'bg': @@ -307,26 +307,26 @@ else: raise InvalidFgBg('Invalid fgBg specified') - def GetThemeDict(self, type, themeName): - """Return {option:value} dict for elements in themeName. - + def GetThemeDict(self,type,themeName): + """ type - string, 'default' or 'user' theme type themeName - string, theme name - Values are loaded over ultimate fallback defaults to guarantee - that all theme elements are present in a newly created theme. + Returns a dictionary which holds {option:value} for each element + in the specified theme. Values are loaded over a set of ultimate last + fallback defaults to guarantee that all theme elements are present in + a newly created theme. """ if type == 'user': - cfgParser = self.userCfg['highlight'] + cfgParser=self.userCfg['highlight'] elif type == 'default': - cfgParser = self.defaultCfg['highlight'] + cfgParser=self.defaultCfg['highlight'] else: raise InvalidTheme('Invalid theme type specified') - # Provide foreground and background colors for each theme - # element (other than cursor) even though some values are not - # yet used by idle, to allow for their use in the future. - # Default values are generally black and white. - # TODO copy theme from a class attribute. - theme ={'normal-foreground':'#000000', + #foreground and background values are provded for each theme element + #(apart from cursor) even though all these values are not yet used + #by idle, to allow for their use in the future. Default values are + #generally black and white. + theme={ 'normal-foreground':'#000000', 'normal-background':'#ffffff', 'keyword-foreground':'#000000', 'keyword-background':'#ffffff', @@ -356,74 +356,52 @@ 'console-foreground':'#000000', 'console-background':'#ffffff' } for element in theme: - if not cfgParser.has_option(themeName, element): - # Print warning that will return a default color - warning = ('\n Warning: configHandler.IdleConf.GetThemeDict' + if not cfgParser.has_option(themeName,element): + #we are going to return a default, print warning + warning=('\n Warning: configHandler.py - IdleConf.GetThemeDict' ' -\n problem retrieving theme element %r' '\n from theme %r.\n' - ' returning default color: %r' % + ' returning default value: %r\n' % (element, themeName, theme[element])) try: - print(warning, file=sys.stderr) - except OSError: + sys.stderr.write(warning) + except IOError: pass - theme[element] = cfgParser.Get( - themeName, element, default=theme[element]) + colour=cfgParser.Get(themeName,element,default=theme[element]) + theme[element]=colour return theme def CurrentTheme(self): - """Return the name of the currently active text color theme. - - idlelib.config-main.def includes this section - [Theme] - default= 1 - name= IDLE Classic - name2= - # name2 set in user config-main.cfg for themes added after 2015 Oct 1 - - Item name2 is needed because setting name to a new builtin - causes older IDLEs to display multiple error messages or quit. - See https://bugs.python.org/issue25313. - When default = True, name2 takes precedence over name, - while older IDLEs will just use name. """ - default = self.GetOption('main', 'Theme', 'default', - type='bool', default=True) - if default: - theme = self.GetOption('main', 'Theme', 'name2', default='') - if default and not theme or not default: - theme = self.GetOption('main', 'Theme', 'name', default='') - source = self.defaultCfg if default else self.userCfg - if source['highlight'].has_section(theme): - return theme - else: - return "IDLE Classic" + Returns the name of the currently active theme + """ + return self.GetOption('main','Theme','name',default='') def CurrentKeys(self): - "Return the name of the currently active key set." - return self.GetOption('main', 'Keys', 'name', default='') + """ + Returns the name of the currently active key set + """ + return self.GetOption('main','Keys','name',default='') def GetExtensions(self, active_only=True, editor_only=False, shell_only=False): - """Return extensions in default and user config-extensions files. - - If active_only True, only return active (enabled) extensions - and optionally only editor or shell extensions. - If active_only False, return all extensions. """ - extns = self.RemoveKeyBindNames( - self.GetSectionList('default', 'extensions')) - userExtns = self.RemoveKeyBindNames( - self.GetSectionList('user', 'extensions')) + Gets a list of all idle extensions declared in the config files. + active_only - boolean, if true only return active (enabled) extensions + """ + extns=self.RemoveKeyBindNames( + self.GetSectionList('default','extensions')) + userExtns=self.RemoveKeyBindNames( + self.GetSectionList('user','extensions')) for extn in userExtns: if extn not in extns: #user has added own extension extns.append(extn) if active_only: - activeExtns = [] + activeExtns=[] for extn in extns: if self.GetOption('extensions', extn, 'enable', default=True, type='bool'): #the extension is enabled - if editor_only or shell_only: # TODO if both, contradictory + if editor_only or shell_only: if editor_only: option = "enable_editor" else: @@ -438,110 +416,106 @@ else: return extns - def RemoveKeyBindNames(self, extnNameList): - "Return extnNameList with keybinding section names removed." - # TODO Easier to return filtered copy with list comp - names = extnNameList - kbNameIndicies = [] + def RemoveKeyBindNames(self,extnNameList): + #get rid of keybinding section names + names=extnNameList + kbNameIndicies=[] for name in names: if name.endswith(('_bindings', '_cfgBindings')): kbNameIndicies.append(names.index(name)) - kbNameIndicies.sort(reverse=True) + kbNameIndicies.sort() + kbNameIndicies.reverse() for index in kbNameIndicies: #delete each keybinding section name del(names[index]) return names - def GetExtnNameForEvent(self, virtualEvent): - """Return the name of the extension binding virtualEvent, or None. - - virtualEvent - string, name of the virtual event to test for, - without the enclosing '<< >>' + def GetExtnNameForEvent(self,virtualEvent): """ - extName = None - vEvent = '<<' + virtualEvent + '>>' + Returns the name of the extension that virtualEvent is bound in, or + None if not bound in any extension. + virtualEvent - string, name of the virtual event to test for, without + the enclosing '<< >>' + """ + extName=None + vEvent='<<'+virtualEvent+'>>' for extn in self.GetExtensions(active_only=0): for event in self.GetExtensionKeys(extn): if event == vEvent: - extName = extn # TODO return here? + extName=extn return extName - def GetExtensionKeys(self, extensionName): - """Return dict: {configurable extensionName event : active keybinding}. - - Events come from default config extension_cfgBindings section. - Keybindings come from GetCurrentKeySet() active key dict, - where previously used bindings are disabled. + def GetExtensionKeys(self,extensionName): """ - keysName = extensionName + '_cfgBindings' - activeKeys = self.GetCurrentKeySet() - extKeys = {} + returns a dictionary of the configurable keybindings for a particular + extension,as they exist in the dictionary returned by GetCurrentKeySet; + that is, where previously used bindings are disabled. + """ + keysName=extensionName+'_cfgBindings' + activeKeys=self.GetCurrentKeySet() + extKeys={} if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) + eventNames=self.defaultCfg['extensions'].GetOptionList(keysName) for eventName in eventNames: - event = '<<' + eventName + '>>' - binding = activeKeys[event] - extKeys[event] = binding + event='<<'+eventName+'>>' + binding=activeKeys[event] + extKeys[event]=binding return extKeys def __GetRawExtensionKeys(self,extensionName): - """Return dict {configurable extensionName event : keybinding list}. - - Events come from default config extension_cfgBindings section. - Keybindings list come from the splitting of GetOption, which - tries user config before default config. """ - keysName = extensionName+'_cfgBindings' - extKeys = {} + returns a dictionary of the configurable keybindings for a particular + extension, as defined in the configuration files, or an empty dictionary + if no bindings are found + """ + keysName=extensionName+'_cfgBindings' + extKeys={} if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) + eventNames=self.defaultCfg['extensions'].GetOptionList(keysName) for eventName in eventNames: - binding = self.GetOption( - 'extensions', keysName, eventName, default='').split() - event = '<<' + eventName + '>>' - extKeys[event] = binding + binding=self.GetOption('extensions',keysName, + eventName,default='').split() + event='<<'+eventName+'>>' + extKeys[event]=binding return extKeys - def GetExtensionBindings(self, extensionName): - """Return dict {extensionName event : active or defined keybinding}. - - Augment self.GetExtensionKeys(extensionName) with mapping of non- - configurable events (from default config) to GetOption splits, - as in self.__GetRawExtensionKeys. + def GetExtensionBindings(self,extensionName): """ - bindsName = extensionName + '_bindings' - extBinds = self.GetExtensionKeys(extensionName) + Returns a dictionary of all the event bindings for a particular + extension. The configurable keybindings are returned as they exist in + the dictionary returned by GetCurrentKeySet; that is, where re-used + keybindings are disabled. + """ + bindsName=extensionName+'_bindings' + extBinds=self.GetExtensionKeys(extensionName) #add the non-configurable bindings if self.defaultCfg['extensions'].has_section(bindsName): - eventNames = self.defaultCfg['extensions'].GetOptionList(bindsName) + eventNames=self.defaultCfg['extensions'].GetOptionList(bindsName) for eventName in eventNames: - binding = self.GetOption( - 'extensions', bindsName, eventName, default='').split() - event = '<<' + eventName + '>>' - extBinds[event] = binding + binding=self.GetOption('extensions',bindsName, + eventName,default='').split() + event='<<'+eventName+'>>' + extBinds[event]=binding return extBinds def GetKeyBinding(self, keySetName, eventStr): - """Return the keybinding list for keySetName eventStr. - - keySetName - name of key binding set (config-keys section). - eventStr - virtual event, including brackets, as in '<>'. """ - eventName = eventStr[2:-2] #trim off the angle brackets - binding = self.GetOption('keys', keySetName, eventName, default='').split() + returns the keybinding for a specific event. + keySetName - string, name of key binding set + eventStr - string, the virtual event we want the binding for, + represented as a string, eg. '<>' + """ + eventName=eventStr[2:-2] #trim off the angle brackets + binding=self.GetOption('keys',keySetName,eventName,default='').split() return binding def GetCurrentKeySet(self): - "Return CurrentKeys with 'darwin' modifications." result = self.GetKeySet(self.CurrentKeys()) - if sys.platform == "darwin": - # OS X Tk variants do not support the "Alt" keyboard modifier. - # So replace all keybingings that use "Alt" with ones that - # use the "Option" keyboard modifier. - # TODO (Ned?): the "Option" modifier does not work properly for - # Cocoa Tk and XQuartz Tk so we should not use it - # in default OS X KeySets. + if macosxSupport.runningAsOSXApp(): + # We're using AquaTk, replace all keybingings that use the + # Alt key by ones that use the Option key because the former + # don't work reliably. for k, v in result.items(): v2 = [ x.replace('>' + def IsCoreBinding(self,virtualEvent): + """ + returns true if the virtual event is bound in the core idle keybindings. + virtualEvent - string, name of the virtual event to test for, without + the enclosing '<< >>' """ return ('<<'+virtualEvent+'>>') in self.GetCoreKeys() -# TODO make keyBindins a file or class attribute used for test above -# and copied in function below - def GetCoreKeys(self, keySetName=None): - """Return dict of core virtual-key keybindings for keySetName. - - The default keySetName None corresponds to the keyBindings base - dict. If keySetName is not None, bindings from the config - file(s) are loaded _over_ these defaults, so if there is a - problem getting any core binding there will be an 'ultimate last - resort fallback' to the CUA-ish bindings defined here. + """ + returns the requested set of core keybindings, with fallbacks if + required. + Keybindings loaded from the config file(s) are loaded _over_ these + defaults, so if there is a problem getting any core binding there will + be an 'ultimate last resort fallback' to the CUA-ish bindings + defined here. """ keyBindings={ '<>': ['', ''], @@ -625,7 +596,7 @@ '<>': [''], '<>': [''], '<>': [''], - '<>': ['', ''], + '<>': [' '], '<>': [''], '<>': [''], '<>': [''], @@ -640,23 +611,23 @@ } if keySetName: for event in keyBindings: - binding = self.GetKeyBinding(keySetName, event) + binding=self.GetKeyBinding(keySetName,event) if binding: - keyBindings[event] = binding + keyBindings[event]=binding else: #we are going to return a default, print warning warning=('\n Warning: configHandler.py - IdleConf.GetCoreKeys' ' -\n problem retrieving key binding for event %r' '\n from key set %r.\n' - ' returning default value: %r' % + ' returning default value: %r\n' % (event, keySetName, keyBindings[event])) try: - print(warning, file=sys.stderr) - except OSError: + sys.stderr.write(warning) + except IOError: pass return keyBindings - def GetExtraHelpSourceList(self, configSet): - """Return list of extra help sources from a given configSet. + def GetExtraHelpSourceList(self,configSet): + """Fetch list of extra help sources from a given configSet. Valid configSets are 'user' or 'default'. Return a list of tuples of the form (menu_item , path_to_help_file , option), or return the empty @@ -665,19 +636,19 @@ therefore the returned list must be sorted by 'option'. """ - helpSources = [] - if configSet == 'user': - cfgParser = self.userCfg['main'] - elif configSet == 'default': - cfgParser = self.defaultCfg['main'] + helpSources=[] + if configSet=='user': + cfgParser=self.userCfg['main'] + elif configSet=='default': + cfgParser=self.defaultCfg['main'] else: raise InvalidConfigSet('Invalid configSet specified') options=cfgParser.GetOptionList('HelpFiles') for option in options: - value=cfgParser.Get('HelpFiles', option, default=';') - if value.find(';') == -1: #malformed config entry with no ';' - menuItem = '' #make these empty - helpPath = '' #so value won't be added to list + value=cfgParser.Get('HelpFiles',option,default=';') + if value.find(';')==-1: #malformed config entry with no ';' + menuItem='' #make these empty + helpPath='' #so value won't be added to list else: #config entry contains ';' as expected value=value.split(';') menuItem=value[0].strip() @@ -688,73 +659,47 @@ return helpSources def GetAllExtraHelpSourcesList(self): - """Return a list of the details of all additional help sources. - - Tuples in the list are those of GetExtraHelpSourceList. """ - allHelpSources = (self.GetExtraHelpSourceList('default') + + Returns a list of tuples containing the details of all additional help + sources configured, or an empty list if there are none. Tuples are of + the format returned by GetExtraHelpSourceList. + """ + allHelpSources=( self.GetExtraHelpSourceList('default')+ self.GetExtraHelpSourceList('user') ) return allHelpSources - def GetFont(self, root, configType, section): - """Retrieve a font from configuration (font, font-size, font-bold) - Intercept the special value 'TkFixedFont' and substitute - the actual font, factoring in some tweaks if needed for - appearance sakes. - - The 'root' parameter can normally be any valid Tkinter widget. - - Return a tuple (family, size, weight) suitable for passing - to tkinter.Font + def LoadCfgFiles(self): """ - family = self.GetOption(configType, section, 'font', default='courier') - size = self.GetOption(configType, section, 'font-size', type='int', - default='10') - bold = self.GetOption(configType, section, 'font-bold', default=0, - type='bool') - if (family == 'TkFixedFont'): - if TkVersion < 8.5: - family = 'Courier' - else: - f = Font(name='TkFixedFont', exists=True, root=root) - actualFont = Font.actual(f) - family = actualFont['family'] - size = actualFont['size'] - if size < 0: - size = 10 # if font in pixels, ignore actual size - bold = actualFont['weight']=='bold' - return (family, size, 'bold' if bold else 'normal') - - def LoadCfgFiles(self): - "Load all configuration files." + load all configuration files. + """ for key in self.defaultCfg: self.defaultCfg[key].Load() self.userCfg[key].Load() #same keys def SaveUserCfgFiles(self): - "Write all loaded user configuration files to disk." + """ + write all loaded user configuration files back to disk + """ for key in self.userCfg: self.userCfg[key].Save() +idleConf=IdleConf() -idleConf = IdleConf() - -# TODO Revise test output, write expanded unittest ### module test if __name__ == '__main__': def dumpCfg(cfg): - print('\n', cfg, '\n') + print('\n',cfg,'\n') for key in cfg: - sections = cfg[key].sections() + sections=cfg[key].sections() print(key) print(sections) for section in sections: - options = cfg[key].options(section) + options=cfg[key].options(section) print(section) print(options) for option in options: - print(option, '=', cfg[key].Get(section, option)) + print(option, '=', cfg[key].Get(section,option)) dumpCfg(idleConf.defaultCfg) dumpCfg(idleConf.userCfg) - print(idleConf.userCfg['main'].Get('Theme', 'name')) + print(idleConf.userCfg['main'].Get('Theme','name')) #print idleConf.userCfg['highlight'].GetDefHighlight('Foo','normal') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/configHelpSourceEdit.py --- a/Lib/idlelib/configHelpSourceEdit.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/configHelpSourceEdit.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,14 +8,13 @@ import tkinter.filedialog as tkFileDialog class GetHelpSourceDialog(Toplevel): - def __init__(self, parent, title, menuItem='', filePath='', _htest=False): + def __init__(self, parent, title, menuItem='', filePath=''): """Get menu entry and url/ local file location for Additional Help User selects a name for the Help resource and provides a web url or a local file as its source. The user can enter a url or browse for the file. - _htest - bool, change box location when running htest """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -32,14 +31,12 @@ self.withdraw() #hide while setting geometry #needs to be done here so that the winfo_reqwidth is valid self.update_idletasks() - #centre dialog over parent. below parent if running htest. - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 150))) + #centre dialog over parent: + self.geometry("+%d+%d" % + ((parent.winfo_rootx() + ((parent.winfo_width()/2) + -(self.winfo_reqwidth()/2)), + parent.winfo_rooty() + ((parent.winfo_height()/2) + -(self.winfo_reqheight()/2))))) self.deiconify() #geometry set, unhide self.bind('', self.Ok) self.wait_window() @@ -162,5 +159,11 @@ self.destroy() if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(GetHelpSourceDialog) + #test the dialog + root = Tk() + def run(): + keySeq = '' + dlg = GetHelpSourceDialog(root, 'Get Help Source') + print(dlg.result) + Button(root,text='Dialog', command=run).pack() + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/configSectionNameDialog.py --- a/Lib/idlelib/configSectionNameDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/configSectionNameDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,98 +1,97 @@ """ Dialog that allows user to specify a new config file section name. Used to get new highlight theme and keybinding set names. -The 'return value' for the dialog, used two placed in configDialog.py, -is the .result attribute set in the Ok and Cancel methods. """ from tkinter import * import tkinter.messagebox as tkMessageBox class GetCfgSectionNameDialog(Toplevel): - def __init__(self, parent, title, message, used_names, _htest=False): + def __init__(self,parent,title,message,usedNames): """ message - string, informational message to display - used_names - string collection, names already in use for validity check - _htest - bool, change box location when running htest + usedNames - list, list of names already in use for validity check """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) - self.resizable(height=FALSE, width=FALSE) + self.resizable(height=FALSE,width=FALSE) self.title(title) self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.Cancel) self.parent = parent - self.message = message - self.used_names = used_names - self.create_widgets() - self.withdraw() #hide while setting geometry + self.message=message + self.usedNames=usedNames + self.result='' + self.CreateWidgets() + self.withdraw() #hide while setting geometry self.update_idletasks() #needs to be done here so that the winfo_reqwidth is valid self.messageInfo.config(width=self.frameMain.winfo_reqwidth()) - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 100) - ) ) #centre dialog over parent (or below htest box) - self.deiconify() #geometry set, unhide + self.geometry("+%d+%d" % + ((parent.winfo_rootx()+((parent.winfo_width()/2) + -(self.winfo_reqwidth()/2)), + parent.winfo_rooty()+((parent.winfo_height()/2) + -(self.winfo_reqheight()/2)) )) ) #centre dialog over parent + self.deiconify() #geometry set, unhide self.wait_window() - def create_widgets(self): - self.name = StringVar(self.parent) - self.fontSize = StringVar(self.parent) - self.frameMain = Frame(self, borderwidth=2, relief=SUNKEN) - self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) - self.messageInfo = Message(self.frameMain, anchor=W, justify=LEFT, - padx=5, pady=5, text=self.message) #,aspect=200) - entryName = Entry(self.frameMain, textvariable=self.name, width=30) + def CreateWidgets(self): + self.name=StringVar(self) + self.fontSize=StringVar(self) + self.frameMain = Frame(self,borderwidth=2,relief=SUNKEN) + self.frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) + self.messageInfo=Message(self.frameMain,anchor=W,justify=LEFT,padx=5,pady=5, + text=self.message)#,aspect=200) + entryName=Entry(self.frameMain,textvariable=self.name,width=30) entryName.focus_set() - self.messageInfo.pack(padx=5, pady=5) #, expand=TRUE, fill=BOTH) - entryName.pack(padx=5, pady=5) + self.messageInfo.pack(padx=5,pady=5)#,expand=TRUE,fill=BOTH) + entryName.pack(padx=5,pady=5) + frameButtons=Frame(self) + frameButtons.pack(side=BOTTOM,fill=X) + self.buttonOk = Button(frameButtons,text='Ok', + width=8,command=self.Ok) + self.buttonOk.grid(row=0,column=0,padx=5,pady=5) + self.buttonCancel = Button(frameButtons,text='Cancel', + width=8,command=self.Cancel) + self.buttonCancel.grid(row=0,column=1,padx=5,pady=5) - frameButtons = Frame(self, pady=2) - frameButtons.pack(side=BOTTOM) - self.buttonOk = Button(frameButtons, text='Ok', - width=8, command=self.Ok) - self.buttonOk.pack(side=LEFT, padx=5) - self.buttonCancel = Button(frameButtons, text='Cancel', - width=8, command=self.Cancel) - self.buttonCancel.pack(side=RIGHT, padx=5) - - def name_ok(self): - ''' After stripping entered name, check that it is a sensible - ConfigParser file section name. Return it if it is, '' if not. - ''' - name = self.name.get().strip() + def NameOk(self): + #simple validity check for a sensible + #ConfigParser file section name + nameOk=1 + name=self.name.get() + name.strip() if not name: #no name specified tkMessageBox.showerror(title='Name Error', message='No name specified.', parent=self) + nameOk=0 elif len(name)>30: #name too long tkMessageBox.showerror(title='Name Error', message='Name too long. It should be no more than '+ '30 characters.', parent=self) - name = '' - elif name in self.used_names: + nameOk=0 + elif name in self.usedNames: tkMessageBox.showerror(title='Name Error', message='This name is already in use.', parent=self) - name = '' - return name + nameOk=0 + return nameOk def Ok(self, event=None): - name = self.name_ok() - if name: - self.result = name + if self.NameOk(): + self.result=self.name.get().strip() self.destroy() def Cancel(self, event=None): - self.result = '' + self.result='' self.destroy() if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_config_name', verbosity=2, exit=False) - - from idlelib.idle_test.htest import run - run(GetCfgSectionNameDialog) + #test the dialog + root=Tk() + def run(): + keySeq='' + dlg=GetCfgSectionNameDialog(root,'Get Name', + 'The information here should need to be word wrapped. Test.') + print(dlg.result) + Button(root,text='Dialog',command=run).pack() + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/dynOptionMenuWidget.py --- a/Lib/idlelib/dynOptionMenuWidget.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/dynOptionMenuWidget.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,15 +2,16 @@ OptionMenu widget modified to allow dynamic menu reconfiguration and setting of highlightthickness """ +from tkinter import OptionMenu +from tkinter import _setit import copy -from tkinter import OptionMenu, _setit, StringVar, Button class DynOptionMenu(OptionMenu): """ unlike OptionMenu, our kwargs can include highlightthickness """ def __init__(self, master, variable, value, *values, **kwargs): - # TODO copy value instead of whole dict + #get a copy of kwargs before OptionMenu.__init__ munges them kwargsCopy=copy.copy(kwargs) if 'highlightthickness' in list(kwargs.keys()): del(kwargs['highlightthickness']) @@ -32,26 +33,3 @@ command=_setit(self.variable,item,self.command)) if value: self.variable.set(value) - -def _dyn_option_menu(parent): # htest # - from tkinter import Toplevel - - top = Toplevel() - top.title("Tets dynamic option menu") - top.geometry("200x100+%d+%d" % (parent.winfo_rootx() + 200, - parent.winfo_rooty() + 150)) - top.focus_set() - - var = StringVar(top) - var.set("Old option set") #Set the default value - dyn = DynOptionMenu(top,var, "old1","old2","old3","old4") - dyn.pack() - - def update(): - dyn.SetMenu(["new1","new2","new3","new4"], value="new option set") - button = Button(top, text="Change option set", command=update) - button.pack() - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_dyn_option_menu) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/extend.txt --- a/Lib/idlelib/extend.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/extend.txt Mon Jan 25 17:05:13 2016 +0100 @@ -54,7 +54,7 @@ implement. (They are also not required to create keybindings, but in that case there must be empty bindings in cofig-extensions.def) -Here is a complete example: +Here is a complete example example: class ZoomHeight: @@ -72,7 +72,7 @@ "...Do what you want here..." The final piece of the puzzle is the file "config-extensions.def", which is -used to configure the loading of extensions and to establish key (or, more +used to to configure the loading of extensions and to establish key (or, more generally, event) bindings to the virtual events defined in the extensions. See the comments at the top of config-extensions.def for information. It's diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/help.html --- a/Lib/idlelib/help.html Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,709 +0,0 @@ - - - - - - - - 25.5. IDLE — Python 3.4.3 documentation - - - - - - - - - - - - - - - - - - - - - - - - -
    -

    Navigation

    - -
    - -
    -
    -
    -
    - -
    -

    25.5. IDLE

    -

    IDLE is Python’s Integrated Development and Learning Environment.

    -

    IDLE has the following features:

    -
      -
    • coded in 100% pure Python, using the tkinter GUI toolkit
      • -
    • cross-platform: works mostly the same on Windows, Unix, and Mac OS X
      • -
    • Python shell window (interactive interpreter) with colorizing -of code input, output, and error messages
      • -
    • multi-window text editor with multiple undo, Python colorizing, -smart indent, call tips, auto completion, and other features
      • -
    • search within any window, replace within editor windows, and search -through multiple files (grep)
      • -
    • debugger with persistent breakpoints, stepping, and viewing -of global and local namespaces
      • -
    • configuration, browsers, and other dialogs
      • -
    - -
    -

    25.5.2. Editing and navigation

    -

    In this section, ‘C’ refers to the Control key on Windows and Unix and -the Command key on Mac OSX.

    -
      -

    • Backspace deletes to the left; Del deletes to the right

      -
      • -

    • C-Backspace delete word left; C-Del delete word to the right

      -
      • -

    • Arrow keys and Page Up/Page Down to move around

      -
      • -

    • C-LeftArrow and C-RightArrow moves by words

      -
      • -

    • Home/End go to begin/end of line

      -
      • -

    • C-Home/C-End go to begin/end of file

      -
      • -

    • Some useful Emacs bindings are inherited from Tcl/Tk:

      -
      -
        -
      • C-a beginning of line
        • -
      • C-e end of line
        • -
      • C-k kill line (but doesn’t put it in clipboard)
        • -
      • C-l center window around the insertion point
        • -
      • C-b go backwards one character without deleting (usually you can -also use the cursor key for this)
        • -
      • C-f go forward one character without deleting (usually you can -also use the cursor key for this)
        • -
      • C-p go up one line (usually you can also use the cursor key for -this)
        • -
      • C-d delete next character
        • -
      -
      -
      • -
    -

    Standard keybindings (like C-c to copy and C-v to paste) -may work. Keybindings are selected in the Configure IDLE dialog.

    -
    -

    25.5.2.1. Automatic indentation

    -

    After a block-opening statement, the next line is indented by 4 spaces (in the -Python Shell window by one tab). After certain keywords (break, return etc.) -the next line is dedented. In leading indentation, Backspace deletes up -to 4 spaces if they are there. Tab inserts spaces (in the Python -Shell window one tab), number depends on Indent width. Currently tabs -are restricted to four spaces due to Tcl/Tk limitations.

    -

    See also the indent/dedent region commands in the edit menu.

    -
    -
    -

    25.5.2.2. Completions

    -

    Completions are supplied for functions, classes, and attributes of classes, -both built-in and user-defined. Completions are also provided for -filenames.

    -

    The AutoCompleteWindow (ACW) will open after a predefined delay (default is -two seconds) after a ‘.’ or (in a string) an os.sep is typed. If after one -of those characters (plus zero or more other characters) a tab is typed -the ACW will open immediately if a possible continuation is found.

    -

    If there is only one possible completion for the characters entered, a -Tab will supply that completion without opening the ACW.

    -

    ‘Show Completions’ will force open a completions window, by default the -C-space will open a completions window. In an empty -string, this will contain the files in the current directory. On a -blank line, it will contain the built-in and user-defined functions and -classes in the current name spaces, plus any modules imported. If some -characters have been entered, the ACW will attempt to be more specific.

    -

    If a string of characters is typed, the ACW selection will jump to the -entry most closely matching those characters. Entering a tab will -cause the longest non-ambiguous match to be entered in the Editor window or -Shell. Two tab in a row will supply the current ACW selection, as -will return or a double click. Cursor keys, Page Up/Down, mouse selection, -and the scroll wheel all operate on the ACW.

    -

    “Hidden” attributes can be accessed by typing the beginning of hidden -name after a ‘.’, e.g. ‘_’. This allows access to modules with -__all__ set, or to class-private attributes.

    -

    Completions and the ‘Expand Word’ facility can save a lot of typing!

    -

    Completions are currently limited to those in the namespaces. Names in -an Editor window which are not via __main__ and sys.modules will -not be found. Run the module once with your imports to correct this situation. -Note that IDLE itself places quite a few modules in sys.modules, so -much can be found by default, e.g. the re module.

    -

    If you don’t like the ACW popping up unbidden, simply make the delay -longer or disable the extension.

    -
    -
    -

    25.5.2.3. Calltips

    -

    A calltip is shown when one types ( after the name of an acccessible -function. A name expression may include dots and subscripts. A calltip -remains until it is clicked, the cursor is moved out of the argument area, -or ) is typed. When the cursor is in the argument part of a definition, -the menu or shortcut display a calltip.

    -

    A calltip consists of the function signature and the first line of the -docstring. For builtins without an accessible signature, the calltip -consists of all lines up the fifth line or the first blank line. These -details may change.

    -

    The set of accessible functions depends on what modules have been imported -into the user process, including those imported by Idle itself, -and what definitions have been run, all since the last restart.

    -

    For example, restart the Shell and enter itertools.count(. A calltip -appears because Idle imports itertools into the user process for its own use. -(This could change.) Enter turtle.write( and nothing appears. Idle does -not import turtle. The menu or shortcut do nothing either. Enter -import turtle and then turtle.write( will work.

    -

    In an editor, import statements have no effect until one runs the file. One -might want to run a file after writing the import statements at the top, -or immediately run an existing file before editing.

    -
    -
    -

    25.5.2.4. Python Shell window

    -
      -

    • C-c interrupts executing command

      -
      • -

    • C-d sends end-of-file; closes window if typed at a >>> prompt

      -
      • -

    • Alt-/ (Expand word) is also useful to reduce typing

      -

      Command history

      -
        -
      • Alt-p retrieves previous command matching what you have typed. On -OS X use C-p.
        • -
      • Alt-n retrieves next. On OS X use C-n.
        • -
      • Return while on any previous command retrieves that command
        • -
      -
      • -
    -
    -
    -

    25.5.2.5. Text colors

    -

    Idle defaults to black on white text, but colors text with special meanings. -For the shell, these are shell output, shell error, user output, and -user error. For Python code, at the shell prompt or in an editor, these are -keywords, builtin class and function names, names following class and -def, strings, and comments. For any text window, these are the cursor (when -present), found text (when possible), and selected text.

    -

    Text coloring is done in the background, so uncolorized text is occasionally -visible. To change the color scheme, use the Configure IDLE dialog -Highlighting tab. The marking of debugger breakpoint lines in the editor and -text in popups and dialogs is not user-configurable.

    -
    -
    -
    -

    25.5.3. Startup and code execution

    -

    Upon startup with the -s option, IDLE will execute the file referenced by -the environment variables IDLESTARTUP or PYTHONSTARTUP. -IDLE first checks for IDLESTARTUP; if IDLESTARTUP is present the file -referenced is run. If IDLESTARTUP is not present, IDLE checks for -PYTHONSTARTUP. Files referenced by these environment variables are -convenient places to store functions that are used frequently from the IDLE -shell, or for executing import statements to import common modules.

    -

    In addition, Tk also loads a startup file if it is present. Note that the -Tk file is loaded unconditionally. This additional file is .Idle.py and is -looked for in the user’s home directory. Statements in this file will be -executed in the Tk namespace, so this file is not useful for importing -functions to be used from IDLE’s Python shell.

    -
    -

    25.5.3.1. Command line usage

    -
    idle.py [-c command] [-d] [-e] [-h] [-i] [-r file] [-s] [-t title] [-] [arg] ...
-
--c command  run command in the shell window
--d          enable debugger and open shell window
--e          open editor window
--h          print help message with legal combinatios and exit
--i          open shell window
--r file     run file in shell window
--s          run $IDLESTARTUP or $PYTHONSTARTUP first, in shell window
--t title    set title of shell window
--           run stdin in shell (- must be last option before args)
-
    -
    -

    If there are arguments:

    -
      -
    • If -, -c, or r is used, all arguments are placed in -sys.argv[1:...] and sys.argv[0] is set to '', '-c', -or '-r'. No editor window is opened, even if that is the default -set in the Options dialog.
      • -
    • Otherwise, arguments are files opened for editing and -sys.argv reflects the arguments passed to IDLE itself.
      • -
    -
    -
    -

    25.5.3.2. IDLE-console differences

    -

    As much as possible, the result of executing Python code with IDLE is the -same as executing the same code in a console window. However, the different -interface and operation occasionally affects results.

    -

    For instance, IDLE normally executes user code in a separate process from -the IDLE GUI itself. The IDLE versions of sys.stdin, .stdout, and .stderr in the -execution process get input from and send output to the GUI process, -which keeps control of the keyboard and screen. This is normally transparent, -but code that access these object will see different attribute values. -Also, functions that directly access the keyboard and screen will not work.

    -

    With IDLE’s Shell, one enters, edits, and recalls complete statements. -Some consoles only work with a single physical line at a time.

    -
    -
    -

    25.5.3.3. Running without a subprocess

    -

    By default, IDLE executes user code in a separate subprocess via a socket, -which uses the internal loopback interface. This connection is not -externally visible and no data is sent to or received from the Internet. -If firewall software complains anyway, you can ignore it.

    -

    If the attempt to make the socket connection fails, Idle will notify you. -Such failures are sometimes transient, but if persistent, the problem -may be either a firewall blocking the connecton or misconfiguration of -a particular system. Until the problem is fixed, one can run Idle with -the -n command line switch.

    -

    If IDLE is started with the -n command line switch it will run in a -single process and will not create the subprocess which runs the RPC -Python execution server. This can be useful if Python cannot create -the subprocess or the RPC socket interface on your platform. However, -in this mode user code is not isolated from IDLE itself. Also, the -environment is not restarted when Run/Run Module (F5) is selected. If -your code has been modified, you must reload() the affected modules and -re-import any specific items (e.g. from foo import baz) if the changes -are to take effect. For these reasons, it is preferable to run IDLE -with the default subprocess if at all possible.

    -
    -

    Deprecated since version 3.4.

    -
    -
    -
    -
    -

    25.5.4. Help and preferences

    -
    -

    25.5.4.1. Additional help sources

    -

    IDLE includes a help menu entry called “Python Docs” that will open the -extensive sources of help, including tutorials, available at docs.python.org. -Selected URLs can be added or removed from the help menu at any time using the -Configure IDLE dialog. See the IDLE help option in the help menu of IDLE for -more information.

    -
    -
    -

    25.5.4.2. Setting preferences

    -

    The font preferences, highlighting, keys, and general preferences can be -changed via Configure IDLE on the Option menu. Keys can be user defined; -IDLE ships with four built in key sets. In addition a user can create a -custom key set in the Configure IDLE dialog under the keys tab.

    -
    -
    -

    25.5.4.3. Extensions

    -

    IDLE contains an extension facility. Peferences for extensions can be -changed with Configure Extensions. See the beginning of config-extensions.def -in the idlelib directory for further information. The default extensions -are currently:

    -
      -
    • FormatParagraph
      • -
    • AutoExpand
      • -
    • ZoomHeight
      • -
    • ScriptBinding
      • -
    • CallTips
      • -
    • ParenMatch
      • -
    • AutoComplete
      • -
    • CodeContext
      • -
    • RstripExtension
      • -
    -
    -
    -
    - - -
    -
    -
    -
    -
    -

    Table Of Contents

    - - -

    Previous topic

    -

    25.4. tkinter.scrolledtext — Scrolled Text Widget

    -

    Next topic

    -

    25.6. Other Graphical User Interface Packages

    -

    This Page

    - - - - -
    -
    -
    -
    -
    -

    Navigation

    - -
    -
    - © Copyright 1990-2015, Python Software Foundation. -
    - The Python Software Foundation is a non-profit corporation. - Please donate. -
    - Last updated on Oct 13, 2015. - Found a bug? -
    - Created using Sphinx 1.2.3. -
    - - - diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/help.py --- a/Lib/idlelib/help.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,249 +0,0 @@ -""" help.py: Implement the Idle help menu. -Contents are subject to revision at any time, without notice. - - -Help => About IDLE: diplay About Idle dialog - - - - -Help => IDLE Help: Display help.html with proper formatting. -Doc/library/idle.rst (Sphinx)=> Doc/build/html/library/idle.html -(help.copy_strip)=> Lib/idlelib/help.html - -HelpParser - Parse help.html and and render to tk Text. - -HelpText - Display formatted help.html. - -HelpFrame - Contain text, scrollbar, and table-of-contents. -(This will be needed for display in a future tabbed window.) - -HelpWindow - Display HelpFrame in a standalone window. - -copy_strip - Copy idle.html to help.html, rstripping each line. - -show_idlehelp - Create HelpWindow. Called in EditorWindow.help_dialog. -""" -from html.parser import HTMLParser -from os.path import abspath, dirname, isdir, isfile, join -from tkinter import Tk, Toplevel, Frame, Text, Scrollbar, Menu, Menubutton -from tkinter import font as tkfont -from idlelib.configHandler import idleConf - -use_ttk = False # until available to import -if use_ttk: - from tkinter.ttk import Menubutton - -## About IDLE ## - - -## IDLE Help ## - -class HelpParser(HTMLParser): - """Render help.html into a text widget. - - The overridden handle_xyz methods handle a subset of html tags. - The supplied text should have the needed tag configurations. - The behavior for unsupported tags, such as table, is undefined. - """ - def __init__(self, text): - HTMLParser.__init__(self, convert_charrefs=True) - self.text = text # text widget we're rendering into - self.tags = '' # current block level text tags to apply - self.chartags = '' # current character level text tags - self.show = False # used so we exclude page navigation - self.hdrlink = False # used so we don't show header links - self.level = 0 # indentation level - self.pre = False # displaying preformatted text - self.hprefix = '' # prefix such as '25.5' to strip from headings - self.nested_dl = False # if we're in a nested
    - self.simplelist = False # simple list (no double spacing) - self.toc = [] # pair headers with text indexes for toc - self.header = '' # text within header tags for toc - - def indent(self, amt=1): - self.level += amt - self.tags = '' if self.level == 0 else 'l'+str(self.level) - - def handle_starttag(self, tag, attrs): - "Handle starttags in help.html." - class_ = '' - for a, v in attrs: - if a == 'class': - class_ = v - s = '' - if tag == 'div' and class_ == 'section': - self.show = True # start of main content - elif tag == 'div' and class_ == 'sphinxsidebar': - self.show = False # end of main content - elif tag == 'p' and class_ != 'first': - s = '\n\n' - elif tag == 'span' and class_ == 'pre': - self.chartags = 'pre' - elif tag == 'span' and class_ == 'versionmodified': - self.chartags = 'em' - elif tag == 'em': - self.chartags = 'em' - elif tag in ['ul', 'ol']: - if class_.find('simple') != -1: - s = '\n' - self.simplelist = True - else: - self.simplelist = False - self.indent() - elif tag == 'dl': - if self.level > 0: - self.nested_dl = True - elif tag == 'li': - s = '\n* ' if self.simplelist else '\n\n* ' - elif tag == 'dt': - s = '\n\n' if not self.nested_dl else '\n' # avoid extra line - self.nested_dl = False - elif tag == 'dd': - self.indent() - s = '\n' - elif tag == 'pre': - self.pre = True - if self.show: - self.text.insert('end', '\n\n') - self.tags = 'preblock' - elif tag == 'a' and class_ == 'headerlink': - self.hdrlink = True - elif tag == 'h1': - self.tags = tag - elif tag in ['h2', 'h3']: - if self.show: - self.header = '' - self.text.insert('end', '\n\n') - self.tags = tag - if self.show: - self.text.insert('end', s, (self.tags, self.chartags)) - - def handle_endtag(self, tag): - "Handle endtags in help.html." - if tag in ['h1', 'h2', 'h3']: - self.indent(0) # clear tag, reset indent - if self.show: - self.toc.append((self.header, self.text.index('insert'))) - elif tag in ['span', 'em']: - self.chartags = '' - elif tag == 'a': - self.hdrlink = False - elif tag == 'pre': - self.pre = False - self.tags = '' - elif tag in ['ul', 'dd', 'ol']: - self.indent(amt=-1) - - def handle_data(self, data): - "Handle date segments in help.html." - if self.show and not self.hdrlink: - d = data if self.pre else data.replace('\n', ' ') - if self.tags == 'h1': - self.hprefix = d[0:d.index(' ')] - if self.tags in ['h1', 'h2', 'h3'] and self.hprefix != '': - if d[0:len(self.hprefix)] == self.hprefix: - d = d[len(self.hprefix):].strip() - self.header += d - self.text.insert('end', d, (self.tags, self.chartags)) - - -class HelpText(Text): - "Display help.html." - def __init__(self, parent, filename): - "Configure tags and feed file to parser." - uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int') - uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int') - uhigh = 3 * uhigh // 4 # lines average 4/3 of editor line height - Text.__init__(self, parent, wrap='word', highlightthickness=0, - padx=5, borderwidth=0, width=uwide, height=uhigh) - - normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) - fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) - self['font'] = (normalfont, 12) - self.tag_configure('em', font=(normalfont, 12, 'italic')) - self.tag_configure('h1', font=(normalfont, 20, 'bold')) - self.tag_configure('h2', font=(normalfont, 18, 'bold')) - self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') - self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') - self.tag_configure('l1', lmargin1=25, lmargin2=25) - self.tag_configure('l2', lmargin1=50, lmargin2=50) - self.tag_configure('l3', lmargin1=75, lmargin2=75) - self.tag_configure('l4', lmargin1=100, lmargin2=100) - - self.parser = HelpParser(self) - with open(filename, encoding='utf-8') as f: - contents = f.read() - self.parser.feed(contents) - self['state'] = 'disabled' - - def findfont(self, names): - "Return name of first font family derived from names." - for name in names: - if name.lower() in (x.lower() for x in tkfont.names(root=self)): - font = tkfont.Font(name=name, exists=True, root=self) - return font.actual()['family'] - elif name.lower() in (x.lower() - for x in tkfont.families(root=self)): - return name - - -class HelpFrame(Frame): - "Display html text, scrollbar, and toc." - def __init__(self, parent, filename): - Frame.__init__(self, parent) - text = HelpText(self, filename) - self['background'] = text['background'] - scroll = Scrollbar(self, command=text.yview) - text['yscrollcommand'] = scroll.set - self.rowconfigure(0, weight=1) - self.columnconfigure(1, weight=1) # text - self.toc_menu(text).grid(column=0, row=0, sticky='nw') - text.grid(column=1, row=0, sticky='nsew') - scroll.grid(column=2, row=0, sticky='ns') - - def toc_menu(self, text): - "Create table of contents as drop-down menu." - toc = Menubutton(self, text='TOC') - drop = Menu(toc, tearoff=False) - for lbl, dex in text.parser.toc: - drop.add_command(label=lbl, command=lambda dex=dex:text.yview(dex)) - toc['menu'] = drop - return toc - - -class HelpWindow(Toplevel): - "Display frame with rendered html." - def __init__(self, parent, filename, title): - Toplevel.__init__(self, parent) - self.wm_title(title) - self.protocol("WM_DELETE_WINDOW", self.destroy) - HelpFrame(self, filename).grid(column=0, row=0, sticky='nsew') - self.grid_columnconfigure(0, weight=1) - self.grid_rowconfigure(0, weight=1) - - -def copy_strip(): - "Copy idle.html to idlelib/help.html, stripping trailing whitespace." - src = join(abspath(dirname(dirname(dirname(__file__)))), - 'Doc', 'build', 'html', 'library', 'idle.html') - dst = join(abspath(dirname(__file__)), 'help.html') - with open(src, 'rb') as inn,\ - open(dst, 'wb') as out: - for line in inn: - out.write(line.rstrip() + b'\n') - print('idle.html copied to help.html') - -def show_idlehelp(parent): - "Create HelpWindow; called from Idle Help event handler." - filename = join(abspath(dirname(__file__)), 'help.html') - if not isfile(filename): - # try copy_strip, present message - return - HelpWindow(parent, filename, 'IDLE Help') - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(show_idlehelp) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/help.txt --- a/Lib/idlelib/help.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/help.txt Mon Jan 25 17:05:13 2016 +0100 @@ -1,189 +1,124 @@ -This file, idlelib/help.txt is out-of-date and no longer used by Idle. -It is deprecated and will be removed in the future, possibly in 3.6 ----------------------------------------------------------------------- - [See the end of this file for ** TIPS ** on using IDLE !!] -IDLE is the Python IDE built with the tkinter GUI toolkit. +Click on the dotted line at the top of a menu to "tear it off": a +separate window containing the menu is created. -IDLE has the following features: --coded in 100% pure Python, using the tkinter GUI toolkit --cross-platform: works on Windows, Unix, and OS X --multi-window text editor with multiple undo, Python colorizing, smart indent, -call tips, and many other features --Python shell window (a.k.a interactive interpreter) --debugger (not complete, but you can set breakpoints, view and step) +File Menu: -Menus: + New Window -- Create a new editing window + Open... -- Open an existing file + Recent Files... -- Open a list of recent files + Open Module... -- Open an existing module (searches sys.path) + Class Browser -- Show classes and methods in current file + Path Browser -- Show sys.path directories, modules, classes + and methods + --- + Save -- Save current window to the associated file (unsaved + windows have a * before and after the window title) -IDLE has two window types the Shell window and the Editor window. It is -possible to have multiple editor windows simultaneously. IDLE's -menus dynamically change based on which window is currently selected. Each menu -documented below indicates which window type it is associated with. + Save As... -- Save current window to new file, which becomes + the associated file + Save Copy As... -- Save current window to different file + without changing the associated file + --- + Print Window -- Print the current window + --- + Close -- Close current window (asks to save if unsaved) + Exit -- Close all windows, quit (asks to save if unsaved) -File Menu (Shell and Editor): +Edit Menu: - New File -- Create a new file editing window - Open... -- Open an existing file - Open Module... -- Open an existing module (searches sys.path) - Recent Files... -- Open a list of recent files - Class Browser -- Show classes and methods in current file - Path Browser -- Show sys.path directories, modules, classes, - and methods - --- - Save -- Save current window to the associated file (unsaved - windows have a * before and after the window title) + Undo -- Undo last change to current window + (A maximum of 1000 changes may be undone) + Redo -- Redo last undone change to current window + --- + Cut -- Copy a selection into system-wide clipboard, + then delete the selection + Copy -- Copy selection into system-wide clipboard + Paste -- Insert system-wide clipboard into window + Select All -- Select the entire contents of the edit buffer + --- + Find... -- Open a search dialog box with many options + Find Again -- Repeat last search + Find Selection -- Search for the string in the selection + Find in Files... -- Open a search dialog box for searching files + Replace... -- Open a search-and-replace dialog box + Go to Line -- Ask for a line number and show that line + Show Calltip -- Open a small window with function param hints + Show Completions -- Open a scroll window allowing selection keywords + and attributes. (see '*TIPS*', below) + Show Parens -- Highlight the surrounding parenthesis + Expand Word -- Expand the word you have typed to match another + word in the same buffer; repeat to get a + different expansion - Save As... -- Save current window to new file, which becomes - the associated file - Save Copy As... -- Save current window to different file - without changing the associated file - --- - Print Window -- Print the current window - --- - Close -- Close current window (asks to save if unsaved) - Exit -- Close all windows, quit (asks to save if unsaved) +Format Menu (only in Edit window): -Edit Menu (Shell and Editor): + Indent Region -- Shift selected lines right 4 spaces + Dedent Region -- Shift selected lines left 4 spaces + Comment Out Region -- Insert ## in front of selected lines + Uncomment Region -- Remove leading # or ## from selected lines + Tabify Region -- Turns *leading* stretches of spaces into tabs + (Note: We recommend using 4 space blocks to indent Python code.) + Untabify Region -- Turn *all* tabs into the right number of spaces + New Indent Width... -- Open dialog to change indent width + Format Paragraph -- Reformat the current blank-line-separated + paragraph - Undo -- Undo last change to current window - (a maximum of 1000 changes may be undone) - Redo -- Redo last undone change to current window - --- - Cut -- Copy a selection into system-wide clipboard, - then delete the selection - Copy -- Copy selection into system-wide clipboard - Paste -- Insert system-wide clipboard into window - Select All -- Select the entire contents of the edit buffer - --- - Find... -- Open a search dialog box with many options - Find Again -- Repeat last search - Find Selection -- Search for the string in the selection - Find in Files... -- Open a search dialog box for searching files - Replace... -- Open a search-and-replace dialog box - Go to Line -- Ask for a line number and show that line - Expand Word -- Expand the word you have typed to match another - word in the same buffer; repeat to get a - different expansion - Show Calltip -- After an unclosed parenthesis for a function, open - a small window with function parameter hints - Show Parens -- Highlight the surrounding parenthesis - Show Completions -- Open a scroll window allowing selection keywords - and attributes. (see '*TIPS*', below) +Run Menu (only in Edit window): -Format Menu (Editor window only): + Python Shell -- Open or wake up the Python shell window + --- + Check Module -- Run a syntax check on the module + Run Module -- Execute the current file in the __main__ namespace - Indent Region -- Shift selected lines right by the indent width - (default 4 spaces) - Dedent Region -- Shift selected lines left by the indent width - (default 4 spaces) - Comment Out Region -- Insert ## in front of selected lines - Uncomment Region -- Remove leading # or ## from selected lines - Tabify Region -- Turns *leading* stretches of spaces into tabs. - (Note: We recommend using 4 space blocks to indent Python code.) - Untabify Region -- Turn *all* tabs into the corrent number of spaces - Toggle tabs -- Open a dialog to switch between indenting with - spaces and tabs. - New Indent Width... -- Open a dialog to change indent width. The - accepted default by the Python community is 4 - spaces. - Format Paragraph -- Reformat the current blank-line-separated - paragraph. All lines in the paragraph will be - formatted to less than 80 columns. - --- - Strip trailing whitespace -- Removed any space characters after the end - of the last non-space character +Shell Menu (only in Shell window): -Run Menu (Editor window only): + View Last Restart -- Scroll the shell window to the last restart + Restart Shell -- Restart the interpreter with a fresh environment - Python Shell -- Open or wake up the Python shell window - --- - Check Module -- Check the syntax of the module currently open in the - Editor window. If the module has not been saved IDLE - will prompt the user to save the code. - Run Module -- Restart the shell to clean the environment, then - execute the currently open module. If the module has - not been saved IDLE will prompt the user to save the - code. +Debug Menu (only in Shell window): -Shell Menu (Shell window only): + Go to File/Line -- look around the insert point for a filename + and linenumber, open the file, and show the line + Debugger (toggle) -- Run commands in the shell under the debugger + Stack Viewer -- Show the stack traceback of the last exception + Auto-open Stack Viewer (toggle) -- Open stack viewer on traceback - View Last Restart -- Scroll the shell window to the last Shell restart - Restart Shell -- Restart the shell to clean the environment +Options Menu: -Debug Menu (Shell window only): + Configure IDLE -- Open a configuration dialog. Fonts, indentation, + keybindings, and color themes may be altered. + Startup Preferences may be set, and Additional Help + Sources can be specified. + + On MacOS X this menu is not present, use + menu 'IDLE -> Preferences...' instead. + --- + Code Context -- Open a pane at the top of the edit window which + shows the block context of the section of code + which is scrolling off the top or the window. + (Not present in Shell window.) - Go to File/Line -- Look around the insert point for a filename - and line number, open the file, and show the line. - Useful to view the source lines referenced in an - exception traceback. Available in the context - menu of the Shell window. - Debugger (toggle) -- This feature is not complete and considered - experimental. Run commands in the shell under the - debugger. - Stack Viewer -- Show the stack traceback of the last exception - Auto-open Stack Viewer (toggle) -- Toggle automatically opening the - stack viewer on unhandled - exception +Windows Menu: -Options Menu (Shell and Editor): - - Configure IDLE -- Open a configuration dialog. Fonts, indentation, - keybindings, and color themes may be altered. - Startup Preferences may be set, and additional Help - sources can be specified. On OS X, open the - configuration dialog by selecting Preferences - in the application menu. - - --- - Code Context (toggle) -- Open a pane at the top of the edit window - which shows the block context of the section - of code which is scrolling off the top or the - window. This is not present in the Shell - window only the Editor window. - -Window Menu (Shell and Editor): - - Zoom Height -- Toggles the window between normal size (40x80 initial - setting) and maximum height. The initial size is in the Configure - IDLE dialog under the general tab. - --- - The rest of this menu lists the names of all open windows; - select one to bring it to the foreground (deiconifying it if - necessary). + Zoom Height -- toggles the window between configured size + and maximum height. + --- + The rest of this menu lists the names of all open windows; + select one to bring it to the foreground (deiconifying it if + necessary). Help Menu: - About IDLE -- Version, copyright, license, credits - --- - IDLE Help -- Display this file which is a help file for IDLE - detailing the menu options, basic editing and navigation, - and other tips. - Python Docs -- Access local Python documentation, if - installed. Or will start a web browser and open - docs.python.org showing the latest Python documentation. - --- - Additional help sources may be added here with the Configure IDLE - dialog under the General tab. - -Editor context menu (Right-click / Control-click on OS X in Edit window): - - Cut -- Copy a selection into system-wide clipboard, - then delete the selection - Copy -- Copy selection into system-wide clipboard - Paste -- Insert system-wide clipboard into window - Set Breakpoint -- Sets a breakpoint. Breakpoints are only enabled - when the debugger is open. - Clear Breakpoint -- Clears the breakpoint on that line - -Shell context menu (Right-click / Control-click on OS X in Shell window): - - Cut -- Copy a selection into system-wide clipboard, - then delete the selection - Copy -- Copy selection into system-wide clipboard - Paste -- Insert system-wide clipboard into window - --- - Go to file/line -- Same as in Debug menu + About IDLE -- Version, copyright, license, credits + IDLE Readme -- Background discussion and change details + --- + IDLE Help -- Display this file + Python Docs -- Access local Python documentation, if + installed. Otherwise, access www.python.org. + --- + (Additional Help Sources may be added here) ** TIPS ** @@ -191,182 +126,160 @@ Additional Help Sources: - Windows users can Google on zopeshelf.chm to access Zope help files in - the Windows help format. The Additional Help Sources feature of the - configuration GUI supports .chm, along with any other filetypes - supported by your browser. Supply a Menu Item title, and enter the - location in the Help File Path slot of the New Help Source dialog. Use - http:// and/or www. to identify external URLs, or download the file and - browse for its path on your machine using the Browse button. + Windows users can Google on zopeshelf.chm to access Zope help files in + the Windows help format. The Additional Help Sources feature of the + configuration GUI supports .chm, along with any other filetypes + supported by your browser. Supply a Menu Item title, and enter the + location in the Help File Path slot of the New Help Source dialog. Use + http:// and/or www. to identify external URLs, or download the file and + browse for its path on your machine using the Browse button. - All users can access the extensive sources of help, including - tutorials, available at docs.python.org. Selected URLs can be added - or removed from the Help menu at any time using Configure IDLE. + All users can access the extensive sources of help, including + tutorials, available at www.python.org/doc. Selected URLs can be added + or removed from the Help menu at any time using Configure IDLE. Basic editing and navigation: - Backspace deletes char to the left; DEL deletes char to the right. - Control-backspace deletes word left, Control-DEL deletes word right. - Arrow keys and Page Up/Down move around. - Control-left/right Arrow moves by words in a strange but useful way. - Home/End go to begin/end of line. - Control-Home/End go to begin/end of file. - Some useful Emacs bindings are inherited from Tcl/Tk: - Control-a beginning of line - Control-e end of line - Control-k kill line (but doesn't put it in clipboard) - Control-l center window around the insertion point - Standard keybindings (like Control-c to copy and Control-v to - paste) may work. Keybindings are selected in the Configure IDLE - dialog. + Backspace deletes char to the left; DEL deletes char to the right. + Control-backspace deletes word left, Control-DEL deletes word right. + Arrow keys and Page Up/Down move around. + Control-left/right Arrow moves by words in a strange but useful way. + Home/End go to begin/end of line. + Control-Home/End go to begin/end of file. + Some useful Emacs bindings are inherited from Tcl/Tk: + Control-a beginning of line + Control-e end of line + Control-k kill line (but doesn't put it in clipboard) + Control-l center window around the insertion point + Standard Windows bindings may work on that platform. + Keybindings are selected in the Settings Dialog, look there. Automatic indentation: - After a block-opening statement, the next line is indented by 4 spaces - (in the Python Shell window by one tab). After certain keywords - (break, return etc.) the next line is dedented. In leading - indentation, Backspace deletes up to 4 spaces if they are there. Tab - inserts spaces (in the Python Shell window one tab), number depends on - Indent Width. Currently tabs are restricted to four spaces due - to Tcl/Tk limitations. + After a block-opening statement, the next line is indented by 4 spaces + (in the Python Shell window by one tab). After certain keywords + (break, return etc.) the next line is dedented. In leading + indentation, Backspace deletes up to 4 spaces if they are there. Tab + inserts spaces (in the Python Shell window one tab), number depends on + Indent Width. (N.B. Currently tabs are restricted to four spaces due + to Tcl/Tk issues.) See also the indent/dedent region commands in the edit menu. Completions: - Completions are supplied for functions, classes, and attributes of - classes, both built-in and user-defined. Completions are also provided - for filenames. + Completions are supplied for functions, classes, and attributes of + classes, both built-in and user-defined. Completions are also provided + for filenames. - The AutoCompleteWindow (ACW) will open after a predefined delay - (default is two seconds) after a '.' or (in a string) an os.sep is - typed. If after one of those characters (plus zero or more other - characters) a tab is typed the ACW will open immediately if a possible - continuation is found. + The AutoCompleteWindow (ACW) will open after a predefined delay + (default is two seconds) after a '.' or (in a string) an os.sep is + typed. If after one of those characters (plus zero or more other + characters) you type a Tab the ACW will open immediately if a possible + continuation is found. - If there is only one possible completion for the characters entered, a - tab will supply that completion without opening the ACW. + If there is only one possible completion for the characters entered, a + Tab will supply that completion without opening the ACW. - 'Show Completions' will force open a completions window, by default the - Control-space keys will open a completions window. In an empty - string, this will contain the files in the current directory. On a - blank line, it will contain the built-in and user-defined functions and - classes in the current name spaces, plus any modules imported. If some - characters have been entered, the ACW will attempt to be more specific. + 'Show Completions' will force open a completions window. In an empty + string, this will contain the files in the current directory. On a + blank line, it will contain the built-in and user-defined functions and + classes in the current name spaces, plus any modules imported. If some + characters have been entered, the ACW will attempt to be more specific. - If string of characters is typed, the ACW selection will jump to the - entry most closely matching those characters. Entering a tab will cause - the longest non-ambiguous match to be entered in the Edit window or - Shell. Two tabs in a row will supply the current ACW selection, as - will return or a double click. Cursor keys, Page Up/Down, mouse - selection, and the scroll wheel all operate on the ACW. + If string of characters is typed, the ACW selection will jump to the + entry most closely matching those characters. Entering a Tab will cause + the longest non-ambiguous match to be entered in the Edit window or + Shell. Two Tabs in a row will supply the current ACW selection, as + will Return or a double click. Cursor keys, Page Up/Down, mouse + selection, and the scrollwheel all operate on the ACW. - "Hidden" attributes can be accessed by typing the beginning of hidden - name after a '.', e.g. '_'. This allows access to modules with - '__all__' set, or to class-private attributes. + 'Hidden' attributes can be accessed by typing the beginning of hidden + name after a '.'. e.g. '_'. This allows access to modules with + '__all__' set, or to class-private attributes. - Completions and the 'Expand Word' facility can save a lot of typing! + Completions and the 'Expand Word' facility can save a lot of typing! - Completions are currently limited to those in the namespaces. Names in - an Editor window which are not via __main__ or sys.modules will not be - found. Run the module once with your imports to correct this - situation. Note that IDLE itself places quite a few modules in - sys.modules, so much can be found by default, e.g. the re module. + Completions are currently limited to those in the namespaces. Names in + an Edit window which are not via __main__ or sys.modules will not be + found. Run the module once with your imports to correct this + situation. Note that IDLE itself places quite a few modules in + sys.modules, so much can be found by default, e.g. the re module. - If you don't like the ACW popping up unbidden, simply make the delay - longer or disable the extension. Or another option is the delay could - be set to zero. Another alternative to preventing ACW popups is to - disable the call tips extension. + If you don't like the ACW popping up unbidden, simply make the delay + longer or disable the extension. OTOH, you could make the delay zero. + + You could also switch off the CallTips extension. (We will be adding + a delay to the call tip window.) Python Shell window: - Control-c interrupts executing command. - Control-d sends end-of-file; closes window if typed at >>> prompt. - Alt-/ expand word is also useful to reduce typing. + Control-c interrupts executing command. + Control-d sends end-of-file; closes window if typed at >>> prompt + (this is Control-z on Windows). Command history: - Alt-p retrieves previous command matching what you have typed. On OS X - use Control-p. - Alt-n retrieves next. On OS X use Control-n. - Return while cursor is on a previous command retrieves that command. + Alt-p retrieves previous command matching what you have typed. + Alt-n retrieves next. + (These are Control-p, Control-n on the Mac) + Return while cursor is on a previous command retrieves that command. + Expand word is also useful to reduce typing. Syntax colors: - The coloring is applied in a background "thread", so you may - occasionally see uncolorized text. To change the color - scheme, use the Configure IDLE / Highlighting dialog. + The coloring is applied in a background "thread", so you may + occasionally see uncolorized text. To change the color + scheme, use the Configure IDLE / Highlighting dialog. Python default syntax colors: - Keywords orange - Builtins royal purple - Strings green - Comments red - Definitions blue + Keywords orange + Builtins royal purple + Strings green + Comments red + Definitions blue Shell default colors: - Console output brown - stdout blue - stderr red - stdin black + Console output brown + stdout blue + stderr red + stdin black Other preferences: - The font preferences, highlighting, keys, and general preferences can - be changed via the Configure IDLE menu option. Be sure to note that - keys can be user defined, IDLE ships with four built in key sets. In - addition a user can create a custom key set in the Configure IDLE - dialog under the keys tab. + The font preferences, keybinding, and startup preferences can + be changed using the Settings dialog. Command line usage: - Enter idle -h at the command prompt to get a usage message. + Enter idle -h at the command prompt to get a usage message. - idle.py [-c command] [-d] [-e] [-s] [-t title] [arg] ... +Running without a subprocess: - -c command run this command - -d enable debugger - -e edit mode; arguments are files to be edited - -s run $IDLESTARTUP or $PYTHONSTARTUP first - -t title set title of shell window - - If there are arguments: - 1. If -e is used, arguments are files opened for editing and sys.argv - reflects the arguments passed to IDLE itself. - 2. Otherwise, if -c is used, all arguments are placed in - sys.argv[1:...], with sys.argv[0] set to -c. - 3. Otherwise, if neither -e nor -c is used, the first argument is a - script which is executed with the remaining arguments in - sys.argv[1:...] and sys.argv[0] set to the script name. If the - script name is -, no script is executed but an interactive Python - session is started; the arguments are still available in sys.argv. - -Running without a subprocess: (DEPRECATED in Python 3.4 see Issue 16123) - - If IDLE is started with the -n command line switch it will run in a - single process and will not create the subprocess which runs the RPC - Python execution server. This can be useful if Python cannot create - the subprocess or the RPC socket interface on your platform. However, - in this mode user code is not isolated from IDLE itself. Also, the - environment is not restarted when Run/Run Module (F5) is selected. If - your code has been modified, you must reload() the affected modules and - re-import any specific items (e.g. from foo import baz) if the changes - are to take effect. For these reasons, it is preferable to run IDLE - with the default subprocess if at all possible. + If IDLE is started with the -n command line switch it will run in a + single process and will not create the subprocess which runs the RPC + Python execution server. This can be useful if Python cannot create + the subprocess or the RPC socket interface on your platform. However, + in this mode user code is not isolated from IDLE itself. Also, the + environment is not restarted when Run/Run Module (F5) is selected. If + your code has been modified, you must reload() the affected modules and + re-import any specific items (e.g. from foo import baz) if the changes + are to take effect. For these reasons, it is preferable to run IDLE + with the default subprocess if at all possible. Extensions: - IDLE contains an extension facility. See the beginning of - config-extensions.def in the idlelib directory for further information. - The default extensions are currently: + IDLE contains an extension facility. See the beginning of + config-extensions.def in the idlelib directory for further information. + The default extensions are currently: - FormatParagraph - AutoExpand - ZoomHeight - ScriptBinding - CallTips - ParenMatch - AutoComplete - CodeContext + FormatParagraph + AutoExpand + ZoomHeight + ScriptBinding + CallTips + ParenMatch + AutoComplete + CodeContext diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle.bat --- a/Lib/idlelib/idle.bat Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/idle.bat Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -@echo off -rem Start IDLE using the appropriate Python interpreter -set CURRDIR=%~dp0 -start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 +@echo off +rem Start IDLE using the appropriate Python interpreter +set CURRDIR=%~dp0 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle.pyw --- a/Lib/idlelib/idle.pyw Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/idle.pyw Mon Jan 25 17:05:13 2016 +0100 @@ -2,16 +2,20 @@ import idlelib.PyShell except ImportError: # IDLE is not installed, but maybe PyShell is on sys.path: - from . import PyShell - import os - idledir = os.path.dirname(os.path.abspath(PyShell.__file__)) - if idledir != os.getcwd(): - # We're not in the IDLE directory, help the subprocess find run.py - pypath = os.environ.get('PYTHONPATH', '') - if pypath: - os.environ['PYTHONPATH'] = pypath + ':' + idledir - else: - os.environ['PYTHONPATH'] = idledir - PyShell.main() + try: + from . import PyShell + except ImportError: + raise + else: + import os + idledir = os.path.dirname(os.path.abspath(PyShell.__file__)) + if idledir != os.getcwd(): + # We're not in the IDLE directory, help the subprocess find run.py + pypath = os.environ.get('PYTHONPATH', '') + if pypath: + os.environ['PYTHONPATH'] = pypath + ':' + idledir + else: + os.environ['PYTHONPATH'] = idledir + PyShell.main() else: idlelib.PyShell.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/README.txt --- a/Lib/idlelib/idle_test/README.txt Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,143 +0,0 @@ -README FOR IDLE TESTS IN IDLELIB.IDLE_TEST - -0. Quick Start - -Automated unit tests were added in 2.7 for Python 2.x and 3.3 for Python 3.x. -To run the tests from a command line: - -python -m test.test_idle - -Human-mediated tests were added later in 2.7 and in 3.4. - -python -m idlelib.idle_test.htest - - -1. Test Files - -The idle directory, idlelib, has over 60 xyz.py files. The idle_test -subdirectory should contain a test_xyz.py for each, where 'xyz' is lowercased -even if xyz.py is not. Here is a possible template, with the blanks after after -'.' and 'as', and before and after '_' to be filled in. - -import unittest -from test.support import requires -import idlelib. as - -class _Test(unittest.TestCase): - - def test_(self): - -if __name__ == '__main__': - unittest.main(verbosity=2) - -Add the following at the end of xyy.py, with the appropriate name added after -'test_'. Some files already have something like this for htest. If so, insert -the import and unittest.main lines before the htest lines. - -if __name__ == "__main__": - import unittest - unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) - - - -2. GUI Tests - -When run as part of the Python test suite, Idle gui tests need to run -test.support.requires('gui') (test.test_support in 2.7). A test is a gui test -if it creates a Tk root or master object either directly or indirectly by -instantiating a tkinter or idle class. For the benefit of test processes that -either have no graphical environment available or are not allowed to use it, gui -tests must be 'guarded' by "requires('gui')" in a setUp function or method. -This will typically be setUpClass. - -To avoid interfering with other gui tests, all gui objects must be destroyed and -deleted by the end of the test. Widgets, such as a Tk root, created in a setUpX -function, should be destroyed in the corresponding tearDownX. Module and class -widget attributes should also be deleted.. - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = tk.Tk() - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - -Requires('gui') causes the test(s) it guards to be skipped if any of -a few conditions are met: - - - The tests are being run by regrtest.py, and it was started without enabling - the "gui" resource with the "-u" command line option. - - - The tests are being run on Windows by a service that is not allowed to - interact with the graphical environment. - - - The tests are being run on Mac OSX in a process that cannot make a window - manager connection. - - - tkinter.Tk cannot be successfully instantiated for some reason. - - - test.support.use_resources has been set by something other than - regrtest.py and does not contain "gui". - -Tests of non-gui operations should avoid creating tk widgets. Incidental uses of -tk variables and messageboxes can be replaced by the mock classes in -idle_test/mock_tk.py. The mock text handles some uses of the tk Text widget. - - -3. Running Unit Tests - -Assume that xyz.py and test_xyz.py both end with a unittest.main() call. -Running either from an Idle editor runs all tests in the test_xyz file with the -version of Python running Idle. Test output appears in the Shell window. The -'verbosity=2' option lists all test methods in the file, which is appropriate -when developing tests. The 'exit=False' option is needed in xyx.py files when an -htest follows. - -The following command lines also run all test methods, including -gui tests, in test_xyz.py. (Both '-m idlelib' and '-m idlelib.idle' start -Idle and so cannot run tests.) - -python -m idlelib.xyz -python -m idlelib.idle_test.test_xyz - -The following runs all idle_test/test_*.py tests interactively. - ->>> import unittest ->>> unittest.main('idlelib.idle_test', verbosity=2) - -The following run all Idle tests at a command line. Option '-v' is the same as -'verbosity=2'. (For 2.7, replace 'test' in the second line with -'test.regrtest'.) - -python -m unittest -v idlelib.idle_test -python -m test -v -ugui test_idle -python -m test.test_idle - -The idle tests are 'discovered' by idlelib.idle_test.__init__.load_tests, -which is also imported into test.test_idle. Normally, neither file should be -changed when working on individual test modules. The third command runs -unittest indirectly through regrtest. The same happens when the entire test -suite is run with 'python -m test'. So that command must work for buildbots -to stay green. Idle tests must not disturb the environment in a way that -makes other tests fail (issue 18081). - -To run an individual Testcase or test method, extend the dotted name given to -unittest on the command line. - -python -m unittest -v idlelib.idle_test.test_xyz.Test_case.test_meth - - -4. Human-mediated Tests - -Human-mediated tests are widget tests that cannot be automated but need human -verification. They are contained in idlelib/idle_test/htest.py, which has -instructions. (Some modules need an auxiliary function, identified with # htest -# on the header line.) The set is about complete, though some tests need -improvement. To run all htests, run the htest file from an editor or from the -command line with: - -python -m idlelib.idle_test.htest diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/__init__.py --- a/Lib/idlelib/idle_test/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -from os.path import dirname - -def load_tests(loader, standard_tests, pattern): - this_dir = dirname(__file__) - top_dir = dirname(dirname(this_dir)) - package_tests = loader.discover(start_dir=this_dir, pattern='test*.py', - top_level_dir=top_dir) - standard_tests.addTests(package_tests) - return standard_tests diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/htest.py --- a/Lib/idlelib/idle_test/htest.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,401 +0,0 @@ -'''Run human tests of Idle's window, dialog, and popup widgets. - -run(*tests) -Create a master Tk window. Within that, run each callable in tests -after finding the matching test spec in this file. If tests is empty, -run an htest for each spec dict in this file after finding the matching -callable in the module named in the spec. Close the window to skip or -end the test. - -In a tested module, let X be a global name bound to a callable (class -or function) whose .__name__ attrubute is also X (the usual situation). -The first parameter of X must be 'parent'. When called, the parent -argument will be the root window. X must create a child Toplevel -window (or subclass thereof). The Toplevel may be a test widget or -dialog, in which case the callable is the corresonding class. Or the -Toplevel may contain the widget to be tested or set up a context in -which a test widget is invoked. In this latter case, the callable is a -wrapper function that sets up the Toplevel and other objects. Wrapper -function names, such as _editor_window', should start with '_'. - - -End the module with - -if __name__ == '__main__': - - from idlelib.idle_test.htest import run - run(X) - -To have wrapper functions and test invocation code ignored by coveragepy -reports, put '# htest #' on the def statement header line. - -def _wrapper(parent): # htest # - -Also make sure that the 'if __name__' line matches the above. Then have -make sure that .coveragerc includes the following. - -[report] -exclude_lines = - .*# htest # - if __name__ == .__main__.: - -(The "." instead of "'" is intentional and necessary.) - - -To run any X, this file must contain a matching instance of the -following template, with X.__name__ prepended to '_spec'. -When all tests are run, the prefix is use to get X. - -_spec = { - 'file': '', - 'kwds': {'title': ''}, - 'msg': "" - } - -file (no .py): run() imports file.py. -kwds: augmented with {'parent':root} and passed to X as **kwds. -title: an example kwd; some widgets need this, delete if not. -msg: master window hints about testing the widget. - - -Modules and classes not being tested at the moment: -PyShell.PyShellEditorWindow -Debugger.Debugger -AutoCompleteWindow.AutoCompleteWindow -OutputWindow.OutputWindow (indirectly being tested with grep test) -''' - -from importlib import import_module -from idlelib.macosxSupport import _initializeTkVariantTests -import tkinter as tk - -AboutDialog_spec = { - 'file': 'aboutDialog', - 'kwds': {'title': 'aboutDialog test', - '_htest': True, - }, - 'msg': "Test every button. Ensure Python, TK and IDLE versions " - "are correctly displayed.\n [Close] to exit.", - } - -_calltip_window_spec = { - 'file': 'CallTipWindow', - 'kwds': {}, - 'msg': "Typing '(' should display a calltip.\n" - "Typing ') should hide the calltip.\n" - } - -_class_browser_spec = { - 'file': 'ClassBrowser', - 'kwds': {}, - 'msg': "Inspect names of module, class(with superclass if " - "applicable), methods and functions.\nToggle nested items.\n" - "Double clicking on items prints a traceback for an exception " - "that is ignored." - } - -_color_delegator_spec = { - 'file': 'ColorDelegator', - 'kwds': {}, - 'msg': "The text is sample Python code.\n" - "Ensure components like comments, keywords, builtins,\n" - "string, definitions, and break are correctly colored.\n" - "The default color scheme is in idlelib/config-highlight.def" - } - -ConfigDialog_spec = { - 'file': 'configDialog', - 'kwds': {'title': 'ConfigDialogTest', - '_htest': True,}, - 'msg': "IDLE preferences dialog.\n" - "In the 'Fonts/Tabs' tab, changing font face, should update the " - "font face of the text in the area below it.\nIn the " - "'Highlighting' tab, try different color schemes. Clicking " - "items in the sample program should update the choices above it." - "\nIn the 'Keys', 'General' and 'Extensions' tabs, test settings" - "of interest." - "\n[Ok] to close the dialog.[Apply] to apply the settings and " - "and [Cancel] to revert all changes.\nRe-run the test to ensure " - "changes made have persisted." - } - -# TODO Improve message -_dyn_option_menu_spec = { - 'file': 'dynOptionMenuWidget', - 'kwds': {}, - 'msg': "Select one of the many options in the 'old option set'.\n" - "Click the button to change the option set.\n" - "Select one of the many options in the 'new option set'." - } - -# TODO edit wrapper -_editor_window_spec = { - 'file': 'EditorWindow', - 'kwds': {}, - 'msg': "Test editor functions of interest.\n" - "Best to close editor first." - } - -GetCfgSectionNameDialog_spec = { - 'file': 'configSectionNameDialog', - 'kwds': {'title':'Get Name', - 'message':'Enter something', - 'used_names': {'abc'}, - '_htest': True}, - 'msg': "After the text entered with [Ok] is stripped, , " - "'abc', or more that 30 chars are errors.\n" - "Close 'Get Name' with a valid entry (printed to Shell), " - "[Cancel], or [X]", - } - -GetHelpSourceDialog_spec = { - 'file': 'configHelpSourceEdit', - 'kwds': {'title': 'Get helpsource', - '_htest': True}, - 'msg': "Enter menu item name and help file path\n " - " and more than 30 chars are invalid menu item names.\n" - ", file does not exist are invalid path items.\n" - "Test for incomplete web address for help file path.\n" - "A valid entry will be printed to shell with [0k].\n" - "[Cancel] will print None to shell", - } - -# Update once issue21519 is resolved. -GetKeysDialog_spec = { - 'file': 'keybindingDialog', - 'kwds': {'title': 'Test keybindings', - 'action': 'find-again', - 'currentKeySequences': [''] , - '_htest': True, - }, - 'msg': "Test for different key modifier sequences.\n" - " is invalid.\n" - "No modifier key is invalid.\n" - "Shift key with [a-z],[0-9], function key, move key, tab, space" - "is invalid.\nNo validity checking if advanced key binding " - "entry is used." - } - -_grep_dialog_spec = { - 'file': 'GrepDialog', - 'kwds': {}, - 'msg': "Click the 'Show GrepDialog' button.\n" - "Test the various 'Find-in-files' functions.\n" - "The results should be displayed in a new '*Output*' window.\n" - "'Right-click'->'Goto file/line' anywhere in the search results " - "should open that file \nin a new EditorWindow." - } - -_io_binding_spec = { - 'file': 'IOBinding', - 'kwds': {}, - 'msg': "Test the following bindings.\n" - " to open file from dialog.\n" - "Edit the file.\n" - " to save the file.\n" - "Check that changes were saved by opening the file elsewhere." - } - -_multi_call_spec = { - 'file': 'MultiCall', - 'kwds': {}, - 'msg': "The following actions should trigger a print to console or IDLE" - " Shell.\nEntering and leaving the text area, key entry, " - ",\n, , " - ", \n, and " - "focusing out of the window\nare sequences to be tested." - } - -_multistatus_bar_spec = { - 'file': 'MultiStatusBar', - 'kwds': {}, - 'msg': "Ensure presence of multi-status bar below text area.\n" - "Click 'Update Status' to change the multi-status text" - } - -_object_browser_spec = { - 'file': 'ObjectBrowser', - 'kwds': {}, - 'msg': "Double click on items upto the lowest level.\n" - "Attributes of the objects and related information " - "will be displayed side-by-side at each level." - } - -_path_browser_spec = { - 'file': 'PathBrowser', - 'kwds': {}, - 'msg': "Test for correct display of all paths in sys.path.\n" - "Toggle nested items upto the lowest level.\n" - "Double clicking on an item prints a traceback\n" - "for an exception that is ignored." - } - -_percolator_spec = { - 'file': 'Percolator', - 'kwds': {}, - 'msg': "There are two tracers which can be toggled using a checkbox.\n" - "Toggling a tracer 'on' by checking it should print tracer" - "output to the console or to the IDLE shell.\n" - "If both the tracers are 'on', the output from the tracer which " - "was switched 'on' later, should be printed first\n" - "Test for actions like text entry, and removal." - } - -_replace_dialog_spec = { - 'file': 'ReplaceDialog', - 'kwds': {}, - 'msg': "Click the 'Replace' button.\n" - "Test various replace options in the 'Replace dialog'.\n" - "Click [Close] or [X] to close the 'Replace Dialog'." - } - -_search_dialog_spec = { - 'file': 'SearchDialog', - 'kwds': {}, - 'msg': "Click the 'Search' button.\n" - "Test various search options in the 'Search dialog'.\n" - "Click [Close] or [X] to close the 'Search Dialog'." - } - -_scrolled_list_spec = { - 'file': 'ScrolledList', - 'kwds': {}, - 'msg': "You should see a scrollable list of items\n" - "Selecting (clicking) or double clicking an item " - "prints the name to the console or Idle shell.\n" - "Right clicking an item will display a popup." - } - -show_idlehelp_spec = { - 'file': 'help', - 'kwds': {}, - 'msg': "If the help text displays, this works.\n" - "Text is selectable. Window is scrollable." - } - -_stack_viewer_spec = { - 'file': 'StackViewer', - 'kwds': {}, - 'msg': "A stacktrace for a NameError exception.\n" - "Expand 'idlelib ...' and ''.\n" - "Check that exc_value, exc_tb, and exc_type are correct.\n" - } - -_tabbed_pages_spec = { - 'file': 'tabbedpages', - 'kwds': {}, - 'msg': "Toggle between the two tabs 'foo' and 'bar'\n" - "Add a tab by entering a suitable name for it.\n" - "Remove an existing tab by entering its name.\n" - "Remove all existing tabs.\n" - " is an invalid add page and remove page name.\n" - } - -TextViewer_spec = { - 'file': 'textView', - 'kwds': {'title': 'Test textView', - 'text':'The quick brown fox jumps over the lazy dog.\n'*35, - '_htest': True}, - 'msg': "Test for read-only property of text.\n" - "Text is selectable. Window is scrollable.", - } - -_tooltip_spec = { - 'file': 'ToolTip', - 'kwds': {}, - 'msg': "Place mouse cursor over both the buttons\n" - "A tooltip should appear with some text." - } - -_tree_widget_spec = { - 'file': 'TreeWidget', - 'kwds': {}, - 'msg': "The canvas is scrollable.\n" - "Click on folders upto to the lowest level." - } - -_undo_delegator_spec = { - 'file': 'UndoDelegator', - 'kwds': {}, - 'msg': "Click [Undo] to undo any action.\n" - "Click [Redo] to redo any action.\n" - "Click [Dump] to dump the current state " - "by printing to the console or the IDLE shell.\n" - } - -_widget_redirector_spec = { - 'file': 'WidgetRedirector', - 'kwds': {}, - 'msg': "Every text insert should be printed to the console." - "or the IDLE shell." - } - -def run(*tests): - root = tk.Tk() - root.title('IDLE htest') - root.resizable(0, 0) - _initializeTkVariantTests(root) - - # a scrollable Label like constant width text widget. - frameLabel = tk.Frame(root, padx=10) - frameLabel.pack() - text = tk.Text(frameLabel, wrap='word') - text.configure(bg=root.cget('bg'), relief='flat', height=4, width=70) - scrollbar = tk.Scrollbar(frameLabel, command=text.yview) - text.config(yscrollcommand=scrollbar.set) - scrollbar.pack(side='right', fill='y', expand=False) - text.pack(side='left', fill='both', expand=True) - - test_list = [] # List of tuples of the form (spec, callable widget) - if tests: - for test in tests: - test_spec = globals()[test.__name__ + '_spec'] - test_spec['name'] = test.__name__ - test_list.append((test_spec, test)) - else: - for k, d in globals().items(): - if k.endswith('_spec'): - test_name = k[:-5] - test_spec = d - test_spec['name'] = test_name - mod = import_module('idlelib.' + test_spec['file']) - test = getattr(mod, test_name) - test_list.append((test_spec, test)) - - test_name = tk.StringVar('') - callable_object = None - test_kwds = None - - def next(): - - nonlocal test_name, callable_object, test_kwds - if len(test_list) == 1: - next_button.pack_forget() - test_spec, callable_object = test_list.pop() - test_kwds = test_spec['kwds'] - test_kwds['parent'] = root - test_name.set('Test ' + test_spec['name']) - - text.configure(state='normal') # enable text editing - text.delete('1.0','end') - text.insert("1.0",test_spec['msg']) - text.configure(state='disabled') # preserve read-only property - - def run_test(): - widget = callable_object(**test_kwds) - try: - print(widget.result) - except AttributeError: - pass - - button = tk.Button(root, textvariable=test_name, command=run_test) - button.pack() - next_button = tk.Button(root, text="Next", command=next) - next_button.pack() - - next() - - root.mainloop() - -if __name__ == '__main__': - run() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/mock_idle.py --- a/Lib/idlelib/idle_test/mock_idle.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,55 +0,0 @@ -'''Mock classes that imitate idlelib modules or classes. - -Attributes and methods will be added as needed for tests. -''' - -from idlelib.idle_test.mock_tk import Text - -class Func: - '''Mock function captures args and returns result set by test. - - Attributes: - self.called - records call even if no args, kwds passed. - self.result - set by init, returned by call. - self.args - captures positional arguments. - self.kwds - captures keyword arguments. - - Most common use will probably be to mock methods. - Mock_tk.Var and Mbox_func are special variants of this. - ''' - def __init__(self, result=None): - self.called = False - self.result = result - self.args = None - self.kwds = None - def __call__(self, *args, **kwds): - self.called = True - self.args = args - self.kwds = kwds - if isinstance(self.result, BaseException): - raise self.result - else: - return self.result - - -class Editor: - '''Minimally imitate EditorWindow.EditorWindow class. - ''' - def __init__(self, flist=None, filename=None, key=None, root=None): - self.text = Text() - self.undo = UndoDelegator() - - def get_selection_indices(self): - first = self.text.index('1.0') - last = self.text.index('end') - return first, last - - -class UndoDelegator: - '''Minimally imitate UndoDelegator,UndoDelegator class. - ''' - # A real undo block is only needed for user interaction. - def undo_block_start(*args): - pass - def undo_block_stop(*args): - pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/mock_tk.py --- a/Lib/idlelib/idle_test/mock_tk.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,298 +0,0 @@ -"""Classes that replace tkinter gui objects used by an object being tested. - -A gui object is anything with a master or parent parameter, which is -typically required in spite of what the doc strings say. -""" - -class Event: - '''Minimal mock with attributes for testing event handlers. - - This is not a gui object, but is used as an argument for callbacks - that access attributes of the event passed. If a callback ignores - the event, other than the fact that is happened, pass 'event'. - - Keyboard, mouse, window, and other sources generate Event instances. - Event instances have the following attributes: serial (number of - event), time (of event), type (of event as number), widget (in which - event occurred), and x,y (position of mouse). There are other - attributes for specific events, such as keycode for key events. - tkinter.Event.__doc__ has more but is still not complete. - ''' - def __init__(self, **kwds): - "Create event with attributes needed for test" - self.__dict__.update(kwds) - -class Var: - "Use for String/Int/BooleanVar: incomplete" - def __init__(self, master=None, value=None, name=None): - self.master = master - self.value = value - self.name = name - def set(self, value): - self.value = value - def get(self): - return self.value - -class Mbox_func: - """Generic mock for messagebox functions, which all have the same signature. - - Instead of displaying a message box, the mock's call method saves the - arguments as instance attributes, which test functions can then examime. - The test can set the result returned to ask function - """ - def __init__(self, result=None): - self.result = result # Return None for all show funcs - def __call__(self, title, message, *args, **kwds): - # Save all args for possible examination by tester - self.title = title - self.message = message - self.args = args - self.kwds = kwds - return self.result # Set by tester for ask functions - -class Mbox: - """Mock for tkinter.messagebox with an Mbox_func for each function. - - This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. - Example usage in test_module.py for testing functions in module.py: - --- -from idlelib.idle_test.mock_tk import Mbox -import module - -orig_mbox = module.tkMessageBox -showerror = Mbox.showerror # example, for attribute access in test methods - -class Test(unittest.TestCase): - - @classmethod - def setUpClass(cls): - module.tkMessageBox = Mbox - - @classmethod - def tearDownClass(cls): - module.tkMessageBox = orig_mbox - --- - For 'ask' functions, set func.result return value before calling the method - that uses the message function. When tkMessageBox functions are the - only gui alls in a method, this replacement makes the method gui-free, - """ - askokcancel = Mbox_func() # True or False - askquestion = Mbox_func() # 'yes' or 'no' - askretrycancel = Mbox_func() # True or False - askyesno = Mbox_func() # True or False - askyesnocancel = Mbox_func() # True, False, or None - showerror = Mbox_func() # None - showinfo = Mbox_func() # None - showwarning = Mbox_func() # None - -from _tkinter import TclError - -class Text: - """A semi-functional non-gui replacement for tkinter.Text text editors. - - The mock's data model is that a text is a list of \n-terminated lines. - The mock adds an empty string at the beginning of the list so that the - index of actual lines start at 1, as with Tk. The methods never see this. - Tk initializes files with a terminal \n that cannot be deleted. It is - invisible in the sense that one cannot move the cursor beyond it. - - This class is only tested (and valid) with strings of ascii chars. - For testing, we are not concerned with Tk Text's treatment of, - for instance, 0-width characters or character + accent. - """ - def __init__(self, master=None, cnf={}, **kw): - '''Initialize mock, non-gui, text-only Text widget. - - At present, all args are ignored. Almost all affect visual behavior. - There are just a few Text-only options that affect text behavior. - ''' - self.data = ['', '\n'] - - def index(self, index): - "Return string version of index decoded according to current text." - return "%s.%s" % self._decode(index, endflag=1) - - def _decode(self, index, endflag=0): - """Return a (line, char) tuple of int indexes into self.data. - - This implements .index without converting the result back to a string. - The result is constrained by the number of lines and linelengths of - self.data. For many indexes, the result is initially (1, 0). - - The input index may have any of several possible forms: - * line.char float: converted to 'line.char' string; - * 'line.char' string, where line and char are decimal integers; - * 'line.char lineend', where lineend='lineend' (and char is ignored); - * 'line.end', where end='end' (same as above); - * 'insert', the positions before terminal \n; - * 'end', whose meaning depends on the endflag passed to ._endex. - * 'sel.first' or 'sel.last', where sel is a tag -- not implemented. - """ - if isinstance(index, (float, bytes)): - index = str(index) - try: - index=index.lower() - except AttributeError: - raise TclError('bad text index "%s"' % index) from None - - lastline = len(self.data) - 1 # same as number of text lines - if index == 'insert': - return lastline, len(self.data[lastline]) - 1 - elif index == 'end': - return self._endex(endflag) - - line, char = index.split('.') - line = int(line) - - # Out of bounds line becomes first or last ('end') index - if line < 1: - return 1, 0 - elif line > lastline: - return self._endex(endflag) - - linelength = len(self.data[line]) -1 # position before/at \n - if char.endswith(' lineend') or char == 'end': - return line, linelength - # Tk requires that ignored chars before ' lineend' be valid int - - # Out of bounds char becomes first or last index of line - char = int(char) - if char < 0: - char = 0 - elif char > linelength: - char = linelength - return line, char - - def _endex(self, endflag): - '''Return position for 'end' or line overflow corresponding to endflag. - - -1: position before terminal \n; for .insert(), .delete - 0: position after terminal \n; for .get, .delete index 1 - 1: same viewed as beginning of non-existent next line (for .index) - ''' - n = len(self.data) - if endflag == 1: - return n, 0 - else: - n -= 1 - return n, len(self.data[n]) + endflag - - - def insert(self, index, chars): - "Insert chars before the character at index." - - if not chars: # ''.splitlines() is [], not [''] - return - chars = chars.splitlines(True) - if chars[-1][-1] == '\n': - chars.append('') - line, char = self._decode(index, -1) - before = self.data[line][:char] - after = self.data[line][char:] - self.data[line] = before + chars[0] - self.data[line+1:line+1] = chars[1:] - self.data[line+len(chars)-1] += after - - - def get(self, index1, index2=None): - "Return slice from index1 to index2 (default is 'index1+1')." - - startline, startchar = self._decode(index1) - if index2 is None: - endline, endchar = startline, startchar+1 - else: - endline, endchar = self._decode(index2) - - if startline == endline: - return self.data[startline][startchar:endchar] - else: - lines = [self.data[startline][startchar:]] - for i in range(startline+1, endline): - lines.append(self.data[i]) - lines.append(self.data[endline][:endchar]) - return ''.join(lines) - - - def delete(self, index1, index2=None): - '''Delete slice from index1 to index2 (default is 'index1+1'). - - Adjust default index2 ('index+1) for line ends. - Do not delete the terminal \n at the very end of self.data ([-1][-1]). - ''' - startline, startchar = self._decode(index1, -1) - if index2 is None: - if startchar < len(self.data[startline])-1: - # not deleting \n - endline, endchar = startline, startchar+1 - elif startline < len(self.data) - 1: - # deleting non-terminal \n, convert 'index1+1 to start of next line - endline, endchar = startline+1, 0 - else: - # do not delete terminal \n if index1 == 'insert' - return - else: - endline, endchar = self._decode(index2, -1) - # restricting end position to insert position excludes terminal \n - - if startline == endline and startchar < endchar: - self.data[startline] = self.data[startline][:startchar] + \ - self.data[startline][endchar:] - elif startline < endline: - self.data[startline] = self.data[startline][:startchar] + \ - self.data[endline][endchar:] - startline += 1 - for i in range(startline, endline+1): - del self.data[startline] - - def compare(self, index1, op, index2): - line1, char1 = self._decode(index1) - line2, char2 = self._decode(index2) - if op == '<': - return line1 < line2 or line1 == line2 and char1 < char2 - elif op == '<=': - return line1 < line2 or line1 == line2 and char1 <= char2 - elif op == '>': - return line1 > line2 or line1 == line2 and char1 > char2 - elif op == '>=': - return line1 > line2 or line1 == line2 and char1 >= char2 - elif op == '==': - return line1 == line2 and char1 == char2 - elif op == '!=': - return line1 != line2 or char1 != char2 - else: - raise TclError('''bad comparison operator "%s":''' - '''must be <, <=, ==, >=, >, or !=''' % op) - - # The following Text methods normally do something and return None. - # Whether doing nothing is sufficient for a test will depend on the test. - - def mark_set(self, name, index): - "Set mark *name* before the character at index." - pass - - def mark_unset(self, *markNames): - "Delete all marks in markNames." - - def tag_remove(self, tagName, index1, index2=None): - "Remove tag tagName from all characters between index1 and index2." - pass - - # The following Text methods affect the graphics screen and return None. - # Doing nothing should always be sufficient for tests. - - def scan_dragto(self, x, y): - "Adjust the view of the text according to scan_mark" - - def scan_mark(self, x, y): - "Remember the current X, Y coordinates." - - def see(self, index): - "Scroll screen to make the character at INDEX is visible." - pass - - # The following is a Misc method inherited by Text. - # It should properly go in a Misc mock, but is included here for now. - - def bind(sequence=None, func=None, add=None): - "Bind to this widget at event sequence a call to function func." - pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_autocomplete.py --- a/Lib/idlelib/idle_test/test_autocomplete.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,143 +0,0 @@ -import unittest -from test.support import requires -from tkinter import Tk, Text - -import idlelib.AutoComplete as ac -import idlelib.AutoCompleteWindow as acw -import idlelib.macosxSupport as mac -from idlelib.idle_test.mock_idle import Func -from idlelib.idle_test.mock_tk import Event - -class AutoCompleteWindow: - def complete(): - return - -class DummyEditwin: - def __init__(self, root, text): - self.root = root - self.text = text - self.indentwidth = 8 - self.tabwidth = 8 - self.context_use_ps1 = True - - -class AutoCompleteTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - mac.setupApp(cls.root, None) - cls.text = Text(cls.root) - cls.editor = DummyEditwin(cls.root, cls.text) - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.text - del cls.editor - del cls.root - - def setUp(self): - self.editor.text.delete('1.0', 'end') - self.autocomplete = ac.AutoComplete(self.editor) - - def test_init(self): - self.assertEqual(self.autocomplete.editwin, self.editor) - - def test_make_autocomplete_window(self): - testwin = self.autocomplete._make_autocomplete_window() - self.assertIsInstance(testwin, acw.AutoCompleteWindow) - - def test_remove_autocomplete_window(self): - self.autocomplete.autocompletewindow = ( - self.autocomplete._make_autocomplete_window()) - self.autocomplete._remove_autocomplete_window() - self.assertIsNone(self.autocomplete.autocompletewindow) - - def test_force_open_completions_event(self): - # Test that force_open_completions_event calls _open_completions - o_cs = Func() - self.autocomplete.open_completions = o_cs - self.autocomplete.force_open_completions_event('event') - self.assertEqual(o_cs.args, (True, False, True)) - - def test_try_open_completions_event(self): - Equal = self.assertEqual - autocomplete = self.autocomplete - trycompletions = self.autocomplete.try_open_completions_event - o_c_l = Func() - autocomplete._open_completions_later = o_c_l - - # _open_completions_later should not be called with no text in editor - trycompletions('event') - Equal(o_c_l.args, None) - - # _open_completions_later should be called with COMPLETE_ATTRIBUTES (1) - self.text.insert('1.0', 're.') - trycompletions('event') - Equal(o_c_l.args, (False, False, False, 1)) - - # _open_completions_later should be called with COMPLETE_FILES (2) - self.text.delete('1.0', 'end') - self.text.insert('1.0', '"./Lib/') - trycompletions('event') - Equal(o_c_l.args, (False, False, False, 2)) - - def test_autocomplete_event(self): - Equal = self.assertEqual - autocomplete = self.autocomplete - - # Test that the autocomplete event is ignored if user is pressing a - # modifier key in addition to the tab key - ev = Event(mc_state=True) - self.assertIsNone(autocomplete.autocomplete_event(ev)) - del ev.mc_state - - # If autocomplete window is open, complete() method is called - self.text.insert('1.0', 're.') - # This must call autocomplete._make_autocomplete_window() - Equal(self.autocomplete.autocomplete_event(ev), 'break') - - # If autocomplete window is not active or does not exist, - # open_completions is called. Return depends on its return. - autocomplete._remove_autocomplete_window() - o_cs = Func() # .result = None - autocomplete.open_completions = o_cs - Equal(self.autocomplete.autocomplete_event(ev), None) - Equal(o_cs.args, (False, True, True)) - o_cs.result = True - Equal(self.autocomplete.autocomplete_event(ev), 'break') - Equal(o_cs.args, (False, True, True)) - - def test_open_completions_later(self): - # Test that autocomplete._delayed_completion_id is set - pass - - def test_delayed_open_completions(self): - # Test that autocomplete._delayed_completion_id set to None and that - # open_completions only called if insertion index is the same as - # _delayed_completion_index - pass - - def test_open_completions(self): - # Test completions of files and attributes as well as non-completion - # of errors - pass - - def test_fetch_completions(self): - # Test that fetch_completions returns 2 lists: - # For attribute completion, a large list containing all variables, and - # a small list containing non-private variables. - # For file completion, a large list containing all files in the path, - # and a small list containing files that do not start with '.' - pass - - def test_get_entity(self): - # Test that a name is in the namespace of sys.modules and - # __main__.__dict__ - pass - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_autoexpand.py --- a/Lib/idlelib/idle_test/test_autoexpand.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,141 +0,0 @@ -"""Unit tests for idlelib.AutoExpand""" -import unittest -from test.support import requires -from tkinter import Text, Tk -#from idlelib.idle_test.mock_tk import Text -from idlelib.AutoExpand import AutoExpand - - -class Dummy_Editwin: - # AutoExpand.__init__ only needs .text - def __init__(self, text): - self.text = text - -class AutoExpandTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - if 'tkinter' in str(Text): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - else: - cls.text = Text() - cls.auto_expand = AutoExpand(Dummy_Editwin(cls.text)) - - @classmethod - def tearDownClass(cls): - if hasattr(cls, 'tk'): - cls.tk.destroy() - del cls.tk - del cls.text, cls.auto_expand - - def tearDown(self): - self.text.delete('1.0', 'end') - - def test_get_prevword(self): - text = self.text - previous = self.auto_expand.getprevword - equal = self.assertEqual - - equal(previous(), '') - - text.insert('insert', 't') - equal(previous(), 't') - - text.insert('insert', 'his') - equal(previous(), 'this') - - text.insert('insert', ' ') - equal(previous(), '') - - text.insert('insert', 'is') - equal(previous(), 'is') - - text.insert('insert', '\nsample\nstring') - equal(previous(), 'string') - - text.delete('3.0', 'insert') - equal(previous(), '') - - text.delete('1.0', 'end') - equal(previous(), '') - - def test_before_only(self): - previous = self.auto_expand.getprevword - expand = self.auto_expand.expand_word_event - equal = self.assertEqual - - self.text.insert('insert', 'ab ac bx ad ab a') - equal(self.auto_expand.getwords(), ['ab', 'ad', 'ac', 'a']) - expand('event') - equal(previous(), 'ab') - expand('event') - equal(previous(), 'ad') - expand('event') - equal(previous(), 'ac') - expand('event') - equal(previous(), 'a') - - def test_after_only(self): - # Also add punctuation 'noise' that should be ignored. - text = self.text - previous = self.auto_expand.getprevword - expand = self.auto_expand.expand_word_event - equal = self.assertEqual - - text.insert('insert', 'a, [ab] ac: () bx"" cd ac= ad ya') - text.mark_set('insert', '1.1') - equal(self.auto_expand.getwords(), ['ab', 'ac', 'ad', 'a']) - expand('event') - equal(previous(), 'ab') - expand('event') - equal(previous(), 'ac') - expand('event') - equal(previous(), 'ad') - expand('event') - equal(previous(), 'a') - - def test_both_before_after(self): - text = self.text - previous = self.auto_expand.getprevword - expand = self.auto_expand.expand_word_event - equal = self.assertEqual - - text.insert('insert', 'ab xy yz\n') - text.insert('insert', 'a ac by ac') - - text.mark_set('insert', '2.1') - equal(self.auto_expand.getwords(), ['ab', 'ac', 'a']) - expand('event') - equal(previous(), 'ab') - expand('event') - equal(previous(), 'ac') - expand('event') - equal(previous(), 'a') - - def test_other_expand_cases(self): - text = self.text - expand = self.auto_expand.expand_word_event - equal = self.assertEqual - - # no expansion candidate found - equal(self.auto_expand.getwords(), []) - equal(expand('event'), 'break') - - text.insert('insert', 'bx cy dz a') - equal(self.auto_expand.getwords(), []) - - # reset state by successfully expanding once - # move cursor to another position and expand again - text.insert('insert', 'ac xy a ac ad a') - text.mark_set('insert', '1.7') - expand('event') - initial_state = self.auto_expand.state - text.mark_set('insert', '1.end') - expand('event') - new_state = self.auto_expand.state - self.assertNotEqual(initial_state, new_state) - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_calltips.py --- a/Lib/idlelib/idle_test/test_calltips.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,177 +0,0 @@ -import unittest -import idlelib.CallTips as ct -import textwrap -import types - -default_tip = ct._default_callable_argspec - -# Test Class TC is used in multiple get_argspec test methods -class TC(): - 'doc' - tip = "(ai=None, *b)" - def __init__(self, ai=None, *b): 'doc' - __init__.tip = "(self, ai=None, *b)" - def t1(self): 'doc' - t1.tip = "(self)" - def t2(self, ai, b=None): 'doc' - t2.tip = "(self, ai, b=None)" - def t3(self, ai, *args): 'doc' - t3.tip = "(self, ai, *args)" - def t4(self, *args): 'doc' - t4.tip = "(self, *args)" - def t5(self, ai, b=None, *args, **kw): 'doc' - t5.tip = "(self, ai, b=None, *args, **kw)" - def t6(no, self): 'doc' - t6.tip = "(no, self)" - def __call__(self, ci): 'doc' - __call__.tip = "(self, ci)" - # attaching .tip to wrapped methods does not work - @classmethod - def cm(cls, a): 'doc' - @staticmethod - def sm(b): 'doc' - -tc = TC() - -signature = ct.get_argspec # 2.7 and 3.x use different functions -class Get_signatureTest(unittest.TestCase): - # The signature function must return a string, even if blank. - # Test a variety of objects to be sure that none cause it to raise - # (quite aside from getting as correct an answer as possible). - # The tests of builtins may break if inspect or the docstrings change, - # but a red buildbot is better than a user crash (as has happened). - # For a simple mismatch, change the expected output to the actual. - - def test_builtins(self): - - # Python class that inherits builtin methods - class List(list): "List() doc" - # Simulate builtin with no docstring for default tip test - class SB: __call__ = None - - def gtest(obj, out): - self.assertEqual(signature(obj), out) - - if List.__doc__ is not None: - gtest(List, List.__doc__) - gtest(list.__new__, - 'Create and return a new object. See help(type) for accurate signature.') - gtest(list.__init__, - 'Initialize self. See help(type(self)) for accurate signature.') - append_doc = "L.append(object) -> None -- append object to end" - gtest(list.append, append_doc) - gtest([].append, append_doc) - gtest(List.append, append_doc) - - gtest(types.MethodType, "method(function, instance)") - gtest(SB(), default_tip) - - def test_signature_wrap(self): - if textwrap.TextWrapper.__doc__ is not None: - self.assertEqual(signature(textwrap.TextWrapper), '''\ -(width=70, initial_indent='', subsequent_indent='', expand_tabs=True, - replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, - drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, - placeholder=' [...]')''') - - def test_docline_truncation(self): - def f(): pass - f.__doc__ = 'a'*300 - self.assertEqual(signature(f), '()\n' + 'a' * (ct._MAX_COLS-3) + '...') - - def test_multiline_docstring(self): - # Test fewer lines than max. - self.assertEqual(signature(list), - "list() -> new empty list\n" - "list(iterable) -> new list initialized from iterable's items") - - # Test max lines - self.assertEqual(signature(bytes), '''\ -bytes(iterable_of_ints) -> bytes -bytes(string, encoding[, errors]) -> bytes -bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer -bytes(int) -> bytes object of size given by the parameter initialized with null bytes -bytes() -> empty bytes object''') - - # Test more than max lines - def f(): pass - f.__doc__ = 'a\n' * 15 - self.assertEqual(signature(f), '()' + '\na' * ct._MAX_LINES) - - def test_functions(self): - def t1(): 'doc' - t1.tip = "()" - def t2(a, b=None): 'doc' - t2.tip = "(a, b=None)" - def t3(a, *args): 'doc' - t3.tip = "(a, *args)" - def t4(*args): 'doc' - t4.tip = "(*args)" - def t5(a, b=None, *args, **kw): 'doc' - t5.tip = "(a, b=None, *args, **kw)" - - doc = '\ndoc' if t1.__doc__ is not None else '' - for func in (t1, t2, t3, t4, t5, TC): - self.assertEqual(signature(func), func.tip + doc) - - def test_methods(self): - doc = '\ndoc' if TC.__doc__ is not None else '' - for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__): - self.assertEqual(signature(meth), meth.tip + doc) - self.assertEqual(signature(TC.cm), "(a)" + doc) - self.assertEqual(signature(TC.sm), "(b)" + doc) - - def test_bound_methods(self): - # test that first parameter is correctly removed from argspec - doc = '\ndoc' if TC.__doc__ is not None else '' - for meth, mtip in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"), - (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),): - self.assertEqual(signature(meth), mtip + doc) - - def test_starred_parameter(self): - # test that starred first parameter is *not* removed from argspec - class C: - def m1(*args): pass - def m2(**kwds): pass - c = C() - for meth, mtip in ((C.m1, '(*args)'), (c.m1, "(*args)"), - (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),): - self.assertEqual(signature(meth), mtip) - - def test_non_ascii_name(self): - # test that re works to delete a first parameter name that - # includes non-ascii chars, such as various forms of A. - uni = "(A\u0391\u0410\u05d0\u0627\u0905\u1e00\u3042, a)" - assert ct._first_param.sub('', uni) == '(a)' - - def test_no_docstring(self): - def nd(s): - pass - TC.nd = nd - self.assertEqual(signature(nd), "(s)") - self.assertEqual(signature(TC.nd), "(s)") - self.assertEqual(signature(tc.nd), "()") - - def test_attribute_exception(self): - class NoCall: - def __getattr__(self, name): - raise BaseException - class Call(NoCall): - def __call__(self, ci): - pass - for meth, mtip in ((NoCall, default_tip), (Call, default_tip), - (NoCall(), ''), (Call(), '(ci)')): - self.assertEqual(signature(meth), mtip) - - def test_non_callables(self): - for obj in (0, 0.0, '0', b'0', [], {}): - self.assertEqual(signature(obj), '') - -class Get_entityTest(unittest.TestCase): - def test_bad_entity(self): - self.assertIsNone(ct.get_entity('1/0')) - def test_good_entity(self): - self.assertIs(ct.get_entity('int'), int) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_config_name.py --- a/Lib/idlelib/idle_test/test_config_name.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,75 +0,0 @@ -"""Unit tests for idlelib.configSectionNameDialog""" -import unittest -from idlelib.idle_test.mock_tk import Var, Mbox -from idlelib import configSectionNameDialog as name_dialog_module - -name_dialog = name_dialog_module.GetCfgSectionNameDialog - -class Dummy_name_dialog: - # Mock for testing the following methods of name_dialog - name_ok = name_dialog.name_ok - Ok = name_dialog.Ok - Cancel = name_dialog.Cancel - # Attributes, constant or variable, needed for tests - used_names = ['used'] - name = Var() - result = None - destroyed = False - def destroy(self): - self.destroyed = True - -# name_ok calls Mbox.showerror if name is not ok -orig_mbox = name_dialog_module.tkMessageBox -showerror = Mbox.showerror - -class ConfigNameTest(unittest.TestCase): - dialog = Dummy_name_dialog() - - @classmethod - def setUpClass(cls): - name_dialog_module.tkMessageBox = Mbox - - @classmethod - def tearDownClass(cls): - name_dialog_module.tkMessageBox = orig_mbox - - def test_blank_name(self): - self.dialog.name.set(' ') - self.assertEqual(self.dialog.name_ok(), '') - self.assertEqual(showerror.title, 'Name Error') - self.assertIn('No', showerror.message) - - def test_used_name(self): - self.dialog.name.set('used') - self.assertEqual(self.dialog.name_ok(), '') - self.assertEqual(showerror.title, 'Name Error') - self.assertIn('use', showerror.message) - - def test_long_name(self): - self.dialog.name.set('good'*8) - self.assertEqual(self.dialog.name_ok(), '') - self.assertEqual(showerror.title, 'Name Error') - self.assertIn('too long', showerror.message) - - def test_good_name(self): - self.dialog.name.set(' good ') - showerror.title = 'No Error' # should not be called - self.assertEqual(self.dialog.name_ok(), 'good') - self.assertEqual(showerror.title, 'No Error') - - def test_ok(self): - self.dialog.destroyed = False - self.dialog.name.set('good') - self.dialog.Ok() - self.assertEqual(self.dialog.result, 'good') - self.assertTrue(self.dialog.destroyed) - - def test_cancel(self): - self.dialog.destroyed = False - self.dialog.Cancel() - self.assertEqual(self.dialog.result, '') - self.assertTrue(self.dialog.destroyed) - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_configdialog.py --- a/Lib/idlelib/idle_test/test_configdialog.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,32 +0,0 @@ -'''Unittests for idlelib/configHandler.py - -Coverage: 46% just by creating dialog. The other half is change code. - -''' -import unittest -from test.support import requires -from tkinter import Tk -from idlelib.configDialog import ConfigDialog -from idlelib.macosxSupport import _initializeTkVariantTests - - -class ConfigDialogTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - _initializeTkVariantTests(cls.root) - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def test_dialog(self): - d=ConfigDialog(self.root, 'Test', _utest=True) - d.destroy() - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_delegator.py --- a/Lib/idlelib/idle_test/test_delegator.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,37 +0,0 @@ -import unittest -from idlelib.Delegator import Delegator - -class DelegatorTest(unittest.TestCase): - - def test_mydel(self): - # test a simple use scenario - - # initialize - mydel = Delegator(int) - self.assertIs(mydel.delegate, int) - self.assertEqual(mydel._Delegator__cache, set()) - - # add an attribute: - self.assertRaises(AttributeError, mydel.__getattr__, 'xyz') - bl = mydel.bit_length - self.assertIs(bl, int.bit_length) - self.assertIs(mydel.__dict__['bit_length'], int.bit_length) - self.assertEqual(mydel._Delegator__cache, {'bit_length'}) - - # add a second attribute - mydel.numerator - self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'}) - - # delete the second (which, however, leaves it in the name cache) - del mydel.numerator - self.assertNotIn('numerator', mydel.__dict__) - self.assertIn('numerator', mydel._Delegator__cache) - - # reset by calling .setdelegate, which calls .resetcache - mydel.setdelegate(float) - self.assertIs(mydel.delegate, float) - self.assertNotIn('bit_length', mydel.__dict__) - self.assertEqual(mydel._Delegator__cache, set()) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_editor.py --- a/Lib/idlelib/idle_test/test_editor.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ -import unittest -from tkinter import Tk, Text -from idlelib.EditorWindow import EditorWindow -from test.support import requires - -class Editor_func_test(unittest.TestCase): - def test_filename_to_unicode(self): - func = EditorWindow._filename_to_unicode - class dummy(): filesystemencoding = 'utf-8' - pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'), - (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc')) - for inp, out in pairs: - self.assertEqual(func(dummy, inp), out) - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_formatparagraph.py --- a/Lib/idlelib/idle_test/test_formatparagraph.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,377 +0,0 @@ -# Test the functions and main class method of FormatParagraph.py -import unittest -from idlelib import FormatParagraph as fp -from idlelib.EditorWindow import EditorWindow -from tkinter import Tk, Text -from test.support import requires - - -class Is_Get_Test(unittest.TestCase): - """Test the is_ and get_ functions""" - test_comment = '# This is a comment' - test_nocomment = 'This is not a comment' - trailingws_comment = '# This is a comment ' - leadingws_comment = ' # This is a comment' - leadingws_nocomment = ' This is not a comment' - - def test_is_all_white(self): - self.assertTrue(fp.is_all_white('')) - self.assertTrue(fp.is_all_white('\t\n\r\f\v')) - self.assertFalse(fp.is_all_white(self.test_comment)) - - def test_get_indent(self): - Equal = self.assertEqual - Equal(fp.get_indent(self.test_comment), '') - Equal(fp.get_indent(self.trailingws_comment), '') - Equal(fp.get_indent(self.leadingws_comment), ' ') - Equal(fp.get_indent(self.leadingws_nocomment), ' ') - - def test_get_comment_header(self): - Equal = self.assertEqual - # Test comment strings - Equal(fp.get_comment_header(self.test_comment), '#') - Equal(fp.get_comment_header(self.trailingws_comment), '#') - Equal(fp.get_comment_header(self.leadingws_comment), ' #') - # Test non-comment strings - Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') - Equal(fp.get_comment_header(self.test_nocomment), '') - - -class FindTest(unittest.TestCase): - """Test the find_paragraph function in FormatParagraph. - - Using the runcase() function, find_paragraph() is called with 'mark' set at - multiple indexes before and inside the test paragraph. - - It appears that code with the same indentation as a quoted string is grouped - as part of the same paragraph, which is probably incorrect behavior. - """ - - @classmethod - def setUpClass(cls): - from idlelib.idle_test.mock_tk import Text - cls.text = Text() - - def runcase(self, inserttext, stopline, expected): - # Check that find_paragraph returns the expected paragraph when - # the mark index is set to beginning, middle, end of each line - # up to but not including the stop line - text = self.text - text.insert('1.0', inserttext) - for line in range(1, stopline): - linelength = int(text.index("%d.end" % line).split('.')[1]) - for col in (0, linelength//2, linelength): - tempindex = "%d.%d" % (line, col) - self.assertEqual(fp.find_paragraph(text, tempindex), expected) - text.delete('1.0', 'end') - - def test_find_comment(self): - comment = ( - "# Comment block with no blank lines before\n" - "# Comment line\n" - "\n") - self.runcase(comment, 3, ('1.0', '3.0', '#', comment[0:58])) - - comment = ( - "\n" - "# Comment block with whitespace line before and after\n" - "# Comment line\n" - "\n") - self.runcase(comment, 4, ('2.0', '4.0', '#', comment[1:70])) - - comment = ( - "\n" - " # Indented comment block with whitespace before and after\n" - " # Comment line\n" - "\n") - self.runcase(comment, 4, ('2.0', '4.0', ' #', comment[1:82])) - - comment = ( - "\n" - "# Single line comment\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:23])) - - comment = ( - "\n" - " # Single line comment with leading whitespace\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:51])) - - comment = ( - "\n" - "# Comment immediately followed by code\n" - "x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:40])) - - comment = ( - "\n" - " # Indented comment immediately followed by code\n" - "x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:53])) - - comment = ( - "\n" - "# Comment immediately followed by indented code\n" - " x = 42\n" - "\n") - self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:49])) - - def test_find_paragraph(self): - teststring = ( - '"""String with no blank lines before\n' - 'String line\n' - '"""\n' - '\n') - self.runcase(teststring, 4, ('1.0', '4.0', '', teststring[0:53])) - - teststring = ( - "\n" - '"""String with whitespace line before and after\n' - 'String line.\n' - '"""\n' - '\n') - self.runcase(teststring, 5, ('2.0', '5.0', '', teststring[1:66])) - - teststring = ( - '\n' - ' """Indented string with whitespace before and after\n' - ' Comment string.\n' - ' """\n' - '\n') - self.runcase(teststring, 5, ('2.0', '5.0', ' ', teststring[1:85])) - - teststring = ( - '\n' - '"""Single line string."""\n' - '\n') - self.runcase(teststring, 3, ('2.0', '3.0', '', teststring[1:27])) - - teststring = ( - '\n' - ' """Single line string with leading whitespace."""\n' - '\n') - self.runcase(teststring, 3, ('2.0', '3.0', ' ', teststring[1:55])) - - -class ReformatFunctionTest(unittest.TestCase): - """Test the reformat_paragraph function without the editor window.""" - - def test_reformat_paragrah(self): - Equal = self.assertEqual - reform = fp.reformat_paragraph - hw = "O hello world" - Equal(reform(' ', 1), ' ') - Equal(reform("Hello world", 20), "Hello world") - - # Test without leading newline - Equal(reform(hw, 1), "O\nhello\nworld") - Equal(reform(hw, 6), "O\nhello\nworld") - Equal(reform(hw, 7), "O hello\nworld") - Equal(reform(hw, 12), "O hello\nworld") - Equal(reform(hw, 13), "O hello world") - - # Test with leading newline - hw = "\nO hello world" - Equal(reform(hw, 1), "\nO\nhello\nworld") - Equal(reform(hw, 6), "\nO\nhello\nworld") - Equal(reform(hw, 7), "\nO hello\nworld") - Equal(reform(hw, 12), "\nO hello\nworld") - Equal(reform(hw, 13), "\nO hello world") - - -class ReformatCommentTest(unittest.TestCase): - """Test the reformat_comment function without the editor window.""" - - def test_reformat_comment(self): - Equal = self.assertEqual - - # reformat_comment formats to a minimum of 20 characters - test_string = ( - " \"\"\"this is a test of a reformat for a triple quoted string" - " will it reformat to less than 70 characters for me?\"\"\"") - result = fp.reformat_comment(test_string, 70, " ") - expected = ( - " \"\"\"this is a test of a reformat for a triple quoted string will it\n" - " reformat to less than 70 characters for me?\"\"\"") - Equal(result, expected) - - test_comment = ( - "# this is a test of a reformat for a triple quoted string will " - "it reformat to less than 70 characters for me?") - result = fp.reformat_comment(test_comment, 70, "#") - expected = ( - "# this is a test of a reformat for a triple quoted string will it\n" - "# reformat to less than 70 characters for me?") - Equal(result, expected) - - -class FormatClassTest(unittest.TestCase): - def test_init_close(self): - instance = fp.FormatParagraph('editor') - self.assertEqual(instance.editwin, 'editor') - instance.close() - self.assertEqual(instance.editwin, None) - - -# For testing format_paragraph_event, Initialize FormatParagraph with -# a mock Editor with .text and .get_selection_indices. The text must -# be a Text wrapper that adds two methods - -# A real EditorWindow creates unneeded, time-consuming baggage and -# sometimes emits shutdown warnings like this: -# "warning: callback failed in WindowList -# : invalid command name ".55131368.windows". -# Calling EditorWindow._close in tearDownClass prevents this but causes -# other problems (windows left open). - -class TextWrapper: - def __init__(self, master): - self.text = Text(master=master) - def __getattr__(self, name): - return getattr(self.text, name) - def undo_block_start(self): pass - def undo_block_stop(self): pass - -class Editor: - def __init__(self, root): - self.text = TextWrapper(root) - get_selection_indices = EditorWindow. get_selection_indices - -class FormatEventTest(unittest.TestCase): - """Test the formatting of text inside a Text widget. - - This is done with FormatParagraph.format.paragraph_event, - which calls functions in the module as appropriate. - """ - test_string = ( - " '''this is a test of a reformat for a triple " - "quoted string will it reformat to less than 70 " - "characters for me?'''\n") - multiline_test_string = ( - " '''The first line is under the max width.\n" - " The second line's length is way over the max width. It goes " - "on and on until it is over 100 characters long.\n" - " Same thing with the third line. It is also way over the max " - "width, but FormatParagraph will fix it.\n" - " '''\n") - multiline_test_comment = ( - "# The first line is under the max width.\n" - "# The second line's length is way over the max width. It goes on " - "and on until it is over 100 characters long.\n" - "# Same thing with the third line. It is also way over the max " - "width, but FormatParagraph will fix it.\n" - "# The fourth line is short like the first line.") - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - editor = Editor(root=cls.root) - cls.text = editor.text.text # Test code does not need the wrapper. - cls.formatter = fp.FormatParagraph(editor).format_paragraph_event - # Sets the insert mark just after the re-wrapped and inserted text. - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - del cls.text - del cls.formatter - - def test_short_line(self): - self.text.insert('1.0', "Short line\n") - self.formatter("Dummy") - self.assertEqual(self.text.get('1.0', 'insert'), "Short line\n" ) - self.text.delete('1.0', 'end') - - def test_long_line(self): - text = self.text - - # Set cursor ('insert' mark) to '1.0', within text. - text.insert('1.0', self.test_string) - text.mark_set('insert', '1.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - # find function includes \n - expected = ( -" '''this is a test of a reformat for a triple quoted string will it\n" -" reformat to less than 70 characters for me?'''\n") # yes - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - # Select from 1.11 to line end. - text.insert('1.0', self.test_string) - text.tag_add('sel', '1.11', '1.end') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - # selection excludes \n - expected = ( -" '''this is a test of a reformat for a triple quoted string will it reformat\n" -" to less than 70 characters for me?'''") # no - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - def test_multiple_lines(self): - text = self.text - # Select 2 long lines. - text.insert('1.0', self.multiline_test_string) - text.tag_add('sel', '2.0', '4.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('2.0', 'insert') - expected = ( -" The second line's length is way over the max width. It goes on and\n" -" on until it is over 100 characters long. Same thing with the third\n" -" line. It is also way over the max width, but FormatParagraph will\n" -" fix it.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - def test_comment_block(self): - text = self.text - - # Set cursor ('insert') to '1.0', within block. - text.insert('1.0', self.multiline_test_comment) - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - expected = ( -"# The first line is under the max width. The second line's length is\n" -"# way over the max width. It goes on and on until it is over 100\n" -"# characters long. Same thing with the third line. It is also way over\n" -"# the max width, but FormatParagraph will fix it. The fourth line is\n" -"# short like the first line.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - - # Select line 2, verify line 1 unaffected. - text.insert('1.0', self.multiline_test_comment) - text.tag_add('sel', '2.0', '3.0') - self.formatter('ParameterDoesNothing', limit=70) - result = text.get('1.0', 'insert') - expected = ( -"# The first line is under the max width.\n" -"# The second line's length is way over the max width. It goes on and\n" -"# on until it is over 100 characters long.\n") - self.assertEqual(result, expected) - text.delete('1.0', 'end') - -# The following block worked with EditorWindow but fails with the mock. -# Lines 2 and 3 get pasted together even though the previous block left -# the previous line alone. More investigation is needed. -## # Select lines 3 and 4 -## text.insert('1.0', self.multiline_test_comment) -## text.tag_add('sel', '3.0', '5.0') -## self.formatter('ParameterDoesNothing') -## result = text.get('3.0', 'insert') -## expected = ( -##"# Same thing with the third line. It is also way over the max width,\n" -##"# but FormatParagraph will fix it. The fourth line is short like the\n" -##"# first line.\n") -## self.assertEqual(result, expected) -## text.delete('1.0', 'end') - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_grep.py --- a/Lib/idlelib/idle_test/test_grep.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,80 +0,0 @@ -""" !Changing this line will break Test_findfile.test_found! -Non-gui unit tests for idlelib.GrepDialog methods. -dummy_command calls grep_it calls findfiles. -An exception raised in one method will fail callers. -Otherwise, tests are mostly independent. -*** Currently only test grep_it. -""" -import unittest -from test.support import captured_stdout -from idlelib.idle_test.mock_tk import Var -from idlelib.GrepDialog import GrepDialog -import re - -class Dummy_searchengine: - '''GrepDialog.__init__ calls parent SearchDiabolBase which attaches the - passed in SearchEngine instance as attribute 'engine'. Only a few of the - many possible self.engine.x attributes are needed here. - ''' - def getpat(self): - return self._pat - -searchengine = Dummy_searchengine() - -class Dummy_grep: - # Methods tested - #default_command = GrepDialog.default_command - grep_it = GrepDialog.grep_it - findfiles = GrepDialog.findfiles - # Other stuff needed - recvar = Var(False) - engine = searchengine - def close(self): # gui method - pass - -grep = Dummy_grep() - -class FindfilesTest(unittest.TestCase): - # findfiles is really a function, not a method, could be iterator - # test that filename return filename - # test that idlelib has many .py files - # test that recursive flag adds idle_test .py files - pass - -class Grep_itTest(unittest.TestCase): - # Test captured reports with 0 and some hits. - # Should test file names, but Windows reports have mixed / and \ separators - # from incomplete replacement, so 'later'. - - def report(self, pat): - grep.engine._pat = pat - with captured_stdout() as s: - grep.grep_it(re.compile(pat), __file__) - lines = s.getvalue().split('\n') - lines.pop() # remove bogus '' after last \n - return lines - - def test_unfound(self): - pat = 'xyz*'*7 - lines = self.report(pat) - self.assertEqual(len(lines), 2) - self.assertIn(pat, lines[0]) - self.assertEqual(lines[1], 'No hits.') - - def test_found(self): - - pat = '""" !Changing this line will break Test_findfile.test_found!' - lines = self.report(pat) - self.assertEqual(len(lines), 5) - self.assertIn(pat, lines[0]) - self.assertIn('py: 1:', lines[1]) # line number 1 - self.assertIn('2', lines[3]) # hits found 2 - self.assertTrue(lines[4].startswith('(Hint:')) - -class Default_commandTest(unittest.TestCase): - # To write this, mode OutputWindow import to top of GrepDialog - # so it can be replaced by captured_stdout in class setup/teardown. - pass - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_hyperparser.py --- a/Lib/idlelib/idle_test/test_hyperparser.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,273 +0,0 @@ -"""Unittest for idlelib.HyperParser""" -import unittest -from test.support import requires -from tkinter import Tk, Text -from idlelib.EditorWindow import EditorWindow -from idlelib.HyperParser import HyperParser - -class DummyEditwin: - def __init__(self, text): - self.text = text - self.indentwidth = 8 - self.tabwidth = 8 - self.context_use_ps1 = True - self.num_context_lines = 50, 500, 1000 - - _build_char_in_string_func = EditorWindow._build_char_in_string_func - is_char_in_string = EditorWindow.is_char_in_string - - -class HyperParserTest(unittest.TestCase): - code = ( - '"""This is a module docstring"""\n' - '# this line is a comment\n' - 'x = "this is a string"\n' - "y = 'this is also a string'\n" - 'l = [i for i in range(10)]\n' - 'm = [py*py for # comment\n' - ' py in l]\n' - 'x.__len__\n' - "z = ((r'asdf')+('a')))\n" - '[x for x in\n' - 'for = False\n' - 'cliché = "this is a string with unicode, what a cliché"' - ) - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - cls.text = Text(cls.root) - cls.editwin = DummyEditwin(cls.text) - - @classmethod - def tearDownClass(cls): - del cls.text, cls.editwin - cls.root.destroy() - del cls.root - - def setUp(self): - self.text.insert('insert', self.code) - - def tearDown(self): - self.text.delete('1.0', 'end') - self.editwin.context_use_ps1 = True - - def get_parser(self, index): - """ - Return a parser object with index at 'index' - """ - return HyperParser(self.editwin, index) - - def test_init(self): - """ - test corner cases in the init method - """ - with self.assertRaises(ValueError) as ve: - self.text.tag_add('console', '1.0', '1.end') - p = self.get_parser('1.5') - self.assertIn('precedes', str(ve.exception)) - - # test without ps1 - self.editwin.context_use_ps1 = False - - # number of lines lesser than 50 - p = self.get_parser('end') - self.assertEqual(p.rawtext, self.text.get('1.0', 'end')) - - # number of lines greater than 50 - self.text.insert('end', self.text.get('1.0', 'end')*4) - p = self.get_parser('54.5') - - def test_is_in_string(self): - get = self.get_parser - - p = get('1.0') - self.assertFalse(p.is_in_string()) - p = get('1.4') - self.assertTrue(p.is_in_string()) - p = get('2.3') - self.assertFalse(p.is_in_string()) - p = get('3.3') - self.assertFalse(p.is_in_string()) - p = get('3.7') - self.assertTrue(p.is_in_string()) - p = get('4.6') - self.assertTrue(p.is_in_string()) - p = get('12.54') - self.assertTrue(p.is_in_string()) - - def test_is_in_code(self): - get = self.get_parser - - p = get('1.0') - self.assertTrue(p.is_in_code()) - p = get('1.1') - self.assertFalse(p.is_in_code()) - p = get('2.5') - self.assertFalse(p.is_in_code()) - p = get('3.4') - self.assertTrue(p.is_in_code()) - p = get('3.6') - self.assertFalse(p.is_in_code()) - p = get('4.14') - self.assertFalse(p.is_in_code()) - - def test_get_surrounding_bracket(self): - get = self.get_parser - - def without_mustclose(parser): - # a utility function to get surrounding bracket - # with mustclose=False - return parser.get_surrounding_brackets(mustclose=False) - - def with_mustclose(parser): - # a utility function to get surrounding bracket - # with mustclose=True - return parser.get_surrounding_brackets(mustclose=True) - - p = get('3.2') - self.assertIsNone(with_mustclose(p)) - self.assertIsNone(without_mustclose(p)) - - p = get('5.6') - self.assertTupleEqual(without_mustclose(p), ('5.4', '5.25')) - self.assertTupleEqual(without_mustclose(p), with_mustclose(p)) - - p = get('5.23') - self.assertTupleEqual(without_mustclose(p), ('5.21', '5.24')) - self.assertTupleEqual(without_mustclose(p), with_mustclose(p)) - - p = get('6.15') - self.assertTupleEqual(without_mustclose(p), ('6.4', '6.end')) - self.assertIsNone(with_mustclose(p)) - - p = get('9.end') - self.assertIsNone(with_mustclose(p)) - self.assertIsNone(without_mustclose(p)) - - def test_get_expression(self): - get = self.get_parser - - p = get('4.2') - self.assertEqual(p.get_expression(), 'y ') - - p = get('4.7') - with self.assertRaises(ValueError) as ve: - p.get_expression() - self.assertIn('is inside a code', str(ve.exception)) - - p = get('5.25') - self.assertEqual(p.get_expression(), 'range(10)') - - p = get('6.7') - self.assertEqual(p.get_expression(), 'py') - - p = get('6.8') - self.assertEqual(p.get_expression(), '') - - p = get('7.9') - self.assertEqual(p.get_expression(), 'py') - - p = get('8.end') - self.assertEqual(p.get_expression(), 'x.__len__') - - p = get('9.13') - self.assertEqual(p.get_expression(), "r'asdf'") - - p = get('9.17') - with self.assertRaises(ValueError) as ve: - p.get_expression() - self.assertIn('is inside a code', str(ve.exception)) - - p = get('10.0') - self.assertEqual(p.get_expression(), '') - - p = get('10.6') - self.assertEqual(p.get_expression(), '') - - p = get('10.11') - self.assertEqual(p.get_expression(), '') - - p = get('11.3') - self.assertEqual(p.get_expression(), '') - - p = get('11.11') - self.assertEqual(p.get_expression(), 'False') - - p = get('12.6') - self.assertEqual(p.get_expression(), 'cliché') - - def test_eat_identifier(self): - def is_valid_id(candidate): - result = HyperParser._eat_identifier(candidate, 0, len(candidate)) - if result == len(candidate): - return True - elif result == 0: - return False - else: - err_msg = "Unexpected result: {} (expected 0 or {}".format( - result, len(candidate) - ) - raise Exception(err_msg) - - # invalid first character which is valid elsewhere in an identifier - self.assertFalse(is_valid_id('2notid')) - - # ASCII-only valid identifiers - self.assertTrue(is_valid_id('valid_id')) - self.assertTrue(is_valid_id('_valid_id')) - self.assertTrue(is_valid_id('valid_id_')) - self.assertTrue(is_valid_id('_2valid_id')) - - # keywords which should be "eaten" - self.assertTrue(is_valid_id('True')) - self.assertTrue(is_valid_id('False')) - self.assertTrue(is_valid_id('None')) - - # keywords which should not be "eaten" - self.assertFalse(is_valid_id('for')) - self.assertFalse(is_valid_id('import')) - self.assertFalse(is_valid_id('return')) - - # valid unicode identifiers - self.assertTrue(is_valid_id('cliche')) - self.assertTrue(is_valid_id('cliché')) - self.assertTrue(is_valid_id('a٢')) - - # invalid unicode identifiers - self.assertFalse(is_valid_id('2a')) - self.assertFalse(is_valid_id('٢a')) - self.assertFalse(is_valid_id('a²')) - - # valid identifier after "punctuation" - self.assertEqual(HyperParser._eat_identifier('+ var', 0, 5), len('var')) - self.assertEqual(HyperParser._eat_identifier('+var', 0, 4), len('var')) - self.assertEqual(HyperParser._eat_identifier('.var', 0, 4), len('var')) - - # invalid identifiers - self.assertFalse(is_valid_id('+')) - self.assertFalse(is_valid_id(' ')) - self.assertFalse(is_valid_id(':')) - self.assertFalse(is_valid_id('?')) - self.assertFalse(is_valid_id('^')) - self.assertFalse(is_valid_id('\\')) - self.assertFalse(is_valid_id('"')) - self.assertFalse(is_valid_id('"a string"')) - - def test_eat_identifier_various_lengths(self): - eat_id = HyperParser._eat_identifier - - for length in range(1, 21): - self.assertEqual(eat_id('a' * length, 0, length), length) - self.assertEqual(eat_id('é' * length, 0, length), length) - self.assertEqual(eat_id('a' + '2' * (length - 1), 0, length), length) - self.assertEqual(eat_id('é' + '2' * (length - 1), 0, length), length) - self.assertEqual(eat_id('é' + 'a' * (length - 1), 0, length), length) - self.assertEqual(eat_id('é' * (length - 1) + 'a', 0, length), length) - self.assertEqual(eat_id('+' * length, 0, length), 0) - self.assertEqual(eat_id('2' + 'a' * (length - 1), 0, length), 0) - self.assertEqual(eat_id('2' + 'é' * (length - 1), 0, length), 0) - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_idlehistory.py --- a/Lib/idlelib/idle_test/test_idlehistory.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,167 +0,0 @@ -import unittest -from test.support import requires - -import tkinter as tk -from tkinter import Text as tkText -from idlelib.idle_test.mock_tk import Text as mkText -from idlelib.IdleHistory import History -from idlelib.configHandler import idleConf - -line1 = 'a = 7' -line2 = 'b = a' - -class StoreTest(unittest.TestCase): - '''Tests History.__init__ and History.store with mock Text''' - - @classmethod - def setUpClass(cls): - cls.text = mkText() - cls.history = History(cls.text) - - def tearDown(self): - self.text.delete('1.0', 'end') - self.history.history = [] - - def test_init(self): - self.assertIs(self.history.text, self.text) - self.assertEqual(self.history.history, []) - self.assertIsNone(self.history.prefix) - self.assertIsNone(self.history.pointer) - self.assertEqual(self.history.cyclic, - idleConf.GetOption("main", "History", "cyclic", 1, "bool")) - - def test_store_short(self): - self.history.store('a') - self.assertEqual(self.history.history, []) - self.history.store(' a ') - self.assertEqual(self.history.history, []) - - def test_store_dup(self): - self.history.store(line1) - self.assertEqual(self.history.history, [line1]) - self.history.store(line2) - self.assertEqual(self.history.history, [line1, line2]) - self.history.store(line1) - self.assertEqual(self.history.history, [line2, line1]) - - def test_store_reset(self): - self.history.prefix = line1 - self.history.pointer = 0 - self.history.store(line2) - self.assertIsNone(self.history.prefix) - self.assertIsNone(self.history.pointer) - - -class TextWrapper: - def __init__(self, master): - self.text = tkText(master=master) - self._bell = False - def __getattr__(self, name): - return getattr(self.text, name) - def bell(self): - self._bell = True - -class FetchTest(unittest.TestCase): - '''Test History.fetch with wrapped tk.Text. - ''' - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = tk.Tk() - - def setUp(self): - self.text = text = TextWrapper(self.root) - text.insert('1.0', ">>> ") - text.mark_set('iomark', '1.4') - text.mark_gravity('iomark', 'left') - self.history = History(text) - self.history.history = [line1, line2] - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def fetch_test(self, reverse, line, prefix, index, *, bell=False): - # Perform one fetch as invoked by Alt-N or Alt-P - # Test the result. The line test is the most important. - # The last two are diagnostic of fetch internals. - History = self.history - History.fetch(reverse) - - Equal = self.assertEqual - Equal(self.text.get('iomark', 'end-1c'), line) - Equal(self.text._bell, bell) - if bell: - self.text._bell = False - Equal(History.prefix, prefix) - Equal(History.pointer, index) - Equal(self.text.compare("insert", '==', "end-1c"), 1) - - def test_fetch_prev_cyclic(self): - prefix = '' - test = self.fetch_test - test(True, line2, prefix, 1) - test(True, line1, prefix, 0) - test(True, prefix, None, None, bell=True) - - def test_fetch_next_cyclic(self): - prefix = '' - test = self.fetch_test - test(False, line1, prefix, 0) - test(False, line2, prefix, 1) - test(False, prefix, None, None, bell=True) - - # Prefix 'a' tests skip line2, which starts with 'b' - def test_fetch_prev_prefix(self): - prefix = 'a' - self.text.insert('iomark', prefix) - self.fetch_test(True, line1, prefix, 0) - self.fetch_test(True, prefix, None, None, bell=True) - - def test_fetch_next_prefix(self): - prefix = 'a' - self.text.insert('iomark', prefix) - self.fetch_test(False, line1, prefix, 0) - self.fetch_test(False, prefix, None, None, bell=True) - - def test_fetch_prev_noncyclic(self): - prefix = '' - self.history.cyclic = False - test = self.fetch_test - test(True, line2, prefix, 1) - test(True, line1, prefix, 0) - test(True, line1, prefix, 0, bell=True) - - def test_fetch_next_noncyclic(self): - prefix = '' - self.history.cyclic = False - test = self.fetch_test - test(False, prefix, None, None, bell=True) - test(True, line2, prefix, 1) - test(False, prefix, None, None, bell=True) - test(False, prefix, None, None, bell=True) - - def test_fetch_cursor_move(self): - # Move cursor after fetch - self.history.fetch(reverse=True) # initialization - self.text.mark_set('insert', 'iomark') - self.fetch_test(True, line2, None, None, bell=True) - - def test_fetch_edit(self): - # Edit after fetch - self.history.fetch(reverse=True) # initialization - self.text.delete('iomark', 'insert', ) - self.text.insert('iomark', 'a =') - self.fetch_test(True, line1, 'a =', 0) # prefix is reset - - def test_history_prev_next(self): - # Minimally test functions bound to events - self.history.history_prev('dummy event') - self.assertEqual(self.history.pointer, 1) - self.history.history_next('dummy event') - self.assertEqual(self.history.pointer, None) - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_io.py --- a/Lib/idlelib/idle_test/test_io.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,233 +0,0 @@ -import unittest -import io -from idlelib.PyShell import PseudoInputFile, PseudoOutputFile - - -class S(str): - def __str__(self): - return '%s:str' % type(self).__name__ - def __unicode__(self): - return '%s:unicode' % type(self).__name__ - def __len__(self): - return 3 - def __iter__(self): - return iter('abc') - def __getitem__(self, *args): - return '%s:item' % type(self).__name__ - def __getslice__(self, *args): - return '%s:slice' % type(self).__name__ - -class MockShell: - def __init__(self): - self.reset() - - def write(self, *args): - self.written.append(args) - - def readline(self): - return self.lines.pop() - - def close(self): - pass - - def reset(self): - self.written = [] - - def push(self, lines): - self.lines = list(lines)[::-1] - - -class PseudeOutputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertFalse(f.readable()) - self.assertTrue(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.read, 0) - self.assertRaises(OSError, f.readline, 0) - - def test_write(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.write('test') - self.assertEqual(shell.written, [('test', 'stdout')]) - shell.reset() - f.write('t\xe8st') - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - shell.reset() - - f.write(S('t\xe8st')) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.write) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, b'test') - self.assertRaises(TypeError, f.write, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.write, 'test', 'spam') - self.assertEqual(shell.written, []) - - def test_writelines(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - f.writelines([]) - self.assertEqual(shell.written, []) - shell.reset() - f.writelines(['one\n', 'two']) - self.assertEqual(shell.written, - [('one\n', 'stdout'), ('two', 'stdout')]) - shell.reset() - f.writelines(['on\xe8\n', 'tw\xf2']) - self.assertEqual(shell.written, - [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) - shell.reset() - - f.writelines([S('t\xe8st')]) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) - self.assertEqual(type(shell.written[0][0]), str) - shell.reset() - - self.assertRaises(TypeError, f.writelines) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, 123) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [b'test']) - self.assertRaises(TypeError, f.writelines, [123]) - self.assertEqual(shell.written, []) - self.assertRaises(TypeError, f.writelines, [], []) - self.assertEqual(shell.written, []) - - def test_close(self): - shell = MockShell() - f = PseudoOutputFile(shell, 'stdout', 'utf-8') - self.assertFalse(f.closed) - f.write('test') - f.close() - self.assertTrue(f.closed) - self.assertRaises(ValueError, f.write, 'x') - self.assertEqual(shell.written, [('test', 'stdout')]) - f.close() - self.assertRaises(TypeError, f.close, 1) - - -class PseudeInputFilesTest(unittest.TestCase): - def test_misc(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertIsInstance(f, io.TextIOBase) - self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) - self.assertIsNone(f.newlines) - self.assertEqual(f.name, '') - self.assertFalse(f.closed) - self.assertTrue(f.isatty()) - self.assertTrue(f.readable()) - self.assertFalse(f.writable()) - self.assertFalse(f.seekable()) - - def test_unsupported(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - self.assertRaises(OSError, f.fileno) - self.assertRaises(OSError, f.tell) - self.assertRaises(OSError, f.seek, 0) - self.assertRaises(OSError, f.write, 'x') - self.assertRaises(OSError, f.writelines, ['x']) - - def test_read(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(-1), 'one\ntwo\n') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.read(None), 'one\ntwo\n') - shell.push(['one\n', 'two\n', 'three\n', '']) - self.assertEqual(f.read(2), 'on') - self.assertEqual(f.read(3), 'e\nt') - self.assertEqual(f.read(10), 'wo\nthree\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.read(0), '') - self.assertRaises(TypeError, f.read, 1.5) - self.assertRaises(TypeError, f.read, '1') - self.assertRaises(TypeError, f.read, 1, 1) - - def test_readline(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', 'three\n', 'four\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(-1), 'two\n') - self.assertEqual(f.readline(None), 'three\n') - shell.push(['one\ntwo\n']) - self.assertEqual(f.readline(), 'one\n') - self.assertEqual(f.readline(), 'two\n') - shell.push(['one', 'two', 'three']) - self.assertEqual(f.readline(), 'one') - self.assertEqual(f.readline(), 'two') - shell.push(['one\n', 'two\n', 'three\n']) - self.assertEqual(f.readline(2), 'on') - self.assertEqual(f.readline(1), 'e') - self.assertEqual(f.readline(1), '\n') - self.assertEqual(f.readline(10), 'two\n') - - shell.push(['one\n', 'two\n']) - self.assertEqual(f.readline(0), '') - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_readlines(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(None), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(0), ['one\n', 'two\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(3), ['one\n']) - shell.push(['one\n', 'two\n', '']) - self.assertEqual(f.readlines(4), ['one\n', 'two\n']) - - shell.push(['one\n', 'two\n', '']) - self.assertRaises(TypeError, f.readlines, 1.5) - self.assertRaises(TypeError, f.readlines, '1') - self.assertRaises(TypeError, f.readlines, 1, 1) - - def test_close(self): - shell = MockShell() - f = PseudoInputFile(shell, 'stdin', 'utf-8') - shell.push(['one\n', 'two\n', '']) - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'one\n') - f.close() - self.assertFalse(f.closed) - self.assertEqual(f.readline(), 'two\n') - self.assertRaises(TypeError, f.close, 1) - - -if __name__ == '__main__': - unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_parenmatch.py --- a/Lib/idlelib/idle_test/test_parenmatch.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,109 +0,0 @@ -"""Test idlelib.ParenMatch.""" -# This must currently be a gui test because ParenMatch methods use -# several text methods not defined on idlelib.idle_test.mock_tk.Text. - -import unittest -from unittest.mock import Mock -from test.support import requires -from tkinter import Tk, Text -from idlelib.ParenMatch import ParenMatch - -class DummyEditwin: - def __init__(self, text): - self.text = text - self.indentwidth = 8 - self.tabwidth = 8 - self.context_use_ps1 = True - - -class ParenMatchTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - cls.text = Text(cls.root) - cls.editwin = DummyEditwin(cls.text) - cls.editwin.text_frame = Mock() - - @classmethod - def tearDownClass(cls): - del cls.text, cls.editwin - cls.root.destroy() - del cls.root - - def tearDown(self): - self.text.delete('1.0', 'end') - - def test_paren_expression(self): - """ - Test ParenMatch with 'expression' style. - """ - text = self.text - pm = ParenMatch(self.editwin) - pm.set_style('expression') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.15')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) - - # paren_closed_event can only be tested as below - pm.paren_closed_event('event') - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.16')) - - def test_paren_default(self): - """ - Test ParenMatch with 'default' style. - """ - text = self.text - pm = ParenMatch(self.editwin) - pm.set_style('default') - - text.insert('insert', 'def foobar(a, b') - pm.flash_paren_event('event') - self.assertIn('<>', text.event_info()) - self.assertTupleEqual(text.tag_prevrange('paren', 'end'), - ('1.10', '1.11')) - text.insert('insert', ')') - pm.restore_event() - self.assertNotIn('<>', text.event_info()) - self.assertEqual(text.tag_prevrange('paren', 'end'), ()) - - def test_paren_corner(self): - """ - Test corner cases in flash_paren_event and paren_closed_event. - - These cases force conditional expression and alternate paths. - """ - text = self.text - pm = ParenMatch(self.editwin) - - text.insert('insert', '# this is a commen)') - self.assertIsNone(pm.paren_closed_event('event')) - - text.insert('insert', '\ndef') - self.assertIsNone(pm.flash_paren_event('event')) - self.assertIsNone(pm.paren_closed_event('event')) - - text.insert('insert', ' a, *arg)') - self.assertIsNone(pm.paren_closed_event('event')) - - def test_handle_restore_timer(self): - pm = ParenMatch(self.editwin) - pm.restore_event = Mock() - pm.handle_restore_timer(0) - self.assertTrue(pm.restore_event.called) - pm.restore_event.reset_mock() - pm.handle_restore_timer(1) - self.assertFalse(pm.restore_event.called) - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_pathbrowser.py --- a/Lib/idlelib/idle_test/test_pathbrowser.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,27 +0,0 @@ -import unittest -import os -import sys -import idlelib -from idlelib import PathBrowser - -class PathBrowserTest(unittest.TestCase): - - def test_DirBrowserTreeItem(self): - # Issue16226 - make sure that getting a sublist works - d = PathBrowser.DirBrowserTreeItem('') - d.GetSubList() - self.assertEqual('', d.GetText()) - - dir = os.path.split(os.path.abspath(idlelib.__file__))[0] - self.assertEqual(d.ispackagedir(dir), True) - self.assertEqual(d.ispackagedir(dir + '/Icons'), False) - - def test_PathBrowserTreeItem(self): - p = PathBrowser.PathBrowserTreeItem() - self.assertEqual(p.GetText(), 'sys.path') - sub = p.GetSubList() - self.assertEqual(len(sub), len(sys.path)) - self.assertEqual(type(sub[0]), PathBrowser.DirBrowserTreeItem) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_rstrip.py --- a/Lib/idlelib/idle_test/test_rstrip.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,49 +0,0 @@ -import unittest -import idlelib.RstripExtension as rs -from idlelib.idle_test.mock_idle import Editor - -class rstripTest(unittest.TestCase): - - def test_rstrip_line(self): - editor = Editor() - text = editor.text - do_rstrip = rs.RstripExtension(editor).do_rstrip - - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '') - text.insert('1.0', ' ') - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '') - text.insert('1.0', ' \n') - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '\n') - - def test_rstrip_multiple(self): - editor = Editor() - # Uncomment following to verify that test passes with real widgets. -## from idlelib.EditorWindow import EditorWindow as Editor -## from tkinter import Tk -## editor = Editor(root=Tk()) - text = editor.text - do_rstrip = rs.RstripExtension(editor).do_rstrip - - original = ( - "Line with an ending tab \n" - "Line ending in 5 spaces \n" - "Linewithnospaces\n" - " indented line\n" - " indented line with trailing space \n" - " ") - stripped = ( - "Line with an ending tab\n" - "Line ending in 5 spaces\n" - "Linewithnospaces\n" - " indented line\n" - " indented line with trailing space\n") - - text.insert('1.0', original) - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), stripped) - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_searchdialogbase.py --- a/Lib/idlelib/idle_test/test_searchdialogbase.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,165 +0,0 @@ -'''Unittests for idlelib/SearchDialogBase.py - -Coverage: 99%. The only thing not covered is inconsequential -- -testing skipping of suite when self.needwrapbutton is false. - -''' -import unittest -from test.support import requires -from tkinter import Tk, Toplevel, Frame ##, BooleanVar, StringVar -from idlelib import SearchEngine as se -from idlelib import SearchDialogBase as sdb -from idlelib.idle_test.mock_idle import Func -## from idlelib.idle_test.mock_tk import Var - -# The ## imports above & following could help make some tests gui-free. -# However, they currently make radiobutton tests fail. -##def setUpModule(): -## # Replace tk objects used to initialize se.SearchEngine. -## se.BooleanVar = Var -## se.StringVar = Var -## -##def tearDownModule(): -## se.BooleanVar = BooleanVar -## se.StringVar = StringVar - -class SearchDialogBaseTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.root = Tk() - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - def setUp(self): - self.engine = se.SearchEngine(self.root) # None also seems to work - self.dialog = sdb.SearchDialogBase(root=self.root, engine=self.engine) - - def tearDown(self): - self.dialog.close() - - def test_open_and_close(self): - # open calls create_widgets, which needs default_command - self.dialog.default_command = None - - # Since text parameter of .open is not used in base class, - # pass dummy 'text' instead of tk.Text(). - self.dialog.open('text') - self.assertEqual(self.dialog.top.state(), 'normal') - self.dialog.close() - self.assertEqual(self.dialog.top.state(), 'withdrawn') - - self.dialog.open('text', searchphrase="hello") - self.assertEqual(self.dialog.ent.get(), 'hello') - self.dialog.close() - - def test_create_widgets(self): - self.dialog.create_entries = Func() - self.dialog.create_option_buttons = Func() - self.dialog.create_other_buttons = Func() - self.dialog.create_command_buttons = Func() - - self.dialog.default_command = None - self.dialog.create_widgets() - - self.assertTrue(self.dialog.create_entries.called) - self.assertTrue(self.dialog.create_option_buttons.called) - self.assertTrue(self.dialog.create_other_buttons.called) - self.assertTrue(self.dialog.create_command_buttons.called) - - def test_make_entry(self): - equal = self.assertEqual - self.dialog.row = 0 - self.dialog.top = Toplevel(self.root) - entry, label = self.dialog.make_entry("Test:", 'hello') - equal(label['text'], 'Test:') - - self.assertIn(entry.get(), 'hello') - egi = entry.grid_info() - equal(int(egi['row']), 0) - equal(int(egi['column']), 1) - equal(int(egi['rowspan']), 1) - equal(int(egi['columnspan']), 1) - equal(self.dialog.row, 1) - - def test_create_entries(self): - self.dialog.row = 0 - self.engine.setpat('hello') - self.dialog.create_entries() - self.assertIn(self.dialog.ent.get(), 'hello') - - def test_make_frame(self): - self.dialog.row = 0 - self.dialog.top = Toplevel(self.root) - frame, label = self.dialog.make_frame() - self.assertEqual(label, '') - self.assertIsInstance(frame, Frame) - - frame, label = self.dialog.make_frame('testlabel') - self.assertEqual(label['text'], 'testlabel') - self.assertIsInstance(frame, Frame) - - def btn_test_setup(self, meth): - self.dialog.top = Toplevel(self.root) - self.dialog.row = 0 - return meth() - - def test_create_option_buttons(self): - e = self.engine - for state in (0, 1): - for var in (e.revar, e.casevar, e.wordvar, e.wrapvar): - var.set(state) - frame, options = self.btn_test_setup( - self.dialog.create_option_buttons) - for spec, button in zip (options, frame.pack_slaves()): - var, label = spec - self.assertEqual(button['text'], label) - self.assertEqual(var.get(), state) - if state == 1: - button.deselect() - else: - button.select() - self.assertEqual(var.get(), 1 - state) - - def test_create_other_buttons(self): - for state in (False, True): - var = self.engine.backvar - var.set(state) - frame, others = self.btn_test_setup( - self.dialog.create_other_buttons) - buttons = frame.pack_slaves() - for spec, button in zip(others, buttons): - val, label = spec - self.assertEqual(button['text'], label) - if val == state: - # hit other button, then this one - # indexes depend on button order - self.assertEqual(var.get(), state) - buttons[val].select() - self.assertEqual(var.get(), 1 - state) - buttons[1-val].select() - self.assertEqual(var.get(), state) - - def test_make_button(self): - self.dialog.top = Toplevel(self.root) - self.dialog.buttonframe = Frame(self.dialog.top) - btn = self.dialog.make_button('Test', self.dialog.close) - self.assertEqual(btn['text'], 'Test') - - def test_create_command_buttons(self): - self.dialog.create_command_buttons() - # Look for close button command in buttonframe - closebuttoncommand = '' - for child in self.dialog.buttonframe.winfo_children(): - if child['text'] == 'close': - closebuttoncommand = child['command'] - self.assertIn('close', closebuttoncommand) - - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_searchengine.py --- a/Lib/idlelib/idle_test/test_searchengine.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,329 +0,0 @@ -'''Test functions and SearchEngine class in SearchEngine.py.''' - -# With mock replacements, the module does not use any gui widgets. -# The use of tk.Text is avoided (for now, until mock Text is improved) -# by patching instances with an index function returning what is needed. -# This works because mock Text.get does not use .index. - -import re -import unittest -# from test.support import requires -from tkinter import BooleanVar, StringVar, TclError # ,Tk, Text -import tkinter.messagebox as tkMessageBox -from idlelib import SearchEngine as se -from idlelib.idle_test.mock_tk import Var, Mbox -from idlelib.idle_test.mock_tk import Text as mockText - -def setUpModule(): - # Replace s-e module tkinter imports other than non-gui TclError. - se.BooleanVar = Var - se.StringVar = Var - se.tkMessageBox = Mbox - -def tearDownModule(): - # Restore 'just in case', though other tests should also replace. - se.BooleanVar = BooleanVar - se.StringVar = StringVar - se.tkMessageBox = tkMessageBox - - -class Mock: - def __init__(self, *args, **kwargs): pass - -class GetTest(unittest.TestCase): - # SearchEngine.get returns singleton created & saved on first call. - def test_get(self): - saved_Engine = se.SearchEngine - se.SearchEngine = Mock # monkey-patch class - try: - root = Mock() - engine = se.get(root) - self.assertIsInstance(engine, se.SearchEngine) - self.assertIs(root._searchengine, engine) - self.assertIs(se.get(root), engine) - finally: - se.SearchEngine = saved_Engine # restore class to module - -class GetLineColTest(unittest.TestCase): - # Test simple text-independent helper function - def test_get_line_col(self): - self.assertEqual(se.get_line_col('1.0'), (1, 0)) - self.assertEqual(se.get_line_col('1.11'), (1, 11)) - - self.assertRaises(ValueError, se.get_line_col, ('1.0 lineend')) - self.assertRaises(ValueError, se.get_line_col, ('end')) - -class GetSelectionTest(unittest.TestCase): - # Test text-dependent helper function. -## # Need gui for text.index('sel.first/sel.last/insert'). -## @classmethod -## def setUpClass(cls): -## requires('gui') -## cls.root = Tk() -## -## @classmethod -## def tearDownClass(cls): -## cls.root.destroy() -## del cls.root - - def test_get_selection(self): - # text = Text(master=self.root) - text = mockText() - text.insert('1.0', 'Hello World!') - - # fix text.index result when called in get_selection - def sel(s): - # select entire text, cursor irrelevant - if s == 'sel.first': return '1.0' - if s == 'sel.last': return '1.12' - raise TclError - text.index = sel # replaces .tag_add('sel', '1.0, '1.12') - self.assertEqual(se.get_selection(text), ('1.0', '1.12')) - - def mark(s): - # no selection, cursor after 'Hello' - if s == 'insert': return '1.5' - raise TclError - text.index = mark # replaces .mark_set('insert', '1.5') - self.assertEqual(se.get_selection(text), ('1.5', '1.5')) - - -class ReverseSearchTest(unittest.TestCase): - # Test helper function that searches backwards within a line. - def test_search_reverse(self): - Equal = self.assertEqual - line = "Here is an 'is' test text." - prog = re.compile('is') - Equal(se.search_reverse(prog, line, len(line)).span(), (12, 14)) - Equal(se.search_reverse(prog, line, 14).span(), (12, 14)) - Equal(se.search_reverse(prog, line, 13).span(), (5, 7)) - Equal(se.search_reverse(prog, line, 7).span(), (5, 7)) - Equal(se.search_reverse(prog, line, 6), None) - - -class SearchEngineTest(unittest.TestCase): - # Test class methods that do not use Text widget. - - def setUp(self): - self.engine = se.SearchEngine(root=None) - # Engine.root is only used to create error message boxes. - # The mock replacement ignores the root argument. - - def test_is_get(self): - engine = self.engine - Equal = self.assertEqual - - Equal(engine.getpat(), '') - engine.setpat('hello') - Equal(engine.getpat(), 'hello') - - Equal(engine.isre(), False) - engine.revar.set(1) - Equal(engine.isre(), True) - - Equal(engine.iscase(), False) - engine.casevar.set(1) - Equal(engine.iscase(), True) - - Equal(engine.isword(), False) - engine.wordvar.set(1) - Equal(engine.isword(), True) - - Equal(engine.iswrap(), True) - engine.wrapvar.set(0) - Equal(engine.iswrap(), False) - - Equal(engine.isback(), False) - engine.backvar.set(1) - Equal(engine.isback(), True) - - def test_setcookedpat(self): - engine = self.engine - engine.setcookedpat('\s') - self.assertEqual(engine.getpat(), '\s') - engine.revar.set(1) - engine.setcookedpat('\s') - self.assertEqual(engine.getpat(), r'\\s') - - def test_getcookedpat(self): - engine = self.engine - Equal = self.assertEqual - - Equal(engine.getcookedpat(), '') - engine.setpat('hello') - Equal(engine.getcookedpat(), 'hello') - engine.wordvar.set(True) - Equal(engine.getcookedpat(), r'\bhello\b') - engine.wordvar.set(False) - - engine.setpat('\s') - Equal(engine.getcookedpat(), r'\\s') - engine.revar.set(True) - Equal(engine.getcookedpat(), '\s') - - def test_getprog(self): - engine = self.engine - Equal = self.assertEqual - - engine.setpat('Hello') - temppat = engine.getprog() - Equal(temppat.pattern, re.compile('Hello', re.IGNORECASE).pattern) - engine.casevar.set(1) - temppat = engine.getprog() - Equal(temppat.pattern, re.compile('Hello').pattern, 0) - - engine.setpat('') - Equal(engine.getprog(), None) - engine.setpat('+') - engine.revar.set(1) - Equal(engine.getprog(), None) - self.assertEqual(Mbox.showerror.message, - 'Error: nothing to repeat at position 0\nPattern: +') - - def test_report_error(self): - showerror = Mbox.showerror - Equal = self.assertEqual - pat = '[a-z' - msg = 'unexpected end of regular expression' - - Equal(self.engine.report_error(pat, msg), None) - Equal(showerror.title, 'Regular expression error') - expected_message = ("Error: " + msg + "\nPattern: [a-z") - Equal(showerror.message, expected_message) - - Equal(self.engine.report_error(pat, msg, 5), None) - Equal(showerror.title, 'Regular expression error') - expected_message += "\nOffset: 5" - Equal(showerror.message, expected_message) - - -class SearchTest(unittest.TestCase): - # Test that search_text makes right call to right method. - - @classmethod - def setUpClass(cls): -## requires('gui') -## cls.root = Tk() -## cls.text = Text(master=cls.root) - cls.text = mockText() - test_text = ( - 'First line\n' - 'Line with target\n' - 'Last line\n') - cls.text.insert('1.0', test_text) - cls.pat = re.compile('target') - - cls.engine = se.SearchEngine(None) - cls.engine.search_forward = lambda *args: ('f', args) - cls.engine.search_backward = lambda *args: ('b', args) - -## @classmethod -## def tearDownClass(cls): -## cls.root.destroy() -## del cls.root - - def test_search(self): - Equal = self.assertEqual - engine = self.engine - search = engine.search_text - text = self.text - pat = self.pat - - engine.patvar.set(None) - #engine.revar.set(pat) - Equal(search(text), None) - - def mark(s): - # no selection, cursor after 'Hello' - if s == 'insert': return '1.5' - raise TclError - text.index = mark - Equal(search(text, pat), ('f', (text, pat, 1, 5, True, False))) - engine.wrapvar.set(False) - Equal(search(text, pat), ('f', (text, pat, 1, 5, False, False))) - engine.wrapvar.set(True) - engine.backvar.set(True) - Equal(search(text, pat), ('b', (text, pat, 1, 5, True, False))) - engine.backvar.set(False) - - def sel(s): - if s == 'sel.first': return '2.10' - if s == 'sel.last': return '2.16' - raise TclError - text.index = sel - Equal(search(text, pat), ('f', (text, pat, 2, 16, True, False))) - Equal(search(text, pat, True), ('f', (text, pat, 2, 10, True, True))) - engine.backvar.set(True) - Equal(search(text, pat), ('b', (text, pat, 2, 10, True, False))) - Equal(search(text, pat, True), ('b', (text, pat, 2, 16, True, True))) - - -class ForwardBackwardTest(unittest.TestCase): - # Test that search_forward method finds the target. -## @classmethod -## def tearDownClass(cls): -## cls.root.destroy() -## del cls.root - - @classmethod - def setUpClass(cls): - cls.engine = se.SearchEngine(None) -## requires('gui') -## cls.root = Tk() -## cls.text = Text(master=cls.root) - cls.text = mockText() - # search_backward calls index('end-1c') - cls.text.index = lambda index: '4.0' - test_text = ( - 'First line\n' - 'Line with target\n' - 'Last line\n') - cls.text.insert('1.0', test_text) - cls.pat = re.compile('target') - cls.res = (2, (10, 16)) # line, slice indexes of 'target' - cls.failpat = re.compile('xyz') # not in text - cls.emptypat = re.compile('\w*') # empty match possible - - def make_search(self, func): - def search(pat, line, col, wrap, ok=0): - res = func(self.text, pat, line, col, wrap, ok) - # res is (line, matchobject) or None - return (res[0], res[1].span()) if res else res - return search - - def test_search_forward(self): - # search for non-empty match - Equal = self.assertEqual - forward = self.make_search(self.engine.search_forward) - pat = self.pat - Equal(forward(pat, 1, 0, True), self.res) - Equal(forward(pat, 3, 0, True), self.res) # wrap - Equal(forward(pat, 3, 0, False), None) # no wrap - Equal(forward(pat, 2, 10, False), self.res) - - Equal(forward(self.failpat, 1, 0, True), None) - Equal(forward(self.emptypat, 2, 9, True, ok=True), (2, (9, 9))) - #Equal(forward(self.emptypat, 2, 9, True), self.res) - # While the initial empty match is correctly ignored, skipping - # the rest of the line and returning (3, (0,4)) seems buggy - tjr. - Equal(forward(self.emptypat, 2, 10, True), self.res) - - def test_search_backward(self): - # search for non-empty match - Equal = self.assertEqual - backward = self.make_search(self.engine.search_backward) - pat = self.pat - Equal(backward(pat, 3, 5, True), self.res) - Equal(backward(pat, 2, 0, True), self.res) # wrap - Equal(backward(pat, 2, 0, False), None) # no wrap - Equal(backward(pat, 2, 16, False), self.res) - - Equal(backward(self.failpat, 3, 9, True), None) - Equal(backward(self.emptypat, 2, 10, True, ok=True), (2, (9,9))) - # Accepted because 9 < 10, not because ok=True. - # It is not clear that ok=True is useful going back - tjr - Equal(backward(self.emptypat, 2, 9, True), (2, (5, 9))) - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_text.py --- a/Lib/idlelib/idle_test/test_text.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,227 +0,0 @@ -# Test mock_tk.Text class against tkinter.Text class by running same tests with both. -import unittest -from test.support import requires - -from _tkinter import TclError - -class TextTest(object): - - hw = 'hello\nworld' # usual initial insert after initialization - hwn = hw+'\n' # \n present at initialization, before insert - - Text = None - def setUp(self): - self.text = self.Text() - - def test_init(self): - self.assertEqual(self.text.get('1.0'), '\n') - self.assertEqual(self.text.get('end'), '') - - def test_index_empty(self): - index = self.text.index - - for dex in (-1.0, 0.3, '1.-1', '1.0', '1.0 lineend', '1.end', '1.33', - 'insert'): - self.assertEqual(index(dex), '1.0') - - for dex in 'end', 2.0, '2.1', '33.44': - self.assertEqual(index(dex), '2.0') - - def test_index_data(self): - index = self.text.index - self.text.insert('1.0', self.hw) - - for dex in -1.0, 0.3, '1.-1', '1.0': - self.assertEqual(index(dex), '1.0') - - for dex in '1.0 lineend', '1.end', '1.33': - self.assertEqual(index(dex), '1.5') - - for dex in 'end', '33.44': - self.assertEqual(index(dex), '3.0') - - def test_get(self): - get = self.text.get - Equal = self.assertEqual - self.text.insert('1.0', self.hw) - - Equal(get('end'), '') - Equal(get('end', 'end'), '') - Equal(get('1.0'), 'h') - Equal(get('1.0', '1.1'), 'h') - Equal(get('1.0', '1.3'), 'hel') - Equal(get('1.1', '1.3'), 'el') - Equal(get('1.0', '1.0 lineend'), 'hello') - Equal(get('1.0', '1.10'), 'hello') - Equal(get('1.0 lineend'), '\n') - Equal(get('1.1', '2.3'), 'ello\nwor') - Equal(get('1.0', '2.5'), self.hw) - Equal(get('1.0', 'end'), self.hwn) - Equal(get('0.0', '5.0'), self.hwn) - - def test_insert(self): - insert = self.text.insert - get = self.text.get - Equal = self.assertEqual - - insert('1.0', self.hw) - Equal(get('1.0', 'end'), self.hwn) - - insert('1.0', '') # nothing - Equal(get('1.0', 'end'), self.hwn) - - insert('1.0', '*') - Equal(get('1.0', 'end'), '*hello\nworld\n') - - insert('1.0 lineend', '*') - Equal(get('1.0', 'end'), '*hello*\nworld\n') - - insert('2.3', '*') - Equal(get('1.0', 'end'), '*hello*\nwor*ld\n') - - insert('end', 'x') - Equal(get('1.0', 'end'), '*hello*\nwor*ldx\n') - - insert('1.4', 'x\n') - Equal(get('1.0', 'end'), '*helx\nlo*\nwor*ldx\n') - - def test_no_delete(self): - # if index1 == 'insert' or 'end' or >= end, there is no deletion - delete = self.text.delete - get = self.text.get - Equal = self.assertEqual - self.text.insert('1.0', self.hw) - - delete('insert') - Equal(get('1.0', 'end'), self.hwn) - - delete('end') - Equal(get('1.0', 'end'), self.hwn) - - delete('insert', 'end') - Equal(get('1.0', 'end'), self.hwn) - - delete('insert', '5.5') - Equal(get('1.0', 'end'), self.hwn) - - delete('1.4', '1.0') - Equal(get('1.0', 'end'), self.hwn) - - delete('1.4', '1.4') - Equal(get('1.0', 'end'), self.hwn) - - def test_delete_char(self): - delete = self.text.delete - get = self.text.get - Equal = self.assertEqual - self.text.insert('1.0', self.hw) - - delete('1.0') - Equal(get('1.0', '1.end'), 'ello') - - delete('1.0', '1.1') - Equal(get('1.0', '1.end'), 'llo') - - # delete \n and combine 2 lines into 1 - delete('1.end') - Equal(get('1.0', '1.end'), 'lloworld') - - self.text.insert('1.3', '\n') - delete('1.10') - Equal(get('1.0', '1.end'), 'lloworld') - - self.text.insert('1.3', '\n') - delete('1.3', '2.0') - Equal(get('1.0', '1.end'), 'lloworld') - - def test_delete_slice(self): - delete = self.text.delete - get = self.text.get - Equal = self.assertEqual - self.text.insert('1.0', self.hw) - - delete('1.0', '1.0 lineend') - Equal(get('1.0', 'end'), '\nworld\n') - - delete('1.0', 'end') - Equal(get('1.0', 'end'), '\n') - - self.text.insert('1.0', self.hw) - delete('1.0', '2.0') - Equal(get('1.0', 'end'), 'world\n') - - delete('1.0', 'end') - Equal(get('1.0', 'end'), '\n') - - self.text.insert('1.0', self.hw) - delete('1.2', '2.3') - Equal(get('1.0', 'end'), 'held\n') - - def test_multiple_lines(self): # insert and delete - self.text.insert('1.0', 'hello') - - self.text.insert('1.3', '1\n2\n3\n4\n5') - self.assertEqual(self.text.get('1.0', 'end'), 'hel1\n2\n3\n4\n5lo\n') - - self.text.delete('1.3', '5.1') - self.assertEqual(self.text.get('1.0', 'end'), 'hello\n') - - def test_compare(self): - compare = self.text.compare - Equal = self.assertEqual - # need data so indexes not squished to 1,0 - self.text.insert('1.0', 'First\nSecond\nThird\n') - - self.assertRaises(TclError, compare, '2.2', 'op', '2.2') - - for op, less1, less0, equal, greater0, greater1 in ( - ('<', True, True, False, False, False), - ('<=', True, True, True, False, False), - ('>', False, False, False, True, True), - ('>=', False, False, True, True, True), - ('==', False, False, True, False, False), - ('!=', True, True, False, True, True), - ): - Equal(compare('1.1', op, '2.2'), less1, op) - Equal(compare('2.1', op, '2.2'), less0, op) - Equal(compare('2.2', op, '2.2'), equal, op) - Equal(compare('2.3', op, '2.2'), greater0, op) - Equal(compare('3.3', op, '2.2'), greater1, op) - - -class MockTextTest(TextTest, unittest.TestCase): - - @classmethod - def setUpClass(cls): - from idlelib.idle_test.mock_tk import Text - cls.Text = Text - - def test_decode(self): - # test endflags (-1, 0) not tested by test_index (which uses +1) - decode = self.text._decode - Equal = self.assertEqual - self.text.insert('1.0', self.hw) - - Equal(decode('end', -1), (2, 5)) - Equal(decode('3.1', -1), (2, 5)) - Equal(decode('end', 0), (2, 6)) - Equal(decode('3.1', 0), (2, 6)) - - -class TkTextTest(TextTest, unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - from tkinter import Tk, Text - cls.Text = Text - cls.root = Tk() - - @classmethod - def tearDownClass(cls): - cls.root.destroy() - del cls.root - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_textview.py --- a/Lib/idlelib/idle_test/test_textview.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,97 +0,0 @@ -'''Test the functions and main class method of textView.py. - -Since all methods and functions create (or destroy) a TextViewer, which -is a widget containing multiple widgets, all tests must be gui tests. -Using mock Text would not change this. Other mocks are used to retrieve -information about calls. - -The coverage is essentially 100%. -''' -from test.support import requires -requires('gui') - -import unittest -import os -from tkinter import Tk -from idlelib import textView as tv -from idlelib.idle_test.mock_idle import Func -from idlelib.idle_test.mock_tk import Mbox - -def setUpModule(): - global root - root = Tk() - -def tearDownModule(): - global root - root.destroy() # pyflakes falsely sees root as undefined - del root - - -class TV(tv.TextViewer): # used by TextViewTest - transient = Func() - grab_set = Func() - wait_window = Func() - -class TextViewTest(unittest.TestCase): - - def setUp(self): - TV.transient.__init__() - TV.grab_set.__init__() - TV.wait_window.__init__() - - def test_init_modal(self): - view = TV(root, 'Title', 'test text') - self.assertTrue(TV.transient.called) - self.assertTrue(TV.grab_set.called) - self.assertTrue(TV.wait_window.called) - view.Ok() - - def test_init_nonmodal(self): - view = TV(root, 'Title', 'test text', modal=False) - self.assertFalse(TV.transient.called) - self.assertFalse(TV.grab_set.called) - self.assertFalse(TV.wait_window.called) - view.Ok() - - def test_ok(self): - view = TV(root, 'Title', 'test text', modal=False) - view.destroy = Func() - view.Ok() - self.assertTrue(view.destroy.called) - del view.destroy # unmask real function - view.destroy - - -class textviewTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.orig_mbox = tv.tkMessageBox - tv.tkMessageBox = Mbox - - @classmethod - def tearDownClass(cls): - tv.tkMessageBox = cls.orig_mbox - del cls.orig_mbox - - def test_view_text(self): - # If modal True, tkinter will error with 'can't invoke "event" command' - view = tv.view_text(root, 'Title', 'test text', modal=False) - self.assertIsInstance(view, tv.TextViewer) - - def test_view_file(self): - test_dir = os.path.dirname(__file__) - testfile = os.path.join(test_dir, 'test_textview.py') - view = tv.view_file(root, 'Title', testfile, modal=False) - self.assertIsInstance(view, tv.TextViewer) - self.assertIn('Test', view.textView.get('1.0', '1.end')) - view.Ok() - - # Mock messagebox will be used and view_file will not return anything - testfile = os.path.join(test_dir, '../notthere.py') - view = tv.view_file(root, 'Title', testfile, modal=False) - self.assertIsNone(view) - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_warning.py --- a/Lib/idlelib/idle_test/test_warning.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,82 +0,0 @@ -'''Test warnings replacement in PyShell.py and run.py. - -This file could be expanded to include traceback overrides -(in same two modules). If so, change name. -Revise if output destination changes (http://bugs.python.org/issue18318). -Make sure warnings module is left unaltered (http://bugs.python.org/issue18081). -''' - -import unittest -from test.support import captured_stderr - -import warnings -# Try to capture default showwarning before Idle modules are imported. -showwarning = warnings.showwarning -# But if we run this file within idle, we are in the middle of the run.main loop -# and default showwarnings has already been replaced. -running_in_idle = 'idle' in showwarning.__name__ - -from idlelib import run -from idlelib import PyShell as shell - -# The following was generated from PyShell.idle_formatwarning -# and checked as matching expectation. -idlemsg = ''' -Warning (from warnings module): - File "test_warning.py", line 99 - Line of code -UserWarning: Test -''' -shellmsg = idlemsg + ">>> " - -class RunWarnTest(unittest.TestCase): - - @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") - def test_showwarnings(self): - self.assertIs(warnings.showwarning, showwarning) - run.capture_warnings(True) - self.assertIs(warnings.showwarning, run.idle_showwarning_subproc) - run.capture_warnings(False) - self.assertIs(warnings.showwarning, showwarning) - - def test_run_show(self): - with captured_stderr() as f: - run.idle_showwarning_subproc( - 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') - # The following uses .splitlines to erase line-ending differences - self.assertEqual(idlemsg.splitlines(), f.getvalue().splitlines()) - -class ShellWarnTest(unittest.TestCase): - - @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") - def test_showwarnings(self): - self.assertIs(warnings.showwarning, showwarning) - shell.capture_warnings(True) - self.assertIs(warnings.showwarning, shell.idle_showwarning) - shell.capture_warnings(False) - self.assertIs(warnings.showwarning, showwarning) - - def test_idle_formatter(self): - # Will fail if format changed without regenerating idlemsg - s = shell.idle_formatwarning( - 'Test', UserWarning, 'test_warning.py', 99, 'Line of code') - self.assertEqual(idlemsg, s) - - def test_shell_show(self): - with captured_stderr() as f: - shell.idle_showwarning( - 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') - self.assertEqual(shellmsg.splitlines(), f.getvalue().splitlines()) - -class ImportWarnTest(unittest.TestCase): - def test_idlever(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - import idlelib.idlever - self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, DeprecationWarning)) - self.assertIn("version", str(w[-1].message)) - - -if __name__ == '__main__': - unittest.main(verbosity=2, exit=False) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idle_test/test_widgetredir.py --- a/Lib/idlelib/idle_test/test_widgetredir.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,122 +0,0 @@ -"""Unittest for idlelib.WidgetRedirector - -100% coverage -""" -from test.support import requires -import unittest -from idlelib.idle_test.mock_idle import Func -from tkinter import Tk, Text, TclError -from idlelib.WidgetRedirector import WidgetRedirector - - -class InitCloseTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - - @classmethod - def tearDownClass(cls): - cls.text.destroy() - cls.tk.destroy() - del cls.text, cls.tk - - def test_init(self): - redir = WidgetRedirector(self.text) - self.assertEqual(redir.widget, self.text) - self.assertEqual(redir.tk, self.text.tk) - self.assertRaises(TclError, WidgetRedirector, self.text) - redir.close() # restore self.tk, self.text - - def test_close(self): - redir = WidgetRedirector(self.text) - redir.register('insert', Func) - redir.close() - self.assertEqual(redir._operations, {}) - self.assertFalse(hasattr(self.text, 'widget')) - - -class WidgetRedirectorTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - cls.tk = Tk() - cls.text = Text(cls.tk) - - @classmethod - def tearDownClass(cls): - cls.text.destroy() - cls.tk.destroy() - del cls.text, cls.tk - - def setUp(self): - self.redir = WidgetRedirector(self.text) - self.func = Func() - self.orig_insert = self.redir.register('insert', self.func) - self.text.insert('insert', 'asdf') # leaves self.text empty - - def tearDown(self): - self.text.delete('1.0', 'end') - self.redir.close() - - def test_repr(self): # partly for 100% coverage - self.assertIn('Redirector', repr(self.redir)) - self.assertIn('Original', repr(self.orig_insert)) - - def test_register(self): - self.assertEqual(self.text.get('1.0', 'end'), '\n') - self.assertEqual(self.func.args, ('insert', 'asdf')) - self.assertIn('insert', self.redir._operations) - self.assertIn('insert', self.text.__dict__) - self.assertEqual(self.text.insert, self.func) - - def test_original_command(self): - self.assertEqual(self.orig_insert.operation, 'insert') - self.assertEqual(self.orig_insert.tk_call, self.text.tk.call) - self.orig_insert('insert', 'asdf') - self.assertEqual(self.text.get('1.0', 'end'), 'asdf\n') - - def test_unregister(self): - self.assertIsNone(self.redir.unregister('invalid operation name')) - self.assertEqual(self.redir.unregister('insert'), self.func) - self.assertNotIn('insert', self.redir._operations) - self.assertNotIn('insert', self.text.__dict__) - - def test_unregister_no_attribute(self): - del self.text.insert - self.assertEqual(self.redir.unregister('insert'), self.func) - - def test_dispatch_intercept(self): - self.func.__init__(True) - self.assertTrue(self.redir.dispatch('insert', False)) - self.assertFalse(self.func.args[0]) - - def test_dispatch_bypass(self): - self.orig_insert('insert', 'asdf') - # tk.call returns '' where Python would return None - self.assertEqual(self.redir.dispatch('delete', '1.0', 'end'), '') - self.assertEqual(self.text.get('1.0', 'end'), '\n') - - def test_dispatch_error(self): - self.func.__init__(TclError()) - self.assertEqual(self.redir.dispatch('insert', False), '') - self.assertEqual(self.redir.dispatch('invalid'), '') - - def test_command_dispatch(self): - # Test that .__init__ causes redirection of tk calls - # through redir.dispatch - self.tk.call(self.text._w, 'insert', 'hello') - self.assertEqual(self.func.args, ('hello',)) - self.assertEqual(self.text.get('1.0', 'end'), '\n') - # Ensure that called through redir .dispatch and not through - # self.text.insert by having mock raise TclError. - self.func.__init__(TclError()) - self.assertEqual(self.tk.call(self.text._w, 'insert', 'boo'), '') - - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/idlever.py --- a/Lib/idlelib/idlever.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/idlever.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,12 +1,1 @@ -""" -The separate Idle version was eliminated years ago; -idlelib.idlever is no longer used by Idle -and will be removed in 3.6 or later. Use - from sys import version - IDLE_VERSION = version[:version.index(' ')] -""" -# Kept for now only for possible existing extension use -import warnings as w -w.warn(__doc__, DeprecationWarning, stacklevel=2) -from sys import version -IDLE_VERSION = version[:version.index(' ')] +IDLE_VERSION = "3.3.0a1" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/keybindingDialog.py --- a/Lib/idlelib/keybindingDialog.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/keybindingDialog.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,16 +4,15 @@ from tkinter import * import tkinter.messagebox as tkMessageBox import string -import sys +from idlelib import macosxSupport class GetKeysDialog(Toplevel): - def __init__(self,parent,title,action,currentKeySequences,_htest=False): + def __init__(self,parent,title,action,currentKeySequences): """ action - string, the name of the virtual event these keys will be mapped to currentKeys - list, a list of all key sequence lists currently mapped to virtual events, for overlap checking - _htest - bool, change box location when running htest """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) @@ -39,14 +38,11 @@ self.LoadFinalKeyList() self.withdraw() #hide while setting geometry self.update_idletasks() - self.geometry( - "+%d+%d" % ( - parent.winfo_rootx() + - (parent.winfo_width()/2 - self.winfo_reqwidth()/2), - parent.winfo_rooty() + - ((parent.winfo_height()/2 - self.winfo_reqheight()/2) - if not _htest else 150) - ) ) #centre dialog over parent (or below htest box) + self.geometry("+%d+%d" % + ((parent.winfo_rootx()+((parent.winfo_width()/2) + -(self.winfo_reqwidth()/2)), + parent.winfo_rooty()+((parent.winfo_height()/2) + -(self.winfo_reqheight()/2)) )) ) #centre dialog over parent self.deiconify() #geometry set, unhide self.wait_window() @@ -137,7 +133,8 @@ order is also important: key binding equality depends on it, so config-keys.def must use the same ordering. """ - if sys.platform == "darwin": + import sys + if macosxSupport.runningAsOSXApp(): self.modifiers = ['Shift', 'Control', 'Option', 'Command'] else: self.modifiers = ['Control', 'Alt', 'Shift'] @@ -262,5 +259,11 @@ return keysOK if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(GetKeysDialog) + #test the dialog + root=Tk() + def run(): + keySeq='' + dlg=GetKeysDialog(root,'Get Keys','find-again',[]) + print(dlg.result) + Button(root,text='Dialog',command=run).pack() + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/macosxSupport.py --- a/Lib/idlelib/macosxSupport.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/macosxSupport.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,88 +1,53 @@ """ -A number of functions that enhance IDLE on Mac OSX. +A number of function that enhance IDLE on MacOSX when it used as a normal +GUI application (as opposed to an X11 application). """ import sys import tkinter from os import path -import warnings + + +_appbundle = None def runningAsOSXApp(): - warnings.warn("runningAsOSXApp() is deprecated, use isAquaTk()", - DeprecationWarning, stacklevel=2) - return isAquaTk() + """ + Returns True if Python is running from within an app on OSX. + If so, assume that Python was built with Aqua Tcl/Tk rather than + X11 Tcl/Tk. + """ + global _appbundle + if _appbundle is None: + _appbundle = (sys.platform == 'darwin' and '.app' in sys.executable) + return _appbundle + +_carbonaquatk = None def isCarbonAquaTk(root): - warnings.warn("isCarbonAquaTk(root) is deprecated, use isCarbonTk()", - DeprecationWarning, stacklevel=2) - return isCarbonTk() - -_tk_type = None - -def _initializeTkVariantTests(root): - """ - Initializes OS X Tk variant values for - isAquaTk(), isCarbonTk(), isCocoaTk(), and isXQuartz(). - """ - global _tk_type - if sys.platform == 'darwin': - ws = root.tk.call('tk', 'windowingsystem') - if 'x11' in ws: - _tk_type = "xquartz" - elif 'aqua' not in ws: - _tk_type = "other" - elif 'AppKit' in root.tk.call('winfo', 'server', '.'): - _tk_type = "cocoa" - else: - _tk_type = "carbon" - else: - _tk_type = "other" - -def isAquaTk(): - """ - Returns True if IDLE is using a native OS X Tk (Cocoa or Carbon). - """ - assert _tk_type is not None - return _tk_type == "cocoa" or _tk_type == "carbon" - -def isCarbonTk(): """ Returns True if IDLE is using a Carbon Aqua Tk (instead of the newer Cocoa Aqua Tk). """ - assert _tk_type is not None - return _tk_type == "carbon" - -def isCocoaTk(): - """ - Returns True if IDLE is using a Cocoa Aqua Tk. - """ - assert _tk_type is not None - return _tk_type == "cocoa" - -def isXQuartz(): - """ - Returns True if IDLE is using an OS X X11 Tk. - """ - assert _tk_type is not None - return _tk_type == "xquartz" + global _carbonaquatk + if _carbonaquatk is None: + _carbonaquatk = (runningAsOSXApp() and + 'aqua' in root.tk.call('tk', 'windowingsystem') and + 'AppKit' not in root.tk.call('winfo', 'server', '.')) + return _carbonaquatk def tkVersionWarning(root): """ Returns a string warning message if the Tk version in use appears to - be one known to cause problems with IDLE. - 1. Apple Cocoa-based Tk 8.5.7 shipped with Mac OS X 10.6 is unusable. - 2. Apple Cocoa-based Tk 8.5.9 in OS X 10.7 and 10.8 is better but - can still crash unexpectedly. + be one known to cause problems with IDLE. The Apple Cocoa-based Tk 8.5 + that was shipped with Mac OS X 10.6. """ - if isCocoaTk(): - patchlevel = root.tk.call('info', 'patchlevel') - if patchlevel not in ('8.5.7', '8.5.9'): - return False - return (r"WARNING: The version of Tcl/Tk ({0}) in use may" + if (runningAsOSXApp() and + ('AppKit' in root.tk.call('winfo', 'server', '.')) and + (root.tk.call('info', 'patchlevel') == '8.5.7') ): + return (r"WARNING: The version of Tcl/Tk (8.5.7) in use may" r" be unstable.\n" r"Visit http://www.python.org/download/mac/tcltk/" - r" for current information.".format(patchlevel)) + r" for current information.") else: return False @@ -109,8 +74,8 @@ def overrideRootMenu(root, flist): """ - Replace the Tk root menu by something that is more appropriate for - IDLE with an Aqua Tk. + Replace the Tk root menu by something that's more appropriate for + IDLE. """ # The menu that is attached to the Tk root (".") is also used by AquaTk for # all windows that don't specify a menu of their own. The default menubar @@ -123,29 +88,17 @@ # # Due to a (mis-)feature of TkAqua the user will also see an empty Help # menu. - from tkinter import Menu + from tkinter import Menu, Text, Text + from idlelib.EditorWindow import prepstr, get_accelerator from idlelib import Bindings from idlelib import WindowList + from idlelib.MultiCall import MultiCallCreator - closeItem = Bindings.menudefs[0][1][-2] - - # Remove the last 3 items of the file menu: a separator, close window and - # quit. Close window will be reinserted just above the save item, where - # it should be according to the HIG. Quit is in the application menu. - del Bindings.menudefs[0][1][-3:] - Bindings.menudefs[0][1].insert(6, closeItem) - - # Remove the 'About' entry from the help menu, it is in the application - # menu - del Bindings.menudefs[-1][1][0:2] - # Remove the 'Configure Idle' entry from the options menu, it is in the - # application menu as 'Preferences' - del Bindings.menudefs[-2][1][0] menubar = Menu(root) root.configure(menu=menubar) menudict = {} - menudict['windows'] = menu = Menu(menubar, name='windows', tearoff=0) + menudict['windows'] = menu = Menu(menubar, name='windows') menubar.add_cascade(label='Window', menu=menu, underline=0) def postwindowsmenu(menu=menu): @@ -159,14 +112,10 @@ WindowList.register_callback(postwindowsmenu) def about_dialog(event=None): - "Handle Help 'About IDLE' event." - # Synchronize with EditorWindow.EditorWindow.about_dialog. from idlelib import aboutDialog aboutDialog.AboutDialog(root, 'About IDLE') def config_dialog(event=None): - "Handle Options 'Configure IDLE' event." - # Synchronize with EditorWindow.EditorWindow.config_dialog. from idlelib import configDialog # Ensure that the root object has an instance_dict attribute, @@ -174,13 +123,13 @@ # on an EditorWindow instance that is then passed as the first # argument to ConfigDialog) root.instance_dict = flist.inversedict + root.instance_dict = flist.inversedict configDialog.ConfigDialog(root, 'Settings') def help_dialog(event=None): - "Handle Help 'IDLE Help' event." - # Synchronize with EditorWindow.EditorWindow.help_dialog. - from idlelib import help - help.show_idlehelp(root) + from idlelib import textView + fn = path.join(path.abspath(path.dirname(__file__)), 'help.txt') + textView.view_file(root, 'Help', fn) root.bind('<>', about_dialog) root.bind('<>', config_dialog) @@ -193,10 +142,9 @@ # right thing for now. root.createcommand('exit', flist.close_all_callback) - if isCarbonTk(): + if isCarbonAquaTk(root): # for Carbon AquaTk, replace the default Tk apple menu - menudict['application'] = menu = Menu(menubar, name='apple', - tearoff=0) + menudict['application'] = menu = Menu(menubar, name='apple') menubar.add_cascade(label='IDLE', menu=menu) Bindings.menudefs.insert(0, ('application', [ @@ -209,7 +157,8 @@ Bindings.menudefs[0][1].append( ('_Preferences....', '<>'), ) - if isCocoaTk(): + else: + # assume Cocoa AquaTk # replace default About dialog with About IDLE one root.createcommand('tkAboutDialog', about_dialog) # replace default "Help" item in Help menu @@ -219,22 +168,10 @@ def setupApp(root, flist): """ - Perform initial OS X customizations if needed. - Called from PyShell.main() after initial calls to Tk() + Perform setup for the OSX application bundle. + """ + if not runningAsOSXApp(): return - There are currently three major versions of Tk in use on OS X: - 1. Aqua Cocoa Tk (native default since OS X 10.6) - 2. Aqua Carbon Tk (original native, 32-bit only, deprecated) - 3. X11 (supported by some third-party distributors, deprecated) - There are various differences among the three that affect IDLE - behavior, primarily with menus, mouse key events, and accelerators. - Some one-time customizations are performed here. - Others are dynamically tested throughout idlelib by calls to the - isAquaTk(), isCarbonTk(), isCocoaTk(), isXQuartz() functions which - are initialized here as well. - """ - _initializeTkVariantTests(root) - if isAquaTk(): - hideTkConsole(root) - overrideRootMenu(root, flist) - addOpenEventSupport(root, flist) + hideTkConsole(root) + overrideRootMenu(root, flist) + addOpenEventSupport(root, flist) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/rpc.py --- a/Lib/idlelib/rpc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/rpc.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,7 +2,7 @@ For security reasons, GvR requested that Idle's Python execution server process connect to the Idle process, which listens for the connection. Since Idle has -only one client per server, this was not a limitation. +has only one client per server, this was not a limitation. +---------------------------------+ +-------------+ | socketserver.BaseRequestHandler | | SocketIO | @@ -29,7 +29,6 @@ import sys import os -import io import socket import select import socketserver @@ -54,15 +53,16 @@ ms = marshal.dumps(co) return unpickle_code, (ms,) -def dumps(obj, protocol=None): - f = io.BytesIO() - p = CodePickler(f, protocol) - p.dump(obj) - return f.getvalue() +# XXX KBK 24Aug02 function pickling capability not used in Idle +# def unpickle_function(ms): +# return ms -class CodePickler(pickle.Pickler): - dispatch_table = {types.CodeType: pickle_code} - dispatch_table.update(copyreg.dispatch_table) +# def pickle_function(fn): +# assert isinstance(fn, type.FunctionType) +# return repr(fn) + +copyreg.pickle(types.CodeType, pickle_code, unpickle_code) +# copyreg.pickle(types.FunctionType, pickle_function, unpickle_function) BUFSIZE = 8*1024 LOCALHOST = '127.0.0.1' @@ -145,7 +145,7 @@ def exithook(self): "override for specific exit action" - os._exit(0) + os._exit() def debug(self, *args): if not self.debugging: @@ -199,7 +199,7 @@ raise except KeyboardInterrupt: raise - except OSError: + except socket.error: raise except Exception as ex: return ("CALLEXC", ex) @@ -329,7 +329,7 @@ def putmessage(self, message): self.debug("putmessage:%d:" % message[0]) try: - s = dumps(message) + s = pickle.dumps(message) except pickle.PicklingError: print("Cannot pickle:", repr(message), file=sys.__stderr__) raise @@ -339,8 +339,11 @@ r, w, x = select.select([], [self.sock], []) n = self.sock.send(s[:BUFSIZE]) except (AttributeError, TypeError): - raise OSError("socket no longer exists") - s = s[n:] + raise IOError("socket no longer exists") + except socket.error: + raise + else: + s = s[n:] buff = b'' bufneed = 4 @@ -354,7 +357,7 @@ return None try: s = self.sock.recv(BUFSIZE) - except OSError: + except socket.error: raise EOFError if len(s) == 0: raise EOFError @@ -534,7 +537,7 @@ SocketIO.__init__(self, working_sock) else: print("** Invalid host: ", address, file=sys.__stderr__) - raise OSError + raise socket.error def get_remote_proxy(self, oid): return RPCProxy(self, oid) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/run.py --- a/Lib/idlelib/run.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/run.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,11 @@ import sys import linecache import time +import socket import traceback import _thread as thread import threading import queue -import tkinter from idlelib import CallTips from idlelib import AutoComplete @@ -14,52 +14,44 @@ from idlelib import RemoteObjectBrowser from idlelib import StackViewer from idlelib import rpc -from idlelib import PyShell -from idlelib import IOBinding import __main__ LOCALHOST = '127.0.0.1' -import warnings +try: + import warnings +except ImportError: + pass +else: + def idle_formatwarning_subproc(message, category, filename, lineno, + line=None): + """Format warnings the IDLE way""" + s = "\nWarning (from warnings module):\n" + s += ' File \"%s\", line %s\n' % (filename, lineno) + if line is None: + line = linecache.getline(filename, lineno) + line = line.strip() + if line: + s += " %s\n" % line + s += "%s: %s\n" % (category.__name__, message) + return s + warnings.formatwarning = idle_formatwarning_subproc -def idle_showwarning_subproc( - message, category, filename, lineno, file=None, line=None): - """Show Idle-format warning after replacing warnings.showwarning. - The only difference is the formatter called. - """ - if file is None: - file = sys.stderr - try: - file.write(PyShell.idle_formatwarning( - message, category, filename, lineno, line)) - except IOError: - pass # the file (probably stderr) is invalid - this warning gets lost. - -_warnings_showwarning = None - -def capture_warnings(capture): - "Replace warning.showwarning with idle_showwarning_subproc, or reverse." - - global _warnings_showwarning - if capture: - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning - warnings.showwarning = idle_showwarning_subproc - else: - if _warnings_showwarning is not None: - warnings.showwarning = _warnings_showwarning - _warnings_showwarning = None - -capture_warnings(True) -tcl = tkinter.Tcl() - -def handle_tk_events(tcl=tcl): +def handle_tk_events(): """Process any tk events that are ready to be dispatched if tkinter has been imported, a tcl interpreter has been created and tk has been loaded.""" - tcl.eval("update") + tkinter = sys.modules.get('tkinter') + if tkinter and tkinter._default_root: + # tkinter has been imported, an Tcl interpreter was created and + # tk has been loaded. + root = tkinter._default_root + while root.tk.dooneevent(tkinter._tkinter.DONT_WAIT): + # Process pending events. + pass + # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global @@ -99,8 +91,6 @@ print("IDLE Subprocess: no IP port passed in sys.argv.", file=sys.__stderr__) return - - capture_warnings(True) sys.argv[:] = [""] sockthread = threading.Thread(target=manage_socket, name='SockThread', @@ -128,7 +118,6 @@ exit_now = True continue except SystemExit: - capture_warnings(False) raise except: type, value, tb = sys.exc_info() @@ -148,8 +137,8 @@ try: server = MyRPCServer(address, MyHandler) break - except OSError as err: - print("IDLE Subprocess: OSError: " + err.args[1] + + except socket.error as err: + print("IDLE Subprocess: socket error: " + err.args[1] + ", retrying....", file=sys.__stderr__) socket_error = err else: @@ -174,7 +163,7 @@ tkMessageBox.showerror("IDLE Subprocess Error", msg, parent=root) else: tkMessageBox.showerror("IDLE Subprocess Error", - "Socket Error: %s" % err.args[1], parent=root) + "Socket Error: %s" % err.args[1]) root.destroy() def print_exception(): @@ -184,34 +173,15 @@ efile = sys.stderr typ, val, tb = excinfo = sys.exc_info() sys.last_type, sys.last_value, sys.last_traceback = excinfo - seen = set() - - def print_exc(typ, exc, tb): - seen.add(exc) - context = exc.__context__ - cause = exc.__cause__ - if cause is not None and cause not in seen: - print_exc(type(cause), cause, cause.__traceback__) - print("\nThe above exception was the direct cause " - "of the following exception:\n", file=efile) - elif (context is not None and - not exc.__suppress_context__ and - context not in seen): - print_exc(type(context), context, context.__traceback__) - print("\nDuring handling of the above exception, " - "another exception occurred:\n", file=efile) - if tb: - tbe = traceback.extract_tb(tb) - print('Traceback (most recent call last):', file=efile) - exclude = ("run.py", "rpc.py", "threading.py", "queue.py", - "RemoteDebugger.py", "bdb.py") - cleanup_traceback(tbe, exclude) - traceback.print_list(tbe, file=efile) - lines = traceback.format_exception_only(typ, exc) - for line in lines: - print(line, end='', file=efile) - - print_exc(typ, val, tb) + tbe = traceback.extract_tb(tb) + print('Traceback (most recent call last):', file=efile) + exclude = ("run.py", "rpc.py", "threading.py", "queue.py", + "RemoteDebugger.py", "bdb.py") + cleanup_traceback(tbe, exclude) + traceback.print_list(tbe, file=efile) + lines = traceback.format_exception_only(typ, val) + for line in lines: + print(line, end='', file=efile) def cleanup_traceback(tb, exclude): "Remove excluded traces from beginning/end of tb; get cached lines" @@ -258,7 +228,6 @@ if no_exitfunc: import atexit atexit._clear() - capture_warnings(False) sys.exit(0) class MyRPCServer(rpc.RPCServer): @@ -291,29 +260,23 @@ quitting = True thread.interrupt_main() + class MyHandler(rpc.RPCHandler): def handle(self): """Override base method""" executive = Executive(self) self.register("exec", executive) - self.console = self.get_remote_proxy("console") - sys.stdin = PyShell.PseudoInputFile(self.console, "stdin", - IOBinding.encoding) - sys.stdout = PyShell.PseudoOutputFile(self.console, "stdout", - IOBinding.encoding) - sys.stderr = PyShell.PseudoOutputFile(self.console, "stderr", - IOBinding.encoding) - + sys.stdin = self.console = self.get_remote_proxy("stdin") + sys.stdout = self.get_remote_proxy("stdout") + sys.stderr = self.get_remote_proxy("stderr") sys.displayhook = rpc.displayhook # page help() text to shell. import pydoc # import must be done here to capture i/o binding pydoc.pager = pydoc.plainpager - - # Keep a reference to stdin so that it won't try to exit IDLE if - # sys.stdin gets changed from within IDLE's shell. See issue17838. - self._keep_stdin = sys.stdin - + from idlelib import IOBinding + sys.stdin.encoding = sys.stdout.encoding = \ + sys.stderr.encoding = IOBinding.encoding self.interp = self.get_remote_proxy("interp") rpc.RPCHandler.getresponse(self, myseq=None, wait=0.05) @@ -351,10 +314,6 @@ exec(code, self.locals) finally: interruptable = False - except SystemExit: - # Scripts that raise SystemExit should just - # return to the interactive prompt - pass except: self.usr_exc_info = sys.exc_info() if quitting: @@ -398,5 +357,3 @@ sys.last_value = val item = StackViewer.StackTreeItem(flist, tb) return RemoteObjectBrowser.remote_object_tree_item(item) - -capture_warnings(False) # Make sure turned off; see issue 18081 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/tabbedpages.py --- a/Lib/idlelib/tabbedpages.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/tabbedpages.py Mon Jan 25 17:05:13 2016 +0100 @@ -78,7 +78,7 @@ def remove_tab(self, tab_name): """Remove the tab named """ if not tab_name in self._tab_names: - raise KeyError("No such Tab: '%s" % tab_name) + raise KeyError("No such Tab: '%s" % page_name) self._tab_names.remove(tab_name) self._arrange_tabs() @@ -88,7 +88,7 @@ if tab_name == self._selected_tab: return if tab_name is not None and tab_name not in self._tabs: - raise KeyError("No such Tab: '%s" % tab_name) + raise KeyError("No such Tab: '%s" % page_name) # deselect the current selected tab if self._selected_tab is not None: @@ -467,12 +467,9 @@ self._tab_set.set_selected_tab(page_name) -def _tabbed_pages(parent): +if __name__ == '__main__': # test dialog root=Tk() - width, height, x, y = list(map(int, re.split('[x+]', parent.geometry()))) - root.geometry("+%d+%d"%(x, y + 175)) - root.title("Test tabbed pages") tabPage=TabbedPageSet(root, page_names=['Foobar','Baz'], n_rows=0, expand_tabs=False, ) @@ -491,8 +488,3 @@ labelPgName.pack(padx=5) entryPgName.pack(padx=5) root.mainloop() - - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tabbed_pages) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/testcode.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/idlelib/testcode.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,31 @@ +import string + +def f(): + a = 0 + b = 1 + c = 2 + d = 3 + e = 4 + g() + +def g(): + h() + +def h(): + i() + +def i(): + j() + +def j(): + k() + +def k(): + l() + +l = lambda: test() + +def test(): + string.capwords(1) + +f() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/idlelib/textView.py --- a/Lib/idlelib/textView.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/idlelib/textView.py Mon Jan 25 17:05:13 2016 +0100 @@ -9,21 +9,15 @@ """A simple text viewer dialog for IDLE """ - def __init__(self, parent, title, text, modal=True, _htest=False): + def __init__(self, parent, title, text, modal=True): """Show the given text in a scrollable window with a 'close' button - If modal option set to False, user can interact with other windows, - otherwise they will be unable to interact with other windows until - the textview window is closed. - - _htest - bool; change box location when running htest. """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) - # place dialog below parent if running htest - self.geometry("=%dx%d+%d+%d" % (750, 500, - parent.winfo_rootx() + 10, - parent.winfo_rooty() + (10 if not _htest else 100))) + self.geometry("=%dx%d+%d+%d" % (625, 500, + parent.winfo_rootx() + 10, + parent.winfo_rooty() + 10)) #elguavas - config placeholders til config stuff completed self.bg = '#ffffff' self.fg = '#000000' @@ -73,14 +67,30 @@ with open(filename, 'r', encoding=encoding) as file: contents = file.read() except IOError: + import tkinter.messagebox as tkMessageBox tkMessageBox.showerror(title='File Load Error', message='Unable to load file %r .' % filename, parent=parent) else: return view_text(parent, title, contents, modal) + if __name__ == '__main__': - import unittest - unittest.main('idlelib.idle_test.test_textview', verbosity=2, exit=False) - from idlelib.idle_test.htest import run - run(TextViewer) + #test the dialog + root=Tk() + root.title('textView test') + filename = './textView.py' + text = file(filename, 'r').read() + btn1 = Button(root, text='view_text', + command=lambda:view_text(root, 'view_text', text)) + btn1.pack(side=LEFT) + btn2 = Button(root, text='view_file', + command=lambda:view_file(root, 'view_file', filename)) + btn2.pack(side=LEFT) + btn3 = Button(root, text='nonmodal view_text', + command=lambda:view_text(root, 'nonmodal view_text', text, + modal=False)) + btn3.pack(side=LEFT) + close = Button(root, text='Close', command=root.destroy) + close.pack(side=RIGHT) + root.mainloop() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/imaplib.py --- a/Lib/imaplib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/imaplib.py Mon Jan 25 17:05:13 2016 +0100 @@ -22,9 +22,7 @@ __version__ = "2.58" -import binascii, errno, random, re, socket, subprocess, sys, time, calendar -from datetime import datetime, timezone, timedelta -from io import DEFAULT_BUFFER_SIZE +import binascii, errno, random, re, socket, subprocess, sys, time try: import ssl @@ -43,16 +41,6 @@ IMAP4_SSL_PORT = 993 AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first -# Maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) -# don't specify a line length. RFC 2683 suggests limiting client -# command lines to 1000 octets and that servers should be prepared -# to accept command lines up to 8000 octets, so we used to use 10K here. -# In the modern world (eg: gmail) the response to, for example, a -# search command can be quite large, so we now use 1M. -_MAXLINE = 1000000 - - # Commands Commands = { @@ -66,7 +54,6 @@ 'CREATE': ('AUTH', 'SELECTED'), 'DELETE': ('AUTH', 'SELECTED'), 'DELETEACL': ('AUTH', 'SELECTED'), - 'ENABLE': ('AUTH', ), 'EXAMINE': ('AUTH', 'SELECTED'), 'EXPUNGE': ('SELECTED',), 'FETCH': ('SELECTED',), @@ -108,25 +95,12 @@ br' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' br' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' br'"') -# Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n|\r|\n') -# We no longer exclude the ']' character from the data portion of the response -# code, even though it violates the RFC. Popular IMAP servers such as Gmail -# allow flags with ']', and there are programs (including imaplib!) that can -# produce them. The problem with this is if the 'text' portion of the response -# includes a ']' we'll parse the response wrong (which is the point of the RFC -# restriction). However, that seems less likely to be a problem in practice -# than being unable to correctly parse flags that include ']' chars, which -# was reported as a real-world problem in issue #21815. -Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') -# Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?', re.ASCII) -# We compile these in _mode_xxx. -_Literal = br'.*{(?P\d+)}$' -_Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' @@ -180,7 +154,7 @@ class abort(error): pass # Service errors - close and retry class readonly(abort): pass # Mailbox status changed to READ-ONLY - def __init__(self, host='', port=IMAP4_PORT): + def __init__(self, host = '', port = IMAP4_PORT): self.debug = Debug self.state = 'LOGOUT' self.literal = None # A literal argument to a command @@ -190,7 +164,6 @@ self.is_readonly = False # READ-ONLY desired state self.tagnum = 0 self._tls_established = False - self._mode_ascii() # Open socket to server. @@ -201,23 +174,10 @@ except Exception: try: self.shutdown() - except OSError: + except socket.error: pass raise - def _mode_ascii(self): - self.utf8_enabled = False - self._encoding = 'ascii' - self.Literal = re.compile(_Literal, re.ASCII) - self.Untagged_status = re.compile(_Untagged_status, re.ASCII) - - - def _mode_utf8(self): - self.utf8_enabled = True - self._encoding = 'utf-8' - self.Literal = re.compile(_Literal) - self.Untagged_status = re.compile(_Untagged_status) - def _connect(self): # Create unique tag for this session, @@ -267,14 +227,6 @@ return getattr(self, attr.lower()) raise AttributeError("Unknown IMAP4 command: '%s'" % attr) - def __enter__(self): - return self - - def __exit__(self, *args): - try: - self.logout() - except OSError: - pass # Overridable methods @@ -302,10 +254,7 @@ def readline(self): """Read line from remote.""" - line = self.file.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise self.error("got more than %d bytes" % _MAXLINE) - return line + return self.file.readline() def send(self, data): @@ -318,7 +267,7 @@ self.file.close() try: self.sock.shutdown(socket.SHUT_RDWR) - except OSError as e: + except socket.error as e: # The server might already have closed the connection if e.errno != errno.ENOTCONN: raise @@ -388,10 +337,7 @@ date_time = Time2Internaldate(date_time) else: date_time = None - literal = MapCRLF.sub(CRLF, message) - if self.utf8_enabled: - literal = b'UTF8 (' + literal + b')' - self.literal = literal + self.literal = MapCRLF.sub(CRLF, message) return self._simple_command(name, mailbox, flags, date_time) @@ -406,10 +352,10 @@ data = authobject(response) - It will be called to process server continuation responses; the - response argument it is passed will be a bytes. It should return bytes - data that will be base64 encoded and sent to the server. It should - return None if the client abort response '*' should be sent instead. + It will be called to process server continuation responses. + It should return data that will be encoded and sent to server. + It should return None if the client abort response '*' should + be sent instead. """ mech = mechanism.upper() # XXX: shouldn't this code be removed, not commented out? @@ -486,18 +432,6 @@ """ return self._simple_command('DELETEACL', mailbox, who) - def enable(self, capability): - """Send an RFC5161 enable string to the server. - - (typ, [data]) = .enable(capability) - """ - if 'ENABLE' not in self.capabilities: - raise IMAP4.error("Server does not support ENABLE") - typ, data = self._simple_command('ENABLE', capability) - if typ == 'OK' and 'UTF8=ACCEPT' in capability.upper(): - self._mode_utf8() - return typ, data - def expunge(self): """Permanently remove deleted items from selected mailbox. @@ -604,9 +538,7 @@ def _CRAM_MD5_AUTH(self, challenge): """ Authobject to use with CRAM-MD5 authentication. """ import hmac - pwd = (self.password.encode('utf-8') if isinstance(self.password, str) - else self.password) - return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() + return self.user + " " + hmac.HMAC(self.password, challenge).hexdigest() def logout(self): @@ -704,12 +636,9 @@ (typ, [data]) = .search(charset, criterion, ...) 'data' is space separated list of matching message numbers. - If UTF8 is enabled, charset MUST be None. """ name = 'SEARCH' if charset: - if self.utf8_enabled: - raise IMAP4.error("Non-None charset not valid in UTF8 mode") typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria) else: typ, dat = self._simple_command(name, *criteria) @@ -797,11 +726,12 @@ raise self.abort('TLS not supported by server') # Generate a default SSL context if none was passed. if ssl_context is None: - ssl_context = ssl._create_stdlib_context() + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + # SSLv2 considered harmful. + ssl_context.options |= ssl.OP_NO_SSLv2 typ, dat = self._simple_command(name) if typ == 'OK': - self.sock = ssl_context.wrap_socket(self.sock, - server_hostname=self.host) + self.sock = ssl_context.wrap_socket(self.sock) self.file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() @@ -923,7 +853,7 @@ def _check_bye(self): bye = self.untagged_responses.get('BYE') if bye: - raise self.abort(bye[-1].decode(self._encoding, 'replace')) + raise self.abort(bye[-1].decode('ascii', 'replace')) def _command(self, name, *args): @@ -944,12 +874,12 @@ raise self.readonly('mailbox status changed to READ-ONLY') tag = self._new_tag() - name = bytes(name, self._encoding) + name = bytes(name, 'ASCII') data = tag + b' ' + name for arg in args: if arg is None: continue if isinstance(arg, str): - arg = bytes(arg, self._encoding) + arg = bytes(arg, "ASCII") data = data + b' ' + arg literal = self.literal @@ -959,7 +889,7 @@ literator = literal else: literator = None - data = data + bytes(' {%s}' % len(literal), self._encoding) + data = data + bytes(' {%s}' % len(literal), 'ASCII') if __debug__: if self.debug >= 4: @@ -969,7 +899,7 @@ try: self.send(data + CRLF) - except OSError as val: + except (socket.error, OSError) as val: raise self.abort('socket error: %s' % val) if literal is None: @@ -994,7 +924,7 @@ try: self.send(literal) self.send(CRLF) - except OSError as val: + except (socket.error, OSError) as val: raise self.abort('socket error: %s' % val) if not literator: @@ -1024,7 +954,7 @@ typ, dat = self.capability() if dat == [None]: raise self.error('no CAPABILITY response from server') - dat = str(dat[-1], self._encoding) + dat = str(dat[-1], "ASCII") dat = dat.upper() self.capabilities = tuple(dat.split()) @@ -1043,10 +973,10 @@ if self._match(self.tagre, resp): tag = self.mo.group('tag') if not tag in self.tagged_commands: - raise self.abort('unexpected tagged response: %r' % resp) + raise self.abort('unexpected tagged response: %s' % resp) typ = self.mo.group('type') - typ = str(typ, self._encoding) + typ = str(typ, 'ASCII') dat = self.mo.group('data') self.tagged_commands[tag] = (typ, [dat]) else: @@ -1055,7 +985,7 @@ # '*' (untagged) responses? if not self._match(Untagged_response, resp): - if self._match(self.Untagged_status, resp): + if self._match(Untagged_status, resp): dat2 = self.mo.group('data2') if self.mo is None: @@ -1065,17 +995,17 @@ self.continuation_response = self.mo.group('data') return None # NB: indicates continuation - raise self.abort("unexpected response: %r" % resp) + raise self.abort("unexpected response: '%s'" % resp) typ = self.mo.group('type') - typ = str(typ, self._encoding) + typ = str(typ, 'ascii') dat = self.mo.group('data') if dat is None: dat = b'' # Null untagged response if dat2: dat = dat + b' ' + dat2 # Is there a literal to come? - while self._match(self.Literal, dat): + while self._match(Literal, dat): # Read literal direct from connection. @@ -1099,7 +1029,7 @@ if typ in ('OK', 'NO', 'BAD') and self._match(Response_code, dat): typ = self.mo.group('type') - typ = str(typ, self._encoding) + typ = str(typ, "ASCII") self._append_untagged(typ, self.mo.group('data')) if __debug__: @@ -1117,11 +1047,6 @@ del self.tagged_commands[tag] return result - # If we've seen a BYE at this point, the socket will be - # closed, so report the BYE now. - - self._check_bye() - # Some have reported "unexpected response" exceptions. # Note that ignoring them here causes loops. # Instead, send me details of the unexpected response and @@ -1144,7 +1069,7 @@ # Protocol mandates all lines terminated by CRLF if not line.endswith(b'\r\n'): - raise self.abort('socket error: unterminated line: %r' % line) + raise self.abort('socket error: unterminated line') line = line[:-2] if __debug__: @@ -1169,7 +1094,7 @@ def _new_tag(self): - tag = self.tagpre + bytes(str(self.tagnum), self._encoding) + tag = self.tagpre + bytes(str(self.tagnum), 'ASCII') self.tagnum = self.tagnum + 1 self.tagged_commands[tag] = None return tag @@ -1253,14 +1178,13 @@ ssl_context - a SSLContext object that contains your certificate chain and private key (default: None) Note: if ssl_context is provided, then parameters keyfile or - certfile should not be set otherwise ValueError is raised. + certfile should not be set otherwise ValueError is thrown. for more documentation see the docstring of the parent class IMAP4. """ - def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, - certfile=None, ssl_context=None): + def __init__(self, host='', port=IMAP4_SSL_PORT, keyfile=None, certfile=None, ssl_context=None): if ssl_context is not None and keyfile is not None: raise ValueError("ssl_context and keyfile arguments are mutually " "exclusive") @@ -1270,16 +1194,15 @@ self.keyfile = keyfile self.certfile = certfile - if ssl_context is None: - ssl_context = ssl._create_stdlib_context(certfile=certfile, - keyfile=keyfile) self.ssl_context = ssl_context IMAP4.__init__(self, host, port) def _create_socket(self): sock = IMAP4._create_socket(self) - return self.ssl_context.wrap_socket(sock, - server_hostname=self.host) + if self.ssl_context: + return self.ssl_context.wrap_socket(sock) + else: + return ssl.wrap_socket(sock, self.keyfile, self.certfile) def open(self, host='', port=IMAP4_SSL_PORT): """Setup connection to remote server on "host:port". @@ -1298,7 +1221,7 @@ Instantiate with: IMAP4_stream(command) - "command" - a string that can be passed to subprocess.Popen() + where "command" is a string that can be passed to subprocess.Popen() for more documentation see the docstring of the parent class IMAP4. """ @@ -1319,7 +1242,6 @@ self.sock = None self.file = None self.process = subprocess.Popen(self.command, - bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, close_fds=True) self.writefile = self.process.stdin @@ -1361,7 +1283,7 @@ def process(self, data): ret = self.mech(self.decode(data)) if ret is None: - return b'*' # Abort conversation + return '*' # Abort conversation return self.encode(ret) def encode(self, inp): @@ -1373,16 +1295,14 @@ # so when it gets to the end of the 8-bit input # there's no partial 6-bit output. # - oup = b'' - if isinstance(inp, str): - inp = inp.encode('utf-8') + oup = '' while inp: if len(inp) > 48: t = inp[:48] inp = inp[48:] else: t = inp - inp = b'' + inp = '' e = binascii.b2a_base64(t) if e: oup = oup + e[:-1] @@ -1390,11 +1310,13 @@ def decode(self, inp): if not inp: - return b'' + return '' return binascii.a2b_base64(inp) -Months = ' Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split(' ') -Mon2num = {s.encode():n+1 for n, s in enumerate(Months[1:])} + + +Mon2num = {b'Jan': 1, b'Feb': 2, b'Mar': 3, b'Apr': 4, b'May': 5, b'Jun': 6, + b'Jul': 7, b'Aug': 8, b'Sep': 9, b'Oct': 10, b'Nov': 11, b'Dec': 12} def Internaldate2tuple(resp): """Parse an IMAP4 INTERNALDATE string. @@ -1425,9 +1347,19 @@ zone = -zone tt = (year, mon, day, hour, min, sec, -1, -1, -1) - utc = calendar.timegm(tt) - zone - return time.localtime(utc) + utc = time.mktime(tt) + + # Following is necessary because the time module has no 'mkgmtime'. + # 'mktime' assumes arg in local timezone, so adds timezone/altzone. + + lt = time.localtime(utc) + if time.daylight and lt[-1]: + zone = zone + time.altzone + else: + zone = zone + time.timezone + + return time.localtime(utc - zone) @@ -1462,37 +1394,28 @@ Return string in form: '"DD-Mmm-YYYY HH:MM:SS +HHMM"'. The date_time argument can be a number (int or float) representing seconds since epoch (as returned by time.time()), a 9-tuple - representing local time, an instance of time.struct_time (as - returned by time.localtime()), an aware datetime instance or a + representing local time (as returned by time.localtime()), or a double-quoted string. In the last case, it is assumed to already be in the correct format. """ + if isinstance(date_time, (int, float)): - dt = datetime.fromtimestamp(date_time, - timezone.utc).astimezone() - elif isinstance(date_time, tuple): - try: - gmtoff = date_time.tm_gmtoff - except AttributeError: - if time.daylight: - dst = date_time[8] - if dst == -1: - dst = time.localtime(time.mktime(date_time))[8] - gmtoff = -(time.timezone, time.altzone)[dst] - else: - gmtoff = -time.timezone - delta = timedelta(seconds=gmtoff) - dt = datetime(*date_time[:6], tzinfo=timezone(delta)) - elif isinstance(date_time, datetime): - if date_time.tzinfo is None: - raise ValueError("date_time must be aware") - dt = date_time + tt = time.localtime(date_time) + elif isinstance(date_time, (tuple, time.struct_time)): + tt = date_time elif isinstance(date_time, str) and (date_time[0],date_time[-1]) == ('"','"'): return date_time # Assume in correct format else: raise ValueError("date_time not of a known type") - fmt = '"%d-{}-%Y %H:%M:%S %z"'.format(Months[dt.month]) - return dt.strftime(fmt) + + dt = time.strftime("%d-%b-%Y %H:%M:%S", tt) + if dt[0] == '0': + dt = ' ' + dt[1:] + if time.daylight and tt[-1]: + zone = -time.altzone + else: + zone = -time.timezone + return '"' + dt + " %+03d%02d" % divmod(zone//60, 60) + '"' diff -r 6db40a9955dc -r 0d413f60cc23 Lib/imghdr.py --- a/Lib/imghdr.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/imghdr.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,16 +7,18 @@ #-------------------------# def what(file, h=None): - f = None + if h is None: + if isinstance(file, str): + f = open(file, 'rb') + h = f.read(32) + else: + location = file.tell() + h = file.read(32) + file.seek(location) + f = None + else: + f = None try: - if h is None: - if isinstance(file, str): - f = open(file, 'rb') - h = f.read(32) - else: - location = file.tell() - h = file.read(32) - file.seek(location) for tf in tests: res = tf(h, f) if res: @@ -110,18 +112,6 @@ tests.append(test_bmp) -def test_webp(h, f): - if h.startswith(b'RIFF') and h[8:12] == b'WEBP': - return 'webp' - -tests.append(test_webp) - -def test_exr(h, f): - if h.startswith(b'\x76\x2f\x31\x01'): - return 'exr' - -tests.append(test_exr) - #--------------------# # Small test program # #--------------------# @@ -159,7 +149,7 @@ sys.stdout.flush() try: print(what(filename)) - except OSError: + except IOError: print('*** not found ***') if __name__ == '__main__': diff -r 6db40a9955dc -r 0d413f60cc23 Lib/imp.py --- a/Lib/imp.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,345 +0,0 @@ -"""This module provides the components needed to build your own __import__ -function. Undocumented functions are obsolete. - -In most cases it is preferred you consider using the importlib module's -functionality over this module. - -""" -# (Probably) need to stay in _imp -from _imp import (lock_held, acquire_lock, release_lock, - get_frozen_object, is_frozen_package, - init_frozen, is_builtin, is_frozen, - _fix_co_filename) -try: - from _imp import create_dynamic -except ImportError: - # Platform doesn't support dynamic loading. - create_dynamic = None - -from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name -from importlib._bootstrap_external import SourcelessFileLoader - -from importlib import machinery -from importlib import util -import importlib -import os -import sys -import tokenize -import types -import warnings - -warnings.warn("the imp module is deprecated in favour of importlib; " - "see the module's documentation for alternative uses", - DeprecationWarning, stacklevel=2) - -# DEPRECATED -SEARCH_ERROR = 0 -PY_SOURCE = 1 -PY_COMPILED = 2 -C_EXTENSION = 3 -PY_RESOURCE = 4 -PKG_DIRECTORY = 5 -C_BUILTIN = 6 -PY_FROZEN = 7 -PY_CODERESOURCE = 8 -IMP_HOOK = 9 - - -def new_module(name): - """**DEPRECATED** - - Create a new module. - - The module is not entered into sys.modules. - - """ - return types.ModuleType(name) - - -def get_magic(): - """**DEPRECATED** - - Return the magic number for .pyc files. - """ - return util.MAGIC_NUMBER - - -def get_tag(): - """Return the magic tag for .pyc files.""" - return sys.implementation.cache_tag - - -def cache_from_source(path, debug_override=None): - """**DEPRECATED** - - Given the path to a .py file, return the path to its .pyc file. - - The .py file does not need to exist; this simply returns the path to the - .pyc file calculated as if the .py file were imported. - - If debug_override is not None, then it must be a boolean and is used in - place of sys.flags.optimize. - - If sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - return util.cache_from_source(path, debug_override) - - -def source_from_cache(path): - """**DEPRECATED** - - Given the path to a .pyc. file, return the path to its .py file. - - The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - return util.source_from_cache(path) - - -def get_suffixes(): - """**DEPRECATED**""" - extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] - source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] - bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] - - return extensions + source + bytecode - - -class NullImporter: - - """**DEPRECATED** - - Null import object. - - """ - - def __init__(self, path): - if path == '': - raise ImportError('empty pathname', path='') - elif os.path.isdir(path): - raise ImportError('existing directory', path=path) - - def find_module(self, fullname): - """Always returns None.""" - return None - - -class _HackedGetData: - - """Compatibility support for 'file' arguments of various load_*() - functions.""" - - def __init__(self, fullname, path, file=None): - super().__init__(fullname, path) - self.file = file - - def get_data(self, path): - """Gross hack to contort loader to deal w/ load_*()'s bad API.""" - if self.file and path == self.path: - if not self.file.closed: - file = self.file - else: - self.file = file = open(self.path, 'r') - - with file: - # Technically should be returning bytes, but - # SourceLoader.get_code() just passed what is returned to - # compile() which can handle str. And converting to bytes would - # require figuring out the encoding to decode to and - # tokenize.detect_encoding() only accepts bytes. - return file.read() - else: - return super().get_data(path) - - -class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): - - """Compatibility support for implementing load_source().""" - - -def load_source(name, pathname, file=None): - loader = _LoadSourceCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = machinery.SourceFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): - - """Compatibility support for implementing load_compiled().""" - - -def load_compiled(name, pathname, file=None): - """**DEPRECATED**""" - loader = _LoadCompiledCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = SourcelessFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -def load_package(name, path): - """**DEPRECATED**""" - if os.path.isdir(path): - extensions = (machinery.SOURCE_SUFFIXES[:] + - machinery.BYTECODE_SUFFIXES[:]) - for extension in extensions: - path = os.path.join(path, '__init__'+extension) - if os.path.exists(path): - break - else: - raise ValueError('{!r} is not a package'.format(path)) - spec = util.spec_from_file_location(name, path, - submodule_search_locations=[]) - if name in sys.modules: - return _exec(spec, sys.modules[name]) - else: - return _load(spec) - - -def load_module(name, file, filename, details): - """**DEPRECATED** - - Load a module, given information returned by find_module(). - - The module name must include the full package name, if any. - - """ - suffix, mode, type_ = details - if mode and (not mode.startswith(('r', 'U')) or '+' in mode): - raise ValueError('invalid file open mode {!r}'.format(mode)) - elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: - msg = 'file object required for import (type code {})'.format(type_) - raise ValueError(msg) - elif type_ == PY_SOURCE: - return load_source(name, filename, file) - elif type_ == PY_COMPILED: - return load_compiled(name, filename, file) - elif type_ == C_EXTENSION and load_dynamic is not None: - if file is None: - with open(filename, 'rb') as opened_file: - return load_dynamic(name, filename, opened_file) - else: - return load_dynamic(name, filename, file) - elif type_ == PKG_DIRECTORY: - return load_package(name, filename) - elif type_ == C_BUILTIN: - return init_builtin(name) - elif type_ == PY_FROZEN: - return init_frozen(name) - else: - msg = "Don't know how to import {} (type code {})".format(name, type_) - raise ImportError(msg, name=name) - - -def find_module(name, path=None): - """**DEPRECATED** - - Search for a module. - - If path is omitted or None, search for a built-in, frozen or special - module and continue search in sys.path. The module name cannot - contain '.'; to search for a submodule of a package, pass the - submodule name and the package's __path__. - - """ - if not isinstance(name, str): - raise TypeError("'name' must be a str, not {}".format(type(name))) - elif not isinstance(path, (type(None), list)): - # Backwards-compatibility - raise RuntimeError("'list' must be None or a list, " - "not {}".format(type(name))) - - if path is None: - if is_builtin(name): - return None, None, ('', '', C_BUILTIN) - elif is_frozen(name): - return None, None, ('', '', PY_FROZEN) - else: - path = sys.path - - for entry in path: - package_directory = os.path.join(entry, name) - for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: - package_file_name = '__init__' + suffix - file_path = os.path.join(package_directory, package_file_name) - if os.path.isfile(file_path): - return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. - else: - raise ImportError(_ERR_MSG.format(name), name=name) - - encoding = None - if 'b' not in mode: - with open(file_path, 'rb') as file: - encoding = tokenize.detect_encoding(file.readline)[0] - file = open(file_path, mode, encoding=encoding) - return file, file_path, (suffix, mode, type_) - - -def reload(module): - """**DEPRECATED** - - Reload the module and return it. - - The module must have been successfully imported before. - - """ - return importlib.reload(module) - - -def init_builtin(name): - """**DEPRECATED** - - Load and return a built-in module by name, or None is such module doesn't - exist - """ - try: - return _builtin_from_name(name) - except ImportError: - return None - - -if create_dynamic: - def load_dynamic(name, path, file=None): - """**DEPRECATED** - - Load an extension module. - """ - import importlib.machinery - loader = importlib.machinery.ExtensionFileLoader(name, path) - - # Issue #24748: Skip the sys.modules check in _load_module_shim; - # always load new extension - spec = importlib.machinery.ModuleSpec( - name=name, loader=loader, origin=path) - return _load(spec) - -else: - load_dynamic = None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/__init__.py --- a/Lib/importlib/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/importlib/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,61 +1,19 @@ """A pure Python implementation of import.""" -__all__ = ['__import__', 'import_module', 'invalidate_caches', 'reload'] +__all__ = ['__import__', 'import_module', 'invalidate_caches'] + +from . import _bootstrap + + +# To simplify imports in test code +_w_long = _bootstrap._w_long +_r_long = _bootstrap._r_long + # Bootstrap help ##################################################### - -# Until bootstrapping is complete, DO NOT import any modules that attempt -# to import importlib._bootstrap (directly or indirectly). Since this -# partially initialised package would be present in sys.modules, those -# modules would get an uninitialised copy of the source version, instead -# of a fully initialised version (either the frozen one or the one -# initialised below if the frozen one is not available). -import _imp # Just the builtin component, NOT the full Python module +import imp import sys -try: - import _frozen_importlib as _bootstrap -except ImportError: - from . import _bootstrap - _bootstrap._setup(sys, _imp) -else: - # importlib._bootstrap is the built-in import, ensure we don't create - # a second copy of the module. - _bootstrap.__name__ = 'importlib._bootstrap' - _bootstrap.__package__ = 'importlib' - try: - _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py') - except NameError: - # __file__ is not guaranteed to be defined, e.g. if this code gets - # frozen by a tool like cx_Freeze. - pass - sys.modules['importlib._bootstrap'] = _bootstrap - -try: - import _frozen_importlib_external as _bootstrap_external -except ImportError: - from . import _bootstrap_external - _bootstrap_external._setup(_bootstrap) - _bootstrap._bootstrap_external = _bootstrap_external -else: - _bootstrap_external.__name__ = 'importlib._bootstrap_external' - _bootstrap_external.__package__ = 'importlib' - try: - _bootstrap_external.__file__ = __file__.replace('__init__.py', '_bootstrap_external.py') - except NameError: - # __file__ is not guaranteed to be defined, e.g. if this code gets - # frozen by a tool like cx_Freeze. - pass - sys.modules['importlib._bootstrap_external'] = _bootstrap_external - -# To simplify imports in test code -_w_long = _bootstrap_external._w_long -_r_long = _bootstrap_external._r_long - -# Fully bootstrapped at this point, import whatever you like, circular -# dependencies and startup overhead minimisation permitting :) - -import types -import warnings +_bootstrap._setup(sys, imp) # Public API ######################################################### @@ -64,47 +22,13 @@ def invalidate_caches(): - """Call the invalidate_caches() method on all meta path finders stored in - sys.meta_path (where implemented).""" - for finder in sys.meta_path: + """Call the invalidate_caches() method on all finders stored in + sys.path_importer_caches (where implemented).""" + for finder in sys.path_importer_cache.values(): if hasattr(finder, 'invalidate_caches'): finder.invalidate_caches() -def find_loader(name, path=None): - """Return the loader for the specified module. - - This is a backward-compatible wrapper around find_spec(). - - This function is deprecated in favor of importlib.util.find_spec(). - - """ - warnings.warn('Use importlib.util.find_spec() instead.', - DeprecationWarning, stacklevel=2) - try: - loader = sys.modules[name].__loader__ - if loader is None: - raise ValueError('{}.__loader__ is None'.format(name)) - else: - return loader - except KeyError: - pass - except AttributeError: - raise ValueError('{}.__loader__ is not set'.format(name)) from None - - spec = _bootstrap._find_spec(name, path) - # We won't worry about malformed specs (missing attributes). - if spec is None: - return None - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError('spec for {} missing loader'.format(name), - name=name) - raise ImportError('namespace packages do not have loaders', - name=name) - return spec.loader - - def import_module(name, package=None): """Import a module. @@ -116,58 +40,9 @@ level = 0 if name.startswith('.'): if not package: - msg = ("the 'package' argument is required to perform a relative " - "import for {!r}") - raise TypeError(msg.format(name)) + raise TypeError("relative imports require the 'package' argument") for character in name: if character != '.': break level += 1 return _bootstrap._gcd_import(name[level:], package, level) - - -_RELOADING = {} - - -def reload(module): - """Reload the module and return it. - - The module must have been successfully imported before. - - """ - if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be module") - try: - name = module.__spec__.name - except AttributeError: - name = module.__name__ - - if sys.modules.get(name) is not module: - msg = "module {} not in sys.modules" - raise ImportError(msg.format(name), name=name) - if name in _RELOADING: - return _RELOADING[name] - _RELOADING[name] = module - try: - parent_name = name.rpartition('.')[0] - if parent_name: - try: - parent = sys.modules[parent_name] - except KeyError: - msg = "parent {!r} not in sys.modules" - raise ImportError(msg.format(parent_name), - name=parent_name) from None - else: - pkgpath = parent.__path__ - else: - pkgpath = None - target = module - spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target) - _bootstrap._exec(spec, module) - # The module may have replaced itself in sys.modules! - return sys.modules[name] - finally: - try: - del _RELOADING[name] - except KeyError: - pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/_bootstrap.py --- a/Lib/importlib/_bootstrap.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/importlib/_bootstrap.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,236 +6,238 @@ work. One should use importlib as the public-facing version of this module. """ + +# Injected modules are '_warnings', 'imp', 'sys', 'marshal', '_io', +# and '_os' (a.k.a. 'posix', 'nt' or 'os2'). +# Injected attribute is path_sep. +# Most injection is handled by _setup(). # -# IMPORTANT: Whenever making changes to this module, be sure to run -# a top-level make in order to get the frozen version of the module -# updated. Not doing so will result in the Makefile to fail for -# all others who don't have a ./python around to freeze the module -# in the early stages of compilation. -# - -# See importlib._setup() for what is injected into the global namespace. - # When editing this code be aware that code executed at import time CANNOT # reference any injected objects! This includes not only global code but also # anything specified at the class level. + # Bootstrap-related code ###################################################### -_bootstrap_external = None +CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' + + +def _make_relax_case(): + if any(map(sys.platform.startswith, CASE_INSENSITIVE_PLATFORMS)): + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return b'PYTHONCASEOK' in _os.environ + else: + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return False + return _relax_case + + +# TODO: Expose from marshal +def _w_long(x): + """Convert a 32-bit integer to little-endian. + + XXX Temporary until marshal's long functions are exposed. + + """ + x = int(x) + int_bytes = [] + int_bytes.append(x & 0xFF) + int_bytes.append((x >> 8) & 0xFF) + int_bytes.append((x >> 16) & 0xFF) + int_bytes.append((x >> 24) & 0xFF) + return bytearray(int_bytes) + + +# TODO: Expose from marshal +def _r_long(int_bytes): + """Convert 4 bytes in little-endian to an integer. + + XXX Temporary until marshal's long function are exposed. + + """ + x = int_bytes[0] + x |= int_bytes[1] << 8 + x |= int_bytes[2] << 16 + x |= int_bytes[3] << 24 + return x + + + +# XXX Could also expose Modules/getpath.c:joinpath() +def _path_join(*args): + """Replacement for os.path.join.""" + return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x + for x in args if x) + + +def _path_exists(path): + """Replacement for os.path.exists.""" + try: + _os.stat(path) + except OSError: + return False + else: + return True + + +def _path_is_mode_type(path, mode): + """Test whether the path is the specified mode type.""" + try: + stat_info = _os.stat(path) + except OSError: + return False + return (stat_info.st_mode & 0o170000) == mode + + +# XXX Could also expose Modules/getpath.c:isfile() +def _path_isfile(path): + """Replacement for os.path.isfile.""" + return _path_is_mode_type(path, 0o100000) + + +# XXX Could also expose Modules/getpath.c:isdir() +def _path_isdir(path): + """Replacement for os.path.isdir.""" + if not path: + path = _os.getcwd() + return _path_is_mode_type(path, 0o040000) + + +def _path_without_ext(path, ext_type): + """Replacement for os.path.splitext()[0].""" + for suffix in _suffix_list(ext_type): + if path.endswith(suffix): + return path[:-len(suffix)] + else: + raise ValueError("path is not of the specified type") + + +def _path_absolute(path): + """Replacement for os.path.abspath.""" + if not path: + path = _os.getcwd() + try: + return _os._getfullpathname(path) + except AttributeError: + if path.startswith('/'): + return path + else: + return _path_join(_os.getcwd(), path) + + +def _write_atomic(path, data): + """Best-effort function to write data to a path atomically. + Be prepared to handle a FileExistsError if concurrent writing of the + temporary file is attempted.""" + # id() is used to generate a pseudo-random filename. + path_tmp = '{}.{}'.format(path, id(path)) + fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, 0o666) + try: + # We first write data to a temporary file, and then use os.replace() to + # perform an atomic rename. + with _io.FileIO(fd, 'wb') as file: + file.write(data) + _os.replace(path_tmp, path) + except OSError: + try: + _os.unlink(path_tmp) + except OSError: + pass + raise + def _wrap(new, old): - """Simple substitute for functools.update_wrapper.""" + """Simple substitute for functools.wraps.""" for replace in ['__module__', '__name__', '__qualname__', '__doc__']: if hasattr(old, replace): setattr(new, replace, getattr(old, replace)) new.__dict__.update(old.__dict__) -def _new_module(name): - return type(sys)(name) +code_type = type(_wrap.__code__) +# Finder/loader utility code ################################################## -class _ManageReload: - """Manages the possible clean-up of sys.modules for load_module().""" +def set_package(fxn): + """Set __package__ on the returned module.""" + def set_package_wrapper(*args, **kwargs): + module = fxn(*args, **kwargs) + if not hasattr(module, '__package__') or module.__package__ is None: + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = module.__package__.rpartition('.')[0] + return module + _wrap(set_package_wrapper, fxn) + return set_package_wrapper - def __init__(self, name): - self._name = name - def __enter__(self): - self._is_reload = self._name in sys.modules +def set_loader(fxn): + """Set __loader__ on the returned module.""" + def set_loader_wrapper(self, *args, **kwargs): + module = fxn(self, *args, **kwargs) + if not hasattr(module, '__loader__'): + module.__loader__ = self + return module + _wrap(set_loader_wrapper, fxn) + return set_loader_wrapper - def __exit__(self, *args): - if any(arg is not None for arg in args) and not self._is_reload: - try: - del sys.modules[self._name] - except KeyError: - pass -# Module-level locking ######################################################## +def module_for_loader(fxn): + """Decorator to handle selecting the proper module for loaders. -# A dict mapping module names to weakrefs of _ModuleLock instances -_module_locks = {} -# A dict mapping thread ids to _ModuleLock instances -_blocking_on = {} + The decorated function is passed the module to use instead of the module + name. The module passed in to the function is either from sys.modules if + it already exists or is a new module which has __name__ set and is inserted + into sys.modules. If an exception is raised and the decorator created the + module it is subsequently removed from sys.modules. + The decorator assumes that the decorated function takes the module name as + the second argument. -class _DeadlockError(RuntimeError): - pass + """ + def module_for_loader_wrapper(self, fullname, *args, **kwargs): + module = sys.modules.get(fullname) + is_reload = bool(module) + if not is_reload: + # This must be done before open() is called as the 'io' module + # implicitly imports 'locale' and would otherwise trigger an + # infinite loop. + module = imp.new_module(fullname) + sys.modules[fullname] = module + try: + return fxn(self, module, *args, **kwargs) + except: + if not is_reload: + del sys.modules[fullname] + raise + _wrap(module_for_loader_wrapper, fxn) + return module_for_loader_wrapper -class _ModuleLock: - """A recursive lock implementation which is able to detect deadlocks - (e.g. thread 1 trying to take locks A then B, and thread 2 trying to - take locks B then A). +def _check_name(method): + """Decorator to verify that the module being requested matches the one the + loader can handle. + + The first argument (self) must define _name which the second argument is + compared against. If the comparison fails then ImportError is raised. + """ - - def __init__(self, name): - self.lock = _thread.allocate_lock() - self.wakeup = _thread.allocate_lock() - self.name = name - self.owner = None - self.count = 0 - self.waiters = 0 - - def has_deadlock(self): - # Deadlock avoidance for concurrent circular imports. - me = _thread.get_ident() - tid = self.owner - while True: - lock = _blocking_on.get(tid) - if lock is None: - return False - tid = lock.owner - if tid == me: - return True - - def acquire(self): - """ - Acquire the module lock. If a potential deadlock is detected, - a _DeadlockError is raised. - Otherwise, the lock is always acquired and True is returned. - """ - tid = _thread.get_ident() - _blocking_on[tid] = self - try: - while True: - with self.lock: - if self.count == 0 or self.owner == tid: - self.owner = tid - self.count += 1 - return True - if self.has_deadlock(): - raise _DeadlockError('deadlock detected by %r' % self) - if self.wakeup.acquire(False): - self.waiters += 1 - # Wait for a release() call - self.wakeup.acquire() - self.wakeup.release() - finally: - del _blocking_on[tid] - - def release(self): - tid = _thread.get_ident() - with self.lock: - if self.owner != tid: - raise RuntimeError('cannot release un-acquired lock') - assert self.count > 0 - self.count -= 1 - if self.count == 0: - self.owner = None - if self.waiters: - self.waiters -= 1 - self.wakeup.release() - - def __repr__(self): - return '_ModuleLock({!r}) at {}'.format(self.name, id(self)) - - -class _DummyModuleLock: - """A simple _ModuleLock equivalent for Python builds without - multi-threading support.""" - - def __init__(self, name): - self.name = name - self.count = 0 - - def acquire(self): - self.count += 1 - return True - - def release(self): - if self.count == 0: - raise RuntimeError('cannot release un-acquired lock') - self.count -= 1 - - def __repr__(self): - return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self)) - - -class _ModuleLockManager: - - def __init__(self, name): - self._name = name - self._lock = None - - def __enter__(self): - try: - self._lock = _get_module_lock(self._name) - finally: - _imp.release_lock() - self._lock.acquire() - - def __exit__(self, *args, **kwargs): - self._lock.release() - - -# The following two functions are for consumption by Python/import.c. - -def _get_module_lock(name): - """Get or create the module lock for a given module name. - - Should only be called with the import lock taken.""" - lock = None - try: - lock = _module_locks[name]() - except KeyError: - pass - if lock is None: - if _thread is None: - lock = _DummyModuleLock(name) - else: - lock = _ModuleLock(name) - def cb(_): - del _module_locks[name] - _module_locks[name] = _weakref.ref(lock, cb) - return lock - -def _lock_unlock_module(name): - """Release the global import lock, and acquires then release the - module lock for a given module name. - This is used to ensure a module is completely initialized, in the - event it is being imported by another thread. - - Should only be called with the import lock taken.""" - lock = _get_module_lock(name) - _imp.release_lock() - try: - lock.acquire() - except _DeadlockError: - # Concurrent circular import, we'll accept a partially initialized - # module object. - pass - else: - lock.release() - -# Frame stripping magic ############################################### -def _call_with_frames_removed(f, *args, **kwds): - """remove_importlib_frames in import.c will always remove sequences - of importlib frames that end with a call to this function - - Use it instead of a normal call in places where including the importlib - frames introduces unwanted noise into the traceback (e.g. when executing - module code) - """ - return f(*args, **kwds) - - -def _verbose_message(message, *args, verbosity=1): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) + def _check_name_wrapper(self, name, *args, **kwargs): + if self._name != name: + raise ImportError("loader cannot handle %s" % name) + return method(self, name, *args, **kwargs) + _wrap(_check_name_wrapper, method) + return _check_name_wrapper def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" def _requires_builtin_wrapper(self, fullname): if fullname not in sys.builtin_module_names: - raise ImportError('{!r} is not a built-in module'.format(fullname), - name=fullname) + raise ImportError("{0} is not a built-in module".format(fullname)) return fxn(self, fullname) _wrap(_requires_builtin_wrapper, fxn) return _requires_builtin_wrapper @@ -244,453 +246,17 @@ def _requires_frozen(fxn): """Decorator to verify the named module is frozen.""" def _requires_frozen_wrapper(self, fullname): - if not _imp.is_frozen(fullname): - raise ImportError('{!r} is not a frozen module'.format(fullname), - name=fullname) + if not imp.is_frozen(fullname): + raise ImportError("{0} is not a frozen module".format(fullname)) return fxn(self, fullname) _wrap(_requires_frozen_wrapper, fxn) return _requires_frozen_wrapper -# Typically used by loader classes as a method replacement. -def _load_module_shim(self, fullname): - """Load the specified module into sys.modules and return it. - - This method is deprecated. Use loader.exec_module instead. - - """ - spec = spec_from_loader(fullname, self) - if fullname in sys.modules: - module = sys.modules[fullname] - _exec(spec, module) - return sys.modules[fullname] - else: - return _load(spec) - -# Module specifications ####################################################### - -def _module_repr(module): - # The implementation of ModuleType__repr__(). - loader = getattr(module, '__loader__', None) - if hasattr(loader, 'module_repr'): - # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader - # drop their implementations for module_repr. we can add a - # deprecation warning here. - try: - return loader.module_repr(module) - except Exception: - pass - try: - spec = module.__spec__ - except AttributeError: - pass - else: - if spec is not None: - return _module_repr_from_spec(spec) - - # We could use module.__class__.__name__ instead of 'module' in the - # various repr permutations. - try: - name = module.__name__ - except AttributeError: - name = '?' - try: - filename = module.__file__ - except AttributeError: - if loader is None: - return ''.format(name) - else: - return ''.format(name, loader) - else: - return ''.format(name, filename) - - -class _installed_safely: - - def __init__(self, module): - self._module = module - self._spec = module.__spec__ - - def __enter__(self): - # This must be done before putting the module in sys.modules - # (otherwise an optimization shortcut in import.c becomes - # wrong) - self._spec._initializing = True - sys.modules[self._spec.name] = self._module - - def __exit__(self, *args): - try: - spec = self._spec - if any(arg is not None for arg in args): - try: - del sys.modules[spec.name] - except KeyError: - pass - else: - _verbose_message('import {!r} # {!r}', spec.name, spec.loader) - finally: - self._spec._initializing = False - - -class ModuleSpec: - """The specification for a module, used for loading. - - A module's spec is the source for information about the module. For - data associated with the module, including source, use the spec's - loader. - - `name` is the absolute name of the module. `loader` is the loader - to use when loading the module. `parent` is the name of the - package the module is in. The parent is derived from the name. - - `is_package` determines if the module is considered a package or - not. On modules this is reflected by the `__path__` attribute. - - `origin` is the specific location used by the loader from which to - load the module, if that information is available. When filename is - set, origin will match. - - `has_location` indicates that a spec's "origin" reflects a location. - When this is True, `__file__` attribute of the module is set. - - `cached` is the location of the cached bytecode file, if any. It - corresponds to the `__cached__` attribute. - - `submodule_search_locations` is the sequence of path entries to - search when importing submodules. If set, is_package should be - True--and False otherwise. - - Packages are simply modules that (may) have submodules. If a spec - has a non-None value in `submodule_search_locations`, the import - system will consider modules loaded from the spec as packages. - - Only finders (see importlib.abc.MetaPathFinder and - importlib.abc.PathEntryFinder) should modify ModuleSpec instances. - - """ - - def __init__(self, name, loader, *, origin=None, loader_state=None, - is_package=None): - self.name = name - self.loader = loader - self.origin = origin - self.loader_state = loader_state - self.submodule_search_locations = [] if is_package else None - - # file-location attributes - self._set_fileattr = False - self._cached = None - - def __repr__(self): - args = ['name={!r}'.format(self.name), - 'loader={!r}'.format(self.loader)] - if self.origin is not None: - args.append('origin={!r}'.format(self.origin)) - if self.submodule_search_locations is not None: - args.append('submodule_search_locations={}' - .format(self.submodule_search_locations)) - return '{}({})'.format(self.__class__.__name__, ', '.join(args)) - - def __eq__(self, other): - smsl = self.submodule_search_locations - try: - return (self.name == other.name and - self.loader == other.loader and - self.origin == other.origin and - smsl == other.submodule_search_locations and - self.cached == other.cached and - self.has_location == other.has_location) - except AttributeError: - return False - - @property - def cached(self): - if self._cached is None: - if self.origin is not None and self._set_fileattr: - if _bootstrap_external is None: - raise NotImplementedError - self._cached = _bootstrap_external._get_cached(self.origin) - return self._cached - - @cached.setter - def cached(self, cached): - self._cached = cached - - @property - def parent(self): - """The name of the module's parent.""" - if self.submodule_search_locations is None: - return self.name.rpartition('.')[0] - else: - return self.name - - @property - def has_location(self): - return self._set_fileattr - - @has_location.setter - def has_location(self, value): - self._set_fileattr = bool(value) - - -def spec_from_loader(name, loader, *, origin=None, is_package=None): - """Return a module spec based on various loader methods.""" - if hasattr(loader, 'get_filename'): - if _bootstrap_external is None: - raise NotImplementedError - spec_from_file_location = _bootstrap_external.spec_from_file_location - - if is_package is None: - return spec_from_file_location(name, loader=loader) - search = [] if is_package else None - return spec_from_file_location(name, loader=loader, - submodule_search_locations=search) - - if is_package is None: - if hasattr(loader, 'is_package'): - try: - is_package = loader.is_package(name) - except ImportError: - is_package = None # aka, undefined - else: - # the default - is_package = False - - return ModuleSpec(name, loader, origin=origin, is_package=is_package) - - -_POPULATE = object() - - -def _spec_from_module(module, loader=None, origin=None): - # This function is meant for use in _setup(). - try: - spec = module.__spec__ - except AttributeError: - pass - else: - if spec is not None: - return spec - - name = module.__name__ - if loader is None: - try: - loader = module.__loader__ - except AttributeError: - # loader will stay None. - pass - try: - location = module.__file__ - except AttributeError: - location = None - if origin is None: - if location is None: - try: - origin = loader._ORIGIN - except AttributeError: - origin = None - else: - origin = location - try: - cached = module.__cached__ - except AttributeError: - cached = None - try: - submodule_search_locations = list(module.__path__) - except AttributeError: - submodule_search_locations = None - - spec = ModuleSpec(name, loader, origin=origin) - spec._set_fileattr = False if location is None else True - spec.cached = cached - spec.submodule_search_locations = submodule_search_locations - return spec - - -def _init_module_attrs(spec, module, *, override=False): - # The passed-in module may be not support attribute assignment, - # in which case we simply don't set the attributes. - # __name__ - if (override or getattr(module, '__name__', None) is None): - try: - module.__name__ = spec.name - except AttributeError: - pass - # __loader__ - if override or getattr(module, '__loader__', None) is None: - loader = spec.loader - if loader is None: - # A backward compatibility hack. - if spec.submodule_search_locations is not None: - if _bootstrap_external is None: - raise NotImplementedError - _NamespaceLoader = _bootstrap_external._NamespaceLoader - - loader = _NamespaceLoader.__new__(_NamespaceLoader) - loader._path = spec.submodule_search_locations - try: - module.__loader__ = loader - except AttributeError: - pass - # __package__ - if override or getattr(module, '__package__', None) is None: - try: - module.__package__ = spec.parent - except AttributeError: - pass - # __spec__ - try: - module.__spec__ = spec - except AttributeError: - pass - # __path__ - if override or getattr(module, '__path__', None) is None: - if spec.submodule_search_locations is not None: - try: - module.__path__ = spec.submodule_search_locations - except AttributeError: - pass - # __file__/__cached__ - if spec.has_location: - if override or getattr(module, '__file__', None) is None: - try: - module.__file__ = spec.origin - except AttributeError: - pass - - if override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass - return module - - -def module_from_spec(spec): - """Create a module based on the provided spec.""" - # Typically loaders will not implement create_module(). - module = None - if hasattr(spec.loader, 'create_module'): - # If create_module() returns `None` then it means default - # module creation should be used. - module = spec.loader.create_module(spec) - elif hasattr(spec.loader, 'exec_module'): - _warnings.warn('starting in Python 3.6, loaders defining exec_module() ' - 'must also define create_module()', - DeprecationWarning, stacklevel=2) - if module is None: - module = _new_module(spec.name) - _init_module_attrs(spec, module) - return module - - -def _module_repr_from_spec(spec): - """Return the repr to use for the module.""" - # We mostly replicate _module_repr() using the spec attributes. - name = '?' if spec.name is None else spec.name - if spec.origin is None: - if spec.loader is None: - return ''.format(name) - else: - return ''.format(name, spec.loader) - else: - if spec.has_location: - return ''.format(name, spec.origin) - else: - return ''.format(spec.name, spec.origin) - - -# Used by importlib.reload() and _load_module_shim(). -def _exec(spec, module): - """Execute the spec in an existing module's namespace.""" - name = spec.name - _imp.acquire_lock() - with _ModuleLockManager(name): - if sys.modules.get(name) is not module: - msg = 'module {!r} not in sys.modules'.format(name) - raise ImportError(msg, name=name) - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError('missing loader', name=spec.name) - # namespace package - _init_module_attrs(spec, module, override=True) - return module - _init_module_attrs(spec, module, override=True) - if not hasattr(spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - spec.loader.load_module(name) - else: - spec.loader.exec_module(module) - return sys.modules[name] - - -def _load_backward_compatible(spec): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - spec.loader.load_module(spec.name) - # The module must be in sys.modules at this point! - module = sys.modules[spec.name] - if getattr(module, '__loader__', None) is None: - try: - module.__loader__ = spec.loader - except AttributeError: - pass - if getattr(module, '__package__', None) is None: - try: - # Since module.__path__ may not line up with - # spec.submodule_search_paths, we can't necessarily rely - # on spec.parent here. - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = spec.name.rpartition('.')[0] - except AttributeError: - pass - if getattr(module, '__spec__', None) is None: - try: - module.__spec__ = spec - except AttributeError: - pass - return module - -def _load_unlocked(spec): - # A helper for direct use by the import system. - if spec.loader is not None: - # not a namespace package - if not hasattr(spec.loader, 'exec_module'): - return _load_backward_compatible(spec) - - module = module_from_spec(spec) - with _installed_safely(module): - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError('missing loader', name=spec.name) - # A namespace package so do nothing. - else: - spec.loader.exec_module(module) - - # We don't ensure that the import-related module attributes get - # set in the sys.modules replacement case. Such modules are on - # their own. - return sys.modules[spec.name] - -# A method used during testing of _load_unlocked() and by -# _load_module_shim(). -def _load(spec): - """Return a new module object, loaded by the spec's loader. - - The module is not added to its parent. - - If a module is already in sys.modules, that existing module gets - clobbered. - - """ - _imp.acquire_lock() - with _ModuleLockManager(spec.name): - return _load_unlocked(spec) +def _suffix_list(suffix_type): + """Return a list of file suffixes based on the imp file type.""" + return [suffix[0] for suffix in imp.get_suffixes() + if suffix[2] == suffix_type] # Loaders ##################################################################### @@ -704,48 +270,30 @@ """ - @staticmethod - def module_repr(module): - """Return repr for the module. - - The method is deprecated. The import machinery does the job itself. - - """ - return ''.format(module.__name__) - - @classmethod - def find_spec(cls, fullname, path=None, target=None): - if path is not None: - return None - if _imp.is_builtin(fullname): - return spec_from_loader(fullname, cls, origin='built-in') - else: - return None - @classmethod def find_module(cls, fullname, path=None): """Find the built-in module. If 'path' is ever specified then the search is considered a failure. - This method is deprecated. Use find_spec() instead. - """ - spec = cls.find_spec(fullname, path) - return spec.loader if spec is not None else None + if path is not None: + return None + return cls if imp.is_builtin(fullname) else None @classmethod - def create_module(self, spec): - """Create a built-in module""" - if spec.name not in sys.builtin_module_names: - raise ImportError('{!r} is not a built-in module'.format(spec.name), - name=spec.name) - return _call_with_frames_removed(_imp.create_builtin, spec) - - @classmethod - def exec_module(self, module): - """Exec a built-in module""" - _call_with_frames_removed(_imp.exec_builtin, module) + @set_package + @set_loader + @_requires_builtin + def load_module(cls, fullname): + """Load a built-in module.""" + is_reload = fullname in sys.modules + try: + return imp.init_builtin(fullname) + except: + if not is_reload and fullname in sys.modules: + del sys.modules[fullname] + raise @classmethod @_requires_builtin @@ -762,11 +310,9 @@ @classmethod @_requires_builtin def is_package(cls, fullname): - """Return False as built-in modules are never packages.""" + """Return None as built-in modules are never packages.""" return False - load_module = classmethod(_load_module_shim) - class FrozenImporter: @@ -777,58 +323,30 @@ """ - @staticmethod - def module_repr(m): - """Return repr for the module. - - The method is deprecated. The import machinery does the job itself. - - """ - return ''.format(m.__name__) + @classmethod + def find_module(cls, fullname, path=None): + """Find a frozen module.""" + return cls if imp.is_frozen(fullname) else None @classmethod - def find_spec(cls, fullname, path=None, target=None): - if _imp.is_frozen(fullname): - return spec_from_loader(fullname, cls, origin='frozen') - else: - return None - - @classmethod - def find_module(cls, fullname, path=None): - """Find a frozen module. - - This method is deprecated. Use find_spec() instead. - - """ - return cls if _imp.is_frozen(fullname) else None - - @classmethod - def create_module(cls, spec): - """Use default semantics for module creation.""" - - @staticmethod - def exec_module(module): - name = module.__spec__.name - if not _imp.is_frozen(name): - raise ImportError('{!r} is not a frozen module'.format(name), - name=name) - code = _call_with_frames_removed(_imp.get_frozen_object, name) - exec(code, module.__dict__) - - @classmethod + @set_package + @set_loader + @_requires_frozen def load_module(cls, fullname): - """Load a frozen module. - - This method is deprecated. Use exec_module() instead. - - """ - return _load_module_shim(cls, fullname) + """Load a frozen module.""" + is_reload = fullname in sys.modules + try: + return imp.init_frozen(fullname) + except: + if not is_reload and fullname in sys.modules: + del sys.modules[fullname] + raise @classmethod @_requires_frozen def get_code(cls, fullname): """Return the code object for the frozen module.""" - return _imp.get_frozen_object(fullname) + return imp.get_frozen_object(fullname) @classmethod @_requires_frozen @@ -839,23 +357,522 @@ @classmethod @_requires_frozen def is_package(cls, fullname): - """Return True if the frozen module is a package.""" - return _imp.is_frozen_package(fullname) + """Return if the frozen module is a package.""" + return imp.is_frozen_package(fullname) + + +class _LoaderBasics: + + """Base class of common code needed by both SourceLoader and + _SourcelessFileLoader.""" + + def is_package(self, fullname): + """Concrete implementation of InspectLoader.is_package by checking if + the path returned by get_filename has a filename of '__init__.py'.""" + filename = self.get_filename(fullname).rpartition(path_sep)[2] + return filename.rsplit('.', 1)[0] == '__init__' + + def _bytes_from_bytecode(self, fullname, data, source_stats): + """Return the marshalled bytes from bytecode, verifying the magic + number, timestamp and source size along the way. + + If source_stats is None then skip the timestamp check. + + """ + magic = data[:4] + raw_timestamp = data[4:8] + raw_size = data[8:12] + if len(magic) != 4 or magic != imp.get_magic(): + raise ImportError("bad magic number in {}".format(fullname)) + elif len(raw_timestamp) != 4: + raise EOFError("bad timestamp in {}".format(fullname)) + elif len(raw_size) != 4: + raise EOFError("bad size in {}".format(fullname)) + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if _r_long(raw_timestamp) != source_mtime: + raise ImportError( + "bytecode is stale for {}".format(fullname)) + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if _r_long(raw_size) != source_size: + raise ImportError( + "bytecode is stale for {}".format(fullname)) + # Can't return the code object as errors from marshal loading need to + # propagate even when source is available. + return data[12:] + + @module_for_loader + def _load_module(self, module, *, sourceless=False): + """Helper for load_module able to handle either source or sourceless + loading.""" + name = module.__name__ + code_object = self.get_code(name) + module.__file__ = self.get_filename(name) + if not sourceless: + module.__cached__ = imp.cache_from_source(module.__file__) + else: + module.__cached__ = module.__file__ + module.__package__ = name + if self.is_package(name): + module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] + else: + module.__package__ = module.__package__.rpartition('.')[0] + module.__loader__ = self + exec(code_object, module.__dict__) + return module + + +class SourceLoader(_LoaderBasics): + + def path_mtime(self, path): + """Optional method that returns the modification time (an int) for the + specified path, where path is a str. + """ + raise NotImplementedError + + def path_stats(self, path): + """Optional method returning a metadata dict for the specified path + to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + + Implementing this method allows the loader to read bytecode files. + """ + return {'mtime': self.path_mtime(path)} + + def set_data(self, path, data): + """Optional method which writes data (bytes) to a file path (a str). + + Implementing this method allows for the writing of bytecode files. + + """ + raise NotImplementedError + + + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + import tokenize + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except IOError: + raise ImportError("source not available through get_data()") + encoding = tokenize.detect_encoding(_io.BytesIO(source_bytes).readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) + + def get_code(self, fullname): + """Concrete implementation of InspectLoader.get_code. + + Reading of bytecode requires path_stats to be implemented. To write + bytecode, set_data must also be implemented. + + """ + source_path = self.get_filename(fullname) + bytecode_path = imp.cache_from_source(source_path) + source_mtime = None + if bytecode_path is not None: + try: + st = self.path_stats(source_path) + except NotImplementedError: + pass + else: + source_mtime = int(st['mtime']) + try: + data = self.get_data(bytecode_path) + except IOError: + pass + else: + try: + bytes_data = self._bytes_from_bytecode(fullname, data, + st) + except (ImportError, EOFError): + pass + else: + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + imp._fix_co_filename(found, source_path) + return found + else: + msg = "Non-code object in {}" + raise ImportError(msg.format(bytecode_path)) + source_bytes = self.get_data(source_path) + code_object = compile(source_bytes, source_path, 'exec', + dont_inherit=True) + if (not sys.dont_write_bytecode and bytecode_path is not None and + source_mtime is not None): + # If e.g. Jython ever implements imp.cache_from_source to have + # their own cached file format, this block of code will most likely + # throw an exception. + data = bytearray(imp.get_magic()) + data.extend(_w_long(source_mtime)) + data.extend(_w_long(len(source_bytes))) + data.extend(marshal.dumps(code_object)) + try: + self.set_data(bytecode_path, data) + except NotImplementedError: + pass + return code_object + + def load_module(self, fullname): + """Concrete implementation of Loader.load_module. + + Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be + implemented to load source code. Use of bytecode is dictated by whether + get_code uses/writes bytecode. + + """ + return self._load_module(fullname) + + +class _FileLoader: + + """Base file loader class which implements the loader protocol methods that + require file system usage.""" + + def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" + self._name = fullname + self._path = path + + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self._path + + def get_data(self, path): + """Return the data from path as raw bytes.""" + with _io.FileIO(path, 'r') as file: + return file.read() + + +class _SourceFileLoader(_FileLoader, SourceLoader): + + """Concrete implementation of SourceLoader using the file system.""" + + def path_stats(self, path): + """Return the metadat for the path.""" + st = _os.stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} + + def set_data(self, path, data): + """Write bytes data to a file.""" + parent, _, filename = path.rpartition(path_sep) + path_parts = [] + # Figure out what directories are missing. + while parent and not _path_isdir(parent): + parent, _, part = parent.rpartition(path_sep) + path_parts.append(part) + # Create needed directories. + for part in reversed(path_parts): + parent = _path_join(parent, part) + try: + _os.mkdir(parent) + except FileExistsError: + # Probably another Python process already created the dir. + continue + except PermissionError: + # If can't get proper access, then just forget about writing + # the data. + return + try: + _write_atomic(path, data) + except (PermissionError, FileExistsError): + # Don't worry if you can't write bytecode or someone is writing + # it at the same time. + pass + + +class _SourcelessFileLoader(_FileLoader, _LoaderBasics): + + """Loader which handles sourceless file imports.""" + + def load_module(self, fullname): + return self._load_module(fullname, sourceless=True) + + def get_code(self, fullname): + path = self.get_filename(fullname) + data = self.get_data(path) + bytes_data = self._bytes_from_bytecode(fullname, data, None) + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + return found + else: + raise ImportError("Non-code object in {}".format(path)) + + def get_source(self, fullname): + """Return None as there is no source code.""" + return None + + +class _ExtensionFileLoader: + + """Loader for extension modules. + + The constructor is designed to work with FileFinder. + + """ + + def __init__(self, name, path): + self._name = name + self._path = path + + @_check_name + @set_package + @set_loader + def load_module(self, fullname): + """Load an extension module.""" + is_reload = fullname in sys.modules + try: + return imp.load_dynamic(fullname, self._path) + except: + if not is_reload and fullname in sys.modules: + del sys.modules[fullname] + raise + + @_check_name + def is_package(self, fullname): + """Return False as an extension module can never be a package.""" + return False + + @_check_name + def get_code(self, fullname): + """Return None as an extension module cannot create a code object.""" + return None + + @_check_name + def get_source(self, fullname): + """Return None as extension modules have no source code.""" + return None + + +# Finders ##################################################################### + +class PathFinder: + + """Meta path finder for sys.(path|path_hooks|path_importer_cache).""" + + @classmethod + def _path_hooks(cls, path, hooks=None): + """Search sequence of hooks for a finder for 'path'. + + If 'hooks' is false then use sys.path_hooks. + + """ + if not hooks: + hooks = sys.path_hooks + for hook in hooks: + try: + return hook(path) + except ImportError: + continue + else: + raise ImportError("no path hook found for {0}".format(path)) + + @classmethod + def _path_importer_cache(cls, path, default=None): + """Get the finder for the path from sys.path_importer_cache. + + If the path is not in the cache, find the appropriate finder and cache + it. If None is cached, get the default finder and cache that + (if applicable). + + Because of NullImporter, some finder should be returned. The only + explicit fail case is if None is cached but the path cannot be used for + the default hook, for which ImportError is raised. + + """ + if path == '': + path = '.' + try: + finder = sys.path_importer_cache[path] + except KeyError: + finder = cls._path_hooks(path) + sys.path_importer_cache[path] = finder + else: + if finder is None and default: + # Raises ImportError on failure. + finder = default(path) + sys.path_importer_cache[path] = finder + return finder + + @classmethod + def find_module(cls, fullname, path=None): + """Find the module on sys.path or 'path' based on sys.path_hooks and + sys.path_importer_cache.""" + if not path: + path = sys.path + for entry in path: + try: + finder = cls._path_importer_cache(entry) + except ImportError: + continue + if finder: + loader = finder.find_module(fullname) + if loader: + return loader + else: + return None + + +class _FileFinder: + + """File-based finder. + + Constructor takes a list of objects detailing what file extensions their + loader supports along with whether it can be used for a package. + + """ + + def __init__(self, path, *details): + """Initialize with finder details.""" + packages = [] + modules = [] + for detail in details: + modules.extend((suffix, detail.loader) for suffix in detail.suffixes) + if detail.supports_packages: + packages.extend((suffix, detail.loader) + for suffix in detail.suffixes) + self.packages = packages + self.modules = modules + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def invalidate_caches(self): + """Invalidate the directory mtime.""" + self._path_mtime = -1 + + def find_module(self, fullname): + """Try to find a loader for the specified module.""" + tail_module = fullname.rpartition('.')[2] + try: + mtime = _os.stat(self.path).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + if cache_module in cache: + base_path = _path_join(self.path, tail_module) + if _path_isdir(base_path): + for suffix, loader in self.packages: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return loader(fullname, full_path) + else: + msg = "Not importing directory {}: missing __init__" + _warnings.warn(msg.format(base_path), ImportWarning) + for suffix, loader in self.modules: + if cache_module + suffix in cache: + full_path = _path_join(self.path, tail_module + suffix) + if _path_isfile(full_path): + return loader(fullname, full_path) + return None + + def _fill_cache(self): + """Fill the cache of potential modules and packages for this directory.""" + path = self.path + contents = _os.listdir(path) + # We store two cached versions, to handle runtime changes of the + # PYTHONCASEOK environment variable. + self._path_cache = set(contents) + self._relaxed_path_cache = set(fn.lower() for fn in contents) + + +class _SourceFinderDetails: + + loader = _SourceFileLoader + supports_packages = True + + def __init__(self): + self.suffixes = _suffix_list(imp.PY_SOURCE) + +class _SourcelessFinderDetails: + + loader = _SourcelessFileLoader + supports_packages = True + + def __init__(self): + self.suffixes = _suffix_list(imp.PY_COMPILED) + + +class _ExtensionFinderDetails: + + loader = _ExtensionFileLoader + supports_packages = False + + def __init__(self): + self.suffixes = _suffix_list(imp.C_EXTENSION) # Import itself ############################################################### +def _file_path_hook(path): + """If the path is a directory, return a file-based finder.""" + if _path_isdir(path): + return _FileFinder(path, _ExtensionFinderDetails(), + _SourceFinderDetails(), + _SourcelessFinderDetails()) + else: + raise ImportError("only directories are supported") + + +_DEFAULT_PATH_HOOK = _file_path_hook + +class _DefaultPathFinder(PathFinder): + + """Subclass of PathFinder that implements implicit semantics for + __import__.""" + + @classmethod + def _path_hooks(cls, path): + """Search sys.path_hooks as well as implicit path hooks.""" + try: + return super()._path_hooks(path) + except ImportError: + implicit_hooks = [_DEFAULT_PATH_HOOK, imp.NullImporter] + return super()._path_hooks(path, implicit_hooks) + + @classmethod + def _path_importer_cache(cls, path): + """Use the default path hook when None is stored in + sys.path_importer_cache.""" + return super()._path_importer_cache(path, _DEFAULT_PATH_HOOK) + + class _ImportLockContext: """Context manager for the import lock.""" def __enter__(self): """Acquire the import lock.""" - _imp.acquire_lock() + imp.acquire_lock() def __exit__(self, exc_type, exc_value, exc_traceback): """Release the import lock regardless of any raised exceptions.""" - _imp.release_lock() + imp.release_lock() def _resolve_name(name, package, level): @@ -864,54 +881,20 @@ if len(bits) < level: raise ValueError('attempted relative import beyond top-level package') base = bits[0] - return '{}.{}'.format(base, name) if name else base + return '{0}.{1}'.format(base, name) if name else base -def _find_spec_legacy(finder, name, path): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. - loader = finder.find_module(name, path) - if loader is None: - return None - return spec_from_loader(name, loader) - - -def _find_spec(name, path, target=None): +def _find_module(name, path): """Find a module's loader.""" - if sys.meta_path is not None and not sys.meta_path: - _warnings.warn('sys.meta_path is empty', ImportWarning) - # We check sys.modules here for the reload case. While a passed-in - # target will usually indicate a reload there is no guarantee, whereas - # sys.modules provides one. - is_reload = name in sys.modules - for finder in sys.meta_path: - with _ImportLockContext(): - try: - find_spec = finder.find_spec - except AttributeError: - spec = _find_spec_legacy(finder, name, path) - if spec is None: - continue + meta_path = sys.meta_path + _IMPLICIT_META_PATH + for finder in meta_path: + loader = finder.find_module(name, path) + if loader is not None: + # The parent import may have already imported this module. + if name not in sys.modules: + return loader else: - spec = find_spec(name, path, target) - if spec is not None: - # The parent import may have already imported this module. - if not is_reload and name in sys.modules: - module = sys.modules[name] - try: - __spec__ = module.__spec__ - except AttributeError: - # We use the found spec since that is the one that - # we would have used if the parent module hadn't - # beaten us to the punch. - return spec - else: - if __spec__ is None: - return spec - else: - return __spec__ - else: - return spec + return sys.modules[name].__loader__ else: return None @@ -919,56 +902,61 @@ def _sanity_check(name, package, level): """Verify arguments are "sane".""" if not isinstance(name, str): - raise TypeError('module name must be str, not {}'.format(type(name))) + raise TypeError("module name must be str, not {}".format(type(name))) if level < 0: raise ValueError('level must be >= 0') if package: if not isinstance(package, str): - raise TypeError('__package__ not set to a string') + raise TypeError("__package__ not set to a string") elif package not in sys.modules: - msg = ('Parent module {!r} not loaded, cannot perform relative ' - 'import') + msg = ("Parent module {0!r} not loaded, cannot perform relative " + "import") raise SystemError(msg.format(package)) if not name and level == 0: - raise ValueError('Empty module name') + raise ValueError("Empty module name") -_ERR_MSG_PREFIX = 'No module named ' -_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' +_IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder] -def _find_and_load_unlocked(name, import_): +_ERR_MSG = 'No module named {!r}' + +def _find_and_load(name, import_): + """Find and load the module.""" path = None parent = name.rpartition('.')[0] if parent: if parent not in sys.modules: - _call_with_frames_removed(import_, parent) - # Crazy side-effects! - if name in sys.modules: - return sys.modules[name] + import_(parent) + # Backwards-compatibility; be nicer to skip the dict lookup. parent_module = sys.modules[parent] try: path = parent_module.__path__ except AttributeError: - msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) - raise ImportError(msg, name=name) from None - spec = _find_spec(name, path) - if spec is None: - raise ImportError(_ERR_MSG.format(name), name=name) - else: - module = _load_unlocked(spec) + msg = (_ERR_MSG + '; {} is not a package').format(name, parent) + raise ImportError(msg) + loader = _find_module(name, path) + if loader is None: + raise ImportError(_ERR_MSG.format(name)) + elif name not in sys.modules: + # The parent import may have already imported this module. + loader.load_module(name) + # Backwards-compatibility; be nicer to skip the dict lookup. + module = sys.modules[name] if parent: # Set the module as an attribute on its parent. parent_module = sys.modules[parent] setattr(parent_module, name.rpartition('.')[2], module) + # Set __package__ if the loader did not. + if not hasattr(module, '__package__') or module.__package__ is None: + try: + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = module.__package__.rpartition('.')[0] + except AttributeError: + pass return module -def _find_and_load(name, import_): - """Find and load the module, and release the import lock.""" - with _ModuleLockManager(name): - return _find_and_load_unlocked(name, import_) - - def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. @@ -981,17 +969,18 @@ _sanity_check(name, package, level) if level > 0: name = _resolve_name(name, package, level) - _imp.acquire_lock() - if name not in sys.modules: + with _ImportLockContext(): + try: + module = sys.modules[name] + if module is None: + message = ("import of {} halted; " + "None in sys.modules".format(name)) + raise ImportError(message) + return module + except KeyError: + pass # Don't want to chain the exception return _find_and_load(name, _gcd_import) - module = sys.modules[name] - if module is None: - _imp.release_lock() - message = ('import of {} halted; ' - 'None in sys.modules'.format(name)) - raise ImportError(message, name=name) - _lock_unlock_module(name) - return module + def _handle_fromlist(module, fromlist, import_): """Figure out what __import__ should return. @@ -1004,24 +993,15 @@ # The hell that is fromlist ... # If a package was imported, try to import stuff from fromlist. if hasattr(module, '__path__'): - if '*' in fromlist: + if '*' in fromlist and hasattr(module, '__all__'): fromlist = list(fromlist) fromlist.remove('*') - if hasattr(module, '__all__'): - fromlist.extend(module.__all__) - for x in fromlist: - if not hasattr(module, x): - from_name = '{}.{}'.format(module.__name__, x) - try: - _call_with_frames_removed(import_, from_name) - except ImportError as exc: - # Backwards-compatibility dictates we ignore failed - # imports triggered by fromlist for modules that don't - # exist. - if str(exc).startswith(_ERR_MSG_PREFIX): - if exc.name == from_name: - continue - raise + fromlist.extend(module.__all__) + for x in (y for y in fromlist if not hasattr(module, y)): + try: + import_('{0}.{1}'.format(module.__name__, x)) + except ImportError: + pass return module @@ -1033,26 +1013,14 @@ """ package = globals.get('__package__') - spec = globals.get('__spec__') - if package is not None: - if spec is not None and package != spec.parent: - _warnings.warn("__package__ != __spec__.parent " - f"({package!r} != {spec.parent!r})", - ImportWarning, stacklevel=3) - return package - elif spec is not None: - return spec.parent - else: - _warnings.warn("can't resolve package from __spec__ or __package__, " - "falling back on __name__ and __path__", - ImportWarning, stacklevel=3) + if package is None: package = globals['__name__'] if '__path__' not in globals: package = package.rpartition('.')[0] return package -def __import__(name, globals=None, locals=None, fromlist=(), level=0): +def __import__(name, globals={}, locals={}, fromlist=[], level=0): """Import a module. The 'globals' argument is used to infer where the import is occuring from @@ -1066,89 +1034,75 @@ if level == 0: module = _gcd_import(name) else: - globals_ = globals if globals is not None else {} - package = _calc___package__(globals_) + package = _calc___package__(globals) module = _gcd_import(name, package, level) if not fromlist: # Return up to the first dot in 'name'. This is complicated by the fact # that 'name' may be relative. if level == 0: - return _gcd_import(name.partition('.')[0]) + return sys.modules[name.partition('.')[0]] elif not name: return module else: - # Figure out where to slice the module's name up to the first dot - # in 'name'. cut_off = len(name) - len(name.partition('.')[0]) - # Slice end needs to be positive to alleviate need to special-case - # when ``'.' not in name``. - return sys.modules[module.__name__[:len(module.__name__)-cut_off]] + return sys.modules[module.__name__[:-cut_off]] else: return _handle_fromlist(module, fromlist, _gcd_import) -def _builtin_from_name(name): - spec = BuiltinImporter.find_spec(name) - if spec is None: - raise ImportError('no built-in module named ' + name) - return _load_unlocked(spec) - - -def _setup(sys_module, _imp_module): +def _setup(sys_module, imp_module): """Setup importlib by importing needed built-in modules and injecting them into the global namespace. - As sys is needed for sys.modules access and _imp is needed to load built-in + As sys is needed for sys.modules access and imp is needed to load built-in modules, those two modules must be explicitly passed in. """ - global _imp, sys - _imp = _imp_module + global imp, sys + imp = imp_module sys = sys_module - # Set up the spec for existing builtin/frozen modules. - module_type = type(sys) - for name, module in sys.modules.items(): - if isinstance(module, module_type): - if name in sys.builtin_module_names: - loader = BuiltinImporter - elif _imp.is_frozen(name): - loader = FrozenImporter - else: - continue - spec = _spec_from_module(module, loader) - _init_module_attrs(spec, module) + for module in (imp, sys): + if not hasattr(module, '__loader__'): + module.__loader__ = BuiltinImporter - # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] - for builtin_name in ('_warnings',): + for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): if builtin_name not in sys.modules: - builtin_module = _builtin_from_name(builtin_name) + builtin_module = BuiltinImporter.load_module(builtin_name) else: builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) - # Directly load the _thread module (needed during bootstrap). - try: - thread_module = _builtin_from_name('_thread') - except ImportError: - # Python was built without threads - thread_module = None - setattr(self_module, '_thread', thread_module) + for builtin_os, path_sep in [('posix', '/'), ('nt', '\\'), ('os2', '\\')]: + if builtin_os in sys.modules: + os_module = sys.modules[builtin_os] + break + else: + try: + os_module = BuiltinImporter.load_module(builtin_os) + # TODO: rip out os2 code after 3.3 is released as per PEP 11 + if builtin_os == 'os2' and 'EMX GCC' in sys.version: + path_sep = '/' + break + except ImportError: + continue + else: + raise ImportError('importlib requires posix or nt') + setattr(self_module, '_os', os_module) + setattr(self_module, 'path_sep', path_sep) + # Constants + setattr(self_module, '_relax_case', _make_relax_case()) - # Directly load the _weakref module (needed during bootstrap). - weakref_module = _builtin_from_name('_weakref') - setattr(self_module, '_weakref', weakref_module) +def _install(sys_module, imp_module): + """Install importlib as the implementation of import. -def _install(sys_module, _imp_module): - """Install importlib as the implementation of import.""" - _setup(sys_module, _imp_module) + It is assumed that imp and sys have been imported and injected into the + global namespace for the module prior to calling this function. - sys.meta_path.append(BuiltinImporter) - sys.meta_path.append(FrozenImporter) - - global _bootstrap_external - import _frozen_importlib_external - _bootstrap_external = _frozen_importlib_external - _frozen_importlib_external._install(sys.modules[__name__]) + """ + _setup(sys_module, imp_module) + orig_import = builtins.__import__ + builtins.__import__ = __import__ + builtins.__original_import__ = orig_import diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/_bootstrap_external.py --- a/Lib/importlib/_bootstrap_external.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1425 +0,0 @@ -"""Core implementation of path-based import. - -This module is NOT meant to be directly imported! It has been designed such -that it can be bootstrapped into Python as the implementation of import. As -such it requires the injection of specific modules and attributes in order to -work. One should use importlib as the public-facing version of this module. - -""" -# -# IMPORTANT: Whenever making changes to this module, be sure to run -# a top-level make in order to get the frozen version of the module -# updated. Not doing so will result in the Makefile to fail for -# all others who don't have a ./python around to freeze the module -# in the early stages of compilation. -# - -# See importlib._setup() for what is injected into the global namespace. - -# When editing this code be aware that code executed at import time CANNOT -# reference any injected objects! This includes not only global code but also -# anything specified at the class level. - -# Bootstrap-related code ###################################################### - -_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' - - -def _make_relax_case(): - if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): - def _relax_case(): - """True if filenames must be checked case-insensitively.""" - return b'PYTHONCASEOK' in _os.environ - else: - def _relax_case(): - """True if filenames must be checked case-insensitively.""" - return False - return _relax_case - - -def _w_long(x): - """Convert a 32-bit integer to little-endian.""" - return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') - - -def _r_long(int_bytes): - """Convert 4 bytes in little-endian to an integer.""" - return int.from_bytes(int_bytes, 'little') - - -def _path_join(*path_parts): - """Replacement for os.path.join().""" - return path_sep.join([part.rstrip(path_separators) - for part in path_parts if part]) - - -def _path_split(path): - """Replacement for os.path.split().""" - if len(path_separators) == 1: - front, _, tail = path.rpartition(path_sep) - return front, tail - for x in reversed(path): - if x in path_separators: - front, tail = path.rsplit(x, maxsplit=1) - return front, tail - return '', path - - -def _path_stat(path): - """Stat the path. - - Made a separate function to make it easier to override in experiments - (e.g. cache stat results). - - """ - return _os.stat(path) - - -def _path_is_mode_type(path, mode): - """Test whether the path is the specified mode type.""" - try: - stat_info = _path_stat(path) - except OSError: - return False - return (stat_info.st_mode & 0o170000) == mode - - -def _path_isfile(path): - """Replacement for os.path.isfile.""" - return _path_is_mode_type(path, 0o100000) - - -def _path_isdir(path): - """Replacement for os.path.isdir.""" - if not path: - path = _os.getcwd() - return _path_is_mode_type(path, 0o040000) - - -def _write_atomic(path, data, mode=0o666): - """Best-effort function to write data to a path atomically. - Be prepared to handle a FileExistsError if concurrent writing of the - temporary file is attempted.""" - # id() is used to generate a pseudo-random filename. - path_tmp = '{}.{}'.format(path, id(path)) - fd = _os.open(path_tmp, - _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666) - try: - # We first write data to a temporary file, and then use os.replace() to - # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: - file.write(data) - _os.replace(path_tmp, path) - except OSError: - try: - _os.unlink(path_tmp) - except OSError: - pass - raise - - -_code_type = type(_write_atomic.__code__) - - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# The magic numbers must be spaced apart at least 2 values, as the -# -U interpeter flag will cause MAGIC+1 being used. They have been -# odd numbers for some time now. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) -# Python 3.1a0: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD) -# Python 3.1a0: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 3.2a0: 3160 (add SETUP_WITH) -# tag: cpython-32 -# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) -# tag: cpython-32 -# Python 3.2a2 3180 (add DELETE_DEREF) -# Python 3.3a0 3190 __class__ super closure changed -# Python 3.3a0 3200 (__qualname__ added) -# 3210 (added size modulo 2**32 to the pyc header) -# Python 3.3a1 3220 (changed PEP 380 implementation) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars) -# Python 3.4a1 3270 (various tweaks to the __class__ closure) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation) -# Python 3.4a4 3300 (more changes to __qualname__ computation) -# Python 3.4rc2 3310 (alter __qualname__ computation) -# Python 3.5a0 3320 (matrix multiplication operator) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483 -# Python 3.6a0 3361 (lineno delta of code.co_lnotab becomes signed) -# -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). - -MAGIC_NUMBER = (3361).to_bytes(2, 'little') + b'\r\n' -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c - -_PYCACHE = '__pycache__' -_OPT = 'opt-' - -SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. - -BYTECODE_SUFFIXES = ['.pyc'] -# Deprecated. -DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES - -def cache_from_source(path, debug_override=None, *, optimization=None): - """Given the path to a .py file, return the path to its .pyc file. - - The .py file does not need to exist; this simply returns the path to the - .pyc file calculated as if the .py file were imported. - - The 'optimization' parameter controls the presumed optimization level of - the bytecode file. If 'optimization' is not None, the string representation - of the argument is taken and verified to be alphanumeric (else ValueError - is raised). - - The debug_override parameter is deprecated. If debug_override is not None, - a True value is the same as setting 'optimization' to the empty string - while a False value is equivalent to setting 'optimization' to '1'. - - If sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - if debug_override is not None: - _warnings.warn('the debug_override parameter is deprecated; use ' - "'optimization' instead", DeprecationWarning) - if optimization is not None: - message = 'debug_override or optimization must be set to None' - raise TypeError(message) - optimization = '' if debug_override else 1 - head, tail = _path_split(path) - base, sep, rest = tail.rpartition('.') - tag = sys.implementation.cache_tag - if tag is None: - raise NotImplementedError('sys.implementation.cache_tag is None') - almost_filename = ''.join([(base if base else rest), sep, tag]) - if optimization is None: - if sys.flags.optimize == 0: - optimization = '' - else: - optimization = sys.flags.optimize - optimization = str(optimization) - if optimization != '': - if not optimization.isalnum(): - raise ValueError('{!r} is not alphanumeric'.format(optimization)) - almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) - return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0]) - - -def source_from_cache(path): - """Given the path to a .pyc. file, return the path to its .py file. - - The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147/488 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - if sys.implementation.cache_tag is None: - raise NotImplementedError('sys.implementation.cache_tag is None') - head, pycache_filename = _path_split(path) - head, pycache = _path_split(head) - if pycache != _PYCACHE: - raise ValueError('{} not bottom-level directory in ' - '{!r}'.format(_PYCACHE, path)) - dot_count = pycache_filename.count('.') - if dot_count not in {2, 3}: - raise ValueError('expected only 2 or 3 dots in ' - '{!r}'.format(pycache_filename)) - elif dot_count == 3: - optimization = pycache_filename.rsplit('.', 2)[-2] - if not optimization.startswith(_OPT): - raise ValueError("optimization portion of filename does not start " - "with {!r}".format(_OPT)) - opt_level = optimization[len(_OPT):] - if not opt_level.isalnum(): - raise ValueError("optimization level {!r} is not an alphanumeric " - "value".format(optimization)) - base_filename = pycache_filename.partition('.')[0] - return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) - - -def _get_sourcefile(bytecode_path): - """Convert a bytecode file path to a source path (if possible). - - This function exists purely for backwards-compatibility for - PyImport_ExecCodeModuleWithFilenames() in the C API. - - """ - if len(bytecode_path) == 0: - return None - rest, _, extension = bytecode_path.rpartition('.') - if not rest or extension.lower()[-3:-1] != 'py': - return bytecode_path - try: - source_path = source_from_cache(bytecode_path) - except (NotImplementedError, ValueError): - source_path = bytecode_path[:-1] - return source_path if _path_isfile(source_path) else bytecode_path - - -def _get_cached(filename): - if filename.endswith(tuple(SOURCE_SUFFIXES)): - try: - return cache_from_source(filename) - except NotImplementedError: - pass - elif filename.endswith(tuple(BYTECODE_SUFFIXES)): - return filename - else: - return None - - -def _calc_mode(path): - """Calculate the mode permissions for a bytecode file.""" - try: - mode = _path_stat(path).st_mode - except OSError: - mode = 0o666 - # We always ensure write access so we can update cached files - # later even when the source files are read-only on Windows (#6074) - mode |= 0o200 - return mode - - -def _check_name(method): - """Decorator to verify that the module being requested matches the one the - loader can handle. - - The first argument (self) must define _name which the second argument is - compared against. If the comparison fails then ImportError is raised. - - """ - def _check_name_wrapper(self, name=None, *args, **kwargs): - if name is None: - name = self.name - elif self.name != name: - raise ImportError('loader for %s cannot handle %s' % - (self.name, name), name=name) - return method(self, name, *args, **kwargs) - try: - _wrap = _bootstrap._wrap - except NameError: - # XXX yuck - def _wrap(new, old): - for replace in ['__module__', '__name__', '__qualname__', '__doc__']: - if hasattr(old, replace): - setattr(new, replace, getattr(old, replace)) - new.__dict__.update(old.__dict__) - _wrap(_check_name_wrapper, method) - return _check_name_wrapper - - -def _find_module_shim(self, fullname): - """Try to find a loader for the specified module by delegating to - self.find_loader(). - - This method is deprecated in favor of finder.find_spec(). - - """ - # Call find_loader(). If it returns a string (indicating this - # is a namespace package portion), generate a warning and - # return None. - loader, portions = self.find_loader(fullname) - if loader is None and len(portions): - msg = 'Not importing directory {}: missing __init__' - _warnings.warn(msg.format(portions[0]), ImportWarning) - return loader - - -def _validate_bytecode_header(data, source_stats=None, name=None, path=None): - """Validate the header of the passed-in bytecode against source_stats (if - given) and returning the bytecode that can be compiled by compile(). - - All other arguments are used to enhance error reporting. - - ImportError is raised when the magic number is incorrect or the bytecode is - found to be stale. EOFError is raised when the data is found to be - truncated. - - """ - exc_details = {} - if name is not None: - exc_details['name'] = name - else: - # To prevent having to make all messages have a conditional name. - name = '' - if path is not None: - exc_details['path'] = path - magic = data[:4] - raw_timestamp = data[4:8] - raw_size = data[8:12] - if magic != MAGIC_NUMBER: - message = 'bad magic number in {!r}: {!r}'.format(name, magic) - _bootstrap._verbose_message('{}', message) - raise ImportError(message, **exc_details) - elif len(raw_timestamp) != 4: - message = 'reached EOF while reading timestamp in {!r}'.format(name) - _bootstrap._verbose_message('{}', message) - raise EOFError(message) - elif len(raw_size) != 4: - message = 'reached EOF while reading size of source in {!r}'.format(name) - _bootstrap._verbose_message('{}', message) - raise EOFError(message) - if source_stats is not None: - try: - source_mtime = int(source_stats['mtime']) - except KeyError: - pass - else: - if _r_long(raw_timestamp) != source_mtime: - message = 'bytecode is stale for {!r}'.format(name) - _bootstrap._verbose_message('{}', message) - raise ImportError(message, **exc_details) - try: - source_size = source_stats['size'] & 0xFFFFFFFF - except KeyError: - pass - else: - if _r_long(raw_size) != source_size: - raise ImportError('bytecode is stale for {!r}'.format(name), - **exc_details) - return data[12:] - - -def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): - """Compile bytecode as returned by _validate_bytecode_header().""" - code = marshal.loads(data) - if isinstance(code, _code_type): - _bootstrap._verbose_message('code object from {!r}', bytecode_path) - if source_path is not None: - _imp._fix_co_filename(code, source_path) - return code - else: - raise ImportError('Non-code object in {!r}'.format(bytecode_path), - name=name, path=bytecode_path) - -def _code_to_bytecode(code, mtime=0, source_size=0): - """Compile a code object into bytecode for writing out to a byte-compiled - file.""" - data = bytearray(MAGIC_NUMBER) - data.extend(_w_long(mtime)) - data.extend(_w_long(source_size)) - data.extend(marshal.dumps(code)) - return data - - -def decode_source(source_bytes): - """Decode bytes representing source code and return the string. - - Universal newline support is used in the decoding. - """ - import tokenize # To avoid bootstrap issues. - source_bytes_readline = _io.BytesIO(source_bytes).readline - encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) - - -# Module specifications ####################################################### - -_POPULATE = object() - - -def spec_from_file_location(name, location=None, *, loader=None, - submodule_search_locations=_POPULATE): - """Return a module spec based on a file location. - - To indicate that the module is a package, set - submodule_search_locations to a list of directory paths. An - empty list is sufficient, though its not otherwise useful to the - import system. - - The loader must take a spec as its only __init__() arg. - - """ - if location is None: - # The caller may simply want a partially populated location- - # oriented spec. So we set the location to a bogus value and - # fill in as much as we can. - location = '' - if hasattr(loader, 'get_filename'): - # ExecutionLoader - try: - location = loader.get_filename(name) - except ImportError: - pass - - # If the location is on the filesystem, but doesn't actually exist, - # we could return None here, indicating that the location is not - # valid. However, we don't have a good way of testing since an - # indirect location (e.g. a zip file or URL) will look like a - # non-existent file relative to the filesystem. - - spec = _bootstrap.ModuleSpec(name, loader, origin=location) - spec._set_fileattr = True - - # Pick a loader if one wasn't provided. - if loader is None: - for loader_class, suffixes in _get_supported_file_loaders(): - if location.endswith(tuple(suffixes)): - loader = loader_class(name, location) - spec.loader = loader - break - else: - return None - - # Set submodule_search_paths appropriately. - if submodule_search_locations is _POPULATE: - # Check the loader. - if hasattr(loader, 'is_package'): - try: - is_package = loader.is_package(name) - except ImportError: - pass - else: - if is_package: - spec.submodule_search_locations = [] - else: - spec.submodule_search_locations = submodule_search_locations - if spec.submodule_search_locations == []: - if location: - dirname = _path_split(location)[0] - spec.submodule_search_locations.append(dirname) - - return spec - - -# Loaders ##################################################################### - -class WindowsRegistryFinder: - - """Meta path finder for modules declared in the Windows registry.""" - - REGISTRY_KEY = ( - 'Software\\Python\\PythonCore\\{sys_version}' - '\\Modules\\{fullname}') - REGISTRY_KEY_DEBUG = ( - 'Software\\Python\\PythonCore\\{sys_version}' - '\\Modules\\{fullname}\\Debug') - DEBUG_BUILD = False # Changed in _setup() - - @classmethod - def _open_registry(cls, key): - try: - return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) - except OSError: - return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) - - @classmethod - def _search_registry(cls, fullname): - if cls.DEBUG_BUILD: - registry_key = cls.REGISTRY_KEY_DEBUG - else: - registry_key = cls.REGISTRY_KEY - key = registry_key.format(fullname=fullname, - sys_version=sys.version[:3]) - try: - with cls._open_registry(key) as hkey: - filepath = _winreg.QueryValue(hkey, '') - except OSError: - return None - return filepath - - @classmethod - def find_spec(cls, fullname, path=None, target=None): - filepath = cls._search_registry(fullname) - if filepath is None: - return None - try: - _path_stat(filepath) - except OSError: - return None - for loader, suffixes in _get_supported_file_loaders(): - if filepath.endswith(tuple(suffixes)): - spec = _bootstrap.spec_from_loader(fullname, - loader(fullname, filepath), - origin=filepath) - return spec - - @classmethod - def find_module(cls, fullname, path=None): - """Find module named in the registry. - - This method is deprecated. Use exec_module() instead. - - """ - spec = cls.find_spec(fullname, path) - if spec is not None: - return spec.loader - else: - return None - - -class _LoaderBasics: - - """Base class of common code needed by both SourceLoader and - SourcelessFileLoader.""" - - def is_package(self, fullname): - """Concrete implementation of InspectLoader.is_package by checking if - the path returned by get_filename has a filename of '__init__.py'.""" - filename = _path_split(self.get_filename(fullname))[1] - filename_base = filename.rsplit('.', 1)[0] - tail_name = fullname.rpartition('.')[2] - return filename_base == '__init__' and tail_name != '__init__' - - def create_module(self, spec): - """Use default semantics for module creation.""" - - def exec_module(self, module): - """Execute the module.""" - code = self.get_code(module.__name__) - if code is None: - raise ImportError('cannot load module {!r} when get_code() ' - 'returns None'.format(module.__name__)) - _bootstrap._call_with_frames_removed(exec, code, module.__dict__) - - def load_module(self, fullname): - """This module is deprecated.""" - return _bootstrap._load_module_shim(self, fullname) - - -class SourceLoader(_LoaderBasics): - - def path_mtime(self, path): - """Optional method that returns the modification time (an int) for the - specified path, where path is a str. - - Raises IOError when the path cannot be handled. - """ - raise IOError - - def path_stats(self, path): - """Optional method returning a metadata dict for the specified path - to by the path (str). - Possible keys: - - 'mtime' (mandatory) is the numeric timestamp of last source - code modification; - - 'size' (optional) is the size in bytes of the source code. - - Implementing this method allows the loader to read bytecode files. - Raises IOError when the path cannot be handled. - """ - return {'mtime': self.path_mtime(path)} - - def _cache_bytecode(self, source_path, cache_path, data): - """Optional method which writes data (bytes) to a file path (a str). - - Implementing this method allows for the writing of bytecode files. - - The source path is needed in order to correctly transfer permissions - """ - # For backwards compatibility, we delegate to set_data() - return self.set_data(cache_path, data) - - def set_data(self, path, data): - """Optional method which writes data (bytes) to a file path (a str). - - Implementing this method allows for the writing of bytecode files. - """ - - - def get_source(self, fullname): - """Concrete implementation of InspectLoader.get_source.""" - path = self.get_filename(fullname) - try: - source_bytes = self.get_data(path) - except OSError as exc: - raise ImportError('source not available through get_data()', - name=fullname) from exc - return decode_source(source_bytes) - - def source_to_code(self, data, path, *, _optimize=-1): - """Return the code object compiled from source. - - The 'data' argument can be any object type that compile() supports. - """ - return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) - - def get_code(self, fullname): - """Concrete implementation of InspectLoader.get_code. - - Reading of bytecode requires path_stats to be implemented. To write - bytecode, set_data must also be implemented. - - """ - source_path = self.get_filename(fullname) - source_mtime = None - try: - bytecode_path = cache_from_source(source_path) - except NotImplementedError: - bytecode_path = None - else: - try: - st = self.path_stats(source_path) - except IOError: - pass - else: - source_mtime = int(st['mtime']) - try: - data = self.get_data(bytecode_path) - except OSError: - pass - else: - try: - bytes_data = _validate_bytecode_header(data, - source_stats=st, name=fullname, - path=bytecode_path) - except (ImportError, EOFError): - pass - else: - _bootstrap._verbose_message('{} matches {}', bytecode_path, - source_path) - return _compile_bytecode(bytes_data, name=fullname, - bytecode_path=bytecode_path, - source_path=source_path) - source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) - _bootstrap._verbose_message('code object from {}', source_path) - if (not sys.dont_write_bytecode and bytecode_path is not None and - source_mtime is not None): - data = _code_to_bytecode(code_object, source_mtime, - len(source_bytes)) - try: - self._cache_bytecode(source_path, bytecode_path, data) - _bootstrap._verbose_message('wrote {!r}', bytecode_path) - except NotImplementedError: - pass - return code_object - - -class FileLoader: - - """Base file loader class which implements the loader protocol methods that - require file system usage.""" - - def __init__(self, fullname, path): - """Cache the module name and the path to the file found by the - finder.""" - self.name = fullname - self.path = path - - def __eq__(self, other): - return (self.__class__ == other.__class__ and - self.__dict__ == other.__dict__) - - def __hash__(self): - return hash(self.name) ^ hash(self.path) - - @_check_name - def load_module(self, fullname): - """Load a module from a file. - - This method is deprecated. Use exec_module() instead. - - """ - # The only reason for this method is for the name check. - # Issue #14857: Avoid the zero-argument form of super so the implementation - # of that form can be updated without breaking the frozen module - return super(FileLoader, self).load_module(fullname) - - @_check_name - def get_filename(self, fullname): - """Return the path to the source file as found by the finder.""" - return self.path - - def get_data(self, path): - """Return the data from path as raw bytes.""" - with _io.FileIO(path, 'r') as file: - return file.read() - - -class SourceFileLoader(FileLoader, SourceLoader): - - """Concrete implementation of SourceLoader using the file system.""" - - def path_stats(self, path): - """Return the metadata for the path.""" - st = _path_stat(path) - return {'mtime': st.st_mtime, 'size': st.st_size} - - def _cache_bytecode(self, source_path, bytecode_path, data): - # Adapt between the two APIs - mode = _calc_mode(source_path) - return self.set_data(bytecode_path, data, _mode=mode) - - def set_data(self, path, data, *, _mode=0o666): - """Write bytes data to a file.""" - parent, filename = _path_split(path) - path_parts = [] - # Figure out what directories are missing. - while parent and not _path_isdir(parent): - parent, part = _path_split(parent) - path_parts.append(part) - # Create needed directories. - for part in reversed(path_parts): - parent = _path_join(parent, part) - try: - _os.mkdir(parent) - except FileExistsError: - # Probably another Python process already created the dir. - continue - except OSError as exc: - # Could be a permission error, read-only filesystem: just forget - # about writing the data. - _bootstrap._verbose_message('could not create {!r}: {!r}', - parent, exc) - return - try: - _write_atomic(path, data, _mode) - _bootstrap._verbose_message('created {!r}', path) - except OSError as exc: - # Same as above: just don't write the bytecode. - _bootstrap._verbose_message('could not create {!r}: {!r}', path, - exc) - - -class SourcelessFileLoader(FileLoader, _LoaderBasics): - - """Loader which handles sourceless file imports.""" - - def get_code(self, fullname): - path = self.get_filename(fullname) - data = self.get_data(path) - bytes_data = _validate_bytecode_header(data, name=fullname, path=path) - return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path) - - def get_source(self, fullname): - """Return None as there is no source code.""" - return None - - -# Filled in by _setup(). -EXTENSION_SUFFIXES = [] - - -class ExtensionFileLoader(FileLoader, _LoaderBasics): - - """Loader for extension modules. - - The constructor is designed to work with FileFinder. - - """ - - def __init__(self, name, path): - self.name = name - self.path = path - - def __eq__(self, other): - return (self.__class__ == other.__class__ and - self.__dict__ == other.__dict__) - - def __hash__(self): - return hash(self.name) ^ hash(self.path) - - def create_module(self, spec): - """Create an unitialized extension module""" - module = _bootstrap._call_with_frames_removed( - _imp.create_dynamic, spec) - _bootstrap._verbose_message('extension module {!r} loaded from {!r}', - spec.name, self.path) - return module - - def exec_module(self, module): - """Initialize an extension module""" - _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) - _bootstrap._verbose_message('extension module {!r} executed from {!r}', - self.name, self.path) - - def is_package(self, fullname): - """Return True if the extension module is a package.""" - file_name = _path_split(self.path)[1] - return any(file_name == '__init__' + suffix - for suffix in EXTENSION_SUFFIXES) - - def get_code(self, fullname): - """Return None as an extension module cannot create a code object.""" - return None - - def get_source(self, fullname): - """Return None as extension modules have no source code.""" - return None - - @_check_name - def get_filename(self, fullname): - """Return the path to the source file as found by the finder.""" - return self.path - - -class _NamespacePath: - """Represents a namespace package's path. It uses the module name - to find its parent module, and from there it looks up the parent's - __path__. When this changes, the module's own path is recomputed, - using path_finder. For top-level modules, the parent module's path - is sys.path.""" - - def __init__(self, name, path, path_finder): - self._name = name - self._path = path - self._last_parent_path = tuple(self._get_parent_path()) - self._path_finder = path_finder - - def _find_parent_path_names(self): - """Returns a tuple of (parent-module-name, parent-path-attr-name)""" - parent, dot, me = self._name.rpartition('.') - if dot == '': - # This is a top-level module. sys.path contains the parent path. - return 'sys', 'path' - # Not a top-level module. parent-module.__path__ contains the - # parent path. - return parent, '__path__' - - def _get_parent_path(self): - parent_module_name, path_attr_name = self._find_parent_path_names() - return getattr(sys.modules[parent_module_name], path_attr_name) - - def _recalculate(self): - # If the parent's path has changed, recalculate _path - parent_path = tuple(self._get_parent_path()) # Make a copy - if parent_path != self._last_parent_path: - spec = self._path_finder(self._name, parent_path) - # Note that no changes are made if a loader is returned, but we - # do remember the new parent path - if spec is not None and spec.loader is None: - if spec.submodule_search_locations: - self._path = spec.submodule_search_locations - self._last_parent_path = parent_path # Save the copy - return self._path - - def __iter__(self): - return iter(self._recalculate()) - - def __len__(self): - return len(self._recalculate()) - - def __repr__(self): - return '_NamespacePath({!r})'.format(self._path) - - def __contains__(self, item): - return item in self._recalculate() - - def append(self, item): - self._path.append(item) - - -# We use this exclusively in module_from_spec() for backward-compatibility. -class _NamespaceLoader: - def __init__(self, name, path, path_finder): - self._path = _NamespacePath(name, path, path_finder) - - @classmethod - def module_repr(cls, module): - """Return repr for the module. - - The method is deprecated. The import machinery does the job itself. - - """ - return ''.format(module.__name__) - - def is_package(self, fullname): - return True - - def get_source(self, fullname): - return '' - - def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) - - def create_module(self, spec): - """Use default semantics for module creation.""" - - def exec_module(self, module): - pass - - def load_module(self, fullname): - """Load a namespace module. - - This method is deprecated. Use exec_module() instead. - - """ - # The import system never calls this method. - _bootstrap._verbose_message('namespace module loaded with path {!r}', - self._path) - return _bootstrap._load_module_shim(self, fullname) - - -# Finders ##################################################################### - -class PathFinder: - - """Meta path finder for sys.path and package __path__ attributes.""" - - @classmethod - def invalidate_caches(cls): - """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" - for finder in sys.path_importer_cache.values(): - if hasattr(finder, 'invalidate_caches'): - finder.invalidate_caches() - - @classmethod - def _path_hooks(cls, path): - """Search sequence of hooks for a finder for 'path'. - - If 'hooks' is false then use sys.path_hooks. - - """ - if sys.path_hooks is not None and not sys.path_hooks: - _warnings.warn('sys.path_hooks is empty', ImportWarning) - for hook in sys.path_hooks: - try: - return hook(path) - except ImportError: - continue - else: - return None - - @classmethod - def _path_importer_cache(cls, path): - """Get the finder for the path entry from sys.path_importer_cache. - - If the path entry is not in the cache, find the appropriate finder - and cache it. If no finder is available, store None. - - """ - if path == '': - try: - path = _os.getcwd() - except FileNotFoundError: - # Don't cache the failure as the cwd can easily change to - # a valid directory later on. - return None - try: - finder = sys.path_importer_cache[path] - except KeyError: - finder = cls._path_hooks(path) - sys.path_importer_cache[path] = finder - return finder - - @classmethod - def _legacy_get_spec(cls, fullname, finder): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. - if hasattr(finder, 'find_loader'): - loader, portions = finder.find_loader(fullname) - else: - loader = finder.find_module(fullname) - portions = [] - if loader is not None: - return _bootstrap.spec_from_loader(fullname, loader) - spec = _bootstrap.ModuleSpec(fullname, None) - spec.submodule_search_locations = portions - return spec - - @classmethod - def _get_spec(cls, fullname, path, target=None): - """Find the loader or namespace_path for this module/package name.""" - # If this ends up being a namespace package, namespace_path is - # the list of paths that will become its __path__ - namespace_path = [] - for entry in path: - if not isinstance(entry, (str, bytes)): - continue - finder = cls._path_importer_cache(entry) - if finder is not None: - if hasattr(finder, 'find_spec'): - spec = finder.find_spec(fullname, target) - else: - spec = cls._legacy_get_spec(fullname, finder) - if spec is None: - continue - if spec.loader is not None: - return spec - portions = spec.submodule_search_locations - if portions is None: - raise ImportError('spec missing loader') - # This is possibly part of a namespace package. - # Remember these path entries (if any) for when we - # create a namespace package, and continue iterating - # on path. - namespace_path.extend(portions) - else: - spec = _bootstrap.ModuleSpec(fullname, None) - spec.submodule_search_locations = namespace_path - return spec - - @classmethod - def find_spec(cls, fullname, path=None, target=None): - """find the module on sys.path or 'path' based on sys.path_hooks and - sys.path_importer_cache.""" - if path is None: - path = sys.path - spec = cls._get_spec(fullname, path, target) - if spec is None: - return None - elif spec.loader is None: - namespace_path = spec.submodule_search_locations - if namespace_path: - # We found at least one namespace path. Return a - # spec which can create the namespace package. - spec.origin = 'namespace' - spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) - return spec - else: - return None - else: - return spec - - @classmethod - def find_module(cls, fullname, path=None): - """find the module on sys.path or 'path' based on sys.path_hooks and - sys.path_importer_cache. - - This method is deprecated. Use find_spec() instead. - - """ - spec = cls.find_spec(fullname, path) - if spec is None: - return None - return spec.loader - - -class FileFinder: - - """File-based finder. - - Interactions with the file system are cached for performance, being - refreshed when the directory the finder is handling has been modified. - - """ - - def __init__(self, path, *loader_details): - """Initialize with the path to search on and a variable number of - 2-tuples containing the loader and the file suffixes the loader - recognizes.""" - loaders = [] - for loader, suffixes in loader_details: - loaders.extend((suffix, loader) for suffix in suffixes) - self._loaders = loaders - # Base (directory) path - self.path = path or '.' - self._path_mtime = -1 - self._path_cache = set() - self._relaxed_path_cache = set() - - def invalidate_caches(self): - """Invalidate the directory mtime.""" - self._path_mtime = -1 - - find_module = _find_module_shim - - def find_loader(self, fullname): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions). - - This method is deprecated. Use find_spec() instead. - - """ - spec = self.find_spec(fullname) - if spec is None: - return None, [] - return spec.loader, spec.submodule_search_locations or [] - - def _get_spec(self, loader_class, fullname, path, smsl, target): - loader = loader_class(fullname, path) - return spec_from_file_location(fullname, path, loader=loader, - submodule_search_locations=smsl) - - def find_spec(self, fullname, target=None): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions).""" - is_namespace = False - tail_module = fullname.rpartition('.')[2] - try: - mtime = _path_stat(self.path or _os.getcwd()).st_mtime - except OSError: - mtime = -1 - if mtime != self._path_mtime: - self._fill_cache() - self._path_mtime = mtime - # tail_module keeps the original casing, for __file__ and friends - if _relax_case(): - cache = self._relaxed_path_cache - cache_module = tail_module.lower() - else: - cache = self._path_cache - cache_module = tail_module - # Check if the module is the name of a directory (and thus a package). - if cache_module in cache: - base_path = _path_join(self.path, tail_module) - for suffix, loader_class in self._loaders: - init_filename = '__init__' + suffix - full_path = _path_join(base_path, init_filename) - if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, [base_path], target) - else: - # If a namespace package, return the path if we don't - # find a module in the next section. - is_namespace = _path_isdir(base_path) - # Check for a file w/ a proper suffix exists. - for suffix, loader_class in self._loaders: - full_path = _path_join(self.path, tail_module + suffix) - _bootstrap._verbose_message('trying {}', full_path, verbosity=2) - if cache_module + suffix in cache: - if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, - None, target) - if is_namespace: - _bootstrap._verbose_message('possible namespace for {}', base_path) - spec = _bootstrap.ModuleSpec(fullname, None) - spec.submodule_search_locations = [base_path] - return spec - return None - - def _fill_cache(self): - """Fill the cache of potential modules and packages for this directory.""" - path = self.path - try: - contents = _os.listdir(path or _os.getcwd()) - except (FileNotFoundError, PermissionError, NotADirectoryError): - # Directory has either been removed, turned into a file, or made - # unreadable. - contents = [] - # We store two cached versions, to handle runtime changes of the - # PYTHONCASEOK environment variable. - if not sys.platform.startswith('win'): - self._path_cache = set(contents) - else: - # Windows users can import modules with case-insensitive file - # suffixes (for legacy reasons). Make the suffix lowercase here - # so it's done once instead of for every import. This is safe as - # the specified suffixes to check against are always specified in a - # case-sensitive manner. - lower_suffix_contents = set() - for item in contents: - name, dot, suffix = item.partition('.') - if dot: - new_name = '{}.{}'.format(name, suffix.lower()) - else: - new_name = name - lower_suffix_contents.add(new_name) - self._path_cache = lower_suffix_contents - if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): - self._relaxed_path_cache = {fn.lower() for fn in contents} - - @classmethod - def path_hook(cls, *loader_details): - """A class method which returns a closure to use on sys.path_hook - which will return an instance using the specified loaders and the path - called on the closure. - - If the path called on the closure is not a directory, ImportError is - raised. - - """ - def path_hook_for_FileFinder(path): - """Path hook for importlib.machinery.FileFinder.""" - if not _path_isdir(path): - raise ImportError('only directories are supported', path=path) - return cls(path, *loader_details) - - return path_hook_for_FileFinder - - def __repr__(self): - return 'FileFinder({!r})'.format(self.path) - - -# Import setup ############################################################### - -def _fix_up_module(ns, name, pathname, cpathname=None): - # This function is used by PyImport_ExecCodeModuleObject(). - loader = ns.get('__loader__') - spec = ns.get('__spec__') - if not loader: - if spec: - loader = spec.loader - elif pathname == cpathname: - loader = SourcelessFileLoader(name, pathname) - else: - loader = SourceFileLoader(name, pathname) - if not spec: - spec = spec_from_file_location(name, pathname, loader=loader) - try: - ns['__spec__'] = spec - ns['__loader__'] = loader - ns['__file__'] = pathname - ns['__cached__'] = cpathname - except Exception: - # Not important enough to report. - pass - - -def _get_supported_file_loaders(): - """Returns a list of file-based module loaders. - - Each item is a tuple (loader, suffixes). - """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() - source = SourceFileLoader, SOURCE_SUFFIXES - bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] - - -def _setup(_bootstrap_module): - """Setup the path-based importers for importlib by importing needed - built-in modules and injecting them into the global namespace. - - Other components are extracted from the core bootstrap module. - - """ - global sys, _imp, _bootstrap - _bootstrap = _bootstrap_module - sys = _bootstrap.sys - _imp = _bootstrap._imp - - # Directly load built-in modules needed during bootstrap. - self_module = sys.modules[__name__] - for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): - if builtin_name not in sys.modules: - builtin_module = _bootstrap._builtin_from_name(builtin_name) - else: - builtin_module = sys.modules[builtin_name] - setattr(self_module, builtin_name, builtin_module) - - # Directly load the os module (needed during bootstrap). - os_details = ('posix', ['/']), ('nt', ['\\', '/']) - for builtin_os, path_separators in os_details: - # Assumption made in _path_join() - assert all(len(sep) == 1 for sep in path_separators) - path_sep = path_separators[0] - if builtin_os in sys.modules: - os_module = sys.modules[builtin_os] - break - else: - try: - os_module = _bootstrap._builtin_from_name(builtin_os) - break - except ImportError: - continue - else: - raise ImportError('importlib requires posix or nt') - setattr(self_module, '_os', os_module) - setattr(self_module, 'path_sep', path_sep) - setattr(self_module, 'path_separators', ''.join(path_separators)) - - # Directly load the _thread module (needed during bootstrap). - try: - thread_module = _bootstrap._builtin_from_name('_thread') - except ImportError: - # Python was built without threads - thread_module = None - setattr(self_module, '_thread', thread_module) - - # Directly load the _weakref module (needed during bootstrap). - weakref_module = _bootstrap._builtin_from_name('_weakref') - setattr(self_module, '_weakref', weakref_module) - - # Directly load the winreg module (needed during bootstrap). - if builtin_os == 'nt': - winreg_module = _bootstrap._builtin_from_name('winreg') - setattr(self_module, '_winreg', winreg_module) - - # Constants - setattr(self_module, '_relax_case', _make_relax_case()) - EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) - if builtin_os == 'nt': - SOURCE_SUFFIXES.append('.pyw') - if '_d.pyd' in EXTENSION_SUFFIXES: - WindowsRegistryFinder.DEBUG_BUILD = True - - -def _install(_bootstrap_module): - """Install the path-based import components.""" - _setup(_bootstrap_module) - supported_loaders = _get_supported_file_loaders() - sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) - if _os.__name__ == 'nt': - sys.meta_path.append(WindowsRegistryFinder) - sys.meta_path.append(PathFinder) - - # XXX We expose a couple of classes in _bootstrap for the sake of - # a setuptools bug (https://bitbucket.org/pypa/setuptools/issue/378). - _bootstrap_module.FileFinder = FileFinder - _bootstrap_module.SourceFileLoader = SourceFileLoader diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/abc.py --- a/Lib/importlib/abc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/importlib/abc.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,171 +1,40 @@ """Abstract base classes related to import.""" from . import _bootstrap -from . import _bootstrap_external from . import machinery -try: - import _frozen_importlib -# import _frozen_importlib_external -except ImportError as exc: - if exc.name != '_frozen_importlib': - raise - _frozen_importlib = None -try: - import _frozen_importlib_external -except ImportError as exc: - _frozen_importlib_external = _bootstrap_external import abc - - -def _register(abstract_cls, *classes): - for cls in classes: - abstract_cls.register(cls) - if _frozen_importlib is not None: - try: - frozen_cls = getattr(_frozen_importlib, cls.__name__) - except AttributeError: - frozen_cls = getattr(_frozen_importlib_external, cls.__name__) - abstract_cls.register(frozen_cls) - - -class Finder(metaclass=abc.ABCMeta): - - """Legacy abstract base class for import finders. - - It may be subclassed for compatibility with legacy third party - reimplementations of the import system. Otherwise, finder - implementations should derive from the more specific MetaPathFinder - or PathEntryFinder ABCs. - """ - - @abc.abstractmethod - def find_module(self, fullname, path=None): - """An abstract method that should find a module. - The fullname is a str and the optional path is a str or None. - Returns a Loader object or None. - """ - - -class MetaPathFinder(Finder): - - """Abstract base class for import finders on sys.meta_path.""" - - # We don't define find_spec() here since that would break - # hasattr checks we do to support backward compatibility. - - def find_module(self, fullname, path): - """Return a loader for the module. - - If no module is found, return None. The fullname is a str and - the path is a list of strings or None. - - This method is deprecated in favor of finder.find_spec(). If find_spec() - exists then backwards-compatible functionality is provided for this - method. - - """ - if not hasattr(self, 'find_spec'): - return None - found = self.find_spec(fullname, path) - return found.loader if found is not None else None - - def invalidate_caches(self): - """An optional method for clearing the finder's cache, if any. - This method is used by importlib.invalidate_caches(). - """ - -_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, - machinery.PathFinder, machinery.WindowsRegistryFinder) - - -class PathEntryFinder(Finder): - - """Abstract base class for path entry finders used by PathFinder.""" - - # We don't define find_spec() here since that would break - # hasattr checks we do to support backward compatibility. - - def find_loader(self, fullname): - """Return (loader, namespace portion) for the path entry. - - The fullname is a str. The namespace portion is a sequence of - path entries contributing to part of a namespace package. The - sequence may be empty. If loader is not None, the portion will - be ignored. - - The portion will be discarded if another path entry finder - locates the module as a normal module or package. - - This method is deprecated in favor of finder.find_spec(). If find_spec() - is provided than backwards-compatible functionality is provided. - - """ - if not hasattr(self, 'find_spec'): - return None, [] - found = self.find_spec(fullname) - if found is not None: - if not found.submodule_search_locations: - portions = [] - else: - portions = found.submodule_search_locations - return found.loader, portions - else: - return None, [] - - find_module = _bootstrap_external._find_module_shim - - def invalidate_caches(self): - """An optional method for clearing the finder's cache, if any. - This method is used by PathFinder.invalidate_caches(). - """ - -_register(PathEntryFinder, machinery.FileFinder) +import imp +import marshal +import sys +import tokenize +import warnings class Loader(metaclass=abc.ABCMeta): """Abstract base class for import loaders.""" - def create_module(self, spec): - """Return a module to initialize and into which to load. + @abc.abstractmethod + def load_module(self, fullname): + """Abstract method which when implemented should load a module. + The fullname is a str.""" + raise NotImplementedError - This method should raise ImportError if anything prevents it - from creating a new module. It may return None to indicate - that the spec should create the new module. + +class Finder(metaclass=abc.ABCMeta): + + """Abstract base class for import finders.""" + + @abc.abstractmethod + def find_module(self, fullname, path=None): + """Abstract method which when implemented should find a module. + The fullname is a str and the optional path is a str or None. + Returns a Loader object. """ - # By default, defer to default semantics for the new module. - return None + raise NotImplementedError - # We don't define exec_module() here since that would break - # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - return _bootstrap._load_module_shim(self, fullname) - - def module_repr(self, module): - """Return a module's repr. - - Used by the module type when the method does not raise - NotImplementedError. - - This method is deprecated. - - """ - # The exception will cause ModuleType.__repr__ to ignore this method. - raise NotImplementedError +Finder.register(machinery.BuiltinImporter) +Finder.register(machinery.FrozenImporter) +Finder.register(machinery.PathFinder) class ResourceLoader(Loader): @@ -181,7 +50,7 @@ def get_data(self, path): """Abstract method which when implemented should return the bytes for the specified path. The path must be a str.""" - raise IOError + raise NotImplementedError class InspectLoader(Loader): @@ -193,48 +62,26 @@ """ + @abc.abstractmethod def is_package(self, fullname): - """Optional method which when implemented should return whether the - module is a package. The fullname is a str. Returns a bool. + """Abstract method which when implemented should return whether the + module is a package. The fullname is a str. Returns a bool.""" + raise NotImplementedError - Raises ImportError if the module cannot be found. - """ - raise ImportError - + @abc.abstractmethod def get_code(self, fullname): - """Method which returns the code object for the module. - - The fullname is a str. Returns a types.CodeType if possible, else - returns None if a code object does not make sense - (e.g. built-in module). Raises ImportError if the module cannot be - found. - """ - source = self.get_source(fullname) - if source is None: - return None - return self.source_to_code(source) + """Abstract method which when implemented should return the code object + for the module. The fullname is a str. Returns a types.CodeType.""" + raise NotImplementedError @abc.abstractmethod def get_source(self, fullname): """Abstract method which should return the source code for the - module. The fullname is a str. Returns a str. + module. The fullname is a str. Returns a str.""" + raise NotImplementedError - Raises ImportError if the module cannot be found. - """ - raise ImportError - - @staticmethod - def source_to_code(data, path=''): - """Compile 'data' into a code object. - - The 'data' argument can be anything that compile() can handle. The'path' - argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) - - exec_module = _bootstrap_external._LoaderBasics.exec_module - load_module = _bootstrap_external._LoaderBasics.load_module - -_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter) +InspectLoader.register(machinery.BuiltinImporter) +InspectLoader.register(machinery.FrozenImporter) class ExecutionLoader(InspectLoader): @@ -249,41 +96,11 @@ @abc.abstractmethod def get_filename(self, fullname): """Abstract method which should return the value that __file__ is to be - set to. + set to.""" + raise NotImplementedError - Raises ImportError if the module cannot be found. - """ - raise ImportError - def get_code(self, fullname): - """Method to return the code object for fullname. - - Should return None if not applicable (e.g. built-in module). - Raise ImportError if the module cannot be found. - """ - source = self.get_source(fullname) - if source is None: - return None - try: - path = self.get_filename(fullname) - except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) - -_register(ExecutionLoader, machinery.ExtensionFileLoader) - - -class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): - - """Abstract base class partially implementing the ResourceLoader and - ExecutionLoader ABCs.""" - -_register(FileLoader, machinery.SourceFileLoader, - machinery.SourcelessFileLoader) - - -class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader): +class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): """Abstract base class for loading source code (and optionally any corresponding bytecode). @@ -303,7 +120,7 @@ def path_mtime(self, path): """Return the (int) modification time for the path (str).""" if self.path_stats.__func__ is SourceLoader.path_stats: - raise IOError + raise NotImplementedError return int(self.path_stats(path)['mtime']) def path_stats(self, path): @@ -314,7 +131,7 @@ - 'size' (optional) is the size in bytes of the source code. """ if self.path_mtime.__func__ is SourceLoader.path_mtime: - raise IOError + raise NotImplementedError return {'mtime': self.path_mtime(path)} def set_data(self, path, data): @@ -325,6 +142,174 @@ Any needed intermediary directories are to be created. If for some reason the file cannot be written because of permissions, fail silently. + """ + raise NotImplementedError -_register(SourceLoader, machinery.SourceFileLoader) + +class PyLoader(SourceLoader): + + """Implement the deprecated PyLoader ABC in terms of SourceLoader. + + This class has been deprecated! It is slated for removal in Python 3.4. + If compatibility with Python 3.1 is not needed then implement the + SourceLoader ABC instead of this class. If Python 3.1 compatibility is + needed, then use the following idiom to have a single class that is + compatible with Python 3.1 onwards:: + + try: + from importlib.abc import SourceLoader + except ImportError: + from importlib.abc import PyLoader as SourceLoader + + + class CustomLoader(SourceLoader): + def get_filename(self, fullname): + # Implement ... + + def source_path(self, fullname): + '''Implement source_path in terms of get_filename.''' + try: + return self.get_filename(fullname) + except ImportError: + return None + + def is_package(self, fullname): + filename = os.path.basename(self.get_filename(fullname)) + return os.path.splitext(filename)[0] == '__init__' + + """ + + @abc.abstractmethod + def is_package(self, fullname): + raise NotImplementedError + + @abc.abstractmethod + def source_path(self, fullname): + """Abstract method. Accepts a str module name and returns the path to + the source code for the module.""" + raise NotImplementedError + + def get_filename(self, fullname): + """Implement get_filename in terms of source_path. + + As get_filename should only return a source file path there is no + chance of the path not existing but loading still being possible, so + ImportError should propagate instead of being turned into returning + None. + + """ + warnings.warn("importlib.abc.PyLoader is deprecated and is " + "slated for removal in Python 3.4; " + "use SourceLoader instead. " + "See the importlib documentation on how to be " + "compatible with Python 3.1 onwards.", + DeprecationWarning) + path = self.source_path(fullname) + if path is None: + raise ImportError + else: + return path + + +class PyPycLoader(PyLoader): + + """Abstract base class to assist in loading source and bytecode by + requiring only back-end storage methods to be implemented. + + This class has been deprecated! Removal is slated for Python 3.4. Implement + the SourceLoader ABC instead. If Python 3.1 compatibility is needed, see + PyLoader. + + The methods get_code, get_source, and load_module are implemented for the + user. + + """ + + def get_filename(self, fullname): + """Return the source or bytecode file path.""" + path = self.source_path(fullname) + if path is not None: + return path + path = self.bytecode_path(fullname) + if path is not None: + return path + raise ImportError("no source or bytecode path available for " + "{0!r}".format(fullname)) + + def get_code(self, fullname): + """Get a code object from source or bytecode.""" + warnings.warn("importlib.abc.PyPycLoader is deprecated and slated for " + "removal in Python 3.4; use SourceLoader instead. " + "If Python 3.1 compatibility is required, see the " + "latest documentation for PyLoader.", + DeprecationWarning) + source_timestamp = self.source_mtime(fullname) + # Try to use bytecode if it is available. + bytecode_path = self.bytecode_path(fullname) + if bytecode_path: + data = self.get_data(bytecode_path) + try: + magic = data[:4] + if len(magic) < 4: + raise ImportError( + "bad magic number in {}".format(fullname)) + raw_timestamp = data[4:8] + if len(raw_timestamp) < 4: + raise EOFError("bad timestamp in {}".format(fullname)) + pyc_timestamp = _bootstrap._r_long(raw_timestamp) + bytecode = data[8:] + # Verify that the magic number is valid. + if imp.get_magic() != magic: + raise ImportError( + "bad magic number in {}".format(fullname)) + # Verify that the bytecode is not stale (only matters when + # there is source to fall back on. + if source_timestamp: + if pyc_timestamp < source_timestamp: + raise ImportError("bytecode is stale") + except (ImportError, EOFError): + # If source is available give it a shot. + if source_timestamp is not None: + pass + else: + raise + else: + # Bytecode seems fine, so try to use it. + return marshal.loads(bytecode) + elif source_timestamp is None: + raise ImportError("no source or bytecode available to create code " + "object for {0!r}".format(fullname)) + # Use the source. + source_path = self.source_path(fullname) + if source_path is None: + message = "a source path must exist to load {0}".format(fullname) + raise ImportError(message) + source = self.get_data(source_path) + code_object = compile(source, source_path, 'exec', dont_inherit=True) + # Generate bytecode and write it out. + if not sys.dont_write_bytecode: + data = bytearray(imp.get_magic()) + data.extend(_bootstrap._w_long(source_timestamp)) + data.extend(marshal.dumps(code_object)) + self.write_bytecode(fullname, data) + return code_object + + @abc.abstractmethod + def source_mtime(self, fullname): + """Abstract method. Accepts a str filename and returns an int + modification time for the source of the module.""" + raise NotImplementedError + + @abc.abstractmethod + def bytecode_path(self, fullname): + """Abstract method. Accepts a str filename and returns the str pathname + to the bytecode for the module.""" + raise NotImplementedError + + @abc.abstractmethod + def write_bytecode(self, fullname, bytecode): + """Abstract method. Accepts a str filename and bytes object + representing the bytecode for the module. Returns a boolean + representing whether the bytecode was written or not.""" + raise NotImplementedError diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/machinery.py --- a/Lib/importlib/machinery.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/importlib/machinery.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,21 +1,5 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" -import _imp - -from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) -from ._bootstrap_external import WindowsRegistryFinder -from ._bootstrap_external import PathFinder -from ._bootstrap_external import FileFinder -from ._bootstrap_external import SourceFileLoader -from ._bootstrap_external import SourcelessFileLoader -from ._bootstrap_external import ExtensionFileLoader - - -def all_suffixes(): - """Returns a list of all recognized module suffixes for this process""" - return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES +from ._bootstrap import PathFinder diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/__main__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/__main__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,30 @@ +"""Run importlib's test suite. + +Specifying the ``--builtin`` flag will run tests, where applicable, with +builtins.__import__ instead of importlib.__import__. + +""" +from importlib.test.import_ import util +import os.path +from test.support import run_unittest +import unittest + + +def test_main(): + start_dir = os.path.dirname(__file__) + top_dir = os.path.dirname(os.path.dirname(start_dir)) + test_loader = unittest.TestLoader() + run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir)) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(description='Execute the importlib test ' + 'suite') + parser.add_argument('-b', '--builtin', action='store_true', default=False, + help='use builtins.__import__() instead of importlib') + args = parser.parse_args() + if args.builtin: + util.using___import__ = True + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/abc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/abc.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,99 @@ +import abc +import unittest + + +class FinderTests(unittest.TestCase, metaclass=abc.ABCMeta): + + """Basic tests for a finder to pass.""" + + @abc.abstractmethod + def test_module(self): + # Test importing a top-level module. + pass + + @abc.abstractmethod + def test_package(self): + # Test importing a package. + pass + + @abc.abstractmethod + def test_module_in_package(self): + # Test importing a module contained within a package. + # A value for 'path' should be used if for a meta_path finder. + pass + + @abc.abstractmethod + def test_package_in_package(self): + # Test importing a subpackage. + # A value for 'path' should be used if for a meta_path finder. + pass + + @abc.abstractmethod + def test_package_over_module(self): + # Test that packages are chosen over modules. + pass + + @abc.abstractmethod + def test_failure(self): + # Test trying to find a module that cannot be handled. + pass + + +class LoaderTests(unittest.TestCase, metaclass=abc.ABCMeta): + + @abc.abstractmethod + def test_module(self): + """A module should load without issue. + + After the loader returns the module should be in sys.modules. + + Attributes to verify: + + * __file__ + * __loader__ + * __name__ + * No __path__ + + """ + pass + + @abc.abstractmethod + def test_package(self): + """Loading a package should work. + + After the loader returns the module should be in sys.modules. + + Attributes to verify: + + * __name__ + * __file__ + * __package__ + * __path__ + * __loader__ + + """ + pass + + @abc.abstractmethod + def test_lacking_parent(self): + """A loader should not be dependent on it's parent package being + imported.""" + pass + + @abc.abstractmethod + def test_module_reuse(self): + """If a module is already in sys.modules, it should be reused.""" + pass + + @abc.abstractmethod + def test_state_after_failure(self): + """If a module is already in sys.modules and a reload fails + (e.g. a SyntaxError), the module should be in the state it was before + the reload began.""" + pass + + @abc.abstractmethod + def test_unloadable(self): + """Test ImportError is raised when the loader is asked to load a module + it can't.""" + pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/benchmark.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/benchmark.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,239 @@ +"""Benchmark some basic import use-cases. + +The assumption is made that this benchmark is run in a fresh interpreter and +thus has no external changes made to import-related attributes in sys. + +""" +from . import util +from .source import util as source_util +import decimal +import imp +import importlib +import json +import os +import py_compile +import sys +import tabnanny +import timeit + + +def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3): + """Bench the given statement as many times as necessary until total + executions take one second.""" + stmt = "__import__({!r})".format(name) + timer = timeit.Timer(stmt) + for x in range(repeat): + total_time = 0 + count = 0 + while total_time < seconds: + try: + total_time += timer.timeit(1) + finally: + cleanup() + count += 1 + else: + # One execution too far + if total_time > seconds: + count -= 1 + yield count // seconds + +def from_cache(seconds, repeat): + """sys.modules""" + name = '' + module = imp.new_module(name) + module.__file__ = '' + module.__package__ = '' + with util.uncache(name): + sys.modules[name] = module + for result in bench(name, repeat=repeat, seconds=seconds): + yield result + + +def builtin_mod(seconds, repeat): + """Built-in module""" + name = 'errno' + if name in sys.modules: + del sys.modules[name] + # Relying on built-in importer being implicit. + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + + +def source_wo_bytecode(seconds, repeat): + """Source w/o bytecode: small""" + sys.dont_write_bytecode = True + try: + name = '__importlib_test_benchmark__' + # Clears out sys.modules and puts an entry at the front of sys.path. + with source_util.create_modules(name) as mapping: + assert not os.path.exists(imp.cache_from_source(mapping[name])) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + finally: + sys.dont_write_bytecode = False + + +def _wo_bytecode(module): + name = module.__name__ + def benchmark_wo_bytecode(seconds, repeat): + """Source w/o bytecode: {}""" + bytecode_path = imp.cache_from_source(module.__file__) + if os.path.exists(bytecode_path): + os.unlink(bytecode_path) + sys.dont_write_bytecode = True + try: + for result in bench(name, lambda: sys.modules.pop(name), + repeat=repeat, seconds=seconds): + yield result + finally: + sys.dont_write_bytecode = False + + benchmark_wo_bytecode.__doc__ = benchmark_wo_bytecode.__doc__.format(name) + return benchmark_wo_bytecode + +tabnanny_wo_bytecode = _wo_bytecode(tabnanny) +decimal_wo_bytecode = _wo_bytecode(decimal) + + +def source_writing_bytecode(seconds, repeat): + """Source writing bytecode: small""" + assert not sys.dont_write_bytecode + name = '__importlib_test_benchmark__' + with source_util.create_modules(name) as mapping: + def cleanup(): + sys.modules.pop(name) + os.unlink(imp.cache_from_source(mapping[name])) + for result in bench(name, cleanup, repeat=repeat, seconds=seconds): + assert not os.path.exists(imp.cache_from_source(mapping[name])) + yield result + + +def _writing_bytecode(module): + name = module.__name__ + def writing_bytecode_benchmark(seconds, repeat): + """Source writing bytecode: {}""" + assert not sys.dont_write_bytecode + def cleanup(): + sys.modules.pop(name) + os.unlink(imp.cache_from_source(module.__file__)) + for result in bench(name, cleanup, repeat=repeat, seconds=seconds): + yield result + + writing_bytecode_benchmark.__doc__ = ( + writing_bytecode_benchmark.__doc__.format(name)) + return writing_bytecode_benchmark + +tabnanny_writing_bytecode = _writing_bytecode(tabnanny) +decimal_writing_bytecode = _writing_bytecode(decimal) + + +def source_using_bytecode(seconds, repeat): + """Source w/ bytecode: small""" + name = '__importlib_test_benchmark__' + with source_util.create_modules(name) as mapping: + py_compile.compile(mapping[name]) + assert os.path.exists(imp.cache_from_source(mapping[name])) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + + +def _using_bytecode(module): + name = module.__name__ + def using_bytecode_benchmark(seconds, repeat): + """Source w/ bytecode: {}""" + py_compile.compile(module.__file__) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + + using_bytecode_benchmark.__doc__ = ( + using_bytecode_benchmark.__doc__.format(name)) + return using_bytecode_benchmark + +tabnanny_using_bytecode = _using_bytecode(tabnanny) +decimal_using_bytecode = _using_bytecode(decimal) + + +def main(import_, options): + if options.source_file: + with options.source_file: + prev_results = json.load(options.source_file) + else: + prev_results = {} + __builtins__.__import__ = import_ + benchmarks = (from_cache, builtin_mod, + source_writing_bytecode, + source_wo_bytecode, source_using_bytecode, + tabnanny_writing_bytecode, + tabnanny_wo_bytecode, tabnanny_using_bytecode, + decimal_writing_bytecode, + decimal_wo_bytecode, decimal_using_bytecode, + ) + if options.benchmark: + for b in benchmarks: + if b.__doc__ == options.benchmark: + benchmarks = [b] + break + else: + print('Unknown benchmark: {!r}'.format(options.benchmark, + file=sys.stderr)) + sys.exit(1) + seconds = 1 + seconds_plural = 's' if seconds > 1 else '' + repeat = 3 + header = ('Measuring imports/second over {} second{}, best out of {}\n' + 'Entire benchmark run should take about {} seconds\n' + 'Using {!r} as __import__\n') + print(header.format(seconds, seconds_plural, repeat, + len(benchmarks) * seconds * repeat, __import__)) + new_results = {} + for benchmark in benchmarks: + print(benchmark.__doc__, "[", end=' ') + sys.stdout.flush() + results = [] + for result in benchmark(seconds=seconds, repeat=repeat): + results.append(result) + print(result, end=' ') + sys.stdout.flush() + assert not sys.dont_write_bytecode + print("]", "best is", format(max(results), ',d')) + new_results[benchmark.__doc__] = results + if prev_results: + print('\n\nComparing new vs. old\n') + for benchmark in benchmarks: + benchmark_name = benchmark.__doc__ + old_result = max(prev_results[benchmark_name]) + new_result = max(new_results[benchmark_name]) + result = '{:,d} vs. {:,d} ({:%})'.format(new_result, + old_result, + new_result/old_result) + print(benchmark_name, ':', result) + if options.dest_file: + with options.dest_file: + json.dump(new_results, options.dest_file, indent=2) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('-b', '--builtin', dest='builtin', action='store_true', + default=False, help="use the built-in __import__") + parser.add_argument('-r', '--read', dest='source_file', + type=argparse.FileType('r'), + help='file to read benchmark data from to compare ' + 'against') + parser.add_argument('-w', '--write', dest='dest_file', + type=argparse.FileType('w'), + help='file to write benchmark data to') + parser.add_argument('--benchmark', dest='benchmark', + help='specific benchmark to run') + options = parser.parse_args() + import_ = __import__ + if not options.builtin: + import_ = importlib.__import__ + + main(import_, options) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/builtin/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/builtin/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,12 @@ +import importlib.test +import os + + +def test_suite(): + directory = os.path.dirname(__file__) + return importlib.test.test_suite('importlib.test.builtin', directory) + + +if __name__ == '__main__': + from test.support import run_unittest + run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/builtin/test_finder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/builtin/test_finder.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,55 @@ +from importlib import machinery +from .. import abc +from .. import util +from . import util as builtin_util + +import sys +import unittest + +class FinderTests(abc.FinderTests): + + """Test find_module() for built-in modules.""" + + def test_module(self): + # Common case. + with util.uncache(builtin_util.NAME): + found = machinery.BuiltinImporter.find_module(builtin_util.NAME) + self.assertTrue(found) + + def test_package(self): + # Built-in modules cannot be a package. + pass + + def test_module_in_package(self): + # Built-in modules cannobt be in a package. + pass + + def test_package_in_package(self): + # Built-in modules cannot be a package. + pass + + def test_package_over_module(self): + # Built-in modules cannot be a package. + pass + + def test_failure(self): + assert 'importlib' not in sys.builtin_module_names + loader = machinery.BuiltinImporter.find_module('importlib') + self.assertTrue(loader is None) + + def test_ignore_path(self): + # The value for 'path' should always trigger a failed import. + with util.uncache(builtin_util.NAME): + loader = machinery.BuiltinImporter.find_module(builtin_util.NAME, + ['pkg']) + self.assertTrue(loader is None) + + + +def test_main(): + from test.support import run_unittest + run_unittest(FinderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/builtin/test_loader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/builtin/test_loader.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,102 @@ +import importlib +from importlib import machinery +from .. import abc +from .. import util +from . import util as builtin_util + +import sys +import types +import unittest + + +class LoaderTests(abc.LoaderTests): + + """Test load_module() for built-in modules.""" + + verification = {'__name__': 'errno', '__package__': '', + '__loader__': machinery.BuiltinImporter} + + def verify(self, module): + """Verify that the module matches against what it should have.""" + self.assertTrue(isinstance(module, types.ModuleType)) + for attr, value in self.verification.items(): + self.assertEqual(getattr(module, attr), value) + self.assertTrue(module.__name__ in sys.modules) + + load_module = staticmethod(lambda name: + machinery.BuiltinImporter.load_module(name)) + + def test_module(self): + # Common case. + with util.uncache(builtin_util.NAME): + module = self.load_module(builtin_util.NAME) + self.verify(module) + + def test_package(self): + # Built-in modules cannot be a package. + pass + + def test_lacking_parent(self): + # Built-in modules cannot be a package. + pass + + def test_state_after_failure(self): + # Not way to force an imoprt failure. + pass + + def test_module_reuse(self): + # Test that the same module is used in a reload. + with util.uncache(builtin_util.NAME): + module1 = self.load_module(builtin_util.NAME) + module2 = self.load_module(builtin_util.NAME) + self.assertTrue(module1 is module2) + + def test_unloadable(self): + name = 'dssdsdfff' + assert name not in sys.builtin_module_names + with self.assertRaises(ImportError): + self.load_module(name) + + def test_already_imported(self): + # Using the name of a module already imported but not a built-in should + # still fail. + assert hasattr(importlib, '__file__') # Not a built-in. + with self.assertRaises(ImportError): + self.load_module('importlib') + + +class InspectLoaderTests(unittest.TestCase): + + """Tests for InspectLoader methods for BuiltinImporter.""" + + def test_get_code(self): + # There is no code object. + result = machinery.BuiltinImporter.get_code(builtin_util.NAME) + self.assertTrue(result is None) + + def test_get_source(self): + # There is no source. + result = machinery.BuiltinImporter.get_source(builtin_util.NAME) + self.assertTrue(result is None) + + def test_is_package(self): + # Cannot be a package. + result = machinery.BuiltinImporter.is_package(builtin_util.NAME) + self.assertTrue(not result) + + def test_not_builtin(self): + # Modules not built-in should raise ImportError. + for meth_name in ('get_code', 'get_source', 'is_package'): + method = getattr(machinery.BuiltinImporter, meth_name) + with self.assertRaises(ImportError): + method(builtin_util.BAD_NAME) + + + +def test_main(): + from test.support import run_unittest + run_unittest(LoaderTests, InspectLoaderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/builtin/util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/builtin/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,7 @@ +import sys + +assert 'errno' in sys.builtin_module_names +NAME = 'errno' + +assert 'importlib' not in sys.builtin_module_names +BAD_NAME = 'importlib' diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,13 @@ +import importlib.test +import os.path +import unittest + + +def test_suite(): + directory = os.path.dirname(__file__) + return importlib.test.test_suite('importlib.test.extension', directory) + + +if __name__ == '__main__': + from test.support import run_unittest + run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/test_case_sensitivity.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/test_case_sensitivity.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,48 @@ +import sys +from test import support +import unittest +from importlib import _bootstrap +from .. import util +from . import util as ext_util + + +@util.case_insensitive_tests +class ExtensionModuleCaseSensitivityTest(unittest.TestCase): + + def find_module(self): + good_name = ext_util.NAME + bad_name = good_name.upper() + assert good_name != bad_name + finder = _bootstrap._FileFinder(ext_util.PATH, + _bootstrap._ExtensionFinderDetails()) + return finder.find_module(bad_name) + + def test_case_sensitive(self): + with support.EnvironmentVarGuard() as env: + env.unset('PYTHONCASEOK') + if b'PYTHONCASEOK' in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + loader = self.find_module() + self.assertIsNone(loader) + + def test_case_insensitivity(self): + with support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + loader = self.find_module() + self.assertTrue(hasattr(loader, 'load_module')) + + + + +def test_main(): + if ext_util.FILENAME is None: + return + support.run_unittest(ExtensionModuleCaseSensitivityTest) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/test_finder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/test_finder.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,47 @@ +from importlib import _bootstrap +from .. import abc +from . import util + +import unittest + +class FinderTests(abc.FinderTests): + + """Test the finder for extension modules.""" + + def find_module(self, fullname): + importer = _bootstrap._FileFinder(util.PATH, + _bootstrap._ExtensionFinderDetails()) + return importer.find_module(fullname) + + def test_module(self): + self.assertTrue(self.find_module(util.NAME)) + + def test_package(self): + # Extension modules cannot be an __init__ for a package. + pass + + def test_module_in_package(self): + # No extension module in a package available for testing. + pass + + def test_package_in_package(self): + # Extension modules cannot be an __init__ for a package. + pass + + def test_package_over_module(self): + # Extension modules cannot be an __init__ for a package. + pass + + def test_failure(self): + self.assertTrue(self.find_module('asdfjkl;') is None) + + # XXX Raise an exception if someone tries to use the 'path' argument? + + +def test_main(): + from test.support import run_unittest + run_unittest(FinderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/test_loader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/test_loader.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,59 @@ +from importlib import _bootstrap +from . import util as ext_util +from .. import abc +from .. import util + +import sys +import unittest + + +class LoaderTests(abc.LoaderTests): + + """Test load_module() for extension modules.""" + + def load_module(self, fullname): + loader = _bootstrap._ExtensionFileLoader(ext_util.NAME, + ext_util.FILEPATH) + return loader.load_module(fullname) + + def test_module(self): + with util.uncache(ext_util.NAME): + module = self.load_module(ext_util.NAME) + for attr, value in [('__name__', ext_util.NAME), + ('__file__', ext_util.FILEPATH), + ('__package__', '')]: + self.assertEqual(getattr(module, attr), value) + self.assertTrue(ext_util.NAME in sys.modules) + self.assertTrue(isinstance(module.__loader__, + _bootstrap._ExtensionFileLoader)) + + def test_package(self): + # Extensions are not found in packages. + pass + + def test_lacking_parent(self): + # Extensions are not found in packages. + pass + + def test_module_reuse(self): + with util.uncache(ext_util.NAME): + module1 = self.load_module(ext_util.NAME) + module2 = self.load_module(ext_util.NAME) + self.assertTrue(module1 is module2) + + def test_state_after_failure(self): + # No easy way to trigger a failure after a successful import. + pass + + def test_unloadable(self): + with self.assertRaises(ImportError): + self.load_module('asdfjkl;') + + +def test_main(): + from test.support import run_unittest + run_unittest(LoaderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/test_path_hook.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/test_path_hook.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,31 @@ +from importlib import _bootstrap +from . import util + +import collections +import imp +import sys +import unittest + + +class PathHookTests(unittest.TestCase): + + """Test the path hook for extension modules.""" + # XXX Should it only succeed for pre-existing directories? + # XXX Should it only work for directories containing an extension module? + + def hook(self, entry): + return _bootstrap._file_path_hook(entry) + + def test_success(self): + # Path hook should handle a directory where a known extension module + # exists. + self.assertTrue(hasattr(self.hook(util.PATH), 'find_module')) + + +def test_main(): + from test.support import run_unittest + run_unittest(PathHookTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/extension/util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/extension/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,21 @@ +import imp +import os +import sys + +PATH = None +EXT = None +FILENAME = None +NAME = '_testcapi' +_file_exts = [x[0] for x in imp.get_suffixes() if x[2] == imp.C_EXTENSION] +try: + for PATH in sys.path: + for EXT in _file_exts: + FILENAME = NAME + EXT + FILEPATH = os.path.join(PATH, FILENAME) + if os.path.exists(os.path.join(PATH, FILENAME)): + raise StopIteration + else: + PATH = EXT = FILENAME = FILEPATH = None +except StopIteration: + pass +del _file_exts diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/frozen/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/frozen/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,13 @@ +import importlib.test +import os.path +import unittest + + +def test_suite(): + directory = os.path.dirname(__file__) + return importlib.test.test_suite('importlib.test.frozen', directory) + + +if __name__ == '__main__': + from test.support import run_unittest + run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/frozen/test_finder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/frozen/test_finder.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,47 @@ +from ... import machinery +from .. import abc + +import unittest + + +class FinderTests(abc.FinderTests): + + """Test finding frozen modules.""" + + def find(self, name, path=None): + finder = machinery.FrozenImporter + return finder.find_module(name, path) + + def test_module(self): + name = '__hello__' + loader = self.find(name) + self.assertTrue(hasattr(loader, 'load_module')) + + def test_package(self): + loader = self.find('__phello__') + self.assertTrue(hasattr(loader, 'load_module')) + + def test_module_in_package(self): + loader = self.find('__phello__.spam', ['__phello__']) + self.assertTrue(hasattr(loader, 'load_module')) + + def test_package_in_package(self): + # No frozen package within another package to test with. + pass + + def test_package_over_module(self): + # No easy way to test. + pass + + def test_failure(self): + loader = self.find('') + self.assertTrue(loader is None) + + +def test_main(): + from test.support import run_unittest + run_unittest(FinderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/frozen/test_loader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/frozen/test_loader.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,105 @@ +from importlib import machinery +import imp +import unittest +from .. import abc +from .. import util +from test.support import captured_stdout + +class LoaderTests(abc.LoaderTests): + + def test_module(self): + with util.uncache('__hello__'), captured_stdout() as stdout: + module = machinery.FrozenImporter.load_module('__hello__') + check = {'__name__': '__hello__', '__file__': '', + '__package__': '', '__loader__': machinery.FrozenImporter} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + + def test_package(self): + with util.uncache('__phello__'), captured_stdout() as stdout: + module = machinery.FrozenImporter.load_module('__phello__') + check = {'__name__': '__phello__', '__file__': '', + '__package__': '__phello__', '__path__': ['__phello__'], + '__loader__': machinery.FrozenImporter} + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + + def test_lacking_parent(self): + with util.uncache('__phello__', '__phello__.spam'), \ + captured_stdout() as stdout: + module = machinery.FrozenImporter.load_module('__phello__.spam') + check = {'__name__': '__phello__.spam', '__file__': '', + '__package__': '__phello__', + '__loader__': machinery.FrozenImporter} + for attr, value in check.items(): + attr_value = getattr(module, attr) + self.assertEqual(attr_value, value, + "for __phello__.spam.%s, %r != %r" % + (attr, attr_value, value)) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + + def test_module_reuse(self): + with util.uncache('__hello__'), captured_stdout() as stdout: + module1 = machinery.FrozenImporter.load_module('__hello__') + module2 = machinery.FrozenImporter.load_module('__hello__') + self.assertTrue(module1 is module2) + self.assertEqual(stdout.getvalue(), + 'Hello world!\nHello world!\n') + + def test_state_after_failure(self): + # No way to trigger an error in a frozen module. + pass + + def test_unloadable(self): + assert machinery.FrozenImporter.find_module('_not_real') is None + with self.assertRaises(ImportError): + machinery.FrozenImporter.load_module('_not_real') + + +class InspectLoaderTests(unittest.TestCase): + + """Tests for the InspectLoader methods for FrozenImporter.""" + + def test_get_code(self): + # Make sure that the code object is good. + name = '__hello__' + with captured_stdout() as stdout: + code = machinery.FrozenImporter.get_code(name) + mod = imp.new_module(name) + exec(code, mod.__dict__) + self.assertTrue(hasattr(mod, 'initialized')) + self.assertEqual(stdout.getvalue(), 'Hello world!\n') + + def test_get_source(self): + # Should always return None. + result = machinery.FrozenImporter.get_source('__hello__') + self.assertTrue(result is None) + + def test_is_package(self): + # Should be able to tell what is a package. + test_for = (('__hello__', False), ('__phello__', True), + ('__phello__.spam', False)) + for name, is_package in test_for: + result = machinery.FrozenImporter.is_package(name) + self.assertTrue(bool(result) == is_package) + + def test_failure(self): + # Raise ImportError for modules that are not frozen. + for meth_name in ('get_code', 'get_source', 'is_package'): + method = getattr(machinery.FrozenImporter, meth_name) + with self.assertRaises(ImportError): + method('importlib') + + +def test_main(): + from test.support import run_unittest + run_unittest(LoaderTests, InspectLoaderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,13 @@ +import importlib.test +import os.path +import unittest + + +def test_suite(): + directory = os.path.dirname(__file__) + return importlib.test.test_suite('importlib.test.import_', directory) + + +if __name__ == '__main__': + from test.support import run_unittest + run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test___package__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test___package__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,119 @@ +"""PEP 366 ("Main module explicit relative imports") specifies the +semantics for the __package__ attribute on modules. This attribute is +used, when available, to detect which package a module belongs to (instead +of using the typical __path__/__name__ test). + +""" +import unittest +from .. import util +from . import util as import_util + + +class Using__package__(unittest.TestCase): + + """Use of __package__ supercedes the use of __name__/__path__ to calculate + what package a module belongs to. The basic algorithm is [__package__]:: + + def resolve_name(name, package, level): + level -= 1 + base = package.rsplit('.', level)[0] + return '{0}.{1}'.format(base, name) + + But since there is no guarantee that __package__ has been set (or not been + set to None [None]), there has to be a way to calculate the attribute's value + [__name__]:: + + def calc_package(caller_name, has___path__): + if has__path__: + return caller_name + else: + return caller_name.rsplit('.', 1)[0] + + Then the normal algorithm for relative name imports can proceed as if + __package__ had been set. + + """ + + def test_using___package__(self): + # [__package__] + with util.mock_modules('pkg.__init__', 'pkg.fake') as importer: + with util.import_state(meta_path=[importer]): + import_util.import_('pkg.fake') + module = import_util.import_('', + globals={'__package__': 'pkg.fake'}, + fromlist=['attr'], level=2) + self.assertEqual(module.__name__, 'pkg') + + def test_using___name__(self, package_as_None=False): + # [__name__] + globals_ = {'__name__': 'pkg.fake', '__path__': []} + if package_as_None: + globals_['__package__'] = None + with util.mock_modules('pkg.__init__', 'pkg.fake') as importer: + with util.import_state(meta_path=[importer]): + import_util.import_('pkg.fake') + module = import_util.import_('', globals= globals_, + fromlist=['attr'], level=2) + self.assertEqual(module.__name__, 'pkg') + + def test_None_as___package__(self): + # [None] + self.test_using___name__(package_as_None=True) + + def test_bad__package__(self): + globals = {'__package__': ''} + with self.assertRaises(SystemError): + import_util.import_('', globals, {}, ['relimport'], 1) + + def test_bunk__package__(self): + globals = {'__package__': 42} + with self.assertRaises(TypeError): + import_util.import_('', globals, {}, ['relimport'], 1) + + +@import_util.importlib_only +class Setting__package__(unittest.TestCase): + + """Because __package__ is a new feature, it is not always set by a loader. + Import will set it as needed to help with the transition to relying on + __package__. + + For a top-level module, __package__ is set to None [top-level]. For a + package __name__ is used for __package__ [package]. For submodules the + value is __name__.rsplit('.', 1)[0] [submodule]. + + """ + + # [top-level] + def test_top_level(self): + with util.mock_modules('top_level') as mock: + with util.import_state(meta_path=[mock]): + del mock['top_level'].__package__ + module = import_util.import_('top_level') + self.assertEqual(module.__package__, '') + + # [package] + def test_package(self): + with util.mock_modules('pkg.__init__') as mock: + with util.import_state(meta_path=[mock]): + del mock['pkg'].__package__ + module = import_util.import_('pkg') + self.assertEqual(module.__package__, 'pkg') + + # [submodule] + def test_submodule(self): + with util.mock_modules('pkg.__init__', 'pkg.mod') as mock: + with util.import_state(meta_path=[mock]): + del mock['pkg.mod'].__package__ + pkg = import_util.import_('pkg.mod') + module = getattr(pkg, 'mod') + self.assertEqual(module.__package__, 'pkg') + + +def test_main(): + from test.support import run_unittest + run_unittest(Using__package__, Setting__package__) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_api.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_api.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,29 @@ +from . import util +import unittest + + +class APITest(unittest.TestCase): + + """Test API-specific details for __import__ (e.g. raising the right + exception when passing in an int for the module name).""" + + def test_name_requires_rparition(self): + # Raise TypeError if a non-string is passed in for the module name. + with self.assertRaises(TypeError): + util.import_(42) + + def test_negative_level(self): + # Raise ValueError when a negative level is specified. + # PEP 328 did away with sys.module None entries and the ambiguity of + # absolute/relative imports. + with self.assertRaises(ValueError): + util.import_('os', globals(), level=-1) + + +def test_main(): + from test.support import run_unittest + run_unittest(APITest) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_caching.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_caching.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,86 @@ +"""Test that sys.modules is used properly by import.""" +from .. import util +from . import util as import_util +import sys +from types import MethodType +import unittest + + +class UseCache(unittest.TestCase): + + """When it comes to sys.modules, import prefers it over anything else. + + Once a name has been resolved, sys.modules is checked to see if it contains + the module desired. If so, then it is returned [use cache]. If it is not + found, then the proper steps are taken to perform the import, but + sys.modules is still used to return the imported module (e.g., not what a + loader returns) [from cache on return]. This also applies to imports of + things contained within a package and thus get assigned as an attribute + [from cache to attribute] or pulled in thanks to a fromlist import + [from cache for fromlist]. But if sys.modules contains None then + ImportError is raised [None in cache]. + + """ + def test_using_cache(self): + # [use cache] + module_to_use = "some module found!" + with util.uncache(module_to_use): + sys.modules['some_module'] = module_to_use + module = import_util.import_('some_module') + self.assertEqual(id(module_to_use), id(module)) + + def test_None_in_cache(self): + #[None in cache] + name = 'using_None' + with util.uncache(name): + sys.modules[name] = None + with self.assertRaises(ImportError): + import_util.import_(name) + + def create_mock(self, *names, return_=None): + mock = util.mock_modules(*names) + original_load = mock.load_module + def load_module(self, fullname): + original_load(fullname) + return return_ + mock.load_module = MethodType(load_module, mock) + return mock + + # __import__ inconsistent between loaders and built-in import when it comes + # to when to use the module in sys.modules and when not to. + @import_util.importlib_only + def test_using_cache_after_loader(self): + # [from cache on return] + with self.create_mock('module') as mock: + with util.import_state(meta_path=[mock]): + module = import_util.import_('module') + self.assertEqual(id(module), id(sys.modules['module'])) + + # See test_using_cache_after_loader() for reasoning. + @import_util.importlib_only + def test_using_cache_for_assigning_to_attribute(self): + # [from cache to attribute] + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg.module') + self.assertTrue(hasattr(module, 'module')) + self.assertTrue(id(module.module), id(sys.modules['pkg.module'])) + + # See test_using_cache_after_loader() for reasoning. + @import_util.importlib_only + def test_using_cache_for_fromlist(self): + # [from cache for fromlist] + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg', fromlist=['module']) + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) + + +def test_main(): + from test.support import run_unittest + run_unittest(UseCache) + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_fromlist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_fromlist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,123 @@ +"""Test that the semantics relating to the 'fromlist' argument are correct.""" +from .. import util +from . import util as import_util +import unittest + +class ReturnValue(unittest.TestCase): + + """The use of fromlist influences what import returns. + + If direct ``import ...`` statement is used, the root module or package is + returned [import return]. But if fromlist is set, then the specified module + is actually returned (whether it is a relative import or not) + [from return]. + + """ + + def test_return_from_import(self): + # [import return] + with util.mock_modules('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg.module') + self.assertEqual(module.__name__, 'pkg') + + def test_return_from_from_import(self): + # [from return] + with util.mock_modules('pkg.__init__', 'pkg.module')as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg.module', fromlist=['attr']) + self.assertEqual(module.__name__, 'pkg.module') + + +class HandlingFromlist(unittest.TestCase): + + """Using fromlist triggers different actions based on what is being asked + of it. + + If fromlist specifies an object on a module, nothing special happens + [object case]. This is even true if the object does not exist [bad object]. + + If a package is being imported, then what is listed in fromlist may be + treated as a module to be imported [module]. But once again, even if + something in fromlist does not exist as a module, no error is thrown + [no module]. And this extends to what is contained in __all__ when '*' is + imported [using *]. And '*' does not need to be the only name in the + fromlist [using * with others]. + + """ + + def test_object(self): + # [object case] + with util.mock_modules('module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('module', fromlist=['attr']) + self.assertEqual(module.__name__, 'module') + + def test_unexistent_object(self): + # [bad object] + with util.mock_modules('module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('module', fromlist=['non_existent']) + self.assertEqual(module.__name__, 'module') + self.assertTrue(not hasattr(module, 'non_existent')) + + def test_module_from_package(self): + # [module] + with util.mock_modules('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg', fromlist=['module']) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.__name__, 'pkg.module') + + def test_no_module_from_package(self): + # [no module] + with util.mock_modules('pkg.__init__') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg', fromlist='non_existent') + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(not hasattr(module, 'non_existent')) + + def test_empty_string(self): + with util.mock_modules('pkg.__init__', 'pkg.mod') as importer: + with util.import_state(meta_path=[importer]): + module = import_util.import_('pkg.mod', fromlist=['']) + self.assertEqual(module.__name__, 'pkg.mod') + + def basic_star_test(self, fromlist=['*']): + # [using *] + with util.mock_modules('pkg.__init__', 'pkg.module') as mock: + with util.import_state(meta_path=[mock]): + mock['pkg'].__all__ = ['module'] + module = import_util.import_('pkg', fromlist=fromlist) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.__name__, 'pkg.module') + + def test_using_star(self): + # [using *] + self.basic_star_test() + + def test_fromlist_as_tuple(self): + self.basic_star_test(('*',)) + + def test_star_with_others(self): + # [using * with others] + context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2') + with context as mock: + with util.import_state(meta_path=[mock]): + mock['pkg'].__all__ = ['module1'] + module = import_util.import_('pkg', fromlist=['module2', '*']) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module1')) + self.assertTrue(hasattr(module, 'module2')) + self.assertEqual(module.module1.__name__, 'pkg.module1') + self.assertEqual(module.module2.__name__, 'pkg.module2') + + +def test_main(): + from test.support import run_unittest + run_unittest(ReturnValue, HandlingFromlist) + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_meta_path.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_meta_path.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,97 @@ +from .. import util +from . import util as import_util +from types import MethodType +import unittest + + +class CallingOrder(unittest.TestCase): + + """Calls to the importers on sys.meta_path happen in order that they are + specified in the sequence, starting with the first importer + [first called], and then continuing on down until one is found that doesn't + return None [continuing].""" + + + def test_first_called(self): + # [first called] + mod = 'top_level' + first = util.mock_modules(mod) + second = util.mock_modules(mod) + with util.mock_modules(mod) as first, util.mock_modules(mod) as second: + first.modules[mod] = 42 + second.modules[mod] = -13 + with util.import_state(meta_path=[first, second]): + self.assertEqual(import_util.import_(mod), 42) + + def test_continuing(self): + # [continuing] + mod_name = 'for_real' + with util.mock_modules('nonexistent') as first, \ + util.mock_modules(mod_name) as second: + first.find_module = lambda self, fullname, path=None: None + second.modules[mod_name] = 42 + with util.import_state(meta_path=[first, second]): + self.assertEqual(import_util.import_(mod_name), 42) + + +class CallSignature(unittest.TestCase): + + """If there is no __path__ entry on the parent module, then 'path' is None + [no path]. Otherwise, the value for __path__ is passed in for the 'path' + argument [path set].""" + + def log(self, fxn): + log = [] + def wrapper(self, *args, **kwargs): + log.append([args, kwargs]) + return fxn(*args, **kwargs) + return log, wrapper + + + def test_no_path(self): + # [no path] + mod_name = 'top_level' + assert '.' not in mod_name + with util.mock_modules(mod_name) as importer: + log, wrapped_call = self.log(importer.find_module) + importer.find_module = MethodType(wrapped_call, importer) + with util.import_state(meta_path=[importer]): + import_util.import_(mod_name) + assert len(log) == 1 + args = log[0][0] + kwargs = log[0][1] + # Assuming all arguments are positional. + self.assertEqual(len(args), 2) + self.assertEqual(len(kwargs), 0) + self.assertEqual(args[0], mod_name) + self.assertTrue(args[1] is None) + + def test_with_path(self): + # [path set] + pkg_name = 'pkg' + mod_name = pkg_name + '.module' + path = [42] + assert '.' in mod_name + with util.mock_modules(pkg_name+'.__init__', mod_name) as importer: + importer.modules[pkg_name].__path__ = path + log, wrapped_call = self.log(importer.find_module) + importer.find_module = MethodType(wrapped_call, importer) + with util.import_state(meta_path=[importer]): + import_util.import_(mod_name) + assert len(log) == 2 + args = log[1][0] + kwargs = log[1][1] + # Assuming all arguments are positional. + self.assertTrue(not kwargs) + self.assertEqual(args[0], mod_name) + self.assertTrue(args[1] is path) + + + +def test_main(): + from test.support import run_unittest + run_unittest(CallingOrder, CallSignature) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_packages.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_packages.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,37 @@ +from .. import util +from . import util as import_util +import sys +import unittest +import importlib + + +class ParentModuleTests(unittest.TestCase): + + """Importing a submodule should import the parent modules.""" + + def test_import_parent(self): + with util.mock_modules('pkg.__init__', 'pkg.module') as mock: + with util.import_state(meta_path=[mock]): + module = import_util.import_('pkg.module') + self.assertTrue('pkg' in sys.modules) + + def test_bad_parent(self): + with util.mock_modules('pkg.module') as mock: + with util.import_state(meta_path=[mock]): + with self.assertRaises(ImportError): + import_util.import_('pkg.module') + + def test_module_not_package(self): + # Try to import a submodule from a non-package should raise ImportError. + assert not hasattr(sys, '__path__') + with self.assertRaises(ImportError): + import_util.import_('sys.no_submodules_here') + + +def test_main(): + from test.support import run_unittest + run_unittest(ParentModuleTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_path.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_path.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,141 @@ +from importlib import _bootstrap +from importlib import machinery +from .. import util +from . import util as import_util +import imp +import os +import sys +import tempfile +from test import support +from types import MethodType +import unittest + + +class FinderTests(unittest.TestCase): + + """Tests for PathFinder.""" + + def test_failure(self): + # Test None returned upon not finding a suitable finder. + module = '' + with util.import_state(): + self.assertTrue(machinery.PathFinder.find_module(module) is None) + + def test_sys_path(self): + # Test that sys.path is used when 'path' is None. + # Implicitly tests that sys.path_importer_cache is used. + module = '' + path = '' + importer = util.mock_modules(module) + with util.import_state(path_importer_cache={path: importer}, + path=[path]): + loader = machinery.PathFinder.find_module(module) + self.assertTrue(loader is importer) + + def test_path(self): + # Test that 'path' is used when set. + # Implicitly tests that sys.path_importer_cache is used. + module = '' + path = '' + importer = util.mock_modules(module) + with util.import_state(path_importer_cache={path: importer}): + loader = machinery.PathFinder.find_module(module, [path]) + self.assertTrue(loader is importer) + + def test_path_hooks(self): + # Test that sys.path_hooks is used. + # Test that sys.path_importer_cache is set. + module = '' + path = '' + importer = util.mock_modules(module) + hook = import_util.mock_path_hook(path, importer=importer) + with util.import_state(path_hooks=[hook]): + loader = machinery.PathFinder.find_module(module, [path]) + self.assertTrue(loader is importer) + self.assertTrue(path in sys.path_importer_cache) + self.assertTrue(sys.path_importer_cache[path] is importer) + + def test_path_importer_cache_has_None(self): + # Test that if sys.path_importer_cache has None that None is returned. + clear_cache = {path: None for path in sys.path} + with util.import_state(path_importer_cache=clear_cache): + for name in ('asynchat', 'sys', ''): + self.assertTrue(machinery.PathFinder.find_module(name) is None) + + def test_path_importer_cache_has_None_continues(self): + # Test that having None in sys.path_importer_cache causes the search to + # continue. + path = '' + module = '' + importer = util.mock_modules(module) + with util.import_state(path=['1', '2'], + path_importer_cache={'1': None, '2': importer}): + loader = machinery.PathFinder.find_module(module) + self.assertTrue(loader is importer) + + def test_path_importer_cache_empty_string(self): + # The empty string should create a finder using the cwd. + path = '' + module = '' + importer = util.mock_modules(module) + hook = import_util.mock_path_hook(os.curdir, importer=importer) + with util.import_state(path=[path], path_hooks=[hook]): + loader = machinery.PathFinder.find_module(module) + self.assertIs(loader, importer) + self.assertIn(os.curdir, sys.path_importer_cache) + + +class DefaultPathFinderTests(unittest.TestCase): + + """Test importlib._bootstrap._DefaultPathFinder.""" + + def test_implicit_hooks(self): + # Test that the implicit path hooks are used. + bad_path = '' + module = '' + assert not os.path.exists(bad_path) + existing_path = tempfile.mkdtemp() + try: + with util.import_state(): + nothing = _bootstrap._DefaultPathFinder.find_module(module, + path=[existing_path]) + self.assertTrue(nothing is None) + self.assertTrue(existing_path in sys.path_importer_cache) + result = isinstance(sys.path_importer_cache[existing_path], + imp.NullImporter) + self.assertFalse(result) + nothing = _bootstrap._DefaultPathFinder.find_module(module, + path=[bad_path]) + self.assertTrue(nothing is None) + self.assertTrue(bad_path in sys.path_importer_cache) + self.assertTrue(isinstance(sys.path_importer_cache[bad_path], + imp.NullImporter)) + finally: + os.rmdir(existing_path) + + + def test_path_importer_cache_has_None(self): + # Test that the default hook is used when sys.path_importer_cache + # contains None for a path. + module = '' + importer = util.mock_modules(module) + path = '' + # XXX Not blackbox. + original_hook = _bootstrap._DEFAULT_PATH_HOOK + mock_hook = import_util.mock_path_hook(path, importer=importer) + _bootstrap._DEFAULT_PATH_HOOK = mock_hook + try: + with util.import_state(path_importer_cache={path: None}): + loader = _bootstrap._DefaultPathFinder.find_module(module, + path=[path]) + self.assertTrue(loader is importer) + finally: + _bootstrap._DEFAULT_PATH_HOOK = original_hook + + +def test_main(): + from test.support import run_unittest + run_unittest(FinderTests, DefaultPathFinderTests) + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/test_relative_imports.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/test_relative_imports.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,203 @@ +"""Test relative imports (PEP 328).""" +from .. import util +from . import util as import_util +import sys +import unittest + +class RelativeImports(unittest.TestCase): + + """PEP 328 introduced relative imports. This allows for imports to occur + from within a package without having to specify the actual package name. + + A simple example is to import another module within the same package + [module from module]:: + + # From pkg.mod1 with pkg.mod2 being a module. + from . import mod2 + + This also works for getting an attribute from a module that is specified + in a relative fashion [attr from module]:: + + # From pkg.mod1. + from .mod2 import attr + + But this is in no way restricted to working between modules; it works + from [package to module],:: + + # From pkg, importing pkg.module which is a module. + from . import module + + [module to package],:: + + # Pull attr from pkg, called from pkg.module which is a module. + from . import attr + + and [package to package]:: + + # From pkg.subpkg1 (both pkg.subpkg[1,2] are packages). + from .. import subpkg2 + + The number of dots used is in no way restricted [deep import]:: + + # Import pkg.attr from pkg.pkg1.pkg2.pkg3.pkg4.pkg5. + from ...... import attr + + To prevent someone from accessing code that is outside of a package, one + cannot reach the location containing the root package itself:: + + # From pkg.__init__ [too high from package] + from .. import top_level + + # From pkg.module [too high from module] + from .. import top_level + + Relative imports are the only type of import that allow for an empty + module name for an import [empty name]. + + """ + + def relative_import_test(self, create, globals_, callback): + """Abstract out boilerplace for setting up for an import test.""" + uncache_names = [] + for name in create: + if not name.endswith('.__init__'): + uncache_names.append(name) + else: + uncache_names.append(name[:-len('.__init__')]) + with util.mock_modules(*create) as importer: + with util.import_state(meta_path=[importer]): + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) + + + def test_module_from_module(self): + # [module from module] + create = 'pkg.__init__', 'pkg.mod2' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'} + def callback(global_): + import_util.import_('pkg') # For __import__(). + module = import_util.import_('', global_, fromlist=['mod2'], level=1) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'mod2')) + self.assertEqual(module.mod2.attr, 'pkg.mod2') + self.relative_import_test(create, globals_, callback) + + def test_attr_from_module(self): + # [attr from module] + create = 'pkg.__init__', 'pkg.mod2' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'} + def callback(global_): + import_util.import_('pkg') # For __import__(). + module = import_util.import_('mod2', global_, fromlist=['attr'], + level=1) + self.assertEqual(module.__name__, 'pkg.mod2') + self.assertEqual(module.attr, 'pkg.mod2') + self.relative_import_test(create, globals_, callback) + + def test_package_to_module(self): + # [package to module] + create = 'pkg.__init__', 'pkg.module' + globals_ = ({'__package__': 'pkg'}, + {'__name__': 'pkg', '__path__': ['blah']}) + def callback(global_): + import_util.import_('pkg') # For __import__(). + module = import_util.import_('', global_, fromlist=['module'], + level=1) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(module.module.attr, 'pkg.module') + self.relative_import_test(create, globals_, callback) + + def test_module_to_package(self): + # [module to package] + create = 'pkg.__init__', 'pkg.module' + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} + def callback(global_): + import_util.import_('pkg') # For __import__(). + module = import_util.import_('', global_, fromlist=['attr'], level=1) + self.assertEqual(module.__name__, 'pkg') + self.relative_import_test(create, globals_, callback) + + def test_package_to_package(self): + # [package to package] + create = ('pkg.__init__', 'pkg.subpkg1.__init__', + 'pkg.subpkg2.__init__') + globals_ = ({'__package__': 'pkg.subpkg1'}, + {'__name__': 'pkg.subpkg1', '__path__': ['blah']}) + def callback(global_): + module = import_util.import_('', global_, fromlist=['subpkg2'], + level=2) + self.assertEqual(module.__name__, 'pkg') + self.assertTrue(hasattr(module, 'subpkg2')) + self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') + + def test_deep_import(self): + # [deep import] + create = ['pkg.__init__'] + for count in range(1,6): + create.append('{0}.pkg{1}.__init__'.format( + create[-1][:-len('.__init__')], count)) + globals_ = ({'__package__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5'}, + {'__name__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5', + '__path__': ['blah']}) + def callback(global_): + import_util.import_(globals_[0]['__package__']) + module = import_util.import_('', global_, fromlist=['attr'], level=6) + self.assertEqual(module.__name__, 'pkg') + self.relative_import_test(create, globals_, callback) + + def test_too_high_from_package(self): + # [too high from package] + create = ['top_level', 'pkg.__init__'] + globals_ = ({'__package__': 'pkg'}, + {'__name__': 'pkg', '__path__': ['blah']}) + def callback(global_): + import_util.import_('pkg') + with self.assertRaises(ValueError): + import_util.import_('', global_, fromlist=['top_level'], + level=2) + self.relative_import_test(create, globals_, callback) + + def test_too_high_from_module(self): + # [too high from module] + create = ['top_level', 'pkg.__init__', 'pkg.module'] + globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} + def callback(global_): + import_util.import_('pkg') + with self.assertRaises(ValueError): + import_util.import_('', global_, fromlist=['top_level'], + level=2) + self.relative_import_test(create, globals_, callback) + + def test_empty_name_w_level_0(self): + # [empty name] + with self.assertRaises(ValueError): + import_util.import_('') + + def test_import_from_different_package(self): + # Test importing from a different package than the caller. + # in pkg.subpkg1.mod + # from ..subpkg2 import mod + create = ['__runpy_pkg__.__init__', + '__runpy_pkg__.__runpy_pkg__.__init__', + '__runpy_pkg__.uncle.__init__', + '__runpy_pkg__.uncle.cousin.__init__', + '__runpy_pkg__.uncle.cousin.nephew'] + globals_ = {'__package__': '__runpy_pkg__.__runpy_pkg__'} + def callback(global_): + import_util.import_('__runpy_pkg__.__runpy_pkg__') + module = import_util.import_('uncle.cousin', globals_, {}, + fromlist=['nephew'], + level=2) + self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin') + self.relative_import_test(create, globals_, callback) + + + +def test_main(): + from test.support import run_unittest + run_unittest(RelativeImports) + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/import_/util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/import_/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,29 @@ +import functools +import importlib +import importlib._bootstrap +import unittest + + +using___import__ = False + + +def import_(*args, **kwargs): + """Delegate to allow for injecting different implementations of import.""" + if using___import__: + return __import__(*args, **kwargs) + else: + return importlib.__import__(*args, **kwargs) + + +def importlib_only(fxn): + """Decorator to skip a test if using __builtins__.__import__.""" + return unittest.skipIf(using___import__, "importlib-specific test")(fxn) + + +def mock_path_hook(*entries, importer): + """A mock sys.path_hooks entry.""" + def hook(entry): + if entry not in entries: + raise ImportError + return importer + return hook diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/regrtest.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/regrtest.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,28 @@ +"""Run Python's standard test suite using importlib.__import__. + +Tests known to fail because of assumptions that importlib (properly) +invalidates are automatically skipped if the entire test suite is run. +Otherwise all command-line options valid for test.regrtest are also valid for +this script. + +""" +import importlib +import sys +from test import regrtest + +if __name__ == '__main__': + __builtins__.__import__ = importlib.__import__ + + exclude = ['--exclude', + 'test_frozen', # Does not expect __loader__ attribute + 'test_pkg', # Does not expect __loader__ attribute + 'test_pydoc', # Does not expect __loader__ attribute + ] + + # Switching on --exclude implies running all test but the ones listed, so + # only use it when one is not running an explicit test + if len(sys.argv) == 1: + # No programmatic way to specify tests to exclude + sys.argv.extend(exclude) + + regrtest.main(quiet=True, verbose2=True) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,13 @@ +import importlib.test +import os.path +import unittest + + +def test_suite(): + directory = os.path.dirname(__file__) + return importlib.test.test_suite('importlib.test.source', directory) + + +if __name__ == '__main__': + from test.support import run_unittest + run_unittest(test_suite()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_abc_loader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_abc_loader.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,879 @@ +import importlib +from importlib import abc + +from .. import abc as testing_abc +from .. import util +from . import util as source_util + +import imp +import inspect +import io +import marshal +import os +import sys +import types +import unittest +import warnings + + +class SourceOnlyLoaderMock(abc.SourceLoader): + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " + b"repr(__loader__)])") + + def __init__(self, path): + self.path = path + + def get_data(self, path): + assert self.path == path + return self.source + + def get_filename(self, fullname): + return self.path + + +class SourceLoaderMock(SourceOnlyLoaderMock): + + source_mtime = 1 + + def __init__(self, path, magic=imp.get_magic()): + super().__init__(path) + self.bytecode_path = imp.cache_from_source(self.path) + self.source_size = len(self.source) + data = bytearray(magic) + data.extend(importlib._w_long(self.source_mtime)) + data.extend(importlib._w_long(self.source_size)) + code_object = compile(self.source, self.path, 'exec', + dont_inherit=True) + data.extend(marshal.dumps(code_object)) + self.bytecode = bytes(data) + self.written = {} + + def get_data(self, path): + if path == self.path: + return super().get_data(path) + elif path == self.bytecode_path: + return self.bytecode + else: + raise IOError + + def path_stats(self, path): + assert path == self.path + return {'mtime': self.source_mtime, 'size': self.source_size} + + def set_data(self, path, data): + self.written[path] = bytes(data) + return path == self.bytecode_path + + +class PyLoaderMock(abc.PyLoader): + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __package__, " + b"repr(__loader__)])") + + def __init__(self, data): + """Take a dict of 'module_name: path' pairings. + + Paths should have no file extension, allowing packages to be denoted by + ending in '__init__'. + + """ + self.module_paths = data + self.path_to_module = {val:key for key,val in data.items()} + + def get_data(self, path): + if path not in self.path_to_module: + raise IOError + return self.source + + def is_package(self, name): + filename = os.path.basename(self.get_filename(name)) + return os.path.splitext(filename)[0] == '__init__' + + def source_path(self, name): + try: + return self.module_paths[name] + except KeyError: + raise ImportError + + def get_filename(self, name): + """Silence deprecation warning.""" + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + path = super().get_filename(name) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + return path + + +class PyLoaderCompatMock(PyLoaderMock): + + """Mock that matches what is suggested to have a loader that is compatible + from Python 3.1 onwards.""" + + def get_filename(self, fullname): + try: + return self.module_paths[fullname] + except KeyError: + raise ImportError + + def source_path(self, fullname): + try: + return self.get_filename(fullname) + except ImportError: + return None + + +class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): + + default_mtime = 1 + + def __init__(self, source, bc={}): + """Initialize mock. + + 'bc' is a dict keyed on a module's name. The value is dict with + possible keys of 'path', 'mtime', 'magic', and 'bc'. Except for 'path', + each of those keys control if any part of created bytecode is to + deviate from default values. + + """ + super().__init__(source) + self.module_bytecode = {} + self.path_to_bytecode = {} + self.bytecode_to_path = {} + for name, data in bc.items(): + self.path_to_bytecode[data['path']] = name + self.bytecode_to_path[name] = data['path'] + magic = data.get('magic', imp.get_magic()) + mtime = importlib._w_long(data.get('mtime', self.default_mtime)) + if 'bc' in data: + bc = data['bc'] + else: + bc = self.compile_bc(name) + self.module_bytecode[name] = magic + mtime + bc + + def compile_bc(self, name): + source_path = self.module_paths.get(name, '') or '' + code = compile(self.source, source_path, 'exec') + return marshal.dumps(code) + + def source_mtime(self, name): + if name in self.module_paths: + return self.default_mtime + elif name in self.module_bytecode: + return None + else: + raise ImportError + + def bytecode_path(self, name): + try: + return self.bytecode_to_path[name] + except KeyError: + if name in self.module_paths: + return None + else: + raise ImportError + + def write_bytecode(self, name, bytecode): + self.module_bytecode[name] = bytecode + return True + + def get_data(self, path): + if path in self.path_to_module: + return super().get_data(path) + elif path in self.path_to_bytecode: + name = self.path_to_bytecode[path] + return self.module_bytecode[name] + else: + raise IOError + + def is_package(self, name): + try: + return super().is_package(name) + except TypeError: + return '__init__' in self.bytecode_to_path[name] + + def get_code(self, name): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + code_object = super().get_code(name) + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + return code_object + +class PyLoaderTests(testing_abc.LoaderTests): + + """Tests for importlib.abc.PyLoader.""" + + mocker = PyLoaderMock + + def eq_attrs(self, ob, **kwargs): + for attr, val in kwargs.items(): + found = getattr(ob, attr) + self.assertEqual(found, val, + "{} attribute: {} != {}".format(attr, found, val)) + + def test_module(self): + name = '' + path = os.path.join('', 'path', 'to', 'module') + mock = self.mocker({name: path}) + with util.uncache(name): + module = mock.load_module(name) + self.assertTrue(name in sys.modules) + self.eq_attrs(module, __name__=name, __file__=path, __package__='', + __loader__=mock) + self.assertTrue(not hasattr(module, '__path__')) + return mock, name + + def test_package(self): + name = '' + path = os.path.join('path', 'to', name, '__init__') + mock = self.mocker({name: path}) + with util.uncache(name): + module = mock.load_module(name) + self.assertTrue(name in sys.modules) + self.eq_attrs(module, __name__=name, __file__=path, + __path__=[os.path.dirname(path)], __package__=name, + __loader__=mock) + return mock, name + + def test_lacking_parent(self): + name = 'pkg.mod' + path = os.path.join('path', 'to', 'pkg', 'mod') + mock = self.mocker({name: path}) + with util.uncache(name): + module = mock.load_module(name) + self.assertIn(name, sys.modules) + self.eq_attrs(module, __name__=name, __file__=path, __package__='pkg', + __loader__=mock) + self.assertFalse(hasattr(module, '__path__')) + return mock, name + + def test_module_reuse(self): + name = 'mod' + path = os.path.join('path', 'to', 'mod') + module = imp.new_module(name) + mock = self.mocker({name: path}) + with util.uncache(name): + sys.modules[name] = module + loaded_module = mock.load_module(name) + self.assertTrue(loaded_module is module) + self.assertTrue(sys.modules[name] is module) + return mock, name + + def test_state_after_failure(self): + name = "mod" + module = imp.new_module(name) + module.blah = None + mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) + mock.source = b"1/0" + with util.uncache(name): + sys.modules[name] = module + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) + self.assertTrue(sys.modules[name] is module) + self.assertTrue(hasattr(module, 'blah')) + return mock + + def test_unloadable(self): + name = "mod" + mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) + mock.source = b"1/0" + with util.uncache(name): + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) + self.assertTrue(name not in sys.modules) + return mock + + +class PyLoaderCompatTests(PyLoaderTests): + + """Test that the suggested code to make a loader that is compatible from + Python 3.1 forward works.""" + + mocker = PyLoaderCompatMock + + +class PyLoaderInterfaceTests(unittest.TestCase): + + """Tests for importlib.abc.PyLoader to make sure that when source_path() + doesn't return a path everything works as expected.""" + + def test_no_source_path(self): + # No source path should lead to ImportError. + name = 'mod' + mock = PyLoaderMock({}) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_source_path_is_None(self): + name = 'mod' + mock = PyLoaderMock({name: None}) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_get_filename_with_source_path(self): + # get_filename() should return what source_path() returns. + name = 'mod' + path = os.path.join('path', 'to', 'source') + mock = PyLoaderMock({name: path}) + with util.uncache(name): + self.assertEqual(mock.get_filename(name), path) + + def test_get_filename_no_source_path(self): + # get_filename() should raise ImportError if source_path returns None. + name = 'mod' + mock = PyLoaderMock({name: None}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) + + +class PyPycLoaderTests(PyLoaderTests): + + """Tests for importlib.abc.PyPycLoader.""" + + mocker = PyPycLoaderMock + + @source_util.writes_bytecode_files + def verify_bytecode(self, mock, name): + assert name in mock.module_paths + self.assertIn(name, mock.module_bytecode) + magic = mock.module_bytecode[name][:4] + self.assertEqual(magic, imp.get_magic()) + mtime = importlib._r_long(mock.module_bytecode[name][4:8]) + self.assertEqual(mtime, 1) + bc = mock.module_bytecode[name][8:] + self.assertEqual(bc, mock.compile_bc(name)) + + def test_module(self): + mock, name = super().test_module() + self.verify_bytecode(mock, name) + + def test_package(self): + mock, name = super().test_package() + self.verify_bytecode(mock, name) + + def test_lacking_parent(self): + mock, name = super().test_lacking_parent() + self.verify_bytecode(mock, name) + + def test_module_reuse(self): + mock, name = super().test_module_reuse() + self.verify_bytecode(mock, name) + + def test_state_after_failure(self): + super().test_state_after_failure() + + def test_unloadable(self): + super().test_unloadable() + + +class PyPycLoaderInterfaceTests(unittest.TestCase): + + """Test for the interface of importlib.abc.PyPycLoader.""" + + def get_filename_check(self, src_path, bc_path, expect): + name = 'mod' + mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}}) + with util.uncache(name): + assert mock.source_path(name) == src_path + assert mock.bytecode_path(name) == bc_path + self.assertEqual(mock.get_filename(name), expect) + + def test_filename_with_source_bc(self): + # When source and bytecode paths present, return the source path. + self.get_filename_check('source_path', 'bc_path', 'source_path') + + def test_filename_with_source_no_bc(self): + # With source but no bc, return source path. + self.get_filename_check('source_path', None, 'source_path') + + def test_filename_with_no_source_bc(self): + # With not source but bc, return the bc path. + self.get_filename_check(None, 'bc_path', 'bc_path') + + def test_filename_with_no_source_or_bc(self): + # With no source or bc, raise ImportError. + name = 'mod' + mock = PyPycLoaderMock({name: None}, {name: {'path': None}}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) + + +class SkipWritingBytecodeTests(unittest.TestCase): + + """Test that bytecode is properly handled based on + sys.dont_write_bytecode.""" + + @source_util.writes_bytecode_files + def run_test(self, dont_write_bytecode): + name = 'mod' + mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) + sys.dont_write_bytecode = dont_write_bytecode + with util.uncache(name): + mock.load_module(name) + self.assertTrue((name in mock.module_bytecode) is not + dont_write_bytecode) + + def test_no_bytecode_written(self): + self.run_test(True) + + def test_bytecode_written(self): + self.run_test(False) + + +class RegeneratedBytecodeTests(unittest.TestCase): + + """Test that bytecode is regenerated as expected.""" + + @source_util.writes_bytecode_files + def test_different_magic(self): + # A different magic number should lead to new bytecode. + name = 'mod' + bad_magic = b'\x00\x00\x00\x00' + assert bad_magic != imp.get_magic() + mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}, + {name: {'path': os.path.join('path', 'to', + 'mod.bytecode'), + 'magic': bad_magic}}) + with util.uncache(name): + mock.load_module(name) + self.assertTrue(name in mock.module_bytecode) + magic = mock.module_bytecode[name][:4] + self.assertEqual(magic, imp.get_magic()) + + @source_util.writes_bytecode_files + def test_old_mtime(self): + # Bytecode with an older mtime should be regenerated. + name = 'mod' + old_mtime = PyPycLoaderMock.default_mtime - 1 + mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}, + {name: {'path': 'path/to/mod.bytecode', 'mtime': old_mtime}}) + with util.uncache(name): + mock.load_module(name) + self.assertTrue(name in mock.module_bytecode) + mtime = importlib._r_long(mock.module_bytecode[name][4:8]) + self.assertEqual(mtime, PyPycLoaderMock.default_mtime) + + +class BadBytecodeFailureTests(unittest.TestCase): + + """Test import failures when there is no source and parts of the bytecode + is bad.""" + + def test_bad_magic(self): + # A bad magic number should lead to an ImportError. + name = 'mod' + bad_magic = b'\x00\x00\x00\x00' + bc = {name: + {'path': os.path.join('path', 'to', 'mod'), + 'magic': bad_magic}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_no_bytecode(self): + # Missing code object bytecode should lead to an EOFError. + name = 'mod' + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(EOFError): + mock.load_module(name) + + def test_bad_bytecode(self): + # Malformed code object bytecode should lead to a ValueError. + name = 'mod' + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ValueError): + mock.load_module(name) + + +def raise_ImportError(*args, **kwargs): + raise ImportError + +class MissingPathsTests(unittest.TestCase): + + """Test what happens when a source or bytecode path does not exist (either + from *_path returning None or raising ImportError).""" + + def test_source_path_None(self): + # Bytecode should be used when source_path returns None, along with + # __file__ being set to the bytecode path. + name = 'mod' + bytecode_path = 'path/to/mod' + mock = PyPycLoaderMock({name: None}, {name: {'path': bytecode_path}}) + with util.uncache(name): + module = mock.load_module(name) + self.assertEqual(module.__file__, bytecode_path) + + # Testing for bytecode_path returning None handled by all tests where no + # bytecode initially exists. + + def test_all_paths_None(self): + # If all *_path methods return None, raise ImportError. + name = 'mod' + mock = PyPycLoaderMock({name: None}) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_source_path_ImportError(self): + # An ImportError from source_path should trigger an ImportError. + name = 'mod' + mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to', + 'mod')}}) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_bytecode_path_ImportError(self): + # An ImportError from bytecode_path should trigger an ImportError. + name = 'mod' + mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) + bad_meth = types.MethodType(raise_ImportError, mock) + mock.bytecode_path = bad_meth + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + +class SourceLoaderTestHarness(unittest.TestCase): + + def setUp(self, *, is_package=True, **kwargs): + self.package = 'pkg' + if is_package: + self.path = os.path.join(self.package, '__init__.py') + self.name = self.package + else: + module_name = 'mod' + self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) + self.name = '.'.join([self.package, module_name]) + self.cached = imp.cache_from_source(self.path) + self.loader = self.loader_mock(self.path, **kwargs) + + def verify_module(self, module): + self.assertEqual(module.__name__, self.name) + self.assertEqual(module.__file__, self.path) + self.assertEqual(module.__cached__, self.cached) + self.assertEqual(module.__package__, self.package) + self.assertEqual(module.__loader__, self.loader) + values = module._.split('::') + self.assertEqual(values[0], self.name) + self.assertEqual(values[1], self.path) + self.assertEqual(values[2], self.cached) + self.assertEqual(values[3], self.package) + self.assertEqual(values[4], repr(self.loader)) + + def verify_code(self, code_object): + module = imp.new_module(self.name) + module.__file__ = self.path + module.__cached__ = self.cached + module.__package__ = self.package + module.__loader__ = self.loader + module.__path__ = [] + exec(code_object, module.__dict__) + self.verify_module(module) + + +class SourceOnlyLoaderTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader for source-only loading. + + Reload testing is subsumed by the tests for + importlib.util.module_for_loader. + + """ + + loader_mock = SourceOnlyLoaderMock + + def test_get_source(self): + # Verify the source code is returned as a string. + # If an IOError is raised by get_data then raise ImportError. + expected_source = self.loader.source.decode('utf-8') + self.assertEqual(self.loader.get_source(self.name), expected_source) + def raise_IOError(path): + raise IOError + self.loader.get_data = raise_IOError + with self.assertRaises(ImportError): + self.loader.get_source(self.name) + + def test_is_package(self): + # Properly detect when loading a package. + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.setUp(is_package=False) + self.assertFalse(self.loader.is_package(self.name)) + + def test_get_code(self): + # Verify the code object is created. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_load_module(self): + # Loading a module should set __name__, __loader__, __package__, + # __path__ (for packages), __file__, and __cached__. + # The module should also be put into sys.modules. + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertTrue(self.name in sys.modules) + + def test_package_settings(self): + # __package__ needs to be set, while __path__ is set on if the module + # is a package. + # Testing the values for a package are covered by test_load_module. + self.setUp(is_package=False) + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertTrue(not hasattr(module, '__path__')) + + def test_get_source_encoding(self): + # Source is considered encoded in UTF-8 by default unless otherwise + # specified by an encoding line. + source = "_ = 'ü'" + self.loader.source = source.encode('utf-8') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + source = "# coding: latin-1\n_ = ü" + self.loader.source = source.encode('latin-1') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + + +@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +class SourceLoaderBytecodeTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader's use of bytecode. + + Source-only testing handled by SourceOnlyLoaderTests. + + """ + + loader_mock = SourceLoaderMock + + def verify_code(self, code_object, *, bytecode_written=False): + super().verify_code(code_object) + if bytecode_written: + self.assertIn(self.cached, self.loader.written) + data = bytearray(imp.get_magic()) + data.extend(importlib._w_long(self.loader.source_mtime)) + data.extend(importlib._w_long(self.loader.source_size)) + data.extend(marshal.dumps(code_object)) + self.assertEqual(self.loader.written[self.cached], bytes(data)) + + def test_code_with_everything(self): + # When everything should work. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_no_bytecode(self): + # If no bytecode exists then move on to the source. + self.loader.bytecode_path = "" + # Sanity check + with self.assertRaises(IOError): + bytecode_path = imp.cache_from_source(self.path) + self.loader.get_data(bytecode_path) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_code_bad_timestamp(self): + # Bytecode is only used when the timestamp matches the source EXACTLY. + for source_mtime in (0, 2): + assert source_mtime != self.loader.source_mtime + original = self.loader.source_mtime + self.loader.source_mtime = source_mtime + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + self.loader.source_mtime = original + + def test_code_bad_magic(self): + # Skip over bytecode with a bad magic number. + self.setUp(magic=b'0000') + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_dont_write_bytecode(self): + # Bytecode is not written if sys.dont_write_bytecode is true. + # Can assume it is false already thanks to the skipIf class decorator. + try: + sys.dont_write_bytecode = True + self.loader.bytecode_path = "" + code_object = self.loader.get_code(self.name) + self.assertNotIn(self.cached, self.loader.written) + finally: + sys.dont_write_bytecode = False + + def test_no_set_data(self): + # If set_data is not defined, one can still read bytecode. + self.setUp(magic=b'0000') + original_set_data = self.loader.__class__.set_data + try: + del self.loader.__class__.set_data + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + finally: + self.loader.__class__.set_data = original_set_data + + def test_set_data_raises_exceptions(self): + # Raising NotImplementedError or IOError is okay for set_data. + def raise_exception(exc): + def closure(*args, **kwargs): + raise exc + return closure + + self.setUp(magic=b'0000') + self.loader.set_data = raise_exception(NotImplementedError) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + +class SourceLoaderGetSourceTests(unittest.TestCase): + + """Tests for importlib.abc.SourceLoader.get_source().""" + + def test_default_encoding(self): + # Should have no problems with UTF-8 text. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = 'x = "ü"' + mock.source = source.encode('utf-8') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_decoded_source(self): + # Decoding should work. + name = 'mod' + mock = SourceOnlyLoaderMock("mod.file") + source = "# coding: Latin-1\nx='ü'" + assert source.encode('latin-1') != source.encode('utf-8') + mock.source = source.encode('latin-1') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_universal_newlines(self): + # PEP 302 says universal newlines should be used. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = "x = 42\r\ny = -13\r\n" + mock.source = source.encode('utf-8') + expect = io.IncrementalNewlineDecoder(None, True).decode(source) + self.assertEqual(mock.get_source(name), expect) + +class AbstractMethodImplTests(unittest.TestCase): + + """Test the concrete abstractmethod implementations.""" + + class Loader(abc.Loader): + def load_module(self, fullname): + super().load_module(fullname) + + class Finder(abc.Finder): + def find_module(self, _): + super().find_module(_) + + class ResourceLoader(Loader, abc.ResourceLoader): + def get_data(self, _): + super().get_data(_) + + class InspectLoader(Loader, abc.InspectLoader): + def is_package(self, _): + super().is_package(_) + + def get_code(self, _): + super().get_code(_) + + def get_source(self, _): + super().get_source(_) + + class ExecutionLoader(InspectLoader, abc.ExecutionLoader): + def get_filename(self, _): + super().get_filename(_) + + class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader): + pass + + class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader): + def source_path(self, _): + super().source_path(_) + + class PyPycLoader(PyLoader, abc.PyPycLoader): + def bytecode_path(self, _): + super().bytecode_path(_) + + def source_mtime(self, _): + super().source_mtime(_) + + def write_bytecode(self, _, _2): + super().write_bytecode(_, _2) + + def raises_NotImplementedError(self, ins, *args): + for method_name in args: + method = getattr(ins, method_name) + arg_count = len(inspect.getfullargspec(method)[0]) - 1 + args = [''] * arg_count + try: + method(*args) + except NotImplementedError: + pass + else: + msg = "{}.{} did not raise NotImplementedError" + self.fail(msg.format(ins.__class__.__name__, method_name)) + + def test_Loader(self): + self.raises_NotImplementedError(self.Loader(), 'load_module') + + # XXX misplaced; should be somewhere else + def test_Finder(self): + self.raises_NotImplementedError(self.Finder(), 'find_module') + + def test_ResourceLoader(self): + self.raises_NotImplementedError(self.ResourceLoader(), 'load_module', + 'get_data') + + def test_InspectLoader(self): + self.raises_NotImplementedError(self.InspectLoader(), 'load_module', + 'is_package', 'get_code', 'get_source') + + def test_ExecutionLoader(self): + self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module', + 'is_package', 'get_code', 'get_source', + 'get_filename') + + def test_SourceLoader(self): + ins = self.SourceLoader() + # Required abstractmethods. + self.raises_NotImplementedError(ins, 'get_filename', 'get_data') + # Optional abstractmethods. + self.raises_NotImplementedError(ins,'path_stats', 'set_data') + + def test_PyLoader(self): + self.raises_NotImplementedError(self.PyLoader(), 'source_path', + 'get_data', 'is_package') + + def test_PyPycLoader(self): + self.raises_NotImplementedError(self.PyPycLoader(), 'source_path', + 'source_mtime', 'bytecode_path', + 'write_bytecode') + + +def test_main(): + from test.support import run_unittest + run_unittest(PyLoaderTests, PyLoaderCompatTests, + PyLoaderInterfaceTests, + PyPycLoaderTests, PyPycLoaderInterfaceTests, + SkipWritingBytecodeTests, RegeneratedBytecodeTests, + BadBytecodeFailureTests, MissingPathsTests, + SourceOnlyLoaderTests, + SourceLoaderBytecodeTests, + SourceLoaderGetSourceTests, + AbstractMethodImplTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_case_sensitivity.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_case_sensitivity.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,66 @@ +"""Test case-sensitivity (PEP 235).""" +from importlib import _bootstrap +from .. import util +from . import util as source_util +import os +import sys +from test import support as test_support +import unittest + + +@util.case_insensitive_tests +class CaseSensitivityTest(unittest.TestCase): + + """PEP 235 dictates that on case-preserving, case-insensitive file systems + that imports are case-sensitive unless the PYTHONCASEOK environment + variable is set.""" + + name = 'MoDuLe' + assert name != name.lower() + + def find(self, path): + finder = _bootstrap._FileFinder(path, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) + return finder.find_module(self.name) + + def sensitivity_test(self): + """Look for a module with matching and non-matching sensitivity.""" + sensitive_pkg = 'sensitive.{0}'.format(self.name) + insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) + context = source_util.create_modules(insensitive_pkg, sensitive_pkg) + with context as mapping: + sensitive_path = os.path.join(mapping['.root'], 'sensitive') + insensitive_path = os.path.join(mapping['.root'], 'insensitive') + return self.find(sensitive_path), self.find(insensitive_path) + + def test_sensitive(self): + with test_support.EnvironmentVarGuard() as env: + env.unset('PYTHONCASEOK') + if b'PYTHONCASEOK' in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + sensitive, insensitive = self.sensitivity_test() + self.assertTrue(hasattr(sensitive, 'load_module')) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertIsNone(insensitive) + + def test_insensitive(self): + with test_support.EnvironmentVarGuard() as env: + env.set('PYTHONCASEOK', '1') + if b'PYTHONCASEOK' not in _bootstrap._os.environ: + self.skipTest('os.environ changes not reflected in ' + '_os.environ') + sensitive, insensitive = self.sensitivity_test() + self.assertTrue(hasattr(sensitive, 'load_module')) + self.assertIn(self.name, sensitive.get_filename(self.name)) + self.assertTrue(hasattr(insensitive, 'load_module')) + self.assertIn(self.name, insensitive.get_filename(self.name)) + + +def test_main(): + test_support.run_unittest(CaseSensitivityTest) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_file_loader.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_file_loader.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,439 @@ +import importlib +from importlib import _bootstrap +from .. import abc +from .. import util +from . import util as source_util + +import errno +import imp +import marshal +import os +import py_compile +import shutil +import stat +import sys +import unittest + +from test.support import make_legacy_pyc + + +class SimpleTest(unittest.TestCase): + + """Should have no issue importing a source module [basic]. And if there is + a syntax error, it should raise a SyntaxError [syntax error]. + + """ + + # [basic] + def test_module(self): + with source_util.create_modules('_temp') as mapping: + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + module = loader.load_module('_temp') + self.assertTrue('_temp' in sys.modules) + check = {'__name__': '_temp', '__file__': mapping['_temp'], + '__package__': ''} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + def test_package(self): + with source_util.create_modules('_pkg.__init__') as mapping: + loader = _bootstrap._SourceFileLoader('_pkg', + mapping['_pkg.__init__']) + module = loader.load_module('_pkg') + self.assertTrue('_pkg' in sys.modules) + check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], + '__path__': [os.path.dirname(mapping['_pkg.__init__'])], + '__package__': '_pkg'} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + + def test_lacking_parent(self): + with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: + loader = _bootstrap._SourceFileLoader('_pkg.mod', + mapping['_pkg.mod']) + module = loader.load_module('_pkg.mod') + self.assertTrue('_pkg.mod' in sys.modules) + check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], + '__package__': '_pkg'} + for attr, value in check.items(): + self.assertEqual(getattr(module, attr), value) + + def fake_mtime(self, fxn): + """Fake mtime to always be higher than expected.""" + return lambda name: fxn(name) + 1 + + def test_module_reuse(self): + with source_util.create_modules('_temp') as mapping: + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + module = loader.load_module('_temp') + module_id = id(module) + module_dict_id = id(module.__dict__) + with open(mapping['_temp'], 'w') as file: + file.write("testing_var = 42\n") + module = loader.load_module('_temp') + self.assertTrue('testing_var' in module.__dict__, + "'testing_var' not in " + "{0}".format(list(module.__dict__.keys()))) + self.assertEqual(module, sys.modules['_temp']) + self.assertEqual(id(module), module_id) + self.assertEqual(id(module.__dict__), module_dict_id) + + def test_state_after_failure(self): + # A failed reload should leave the original module intact. + attributes = ('__file__', '__path__', '__package__') + value = '' + name = '_temp' + with source_util.create_modules(name) as mapping: + orig_module = imp.new_module(name) + for attr in attributes: + setattr(orig_module, attr, value) + with open(mapping[name], 'w') as file: + file.write('+++ bad syntax +++') + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module(name) + for attr in attributes: + self.assertEqual(getattr(orig_module, attr), value) + + # [syntax error] + def test_bad_syntax(self): + with source_util.create_modules('_temp') as mapping: + with open(mapping['_temp'], 'w') as file: + file.write('=') + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module('_temp') + self.assertTrue('_temp' not in sys.modules) + + def test_file_from_empty_string_dir(self): + # Loading a module found from an empty string entry on sys.path should + # not only work, but keep all attributes relative. + file_path = '_temp.py' + with open(file_path, 'w') as file: + file.write("# test file for importlib") + try: + with util.uncache('_temp'): + loader = _bootstrap._SourceFileLoader('_temp', file_path) + mod = loader.load_module('_temp') + self.assertEqual(file_path, mod.__file__) + self.assertEqual(imp.cache_from_source(file_path), + mod.__cached__) + finally: + os.unlink(file_path) + pycache = os.path.dirname(imp.cache_from_source(file_path)) + shutil.rmtree(pycache) + + def test_timestamp_overflow(self): + # When a modification timestamp is larger than 2**32, it should be + # truncated rather than raise an OverflowError. + with source_util.create_modules('_temp') as mapping: + source = mapping['_temp'] + compiled = imp.cache_from_source(source) + with open(source, 'w') as f: + f.write("x = 5") + try: + os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) + except OverflowError: + self.skipTest("cannot set modification time to large integer") + except OSError as e: + if e.errno != getattr(errno, 'EOVERFLOW', None): + raise + self.skipTest("cannot set modification time to large integer ({})".format(e)) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + mod = loader.load_module('_temp') + # Sanity checks. + self.assertEqual(mod.__cached__, compiled) + self.assertEqual(mod.x, 5) + # The pyc file was created. + os.stat(compiled) + + +class BadBytecodeTest(unittest.TestCase): + + def import_(self, file, module_name): + loader = self.loader(module_name, file) + module = loader.load_module(module_name) + self.assertTrue(module_name in sys.modules) + + def manipulate_bytecode(self, name, mapping, manipulator, *, + del_source=False): + """Manipulate the bytecode of a module by passing it into a callable + that returns what to use as the new bytecode.""" + try: + del sys.modules['_temp'] + except KeyError: + pass + py_compile.compile(mapping[name]) + if not del_source: + bytecode_path = imp.cache_from_source(mapping[name]) + else: + os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) + if manipulator: + with open(bytecode_path, 'rb') as file: + bc = file.read() + new_bc = manipulator(bc) + with open(bytecode_path, 'wb') as file: + if new_bc is not None: + file.write(new_bc) + return bytecode_path + + def _test_empty_file(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'', + del_source=del_source) + test('_temp', mapping, bc_path) + + @source_util.writes_bytecode_files + def _test_partial_magic(self, test, *, del_source=False): + # When their are less than 4 bytes to a .pyc, regenerate it if + # possible, else raise ImportError. + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:3], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_magic_only(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:4], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_timestamp(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:7], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_size(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:11], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_no_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:12], + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bc_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_non_code_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:12] + marshal.dumps(b'abcd'), + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(ImportError): + self.import_(file_path, '_temp') + + def _test_bad_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:12] + b'', + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_bad_magic(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'\x00\x00\x00\x00' + bc[4:]) + test('_temp', mapping, bc_path) + + +class SourceLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourceFileLoader + + @source_util.writes_bytecode_files + def test_empty_file(self): + # When a .pyc is empty, regenerate it if possible, else raise + # ImportError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_empty_file(test) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_partial_magic(test) + + @source_util.writes_bytecode_files + def test_magic_only(self): + # When there is only the magic number, regenerate the .pyc if possible, + # else raise EOFError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_magic_only(test) + + @source_util.writes_bytecode_files + def test_bad_magic(self): + # When the magic number is different, the bytecode should be + # regenerated. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as bytecode_file: + self.assertEqual(bytecode_file.read(4), imp.get_magic()) + + self._test_bad_magic(test) + + @source_util.writes_bytecode_files + def test_partial_timestamp(self): + # When the timestamp is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_partial_timestamp(test) + + @source_util.writes_bytecode_files + def test_partial_size(self): + # When the size is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_partial_size(test) + + @source_util.writes_bytecode_files + def test_no_marshal(self): + # When there is only the magic number and timestamp, raise EOFError. + self._test_no_marshal() + + @source_util.writes_bytecode_files + def test_non_code_marshal(self): + self._test_non_code_marshal() + # XXX ImportError when sourceless + + # [bad marshal] + @source_util.writes_bytecode_files + def test_bad_marshal(self): + # Bad marshal data should raise a ValueError. + self._test_bad_marshal() + + # [bad timestamp] + @source_util.writes_bytecode_files + def test_old_timestamp(self): + # When the timestamp is older than the source, bytecode should be + # regenerated. + zeros = b'\x00\x00\x00\x00' + with source_util.create_modules('_temp') as mapping: + py_compile.compile(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) + with open(bytecode_path, 'r+b') as bytecode_file: + bytecode_file.seek(4) + bytecode_file.write(zeros) + self.import_(mapping['_temp'], '_temp') + source_mtime = os.path.getmtime(mapping['_temp']) + source_timestamp = importlib._w_long(source_mtime) + with open(bytecode_path, 'rb') as bytecode_file: + bytecode_file.seek(4) + self.assertEqual(bytecode_file.read(4), source_timestamp) + + # [bytecode read-only] + @source_util.writes_bytecode_files + def test_read_only_bytecode(self): + # When bytecode is read-only but should be rewritten, fail silently. + with source_util.create_modules('_temp') as mapping: + # Create bytecode that will need to be re-created. + py_compile.compile(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) + with open(bytecode_path, 'r+b') as bytecode_file: + bytecode_file.seek(0) + bytecode_file.write(b'\x00\x00\x00\x00') + # Make the bytecode read-only. + os.chmod(bytecode_path, + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + try: + # Should not raise IOError! + self.import_(mapping['_temp'], '_temp') + finally: + # Make writable for eventual clean-up. + os.chmod(bytecode_path, stat.S_IWUSR) + + +class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourcelessFileLoader + + def test_empty_file(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_empty_file(test, del_source=True) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + self._test_partial_magic(test, del_source=True) + + def test_magic_only(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_magic_only(test, del_source=True) + + def test_bad_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_bad_magic(test, del_source=True) + + def test_partial_timestamp(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_timestamp(test, del_source=True) + + def test_partial_size(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_size(test, del_source=True) + + def test_no_marshal(self): + self._test_no_marshal(del_source=True) + + def test_non_code_marshal(self): + self._test_non_code_marshal(del_source=True) + + +def test_main(): + from test.support import run_unittest + run_unittest(SimpleTest, + SourceLoaderBadBytecodeTest, + SourcelessLoaderBadBytecodeTest + ) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_finder.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_finder.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,160 @@ +from importlib import _bootstrap +from .. import abc +from . import util as source_util +from test.support import make_legacy_pyc +import os +import errno +import py_compile +import unittest +import warnings + + +class FinderTests(abc.FinderTests): + + """For a top-level module, it should just be found directly in the + directory being searched. This is true for a directory with source + [top-level source], bytecode [top-level bc], or both [top-level both]. + There is also the possibility that it is a package [top-level package], in + which case there will be a directory with the module name and an + __init__.py file. If there is a directory without an __init__.py an + ImportWarning is returned [empty dir]. + + For sub-modules and sub-packages, the same happens as above but only use + the tail end of the name [sub module] [sub package] [sub empty]. + + When there is a conflict between a package and module having the same name + in the same directory, the package wins out [package over module]. This is + so that imports of modules within the package can occur rather than trigger + an import error. + + When there is a package and module with the same name, always pick the + package over the module [package over module]. This is so that imports from + the package have the possibility of succeeding. + + """ + + def import_(self, root, module): + finder = _bootstrap._FileFinder(root, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) + return finder.find_module(module) + + def run_test(self, test, create=None, *, compile_=None, unlink=None): + """Test the finding of 'test' with the creation of modules listed in + 'create'. + + Any names listed in 'compile_' are byte-compiled. Modules + listed in 'unlink' have their source files deleted. + + """ + if create is None: + create = {test} + with source_util.create_modules(*create) as mapping: + if compile_: + for name in compile_: + py_compile.compile(mapping[name]) + if unlink: + for name in unlink: + os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise + loader = self.import_(mapping['.root'], test) + self.assertTrue(hasattr(loader, 'load_module')) + return loader + + def test_module(self): + # [top-level source] + self.run_test('top_level') + # [top-level bc] + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) + # [top-level both] + self.run_test('top_level', compile_={'top_level'}) + + # [top-level package] + def test_package(self): + # Source. + self.run_test('pkg', {'pkg.__init__'}) + # Bytecode. + self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'}, + unlink={'pkg.__init__'}) + # Both. + self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'}) + + # [sub module] + def test_module_in_package(self): + with source_util.create_modules('pkg.__init__', 'pkg.sub') as mapping: + pkg_dir = os.path.dirname(mapping['pkg.__init__']) + loader = self.import_(pkg_dir, 'pkg.sub') + self.assertTrue(hasattr(loader, 'load_module')) + + # [sub package] + def test_package_in_package(self): + context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__') + with context as mapping: + pkg_dir = os.path.dirname(mapping['pkg.__init__']) + loader = self.import_(pkg_dir, 'pkg.sub') + self.assertTrue(hasattr(loader, 'load_module')) + + # [sub empty] + def test_empty_sub_directory(self): + context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__') + with warnings.catch_warnings(): + warnings.simplefilter("error", ImportWarning) + with context as mapping: + os.unlink(mapping['pkg.sub.__init__']) + pkg_dir = os.path.dirname(mapping['pkg.__init__']) + with self.assertRaises(ImportWarning): + self.import_(pkg_dir, 'pkg.sub') + + # [package over modules] + def test_package_over_module(self): + name = '_temp' + loader = self.run_test(name, {'{0}.__init__'.format(name), name}) + self.assertTrue('__init__' in loader.get_filename(name)) + + + def test_failure(self): + with source_util.create_modules('blah') as mapping: + nothing = self.import_(mapping['.root'], 'sdfsadsadf') + self.assertTrue(nothing is None) + + # [empty dir] + def test_empty_dir(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", ImportWarning) + with self.assertRaises(ImportWarning): + self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'}) + + def test_empty_string_for_dir(self): + # The empty string from sys.path means to search in the cwd. + finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails()) + with open('mod.py', 'w') as file: + file.write("# test file for importlib") + try: + loader = finder.find_module('mod') + self.assertTrue(hasattr(loader, 'load_module')) + finally: + os.unlink('mod.py') + + def test_invalidate_caches(self): + # invalidate_caches() should reset the mtime. + finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails()) + finder._path_mtime = 42 + finder.invalidate_caches() + self.assertEqual(finder._path_mtime, -1) + + +def test_main(): + from test.support import run_unittest + run_unittest(FinderTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_path_hook.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_path_hook.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,26 @@ +from importlib import _bootstrap +from . import util as source_util +import unittest + + +class PathHookTest(unittest.TestCase): + + """Test the path hook for source.""" + + def test_success(self): + with source_util.create_modules('dummy') as mapping: + self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']), + 'find_module')) + + def test_empty_string(self): + # The empty string represents the cwd. + self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module')) + + +def test_main(): + from test.support import run_unittest + run_unittest(PathHookTest) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/test_source_encoding.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/test_source_encoding.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,123 @@ +from importlib import _bootstrap +from . import util as source_util + +import codecs +import re +import sys +# Because sys.path gets essentially blanked, need to have unicodedata already +# imported for the parser to use. +import unicodedata +import unittest + + +CODING_RE = re.compile(r'coding[:=]\s*([-\w.]+)') + + +class EncodingTest(unittest.TestCase): + + """PEP 3120 makes UTF-8 the default encoding for source code + [default encoding]. + + PEP 263 specifies how that can change on a per-file basis. Either the first + or second line can contain the encoding line [encoding first line] + encoding second line]. If the file has the BOM marker it is considered UTF-8 + implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is + an error [BOM and utf-8][BOM conflict]. + + """ + + variable = '\u00fc' + character = '\u00c9' + source_line = "{0} = '{1}'\n".format(variable, character) + module_name = '_temp' + + def run_test(self, source): + with source_util.create_modules(self.module_name) as mapping: + with open(mapping[self.module_name], 'wb') as file: + file.write(source) + loader = _bootstrap._SourceFileLoader(self.module_name, + mapping[self.module_name]) + return loader.load_module(self.module_name) + + def create_source(self, encoding): + encoding_line = "# coding={0}".format(encoding) + assert CODING_RE.search(encoding_line) + source_lines = [encoding_line.encode('utf-8')] + source_lines.append(self.source_line.encode(encoding)) + return b'\n'.join(source_lines) + + def test_non_obvious_encoding(self): + # Make sure that an encoding that has never been a standard one for + # Python works. + encoding_line = "# coding=koi8-r" + assert CODING_RE.search(encoding_line) + source = "{0}\na=42\n".format(encoding_line).encode("koi8-r") + self.run_test(source) + + # [default encoding] + def test_default_encoding(self): + self.run_test(self.source_line.encode('utf-8')) + + # [encoding first line] + def test_encoding_on_first_line(self): + encoding = 'Latin-1' + source = self.create_source(encoding) + self.run_test(source) + + # [encoding second line] + def test_encoding_on_second_line(self): + source = b"#/usr/bin/python\n" + self.create_source('Latin-1') + self.run_test(source) + + # [BOM] + def test_bom(self): + self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8')) + + # [BOM and utf-8] + def test_bom_and_utf_8(self): + source = codecs.BOM_UTF8 + self.create_source('utf-8') + self.run_test(source) + + # [BOM conflict] + def test_bom_conflict(self): + source = codecs.BOM_UTF8 + self.create_source('latin-1') + with self.assertRaises(SyntaxError): + self.run_test(source) + + +class LineEndingTest(unittest.TestCase): + + r"""Source written with the three types of line endings (\n, \r\n, \r) + need to be readable [cr][crlf][lf].""" + + def run_test(self, line_ending): + module_name = '_temp' + source_lines = [b"a = 42", b"b = -13", b''] + source = line_ending.join(source_lines) + with source_util.create_modules(module_name) as mapping: + with open(mapping[module_name], 'wb') as file: + file.write(source) + loader = _bootstrap._SourceFileLoader(module_name, + mapping[module_name]) + return loader.load_module(module_name) + + # [cr] + def test_cr(self): + self.run_test(b'\r') + + # [crlf] + def test_crlf(self): + self.run_test(b'\r\n') + + # [lf] + def test_lf(self): + self.run_test(b'\n') + + +def test_main(): + from test.support import run_unittest + run_unittest(EncodingTest, LineEndingTest) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/source/util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/source/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,97 @@ +from .. import util +import contextlib +import errno +import functools +import imp +import os +import os.path +import sys +import tempfile +from test import support + + +def writes_bytecode_files(fxn): + """Decorator to protect sys.dont_write_bytecode from mutation and to skip + tests that require it to be set to False.""" + if sys.dont_write_bytecode: + return lambda *args, **kwargs: None + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + original = sys.dont_write_bytecode + sys.dont_write_bytecode = False + try: + to_return = fxn(*args, **kwargs) + finally: + sys.dont_write_bytecode = original + return to_return + return wrapper + + +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + +@contextlib.contextmanager +def create_modules(*names): + """Temporarily create each named module with an attribute (named 'attr') + that contains the name passed into the context manager that caused the + creation of the module. + + All files are created in a temporary directory returned by + tempfile.mkdtemp(). This directory is inserted at the beginning of + sys.path. When the context manager exits all created files (source and + bytecode) are explicitly deleted. + + No magic is performed when creating packages! This means that if you create + a module within a package you must also create the package's __init__ as + well. + + """ + source = 'attr = {0!r}' + created_paths = [] + mapping = {} + state_manager = None + uncache_manager = None + try: + temp_dir = tempfile.mkdtemp() + mapping['.root'] = temp_dir + import_names = set() + for name in names: + if not name.endswith('__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + import_names.add(import_name) + if import_name in sys.modules: + del sys.modules[import_name] + name_parts = name.split('.') + file_path = temp_dir + for directory in name_parts[:-1]: + file_path = os.path.join(file_path, directory) + if not os.path.exists(file_path): + os.mkdir(file_path) + created_paths.append(file_path) + file_path = os.path.join(file_path, name_parts[-1] + '.py') + with open(file_path, 'w') as file: + file.write(source.format(name)) + created_paths.append(file_path) + mapping[name] = file_path + uncache_manager = util.uncache(*import_names) + uncache_manager.__enter__() + state_manager = util.import_state(path=[temp_dir]) + state_manager.__enter__() + yield mapping + finally: + if state_manager is not None: + state_manager.__exit__(None, None, None) + if uncache_manager is not None: + uncache_manager.__exit__(None, None, None) + support.rmtree(temp_dir) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/test_abc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/test_abc.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,89 @@ +from importlib import abc +from importlib import machinery +import inspect +import unittest + + +class InheritanceTests: + + """Test that the specified class is a subclass/superclass of the expected + classes.""" + + subclasses = [] + superclasses = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.subclasses or self.superclasses, self.__class__ + self.__test = getattr(abc, self.__class__.__name__) + + def test_subclasses(self): + # Test that the expected subclasses inherit. + for subclass in self.subclasses: + self.assertTrue(issubclass(subclass, self.__test), + "{0} is not a subclass of {1}".format(subclass, self.__test)) + + def test_superclasses(self): + # Test that the class inherits from the expected superclasses. + for superclass in self.superclasses: + self.assertTrue(issubclass(self.__test, superclass), + "{0} is not a superclass of {1}".format(superclass, self.__test)) + + +class Finder(InheritanceTests, unittest.TestCase): + + subclasses = [machinery.BuiltinImporter, machinery.FrozenImporter, + machinery.PathFinder] + + +class Loader(InheritanceTests, unittest.TestCase): + + subclasses = [abc.PyLoader] + + +class ResourceLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.Loader] + + +class InspectLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.Loader] + subclasses = [abc.PyLoader, machinery.BuiltinImporter, + machinery.FrozenImporter] + + +class ExecutionLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.InspectLoader] + subclasses = [abc.PyLoader] + + +class SourceLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.ResourceLoader, abc.ExecutionLoader] + + +class PyLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.Loader, abc.ResourceLoader, abc.ExecutionLoader] + + +class PyPycLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.PyLoader] + + +def test_main(): + from test.support import run_unittest + classes = [] + for class_ in globals().values(): + if (inspect.isclass(class_) and + issubclass(class_, unittest.TestCase) and + issubclass(class_, InheritanceTests)): + classes.append(class_) + run_unittest(*classes) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/test_api.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/test_api.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,121 @@ +from . import util +import imp +import importlib +import sys +import unittest + + +class ImportModuleTests(unittest.TestCase): + + """Test importlib.import_module.""" + + def test_module_import(self): + # Test importing a top-level module. + with util.mock_modules('top_level') as mock: + with util.import_state(meta_path=[mock]): + module = importlib.import_module('top_level') + self.assertEqual(module.__name__, 'top_level') + + def test_absolute_package_import(self): + # Test importing a module from a package with an absolute name. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + name = '{0}.mod'.format(pkg_name) + with util.mock_modules(pkg_long_name, name) as mock: + with util.import_state(meta_path=[mock]): + module = importlib.import_module(name) + self.assertEqual(module.__name__, name) + + def test_shallow_relative_package_import(self): + # Test importing a module from a package through a relative import. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + module_name = 'mod' + absolute_name = '{0}.{1}'.format(pkg_name, module_name) + relative_name = '.{0}'.format(module_name) + with util.mock_modules(pkg_long_name, absolute_name) as mock: + with util.import_state(meta_path=[mock]): + importlib.import_module(pkg_name) + module = importlib.import_module(relative_name, pkg_name) + self.assertEqual(module.__name__, absolute_name) + + def test_deep_relative_package_import(self): + modules = ['a.__init__', 'a.b.__init__', 'a.c'] + with util.mock_modules(*modules) as mock: + with util.import_state(meta_path=[mock]): + importlib.import_module('a') + importlib.import_module('a.b') + module = importlib.import_module('..c', 'a.b') + self.assertEqual(module.__name__, 'a.c') + + def test_absolute_import_with_package(self): + # Test importing a module from a package with an absolute name with + # the 'package' argument given. + pkg_name = 'pkg' + pkg_long_name = '{0}.__init__'.format(pkg_name) + name = '{0}.mod'.format(pkg_name) + with util.mock_modules(pkg_long_name, name) as mock: + with util.import_state(meta_path=[mock]): + importlib.import_module(pkg_name) + module = importlib.import_module(name, pkg_name) + self.assertEqual(module.__name__, name) + + def test_relative_import_wo_package(self): + # Relative imports cannot happen without the 'package' argument being + # set. + with self.assertRaises(TypeError): + importlib.import_module('.support') + + + def test_loaded_once(self): + # Issue #13591: Modules should only be loaded once when + # initializing the parent package attempts to import the + # module currently being imported. + b_load_count = 0 + def load_a(): + importlib.import_module('a.b') + def load_b(): + nonlocal b_load_count + b_load_count += 1 + code = {'a': load_a, 'a.b': load_b} + modules = ['a.__init__', 'a.b'] + with util.mock_modules(*modules, module_code=code) as mock: + with util.import_state(meta_path=[mock]): + importlib.import_module('a.b') + self.assertEqual(b_load_count, 1) + + +class InvalidateCacheTests(unittest.TestCase): + + def test_method_called(self): + # If defined the method should be called. + class InvalidatingNullFinder: + def __init__(self, *ignored): + self.called = False + def find_module(self, *args): + return None + def invalidate_caches(self): + self.called = True + + key = 'gobledeegook' + ins = InvalidatingNullFinder() + sys.path_importer_cache[key] = ins + self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) + importlib.invalidate_caches() + self.assertTrue(ins.called) + + def test_method_lacking(self): + # There should be no issues if the method is not defined. + key = 'gobbledeegook' + sys.path_importer_cache[key] = imp.NullImporter('abc') + self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key)) + importlib.invalidate_caches() # Shouldn't trigger an exception. + + +def test_main(): + from test.support import run_unittest + run_unittest(ImportModuleTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/test_util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/test_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,128 @@ +from importlib import util +from . import util as test_util +import imp +import sys +import types +import unittest + + +class ModuleForLoaderTests(unittest.TestCase): + + """Tests for importlib.util.module_for_loader.""" + + def return_module(self, name): + fxn = util.module_for_loader(lambda self, module: module) + return fxn(self, name) + + def raise_exception(self, name): + def to_wrap(self, module): + raise ImportError + fxn = util.module_for_loader(to_wrap) + try: + fxn(self, name) + except ImportError: + pass + + def test_new_module(self): + # Test that when no module exists in sys.modules a new module is + # created. + module_name = 'a.b.c' + with test_util.uncache(module_name): + module = self.return_module(module_name) + self.assertTrue(module_name in sys.modules) + self.assertTrue(isinstance(module, types.ModuleType)) + self.assertEqual(module.__name__, module_name) + + def test_reload(self): + # Test that a module is reused if already in sys.modules. + name = 'a.b.c' + module = imp.new_module('a.b.c') + with test_util.uncache(name): + sys.modules[name] = module + returned_module = self.return_module(name) + self.assertIs(returned_module, sys.modules[name]) + + def test_new_module_failure(self): + # Test that a module is removed from sys.modules if added but an + # exception is raised. + name = 'a.b.c' + with test_util.uncache(name): + self.raise_exception(name) + self.assertTrue(name not in sys.modules) + + def test_reload_failure(self): + # Test that a failure on reload leaves the module in-place. + name = 'a.b.c' + module = imp.new_module(name) + with test_util.uncache(name): + sys.modules[name] = module + self.raise_exception(name) + self.assertIs(module, sys.modules[name]) + + def test_decorator_attrs(self): + def fxn(self, module): pass + wrapped = util.module_for_loader(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + +class SetPackageTests(unittest.TestCase): + + + """Tests for importlib.util.set_package.""" + + def verify(self, module, expect): + """Verify the module has the expected value for __package__ after + passing through set_package.""" + fxn = lambda: module + wrapped = util.set_package(fxn) + wrapped() + self.assertTrue(hasattr(module, '__package__')) + self.assertEqual(expect, module.__package__) + + def test_top_level(self): + # __package__ should be set to the empty string if a top-level module. + # Implicitly tests when package is set to None. + module = imp.new_module('module') + module.__package__ = None + self.verify(module, '') + + def test_package(self): + # Test setting __package__ for a package. + module = imp.new_module('pkg') + module.__path__ = [''] + module.__package__ = None + self.verify(module, 'pkg') + + def test_submodule(self): + # Test __package__ for a module in a package. + module = imp.new_module('pkg.mod') + module.__package__ = None + self.verify(module, 'pkg') + + def test_setting_if_missing(self): + # __package__ should be set if it is missing. + module = imp.new_module('mod') + if hasattr(module, '__package__'): + delattr(module, '__package__') + self.verify(module, '') + + def test_leaving_alone(self): + # If __package__ is set and not None then leave it alone. + for value in (True, False): + module = imp.new_module('mod') + module.__package__ = value + self.verify(module, value) + + def test_decorator_attrs(self): + def fxn(module): pass + wrapped = util.set_package(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) + +def test_main(): + from test import support + support.run_unittest(ModuleForLoaderTests, SetPackageTests) + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/test/util.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/importlib/test/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,136 @@ +from contextlib import contextmanager +import imp +import os.path +from test import support +import unittest +import sys + + +CASE_INSENSITIVE_FS = True +# Windows is the only OS that is *always* case-insensitive +# (OS X *can* be case-sensitive). +if sys.platform not in ('win32', 'cygwin'): + changed_name = __file__.upper() + if changed_name == __file__: + changed_name = __file__.lower() + if not os.path.exists(changed_name): + CASE_INSENSITIVE_FS = False + + +def case_insensitive_tests(test): + """Class decorator that nullifies tests requiring a case-insensitive + file system.""" + return unittest.skipIf(not CASE_INSENSITIVE_FS, + "requires a case-insensitive filesystem")(test) + + +@contextmanager +def uncache(*names): + """Uncache a module from sys.modules. + + A basic sanity check is performed to prevent uncaching modules that either + cannot/shouldn't be uncached. + + """ + for name in names: + if name in ('sys', 'marshal', 'imp'): + raise ValueError( + "cannot uncache {0} as it will break _importlib".format(name)) + try: + del sys.modules[name] + except KeyError: + pass + try: + yield + finally: + for name in names: + try: + del sys.modules[name] + except KeyError: + pass + +@contextmanager +def import_state(**kwargs): + """Context manager to manage the various importers and stored state in the + sys module. + + The 'modules' attribute is not supported as the interpreter state stores a + pointer to the dict that the interpreter uses internally; + reassigning to sys.modules does not have the desired effect. + + """ + originals = {} + try: + for attr, default in (('meta_path', []), ('path', []), + ('path_hooks', []), + ('path_importer_cache', {})): + originals[attr] = getattr(sys, attr) + if attr in kwargs: + new_value = kwargs[attr] + del kwargs[attr] + else: + new_value = default + setattr(sys, attr, new_value) + if len(kwargs): + raise ValueError( + 'unrecognized arguments: {0}'.format(kwargs.keys())) + yield + finally: + for attr, value in originals.items(): + setattr(sys, attr, value) + + +class mock_modules: + + """A mock importer/loader.""" + + def __init__(self, *names, module_code={}): + self.modules = {} + self.module_code = {} + for name in names: + if not name.endswith('.__init__'): + import_name = name + else: + import_name = name[:-len('.__init__')] + if '.' not in name: + package = None + elif import_name == name: + package = name.rsplit('.', 1)[0] + else: + package = import_name + module = imp.new_module(import_name) + module.__loader__ = self + module.__file__ = '' + module.__package__ = package + module.attr = name + if import_name != name: + module.__path__ = [''] + self.modules[import_name] = module + if import_name in module_code: + self.module_code[import_name] = module_code[import_name] + + def __getitem__(self, name): + return self.modules[name] + + def find_module(self, fullname, path=None): + if fullname not in self.modules: + return None + else: + return self + + def load_module(self, fullname): + if fullname not in self.modules: + raise ImportError + else: + sys.modules[fullname] = self.modules[fullname] + if fullname in self.module_code: + self.module_code[fullname]() + return self.modules[fullname] + + def __enter__(self): + self._uncache = uncache(*self.modules.keys()) + self._uncache.__enter__() + return self + + def __exit__(self, *exc_info): + self._uncache.__exit__(None, None, None) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/importlib/util.py --- a/Lib/importlib/util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/importlib/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,295 +1,5 @@ """Utility code for constructing importers, etc.""" -from . import abc -from ._bootstrap import module_from_spec -from ._bootstrap import _resolve_name -from ._bootstrap import spec_from_loader -from ._bootstrap import _find_spec -from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import cache_from_source -from ._bootstrap_external import decode_source -from ._bootstrap_external import source_from_cache -from ._bootstrap_external import spec_from_file_location -from contextlib import contextmanager -import functools -import sys -import types -import warnings - - -def resolve_name(name, package): - """Resolve a relative module name to an absolute one.""" - if not name.startswith('.'): - return name - elif not package: - raise ValueError(f'no package specified for {repr(name)} ' - '(required for relative module names)') - level = 0 - for character in name: - if character != '.': - break - level += 1 - return _resolve_name(name[level:], package, level) - - -def _find_spec_from_path(name, path=None): - """Return the spec for the specified module. - - First, sys.modules is checked to see if the module was already imported. If - so, then sys.modules[name].__spec__ is returned. If that happens to be - set to None, then ValueError is raised. If the module is not in - sys.modules, then sys.meta_path is searched for a suitable spec with the - value of 'path' given to the finders. None is returned if no spec could - be found. - - Dotted names do not have their parent packages implicitly imported. You will - most likely need to explicitly import all parent packages in the proper - order for a submodule to get the correct spec. - - """ - if name not in sys.modules: - return _find_spec(name, path) - else: - module = sys.modules[name] - if module is None: - return None - try: - spec = module.__spec__ - except AttributeError: - raise ValueError('{}.__spec__ is not set'.format(name)) from None - else: - if spec is None: - raise ValueError('{}.__spec__ is None'.format(name)) - return spec - - -def find_spec(name, package=None): - """Return the spec for the specified module. - - First, sys.modules is checked to see if the module was already imported. If - so, then sys.modules[name].__spec__ is returned. If that happens to be - set to None, then ValueError is raised. If the module is not in - sys.modules, then sys.meta_path is searched for a suitable spec with the - value of 'path' given to the finders. None is returned if no spec could - be found. - - If the name is for submodule (contains a dot), the parent module is - automatically imported. - - The name and package arguments work the same as importlib.import_module(). - In other words, relative module names (with leading dots) work. - - """ - fullname = resolve_name(name, package) if name.startswith('.') else name - if fullname not in sys.modules: - parent_name = fullname.rpartition('.')[0] - if parent_name: - # Use builtins.__import__() in case someone replaced it. - parent = __import__(parent_name, fromlist=['__path__']) - return _find_spec(fullname, parent.__path__) - else: - return _find_spec(fullname, None) - else: - module = sys.modules[fullname] - if module is None: - return None - try: - spec = module.__spec__ - except AttributeError: - raise ValueError('{}.__spec__ is not set'.format(name)) from None - else: - if spec is None: - raise ValueError('{}.__spec__ is None'.format(name)) - return spec - - -@contextmanager -def _module_to_load(name): - is_reload = name in sys.modules - - module = sys.modules.get(name) - if not is_reload: - # This must be done before open() is called as the 'io' module - # implicitly imports 'locale' and would otherwise trigger an - # infinite loop. - module = type(sys)(name) - # This must be done before putting the module in sys.modules - # (otherwise an optimization shortcut in import.c becomes wrong) - module.__initializing__ = True - sys.modules[name] = module - try: - yield module - except Exception: - if not is_reload: - try: - del sys.modules[name] - except KeyError: - pass - finally: - module.__initializing__ = False - - -def set_package(fxn): - """Set __package__ on the returned module. - - This function is deprecated. - - """ - @functools.wraps(fxn) - def set_package_wrapper(*args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', - DeprecationWarning, stacklevel=2) - module = fxn(*args, **kwargs) - if getattr(module, '__package__', None) is None: - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = module.__package__.rpartition('.')[0] - return module - return set_package_wrapper - - -def set_loader(fxn): - """Set __loader__ on the returned module. - - This function is deprecated. - - """ - @functools.wraps(fxn) - def set_loader_wrapper(self, *args, **kwargs): - warnings.warn('The import system now takes care of this automatically.', - DeprecationWarning, stacklevel=2) - module = fxn(self, *args, **kwargs) - if getattr(module, '__loader__', None) is None: - module.__loader__ = self - return module - return set_loader_wrapper - - -def module_for_loader(fxn): - """Decorator to handle selecting the proper module for loaders. - - The decorated function is passed the module to use instead of the module - name. The module passed in to the function is either from sys.modules if - it already exists or is a new module. If the module is new, then __name__ - is set the first argument to the method, __loader__ is set to self, and - __package__ is set accordingly (if self.is_package() is defined) will be set - before it is passed to the decorated function (if self.is_package() does - not work for the module it will be set post-load). - - If an exception is raised and the decorator created the module it is - subsequently removed from sys.modules. - - The decorator assumes that the decorated function takes the module name as - the second argument. - - """ - warnings.warn('The import system now takes care of this automatically.', - DeprecationWarning, stacklevel=2) - @functools.wraps(fxn) - def module_for_loader_wrapper(self, fullname, *args, **kwargs): - with _module_to_load(fullname) as module: - module.__loader__ = self - try: - is_package = self.is_package(fullname) - except (ImportError, AttributeError): - pass - else: - if is_package: - module.__package__ = fullname - else: - module.__package__ = fullname.rpartition('.')[0] - # If __package__ was not set above, __import__() will do it later. - return fxn(self, module, *args, **kwargs) - - return module_for_loader_wrapper - - -class _Module(types.ModuleType): - - """A subclass of the module type to allow __class__ manipulation.""" - - -class _LazyModule(types.ModuleType): - - """A subclass of the module type which triggers loading upon attribute access.""" - - def __getattribute__(self, attr): - """Trigger the load of the module and return the attribute.""" - # All module metadata must be garnered from __spec__ in order to avoid - # using mutated values. - # Stop triggering this method. - self.__class__ = _Module - # Get the original name to make sure no object substitution occurred - # in sys.modules. - original_name = self.__spec__.name - # Figure out exactly what attributes were mutated between the creation - # of the module and now. - attrs_then = self.__spec__.loader_state - attrs_now = self.__dict__ - attrs_updated = {} - for key, value in attrs_now.items(): - # Code that set the attribute may have kept a reference to the - # assigned object, making identity more important than equality. - if key not in attrs_then: - attrs_updated[key] = value - elif id(attrs_now[key]) != id(attrs_then[key]): - attrs_updated[key] = value - self.__spec__.loader.exec_module(self) - # If exec_module() was used directly there is no guarantee the module - # object was put into sys.modules. - if original_name in sys.modules: - if id(self) != id(sys.modules[original_name]): - msg = ('module object for {!r} substituted in sys.modules ' - 'during a lazy load') - raise ValueError(msg.format(original_name)) - # Update after loading since that's what would happen in an eager - # loading situation. - self.__dict__.update(attrs_updated) - return getattr(self, attr) - - def __delattr__(self, attr): - """Trigger the load and then perform the deletion.""" - # To trigger the load and raise an exception if the attribute - # doesn't exist. - self.__getattribute__(attr) - delattr(self, attr) - - -class LazyLoader(abc.Loader): - - """A loader that creates a module which defers loading until attribute access.""" - - @staticmethod - def __check_eager_loader(loader): - if not hasattr(loader, 'exec_module'): - raise TypeError('loader must define exec_module()') - elif hasattr(loader.__class__, 'create_module'): - if abc.Loader.create_module != loader.__class__.create_module: - # Only care if create_module() is overridden in a subclass of - # importlib.abc.Loader. - raise TypeError('loader cannot define create_module()') - - @classmethod - def factory(cls, loader): - """Construct a callable which returns the eager loader made lazy.""" - cls.__check_eager_loader(loader) - return lambda *args, **kwargs: cls(loader(*args, **kwargs)) - - def __init__(self, loader): - self.__check_eager_loader(loader) - self.loader = loader - - def create_module(self, spec): - """Create a module which can have its __class__ manipulated.""" - return _Module(spec.name) - - def exec_module(self, module): - """Make the module load lazily.""" - module.__spec__.loader = self.loader - module.__loader__ = self.loader - # Don't need to worry about deep-copying as trying to set an attribute - # on an object would have triggered the load, - # e.g. ``module.__spec__.loader = None`` would trigger a load from - # trying to access module.__spec__. - module.__spec__.loader_state = module.__dict__.copy() - module.__class__ = _LazyModule +from ._bootstrap import module_for_loader +from ._bootstrap import set_loader +from ._bootstrap import set_package diff -r 6db40a9955dc -r 0d413f60cc23 Lib/inspect.py --- a/Lib/inspect.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/inspect.py Mon Jan 25 17:05:13 2016 +0100 @@ -16,45 +16,43 @@ getmodule() - determine the module that an object came from getclasstree() - arrange classes so as to represent their hierarchy - getargvalues(), getcallargs() - get info about function arguments - getfullargspec() - same, with support for Python 3 features + getargspec(), getargvalues(), getcallargs() - get info about function arguments + getfullargspec() - same, with support for Python-3000 features formatargspec(), formatargvalues() - format an argument spec getouterframes(), getinnerframes() - get info about frames currentframe() - get the current stack frame stack(), trace() - get info about frames on the stack or in a traceback - - signature() - get a Signature object for the callable """ # This module is in the public domain. No warranties. -__author__ = ('Ka-Ping Yee ', - 'Yury Selivanov ') +__author__ = 'Ka-Ping Yee ' +__date__ = '1 Jan 2001' -import ast -import dis -import collections.abc -import enum -import importlib.machinery +import sys +import os +import types import itertools +import re +import imp +import tokenize import linecache -import os -import re -import sys -import tokenize -import token -import types -import warnings -import functools -import builtins from operator import attrgetter -from collections import namedtuple, OrderedDict +from collections import namedtuple # Create constants for the compiler flags in Include/code.h -# We try to get them from dis to avoid duplication -mod_dict = globals() -for k, v in dis.COMPILER_FLAG_NAMES.items(): - mod_dict["CO_" + v] = k +# We try to get them from dis to avoid duplication, but fall +# back to hardcording so the dependency is optional +try: + from dis import COMPILER_FLAG_NAMES as _flag_names +except ImportError: + CO_OPTIMIZED, CO_NEWLOCALS = 0x1, 0x2 + CO_VARARGS, CO_VARKEYWORDS = 0x4, 0x8 + CO_NESTED, CO_GENERATOR, CO_NOFREE = 0x10, 0x20, 0x40 +else: + mod_dict = globals() + for k, v in _flag_names.items(): + mod_dict["CO_" + v] = k # See Include/object.h TPFLAGS_IS_ABSTRACT = 1 << 20 @@ -177,20 +175,11 @@ return bool((isfunction(object) or ismethod(object)) and object.__code__.co_flags & CO_GENERATOR) -def iscoroutinefunction(object): - """Return true if the object is a coroutine function. - - Coroutine functions are defined with "async def" syntax, - or generators decorated with "types.coroutine". - """ - return bool((isfunction(object) or ismethod(object)) and - object.__code__.co_flags & CO_COROUTINE) - def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support iteration over container + __iter__ defined to support interation over container close raises a new GeneratorExit exception inside the generator to terminate the iteration gi_code code object @@ -203,17 +192,6 @@ throw used to raise an exception inside the generator""" return isinstance(object, types.GeneratorType) -def iscoroutine(object): - """Return true if the object is a coroutine.""" - return isinstance(object, types.CoroutineType) - -def isawaitable(object): - """Return true is object can be passed to an ``await`` expression.""" - return (isinstance(object, types.CoroutineType) or - isinstance(object, types.GeneratorType) and - object.gi_code.co_flags & CO_ITERABLE_COROUTINE or - isinstance(object, collections.abc.Awaitable)) - def istraceback(object): """Return true if the object is a traceback. @@ -284,40 +262,21 @@ else: mro = () results = [] - processed = set() - names = dir(object) - # :dd any DynamicClassAttributes to the list of names if object is a class; - # this may result in duplicate entries if, for example, a virtual - # attribute with the same name as a DynamicClassAttribute exists - try: - for base in object.__bases__: - for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): - names.append(k) - except AttributeError: - pass - for key in names: - # First try to get the value via getattr. Some descriptors don't - # like calling their __get__ (see bug #1785), so fall back to - # looking in the __dict__. - try: - value = getattr(object, key) - # handle the duplicate key - if key in processed: - raise AttributeError - except AttributeError: - for base in mro: - if key in base.__dict__: - value = base.__dict__[key] - break - else: - # could be a (currently) missing slot member, or a buggy - # __dir__; discard and move on + for key in dir(object): + # First try to get the value via __dict__. Some descriptors don't + # like calling their __get__ (see bug #1785). + for base in mro: + if key in base.__dict__: + value = base.__dict__[key] + break + else: + try: + value = getattr(object, key) + except AttributeError: continue if not predicate or predicate(value): results.append((key, value)) - processed.add(key) - results.sort(key=lambda pair: pair[0]) + results.sort() return results Attribute = namedtuple('Attribute', 'name kind defining_class object') @@ -334,106 +293,60 @@ 'class method' created via classmethod() 'static method' created via staticmethod() 'property' created via property() - 'method' any other flavor of method or descriptor + 'method' any other flavor of method 'data' not a method 2. The class which defined this attribute (a class). - 3. The object as obtained by calling getattr; if this fails, or if the - resulting object does not live anywhere in the class' mro (including - metaclasses) then the object is looked up in the defining class's - dict (found by walking the mro). - - If one of the items in dir(cls) is stored in the metaclass it will now - be discovered and not have None be listed as the class in which it was - defined. Any items whose home class cannot be discovered are skipped. + 3. The object as obtained directly from the defining class's + __dict__, not via getattr. This is especially important for + data attributes: C.data is just a data object, but + C.__dict__['data'] may be a data descriptor with additional + info, like a __doc__ string. """ mro = getmro(cls) - metamro = getmro(type(cls)) # for attributes stored in the metaclass - metamro = tuple([cls for cls in metamro if cls not in (type, object)]) - class_bases = (cls,) + mro - all_bases = class_bases + metamro names = dir(cls) - # :dd any DynamicClassAttributes to the list of names; - # this may result in duplicate entries if, for example, a virtual - # attribute with the same name as a DynamicClassAttribute exists. - for base in mro: - for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): - names.append(k) result = [] - processed = set() - for name in names: # Get the object associated with the name, and where it was defined. - # Normal objects will be looked up with both getattr and directly in - # its class' dict (in case getattr fails [bug #1785], and also to look - # for a docstring). - # For DynamicClassAttributes on the second pass we only look in the - # class's dict. - # # Getting an obj from the __dict__ sometimes reveals more than # using getattr. Static and class methods are dramatic examples. + # Furthermore, some objects may raise an Exception when fetched with + # getattr(). This is the case with some descriptors (bug #1785). + # Thus, we only use getattr() as a last resort. homecls = None - get_obj = None - dict_obj = None - if name not in processed: - try: - if name == '__dict__': - raise Exception("__dict__ is special, don't want the proxy") - get_obj = getattr(cls, name) - except Exception as exc: - pass + for base in (cls,) + mro: + if name in base.__dict__: + obj = base.__dict__[name] + homecls = base + break + else: + obj = getattr(cls, name) + homecls = getattr(obj, "__objclass__", homecls) + + # Classify the object. + if isinstance(obj, staticmethod): + kind = "static method" + elif isinstance(obj, classmethod): + kind = "class method" + elif isinstance(obj, property): + kind = "property" + elif ismethoddescriptor(obj): + kind = "method" + elif isdatadescriptor(obj): + kind = "data" + else: + obj_via_getattr = getattr(cls, name) + if (isfunction(obj_via_getattr) or + ismethoddescriptor(obj_via_getattr)): + kind = "method" else: - homecls = getattr(get_obj, "__objclass__", homecls) - if homecls not in class_bases: - # if the resulting object does not live somewhere in the - # mro, drop it and search the mro manually - homecls = None - last_cls = None - # first look in the classes - for srch_cls in class_bases: - srch_obj = getattr(srch_cls, name, None) - if srch_obj is get_obj: - last_cls = srch_cls - # then check the metaclasses - for srch_cls in metamro: - try: - srch_obj = srch_cls.__getattr__(cls, name) - except AttributeError: - continue - if srch_obj is get_obj: - last_cls = srch_cls - if last_cls is not None: - homecls = last_cls - for base in all_bases: - if name in base.__dict__: - dict_obj = base.__dict__[name] - if homecls not in metamro: - homecls = base - break - if homecls is None: - # unable to locate the attribute anywhere, most likely due to - # buggy custom __dir__; discard and move on - continue - obj = get_obj if get_obj is not None else dict_obj - # Classify the object or its descriptor. - if isinstance(dict_obj, staticmethod): - kind = "static method" - obj = dict_obj - elif isinstance(dict_obj, classmethod): - kind = "class method" - obj = dict_obj - elif isinstance(dict_obj, property): - kind = "property" - obj = dict_obj - elif isroutine(obj): - kind = "method" - else: - kind = "data" + kind = "data" + obj = obj_via_getattr + result.append(Attribute(name, kind, homecls, obj)) - processed.add(name) + return result # ----------------------------------------------------------- class helpers @@ -442,115 +355,12 @@ "Return tuple of base classes (including cls) in method resolution order." return cls.__mro__ -# -------------------------------------------------------- function helpers - -def unwrap(func, *, stop=None): - """Get the object wrapped by *func*. - - Follows the chain of :attr:`__wrapped__` attributes returning the last - object in the chain. - - *stop* is an optional callback accepting an object in the wrapper chain - as its sole argument that allows the unwrapping to be terminated early if - the callback returns a true value. If the callback never returns a true - value, the last object in the chain is returned as usual. For example, - :func:`signature` uses this to stop unwrapping if any object in the - chain has a ``__signature__`` attribute defined. - - :exc:`ValueError` is raised if a cycle is encountered. - - """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) - f = func # remember the original func for error reporting - memo = {id(f)} # Memoise by id to tolerate non-hashable objects - while _is_wrapper(func): - func = func.__wrapped__ - id_func = id(func) - if id_func in memo: - raise ValueError('wrapper loop when unwrapping {!r}'.format(f)) - memo.add(id_func) - return func - # -------------------------------------------------- source code extraction def indentsize(line): """Return the indent size, in spaces, at the start of a line of text.""" expline = line.expandtabs() return len(expline) - len(expline.lstrip()) -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not isclass(cls): - return None - return cls - -def _finddoc(obj): - if isclass(obj): - for base in obj.__mro__: - if base is not object: - try: - doc = base.__doc__ - except AttributeError: - continue - if doc is not None: - return doc - return None - - if ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) - if cls is None or getattr(cls, name) is not obj: - return None - elif ismethoddescriptor(obj) or isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - else: - return None - - for base in cls.__mro__: - try: - doc = getattr(base, name).__doc__ - except AttributeError: - continue - if doc is not None: - return doc - return None - def getdoc(object): """Get the documentation string for an object. @@ -561,11 +371,6 @@ doc = object.__doc__ except AttributeError: return None - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None if not isinstance(doc, str): return None return cleandoc(doc) @@ -606,10 +411,9 @@ return object.__file__ raise TypeError('{!r} is a built-in module'.format(object)) if isclass(object): - if hasattr(object, '__module__'): - object = sys.modules.get(object.__module__) - if hasattr(object, '__file__'): - return object.__file__ + object = sys.modules.get(object.__module__) + if hasattr(object, '__file__'): + return object.__file__ raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): object = object.__func__ @@ -624,35 +428,38 @@ raise TypeError('{!r} is not a module, class, method, ' 'function, traceback, frame, or code object'.format(object)) +ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') + +def getmoduleinfo(path): + """Get the module name, suffix, mode, and module type for a given file.""" + filename = os.path.basename(path) + suffixes = [(-len(suffix), suffix, mode, mtype) + for suffix, mode, mtype in imp.get_suffixes()] + suffixes.sort() # try longest suffixes first, in case they overlap + for neglen, suffix, mode, mtype in suffixes: + if filename[neglen:] == suffix: + return ModuleInfo(filename[:neglen], suffix, mode, mtype) + def getmodulename(path): """Return the module name for a given file, or None.""" - fname = os.path.basename(path) - # Check for paths that look like an actual module file - suffixes = [(-len(suffix), suffix) - for suffix in importlib.machinery.all_suffixes()] - suffixes.sort() # try longest suffixes first, in case they overlap - for neglen, suffix in suffixes: - if fname.endswith(suffix): - return fname[:neglen] - return None + info = getmoduleinfo(path) + if info: return info[0] def getsourcefile(object): """Return the filename that can be used to locate an object's source. Return None if no way can be identified to get the source. """ filename = getfile(object) - all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:] - all_bytecode_suffixes += importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES[:] - if any(filename.endswith(s) for s in all_bytecode_suffixes): - filename = (os.path.splitext(filename)[0] + - importlib.machinery.SOURCE_SUFFIXES[0]) - elif any(filename.endswith(s) for s in - importlib.machinery.EXTENSION_SUFFIXES): - return None + if filename[-4:].lower() in ('.pyc', '.pyo'): + filename = filename[:-4] + '.py' + for suffix, mode, kind in imp.get_suffixes(): + if 'b' in mode and filename[-len(suffix):].lower() == suffix: + # Looks like a binary file. We want to only return a text file. + return None if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + if hasattr(getmodule(object, filename), '__loader__'): return filename # or it is in the linecache if filename in linecache.cache: @@ -721,20 +528,14 @@ The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of all the lines - in the file and the line number indexes a line in that list. An OSError + in the file and the line number indexes a line in that list. An IOError is raised if the source code cannot be retrieved.""" - file = getsourcefile(object) - if file: - # Invalidate cache if needed. - linecache.checkcache(file) - else: - file = getfile(object) - # Allow filenames in form of "" to pass through. - # `doctest` monkeypatches `linecache` module to enable - # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): - raise OSError('source code not available') + file = getfile(object) + sourcefile = getsourcefile(object) + if not sourcefile and file[0] + file[-1] != '<>': + raise IOError('source code not available') + file = sourcefile if sourcefile else file module = getmodule(object, file) if module: @@ -742,7 +543,7 @@ else: lines = linecache.getlines(file) if not lines: - raise OSError('could not get source code') + raise IOError('could not get source code') if ismodule(object): return lines, 0 @@ -768,7 +569,7 @@ candidates.sort() return lines, candidates[0][1] else: - raise OSError('could not find class definition') + raise IOError('could not find class definition') if ismethod(object): object = object.__func__ @@ -780,14 +581,14 @@ object = object.f_code if iscode(object): if not hasattr(object, 'co_firstlineno'): - raise OSError('could not find function definition') + raise IOError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: if pat.match(lines[lnum]): break lnum = lnum - 1 return lines, lnum - raise OSError('could not find code object') + raise IOError('could not find code object') def getcomments(object): """Get lines of comments immediately preceding an object's source code. @@ -796,7 +597,7 @@ """ try: lines, lnum = findsource(object) - except (OSError, TypeError): + except (IOError, TypeError): return None if ismodule(object): @@ -843,37 +644,21 @@ self.islambda = False self.started = False self.passline = False - self.indecorator = False - self.decoratorhasargs = False self.last = 1 def tokeneater(self, type, token, srowcol, erowcol, line): - if not self.started and not self.indecorator: - # skip any decorators - if token == "@": - self.indecorator = True + if not self.started: # look for the first "def", "class" or "lambda" - elif token in ("def", "class", "lambda"): + if token in ("def", "class", "lambda"): if token == "lambda": self.islambda = True self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] if self.islambda: # lambdas always end at the first NEWLINE raise EndOfBlock - # hitting a NEWLINE when in a decorator without args - # ends the decorator - if self.indecorator and not self.decoratorhasargs: - self.indecorator = False elif self.passline: pass elif type == tokenize.INDENT: @@ -908,22 +693,19 @@ The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a list of the lines corresponding to the object and the line number indicates where in the - original source file the first line of code was found. An OSError is + original source file the first line of code was found. An IOError is raised if the source code cannot be retrieved.""" - object = unwrap(object) lines, lnum = findsource(object) - if ismodule(object): - return lines, 0 - else: - return getblock(lines[lnum:]), lnum + 1 + if ismodule(object): return lines, 0 + else: return getblock(lines[lnum:]), lnum + 1 def getsource(object): """Return the text of the source code for an object. The argument may be a module, class, method, function, traceback, frame, or code object. The source code is returned as a single string. An - OSError is raised if the source code cannot be retrieved.""" + IOError is raised if the source code cannot be retrieved.""" lines, lnum = getsourcelines(object) return ''.join(lines) @@ -954,8 +736,7 @@ for parent in c.__bases__: if not parent in children: children[parent] = [] - if c not in children[parent]: - children[parent].append(c) + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) @@ -1010,18 +791,17 @@ def getargspec(func): """Get the names and default values of a function's arguments. - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** arguments or None. + A tuple of four things is returned: (args, varargs, varkw, defaults). + 'args' is a list of the argument names. + 'args' will include keyword-only argument names. + 'varargs' and 'varkw' are the names of the * and ** arguments or None. 'defaults' is an n-tuple of the default values of the last n arguments. - Use the getfullargspec() API for Python 3 code, as annotations + Use the getfullargspec() API for Python-3000 code, as annotations and keyword arguments are supported. getargspec() will raise ValueError if the func has either annotations or keyword arguments. """ - warnings.warn("inspect.getargspec() is deprecated, " - "use inspect.signature() instead", DeprecationWarning, - stacklevel=2) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ getfullargspec(func) if kwonlyargs or ann: @@ -1033,7 +813,7 @@ 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') def getfullargspec(func): - """Get the names and default values of a callable object's arguments. + """Get the names and default values of a function's arguments. A tuple of seven things is returned: (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults annotations). @@ -1044,82 +824,16 @@ 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping argument names to annotations. - This function is deprecated, use inspect.signature() instead. + The first four items in the tuple correspond to getargspec(). """ - try: - # Re: `skip_bound_arg=False` - # - # There is a notable difference in behaviour between getfullargspec - # and Signature: the former always returns 'self' parameter for bound - # methods, whereas the Signature always shows the actual calling - # signature of the passed object. - # - # To simulate this behaviour, we "unbind" bound methods, to trick - # inspect.signature to always return their first parameter ("self", - # usually) - - # Re: `follow_wrapper_chains=False` - # - # getfullargspec() historically ignored __wrapped__ attributes, - # so we ensure that remains the case in 3.3+ - - sig = _signature_from_callable(func, - follow_wrapper_chains=False, - skip_bound_arg=False, - sigcls=Signature) - except Exception as ex: - # Most of the times 'signature' will raise ValueError. - # But, it can also raise AttributeError, and, maybe something - # else. So to be fully backwards compatible, we catch all - # possible exceptions here, and reraise a TypeError. - raise TypeError('unsupported callable') from ex - - args = [] - varargs = None - varkw = None - kwonlyargs = [] - defaults = () - annotations = {} - defaults = () - kwdefaults = {} - - if sig.return_annotation is not sig.empty: - annotations['return'] = sig.return_annotation - - for param in sig.parameters.values(): - kind = param.kind - name = param.name - - if kind is _POSITIONAL_ONLY: - args.append(name) - elif kind is _POSITIONAL_OR_KEYWORD: - args.append(name) - if param.default is not param.empty: - defaults += (param.default,) - elif kind is _VAR_POSITIONAL: - varargs = name - elif kind is _KEYWORD_ONLY: - kwonlyargs.append(name) - if param.default is not param.empty: - kwdefaults[name] = param.default - elif kind is _VAR_KEYWORD: - varkw = name - - if param.annotation is not param.empty: - annotations[name] = param.annotation - - if not kwdefaults: - # compatibility with 'func.__kwdefaults__' - kwdefaults = None - - if not defaults: - # compatibility with 'func.__defaults__' - defaults = None - - return FullArgSpec(args, varargs, varkw, defaults, - kwonlyargs, kwdefaults, annotations) - + if ismethod(func): + func = func.__func__ + if not isfunction(func): + raise TypeError('{!r} is not a Python function'.format(func)) + args, varargs, kwonlyargs, varkw = _getfullargs(func.__code__) + return FullArgSpec(args, varargs, varkw, func.__defaults__, + kwonlyargs, func.__kwdefaults__, func.__annotations__) ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals') @@ -1136,8 +850,8 @@ def formatannotation(annotation, base_module=None): if isinstance(annotation, type): if annotation.__module__ in ('builtins', base_module): - return annotation.__qualname__ - return annotation.__module__+'.'+annotation.__qualname__ + return annotation.__name__ + return annotation.__module__+'.'+annotation.__name__ return repr(annotation) def formatannotationrelativeto(object): @@ -1154,7 +868,8 @@ formatvalue=lambda value: '=' + repr(value), formatreturns=lambda text: ' -> ' + text, formatannotation=formatannotation): - """Format an argument spec from the values returned by getfullargspec. + """Format an argument spec from the values returned by getargspec + or getfullargspec. The first seven arguments are (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). The other five arguments @@ -1223,7 +938,7 @@ elif missing == 2: s = "{} and {}".format(*names) else: - tail = ", {} and {}".format(*names[-2:]) + tail = ", {} and {}".format(names[-2:]) del names[-2:] s = ", ".join(names) + tail raise TypeError("%s() missing %i required %s argument%s: %s" % @@ -1252,14 +967,12 @@ (f_name, sig, "s" if plural else "", given, kwonly_sig, "was" if given == 1 and not kwonly_given else "were")) -def getcallargs(*func_and_positional, **named): +def getcallargs(func, *positional, **named): """Get the mapping of arguments to values. A dict is returned, with keys the function argument names (including the names of the * and ** arguments, if any), and values the respective bound values from 'positional' and 'named'.""" - func = func_and_positional[0] - positional = func_and_positional[1:] spec = getfullargspec(func) args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = spec f_name = func.__name__ @@ -1306,7 +1019,7 @@ missing = 0 for kwarg in kwonlyargs: if kwarg not in arg2value: - if kwonlydefaults and kwarg in kwonlydefaults: + if kwarg in kwonlydefaults: arg2value[kwarg] = kwonlydefaults[kwarg] else: missing += 1 @@ -1314,59 +1027,6 @@ _missing_arguments(f_name, kwonlyargs, False, arg2value) return arg2value -ClosureVars = namedtuple('ClosureVars', 'nonlocals globals builtins unbound') - -def getclosurevars(func): - """ - Get the mapping of free variables to their current values. - - Returns a named tuple of dicts mapping the current nonlocal, global - and builtin references as seen by the body of the function. A final - set of unbound names that could not be resolved is also provided. - """ - - if ismethod(func): - func = func.__func__ - - if not isfunction(func): - raise TypeError("'{!r}' is not a Python function".format(func)) - - code = func.__code__ - # Nonlocal references are named in co_freevars and resolved - # by looking them up in __closure__ by positional index - if func.__closure__ is None: - nonlocal_vars = {} - else: - nonlocal_vars = { - var : cell.cell_contents - for var, cell in zip(code.co_freevars, func.__closure__) - } - - # Global and builtin references are named in co_names and resolved - # by looking them up in __globals__ or __builtins__ - global_ns = func.__globals__ - builtin_ns = global_ns.get("__builtins__", builtins.__dict__) - if ismodule(builtin_ns): - builtin_ns = builtin_ns.__dict__ - global_vars = {} - builtin_vars = {} - unbound_names = set() - for name in code.co_names: - if name in ("None", "True", "False"): - # Because these used to be builtins instead of keywords, they - # may still show up as name references. We ignore them. - continue - try: - global_vars[name] = global_ns[name] - except KeyError: - try: - builtin_vars[name] = builtin_ns[name] - except KeyError: - unbound_names.add(name) - - return ClosureVars(nonlocal_vars, global_vars, - builtin_vars, unbound_names) - # -------------------------------------------------- stack frame extraction Traceback = namedtuple('Traceback', 'filename lineno function code_context index') @@ -1392,7 +1052,7 @@ start = lineno - 1 - context//2 try: lines, lnum = findsource(frame) - except OSError: + except IOError: lines = index = None else: start = max(start, 1) @@ -1409,8 +1069,6 @@ # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno -FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields) - def getouterframes(frame, context=1): """Get a list of records for a frame and all higher (calling) frames. @@ -1418,8 +1076,7 @@ name, a list of lines of context, and index within the context.""" framelist = [] while frame: - frameinfo = (frame,) + getframeinfo(frame, context) - framelist.append(FrameInfo(*frameinfo)) + framelist.append((frame,) + getframeinfo(frame, context)) frame = frame.f_back return framelist @@ -1430,8 +1087,7 @@ name, a list of lines of context, and index within the context.""" framelist = [] while tb: - frameinfo = (tb.tb_frame,) + getframeinfo(tb, context) - framelist.append(FrameInfo(*frameinfo)) + framelist.append((tb.tb_frame,) + getframeinfo(tb, context)) tb = tb.tb_next return framelist @@ -1540,8 +1196,6 @@ raise AttributeError(attr) -# ------------------------------------------------ generator introspection - GEN_CREATED = 'GEN_CREATED' GEN_RUNNING = 'GEN_RUNNING' GEN_SUSPENDED = 'GEN_SUSPENDED' @@ -1563,1469 +1217,3 @@ if generator.gi_frame.f_lasti == -1: return GEN_CREATED return GEN_SUSPENDED - - -def getgeneratorlocals(generator): - """ - Get the mapping of generator local variables to their current values. - - A dict is returned, with the keys the local variable names and values the - bound values.""" - - if not isgenerator(generator): - raise TypeError("'{!r}' is not a Python generator".format(generator)) - - frame = getattr(generator, "gi_frame", None) - if frame is not None: - return generator.gi_frame.f_locals - else: - return {} - - -# ------------------------------------------------ coroutine introspection - -CORO_CREATED = 'CORO_CREATED' -CORO_RUNNING = 'CORO_RUNNING' -CORO_SUSPENDED = 'CORO_SUSPENDED' -CORO_CLOSED = 'CORO_CLOSED' - -def getcoroutinestate(coroutine): - """Get current state of a coroutine object. - - Possible states are: - CORO_CREATED: Waiting to start execution. - CORO_RUNNING: Currently being executed by the interpreter. - CORO_SUSPENDED: Currently suspended at an await expression. - CORO_CLOSED: Execution has completed. - """ - if coroutine.cr_running: - return CORO_RUNNING - if coroutine.cr_frame is None: - return CORO_CLOSED - if coroutine.cr_frame.f_lasti == -1: - return CORO_CREATED - return CORO_SUSPENDED - - -def getcoroutinelocals(coroutine): - """ - Get the mapping of coroutine local variables to their current values. - - A dict is returned, with the keys the local variable names and values the - bound values.""" - frame = getattr(coroutine, "cr_frame", None) - if frame is not None: - return frame.f_locals - else: - return {} - - -############################################################################### -### Function Signature Object (PEP 362) -############################################################################### - - -_WrapperDescriptor = type(type.__call__) -_MethodWrapper = type(all.__call__) -_ClassMethodWrapper = type(int.__dict__['from_bytes']) - -_NonUserDefinedCallables = (_WrapperDescriptor, - _MethodWrapper, - _ClassMethodWrapper, - types.BuiltinFunctionType) - - -def _signature_get_user_defined_method(cls, method_name): - """Private helper. Checks if ``cls`` has an attribute - named ``method_name`` and returns it only if it is a - pure python function. - """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return - else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth - - -def _signature_get_partial(wrapped_sig, partial, extra_args=()): - """Private helper to calculate how 'wrapped_sig' signature will - look like after applying a 'functools.partial' object (or alike) - on it. - """ - - old_params = wrapped_sig.parameters - new_params = OrderedDict(old_params.items()) - - partial_args = partial.args or () - partial_keywords = partial.keywords or {} - - if extra_args: - partial_args = extra_args + partial_args - - try: - ba = wrapped_sig.bind_partial(*partial_args, **partial_keywords) - except TypeError as ex: - msg = 'partial object {!r} has incorrect arguments'.format(partial) - raise ValueError(msg) from ex - - - transform_to_kwonly = False - for param_name, param in old_params.items(): - try: - arg_value = ba.arguments[param_name] - except KeyError: - pass - else: - if param.kind is _POSITIONAL_ONLY: - # If positional-only parameter is bound by partial, - # it effectively disappears from the signature - new_params.pop(param_name) - continue - - if param.kind is _POSITIONAL_OR_KEYWORD: - if param_name in partial_keywords: - # This means that this parameter, and all parameters - # after it should be keyword-only (and var-positional - # should be removed). Here's why. Consider the following - # function: - # foo(a, b, *args, c): - # pass - # - # "partial(foo, a='spam')" will have the following - # signature: "(*, a='spam', b, c)". Because attempting - # to call that partial with "(10, 20)" arguments will - # raise a TypeError, saying that "a" argument received - # multiple values. - transform_to_kwonly = True - # Set the new default value - new_params[param_name] = param.replace(default=arg_value) - else: - # was passed as a positional argument - new_params.pop(param.name) - continue - - if param.kind is _KEYWORD_ONLY: - # Set the new default value - new_params[param_name] = param.replace(default=arg_value) - - if transform_to_kwonly: - assert param.kind is not _POSITIONAL_ONLY - - if param.kind is _POSITIONAL_OR_KEYWORD: - new_param = new_params[param_name].replace(kind=_KEYWORD_ONLY) - new_params[param_name] = new_param - new_params.move_to_end(param_name) - elif param.kind in (_KEYWORD_ONLY, _VAR_KEYWORD): - new_params.move_to_end(param_name) - elif param.kind is _VAR_POSITIONAL: - new_params.pop(param.name) - - return wrapped_sig.replace(parameters=new_params.values()) - - -def _signature_bound_method(sig): - """Private helper to transform signatures for unbound - functions to bound methods. - """ - - params = tuple(sig.parameters.values()) - - if not params or params[0].kind in (_VAR_KEYWORD, _KEYWORD_ONLY): - raise ValueError('invalid method signature') - - kind = params[0].kind - if kind in (_POSITIONAL_OR_KEYWORD, _POSITIONAL_ONLY): - # Drop first parameter: - # '(p1, p2[, ...])' -> '(p2[, ...])' - params = params[1:] - else: - if kind is not _VAR_POSITIONAL: - # Unless we add a new parameter type we never - # get here - raise ValueError('invalid argument type') - # It's a var-positional parameter. - # Do nothing. '(*args[, ...])' -> '(*args[, ...])' - - return sig.replace(parameters=params) - - -def _signature_is_builtin(obj): - """Private helper to test if `obj` is a callable that might - support Argument Clinic's __text_signature__ protocol. - """ - return (isbuiltin(obj) or - ismethoddescriptor(obj) or - isinstance(obj, _NonUserDefinedCallables) or - # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) - - -def _signature_is_functionlike(obj): - """Private helper to test if `obj` is a duck type of FunctionType. - A good example of such objects are functions compiled with - Cython, which have all attributes that a pure Python function - would have, but have their code statically compiled. - """ - - if not callable(obj) or isclass(obj): - # All function-like objects are obviously callables, - # and not classes. - return False - - name = getattr(obj, '__name__', None) - code = getattr(obj, '__code__', None) - defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... - kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here - annotations = getattr(obj, '__annotations__', None) - - return (isinstance(code, types.CodeType) and - isinstance(name, str) and - (defaults is None or isinstance(defaults, tuple)) and - (kwdefaults is None or isinstance(kwdefaults, dict)) and - isinstance(annotations, dict)) - - -def _signature_get_bound_param(spec): - """ Private helper to get first parameter name from a - __text_signature__ of a builtin method, which should - be in the following format: '($param1, ...)'. - Assumptions are that the first argument won't have - a default value or an annotation. - """ - - assert spec.startswith('($') - - pos = spec.find(',') - if pos == -1: - pos = spec.find(')') - - cpos = spec.find(':') - assert cpos == -1 or cpos > pos - - cpos = spec.find('=') - assert cpos == -1 or cpos > pos - - return spec[2:pos] - - -def _signature_strip_non_python_syntax(signature): - """ - Private helper function. Takes a signature in Argument Clinic's - extended signature format. - - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, - * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. - """ - - if not signature: - return signature, None, None - - self_parameter = None - last_positional_only = None - - lines = [l.encode('ascii') for l in signature.split('\n')] - generator = iter(lines).__next__ - token_stream = tokenize.tokenize(generator) - - delayed_comma = False - skip_next_comma = False - text = [] - add = text.append - - current_parameter = 0 - OP = token.OP - ERRORTOKEN = token.ERRORTOKEN - - # token stream always starts with ENCODING token, skip it - t = next(token_stream) - assert t.type == tokenize.ENCODING - - for t in token_stream: - type, string = t.type, t.string - - if type == OP: - if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue - - if (type == ERRORTOKEN) and (string == '$'): - assert self_parameter is None - self_parameter = current_parameter - continue - - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') - add(string) - if (string == ','): - add(' ') - clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only - - -def _signature_fromstr(cls, obj, s, skip_bound_arg=True): - """Private helper to parse content of '__text_signature__' - and return a Signature based on it. - """ - - Parameter = cls._parameter_cls - - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) - - program = "def foo" + clean_signature + ": pass" - - try: - module = ast.parse(program) - except SyntaxError: - module = None - - if not isinstance(module, ast.Module): - raise ValueError("{!r} builtin has invalid signature".format(obj)) - - f = module.body[0] - - parameters = [] - empty = Parameter.empty - invalid = object() - - module = None - module_dict = {} - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - module_dict = module.__dict__ - sys_module_dict = sys.modules - - def parse_name(node): - assert isinstance(node, ast.arg) - if node.annotation != None: - raise ValueError("Annotations are not currently supported") - return node.arg - - def wrap_value(s): - try: - value = eval(s, module_dict) - except NameError: - try: - value = eval(s, sys_module_dict) - except NameError: - raise RuntimeError() - - if isinstance(value, str): - return ast.Str(value) - if isinstance(value, (int, float)): - return ast.Num(value) - if isinstance(value, bytes): - return ast.Bytes(value) - if value in (True, False, None): - return ast.NameConstant(value) - raise RuntimeError() - - class RewriteSymbolics(ast.NodeTransformer): - def visit_Attribute(self, node): - a = [] - n = node - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - raise RuntimeError() - a.append(n.id) - value = ".".join(reversed(a)) - return wrap_value(value) - - def visit_Name(self, node): - if not isinstance(node.ctx, ast.Load): - raise ValueError() - return wrap_value(node.id) - - def p(name_node, default_node, default=empty): - name = parse_name(name_node) - if name is invalid: - return None - if default_node and default_node is not _empty: - try: - default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) - except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default - parameters.append(Parameter(name, kind, default=default, annotation=empty)) - - # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): - p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD - - # *args - if f.args.vararg: - kind = Parameter.VAR_POSITIONAL - p(f.args.vararg, empty) - - # keyword-only arguments - kind = Parameter.KEYWORD_ONLY - for name, default in zip(f.args.kwonlyargs, f.args.kw_defaults): - p(name, default) - - # **kwargs - if f.args.kwarg: - kind = Parameter.VAR_KEYWORD - p(f.args.kwarg, empty) - - if self_parameter is not None: - # Possibly strip the bound argument: - # - We *always* strip first bound argument if - # it is a module. - # - We don't strip first bound argument if - # skip_bound_arg is False. - assert parameters - _self = getattr(obj, '__self__', None) - self_isbound = _self is not None - self_ismodule = ismodule(_self) - if self_isbound and (self_ismodule or skip_bound_arg): - parameters.pop(0) - else: - # for builtins, self parameter is always positional-only! - p = parameters[0].replace(kind=Parameter.POSITIONAL_ONLY) - parameters[0] = p - - return cls(parameters, return_annotation=cls.empty) - - -def _signature_from_builtin(cls, func, skip_bound_arg=True): - """Private helper function to get signature for - builtin callables. - """ - - if not _signature_is_builtin(func): - raise TypeError("{!r} is not a Python builtin " - "function".format(func)) - - s = getattr(func, "__text_signature__", None) - if not s: - raise ValueError("no signature found for builtin {!r}".format(func)) - - return _signature_fromstr(cls, func, s, skip_bound_arg) - - -def _signature_from_function(cls, func): - """Private helper: constructs Signature for the given python function.""" - - is_duck_function = False - if not isfunction(func): - if _signature_is_functionlike(func): - is_duck_function = True - else: - # If it's not a pure Python function, and not a duck type - # of pure function: - raise TypeError('{!r} is not a Python function'.format(func)) - - Parameter = cls._parameter_cls - - # Parameter information. - func_code = func.__code__ - pos_count = func_code.co_argcount - arg_names = func_code.co_varnames - positional = tuple(arg_names[:pos_count]) - keyword_only_count = func_code.co_kwonlyargcount - keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)] - annotations = func.__annotations__ - defaults = func.__defaults__ - kwdefaults = func.__kwdefaults__ - - if defaults: - pos_default_count = len(defaults) - else: - pos_default_count = 0 - - parameters = [] - - # Non-keyword-only parameters w/o defaults. - non_default_count = pos_count - pos_default_count - for name in positional[:non_default_count]: - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_POSITIONAL_OR_KEYWORD)) - - # ... w/ defaults. - for offset, name in enumerate(positional[non_default_count:]): - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_POSITIONAL_OR_KEYWORD, - default=defaults[offset])) - - # *args - if func_code.co_flags & CO_VARARGS: - name = arg_names[pos_count + keyword_only_count] - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_VAR_POSITIONAL)) - - # Keyword-only parameters. - for name in keyword_only: - default = _empty - if kwdefaults is not None: - default = kwdefaults.get(name, _empty) - - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_KEYWORD_ONLY, - default=default)) - # **kwargs - if func_code.co_flags & CO_VARKEYWORDS: - index = pos_count + keyword_only_count - if func_code.co_flags & CO_VARARGS: - index += 1 - - name = arg_names[index] - annotation = annotations.get(name, _empty) - parameters.append(Parameter(name, annotation=annotation, - kind=_VAR_KEYWORD)) - - # Is 'func' is a pure Python function - don't validate the - # parameters list (for correct order and defaults), it should be OK. - return cls(parameters, - return_annotation=annotations.get('return', _empty), - __validate_parameters__=is_duck_function) - - -def _signature_from_callable(obj, *, - follow_wrapper_chains=True, - skip_bound_arg=True, - sigcls): - - """Private helper function to get signature for arbitrary - callable objects. - """ - - if not callable(obj): - raise TypeError('{!r} is not a callable object'.format(obj)) - - if isinstance(obj, types.MethodType): - # In this case we skip the first parameter of the underlying - # function (usually `self` or `cls`). - sig = _signature_from_callable( - obj.__func__, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - # Was this function wrapped by a decorator? - if follow_wrapper_chains: - obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__"))) - if isinstance(obj, types.MethodType): - # If the unwrapped object is a *method*, we might want to - # skip its first parameter (self). - # See test_signature_wrapped_bound_method for details. - return _signature_from_callable( - obj, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - try: - sig = obj.__signature__ - except AttributeError: - pass - else: - if sig is not None: - if not isinstance(sig, Signature): - raise TypeError( - 'unexpected object {!r} in __signature__ ' - 'attribute'.format(sig)) - return sig - - try: - partialmethod = obj._partialmethod - except AttributeError: - pass - else: - if isinstance(partialmethod, functools.partialmethod): - # Unbound partialmethod (see functools.partialmethod) - # This means, that we need to calculate the signature - # as if it's a regular partial object, but taking into - # account that the first positional argument - # (usually `self`, or `cls`) will not be passed - # automatically (as for boundmethods) - - wrapped_sig = _signature_from_callable( - partialmethod.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) - - first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] - new_params = (first_wrapped_param,) + tuple(sig.parameters.values()) - - return sig.replace(parameters=new_params) - - if isfunction(obj) or _signature_is_functionlike(obj): - # If it's a pure Python function, or an object that is duck type - # of a Python function (Cython functions, for instance), then: - return _signature_from_function(sigcls, obj) - - if _signature_is_builtin(obj): - return _signature_from_builtin(sigcls, obj, - skip_bound_arg=skip_bound_arg) - - if isinstance(obj, functools.partial): - wrapped_sig = _signature_from_callable( - obj.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - return _signature_get_partial(wrapped_sig, obj) - - sig = None - if isinstance(obj, type): - # obj is a class or a metaclass - - # First, let's see if it has an overloaded __call__ defined - # in its metaclass - call = _signature_get_user_defined_method(type(obj), '__call__') - if call is not None: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - else: - # Now we check if the 'obj' class has a '__new__' method - new = _signature_get_user_defined_method(obj, '__new__') - if new is not None: - sig = _signature_from_callable( - new, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - else: - # Finally, we should have at least __init__ implemented - init = _signature_get_user_defined_method(obj, '__init__') - if init is not None: - sig = _signature_from_callable( - init, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'obj' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, obj, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return signature(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) - - elif not isinstance(obj, _NonUserDefinedCallables): - # An object with __call__ - # We also check that the 'obj' is not an instance of - # _WrapperDescriptor or _MethodWrapper to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') - if call is not None: - try: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) - - raise ValueError('callable {!r} is not supported by signature'.format(obj)) - - -class _void: - """A private marker - used in Parameter & Signature.""" - - -class _empty: - """Marker object for Signature.empty and Parameter.empty.""" - - -class _ParameterKind(enum.IntEnum): - POSITIONAL_ONLY = 0 - POSITIONAL_OR_KEYWORD = 1 - VAR_POSITIONAL = 2 - KEYWORD_ONLY = 3 - VAR_KEYWORD = 4 - - def __str__(self): - return self._name_ - - -_POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY -_POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD -_VAR_POSITIONAL = _ParameterKind.VAR_POSITIONAL -_KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY -_VAR_KEYWORD = _ParameterKind.VAR_KEYWORD - - -class Parameter: - """Represents a parameter in a function signature. - - Has the following public attributes: - - * name : str - The name of the parameter as a string. - * default : object - The default value for the parameter if specified. If the - parameter has no default value, this attribute is set to - `Parameter.empty`. - * annotation - The annotation for the parameter if specified. If the - parameter has no annotation, this attribute is set to - `Parameter.empty`. - * kind : str - Describes how argument values are bound to the parameter. - Possible values: `Parameter.POSITIONAL_ONLY`, - `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, - `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. - """ - - __slots__ = ('_name', '_kind', '_default', '_annotation') - - POSITIONAL_ONLY = _POSITIONAL_ONLY - POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD - VAR_POSITIONAL = _VAR_POSITIONAL - KEYWORD_ONLY = _KEYWORD_ONLY - VAR_KEYWORD = _VAR_KEYWORD - - empty = _empty - - def __init__(self, name, kind, *, default=_empty, annotation=_empty): - - if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD, - _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD): - raise ValueError("invalid value for 'Parameter.kind' attribute") - self._kind = kind - - if default is not _empty: - if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): - msg = '{} parameters cannot have default values'.format(kind) - raise ValueError(msg) - self._default = default - self._annotation = annotation - - if name is _empty: - raise ValueError('name is a required attribute for Parameter') - - if not isinstance(name, str): - raise TypeError("name must be a str, not a {!r}".format(name)) - - if not name.isidentifier(): - raise ValueError('{!r} is not a valid parameter name'.format(name)) - - self._name = name - - def __reduce__(self): - return (type(self), - (self._name, self._kind), - {'_default': self._default, - '_annotation': self._annotation}) - - def __setstate__(self, state): - self._default = state['_default'] - self._annotation = state['_annotation'] - - @property - def name(self): - return self._name - - @property - def default(self): - return self._default - - @property - def annotation(self): - return self._annotation - - @property - def kind(self): - return self._kind - - def replace(self, *, name=_void, kind=_void, - annotation=_void, default=_void): - """Creates a customized copy of the Parameter.""" - - if name is _void: - name = self._name - - if kind is _void: - kind = self._kind - - if annotation is _void: - annotation = self._annotation - - if default is _void: - default = self._default - - return type(self)(name, kind, default=default, annotation=annotation) - - def __str__(self): - kind = self.kind - formatted = self._name - - # Add annotation and default value - if self._annotation is not _empty: - formatted = '{}:{}'.format(formatted, - formatannotation(self._annotation)) - - if self._default is not _empty: - formatted = '{}={}'.format(formatted, repr(self._default)) - - if kind == _VAR_POSITIONAL: - formatted = '*' + formatted - elif kind == _VAR_KEYWORD: - formatted = '**' + formatted - - return formatted - - def __repr__(self): - return '<{} "{}">'.format(self.__class__.__name__, self) - - def __hash__(self): - return hash((self.name, self.kind, self.annotation, self.default)) - - def __eq__(self, other): - if self is other: - return True - if not isinstance(other, Parameter): - return NotImplemented - return (self._name == other._name and - self._kind == other._kind and - self._default == other._default and - self._annotation == other._annotation) - - -class BoundArguments: - """Result of `Signature.bind` call. Holds the mapping of arguments - to the function's parameters. - - Has the following public attributes: - - * arguments : OrderedDict - An ordered mutable mapping of parameters' names to arguments' values. - Does not contain arguments' default values. - * signature : Signature - The Signature object that created this instance. - * args : tuple - Tuple of positional arguments values. - * kwargs : dict - Dict of keyword arguments values. - """ - - __slots__ = ('arguments', '_signature', '__weakref__') - - def __init__(self, signature, arguments): - self.arguments = arguments - self._signature = signature - - @property - def signature(self): - return self._signature - - @property - def args(self): - args = [] - for param_name, param in self._signature.parameters.items(): - if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): - break - - try: - arg = self.arguments[param_name] - except KeyError: - # We're done here. Other arguments - # will be mapped in 'BoundArguments.kwargs' - break - else: - if param.kind == _VAR_POSITIONAL: - # *args - args.extend(arg) - else: - # plain argument - args.append(arg) - - return tuple(args) - - @property - def kwargs(self): - kwargs = {} - kwargs_started = False - for param_name, param in self._signature.parameters.items(): - if not kwargs_started: - if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): - kwargs_started = True - else: - if param_name not in self.arguments: - kwargs_started = True - continue - - if not kwargs_started: - continue - - try: - arg = self.arguments[param_name] - except KeyError: - pass - else: - if param.kind == _VAR_KEYWORD: - # **kwargs - kwargs.update(arg) - else: - # plain keyword argument - kwargs[param_name] = arg - - return kwargs - - def apply_defaults(self): - """Set default values for missing arguments. - - For variable-positional arguments (*args) the default is an - empty tuple. - - For variable-keyword arguments (**kwargs) the default is an - empty dict. - """ - arguments = self.arguments - if not arguments: - return - new_arguments = [] - for name, param in self._signature.parameters.items(): - try: - new_arguments.append((name, arguments[name])) - except KeyError: - if param.default is not _empty: - val = param.default - elif param.kind is _VAR_POSITIONAL: - val = () - elif param.kind is _VAR_KEYWORD: - val = {} - else: - # This BoundArguments was likely produced by - # Signature.bind_partial(). - continue - new_arguments.append((name, val)) - self.arguments = OrderedDict(new_arguments) - - def __eq__(self, other): - if self is other: - return True - if not isinstance(other, BoundArguments): - return NotImplemented - return (self.signature == other.signature and - self.arguments == other.arguments) - - def __setstate__(self, state): - self._signature = state['_signature'] - self.arguments = state['arguments'] - - def __getstate__(self): - return {'_signature': self._signature, 'arguments': self.arguments} - - def __repr__(self): - args = [] - for arg, value in self.arguments.items(): - args.append('{}={!r}'.format(arg, value)) - return '<{} ({})>'.format(self.__class__.__name__, ', '.join(args)) - - -class Signature: - """A Signature object represents the overall signature of a function. - It stores a Parameter object for each parameter accepted by the - function, as well as information specific to the function itself. - - A Signature object has the following public attributes and methods: - - * parameters : OrderedDict - An ordered mapping of parameters' names to the corresponding - Parameter objects (keyword-only arguments are in the same order - as listed in `code.co_varnames`). - * return_annotation : object - The annotation for the return type of the function if specified. - If the function has no annotation for its return type, this - attribute is set to `Signature.empty`. - * bind(*args, **kwargs) -> BoundArguments - Creates a mapping from positional and keyword arguments to - parameters. - * bind_partial(*args, **kwargs) -> BoundArguments - Creates a partial mapping from positional and keyword arguments - to parameters (simulating 'functools.partial' behavior.) - """ - - __slots__ = ('_return_annotation', '_parameters') - - _parameter_cls = Parameter - _bound_arguments_cls = BoundArguments - - empty = _empty - - def __init__(self, parameters=None, *, return_annotation=_empty, - __validate_parameters__=True): - """Constructs Signature from the given list of Parameter - objects and 'return_annotation'. All arguments are optional. - """ - - if parameters is None: - params = OrderedDict() - else: - if __validate_parameters__: - params = OrderedDict() - top_kind = _POSITIONAL_ONLY - kind_defaults = False - - for idx, param in enumerate(parameters): - kind = param.kind - name = param.name - - if kind < top_kind: - msg = 'wrong parameter order: {!r} before {!r}' - msg = msg.format(top_kind, kind) - raise ValueError(msg) - elif kind > top_kind: - kind_defaults = False - top_kind = kind - - if kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD): - if param.default is _empty: - if kind_defaults: - # No default for this parameter, but the - # previous parameter of the same kind had - # a default - msg = 'non-default argument follows default ' \ - 'argument' - raise ValueError(msg) - else: - # There is a default for this parameter. - kind_defaults = True - - if name in params: - msg = 'duplicate parameter name: {!r}'.format(name) - raise ValueError(msg) - - params[name] = param - else: - params = OrderedDict(((param.name, param) - for param in parameters)) - - self._parameters = types.MappingProxyType(params) - self._return_annotation = return_annotation - - @classmethod - def from_function(cls, func): - """Constructs Signature for the given python function.""" - - warnings.warn("inspect.Signature.from_function() is deprecated, " - "use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_function(cls, func) - - @classmethod - def from_builtin(cls, func): - """Constructs Signature for the given builtin function.""" - - warnings.warn("inspect.Signature.from_builtin() is deprecated, " - "use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_builtin(cls, func) - - @classmethod - def from_callable(cls, obj, *, follow_wrapped=True): - """Constructs Signature for the given callable object.""" - return _signature_from_callable(obj, sigcls=cls, - follow_wrapper_chains=follow_wrapped) - - @property - def parameters(self): - return self._parameters - - @property - def return_annotation(self): - return self._return_annotation - - def replace(self, *, parameters=_void, return_annotation=_void): - """Creates a customized copy of the Signature. - Pass 'parameters' and/or 'return_annotation' arguments - to override them in the new copy. - """ - - if parameters is _void: - parameters = self.parameters.values() - - if return_annotation is _void: - return_annotation = self._return_annotation - - return type(self)(parameters, - return_annotation=return_annotation) - - def _hash_basis(self): - params = tuple(param for param in self.parameters.values() - if param.kind != _KEYWORD_ONLY) - - kwo_params = {param.name: param for param in self.parameters.values() - if param.kind == _KEYWORD_ONLY} - - return params, kwo_params, self.return_annotation - - def __hash__(self): - params, kwo_params, return_annotation = self._hash_basis() - kwo_params = frozenset(kwo_params.values()) - return hash((params, kwo_params, return_annotation)) - - def __eq__(self, other): - if self is other: - return True - if not isinstance(other, Signature): - return NotImplemented - return self._hash_basis() == other._hash_basis() - - def _bind(self, args, kwargs, *, partial=False): - """Private method. Don't use directly.""" - - arguments = OrderedDict() - - parameters = iter(self.parameters.values()) - parameters_ex = () - arg_vals = iter(args) - - while True: - # Let's iterate through the positional arguments and corresponding - # parameters - try: - arg_val = next(arg_vals) - except StopIteration: - # No more positional arguments - try: - param = next(parameters) - except StopIteration: - # No more parameters. That's it. Just need to check that - # we have no `kwargs` after this while loop - break - else: - if param.kind == _VAR_POSITIONAL: - # That's OK, just empty *args. Let's start parsing - # kwargs - break - elif param.name in kwargs: - if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None - parameters_ex = (param,) - break - elif (param.kind == _VAR_KEYWORD or - param.default is not _empty): - # That's fine too - we have a default value for this - # parameter. So, lets start parsing `kwargs`, starting - # with the current parameter - parameters_ex = (param,) - break - else: - # No default, not VAR_KEYWORD, not VAR_POSITIONAL, - # not in `kwargs` - if partial: - parameters_ex = (param,) - break - else: - msg = 'missing a required argument: {arg!r}' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None - else: - # We have a positional argument to process - try: - param = next(parameters) - except StopIteration: - raise TypeError('too many positional arguments') from None - else: - if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): - # Looks like we have no parameter for this positional - # argument - raise TypeError( - 'too many positional arguments') from None - - if param.kind == _VAR_POSITIONAL: - # We have an '*args'-like argument, let's fill it with - # all positional arguments we have left and move on to - # the next phase - values = [arg_val] - values.extend(arg_vals) - arguments[param.name] = tuple(values) - break - - if param.name in kwargs: - raise TypeError( - 'multiple values for argument {arg!r}'.format( - arg=param.name)) from None - - arguments[param.name] = arg_val - - # Now, we iterate through the remaining parameters to process - # keyword arguments - kwargs_param = None - for param in itertools.chain(parameters_ex, parameters): - if param.kind == _VAR_KEYWORD: - # Memorize that we have a '**kwargs'-like parameter - kwargs_param = param - continue - - if param.kind == _VAR_POSITIONAL: - # Named arguments don't refer to '*args'-like parameters. - # We only arrive here if the positional arguments ended - # before reaching the last parameter before *args. - continue - - param_name = param.name - try: - arg_val = kwargs.pop(param_name) - except KeyError: - # We have no value for this parameter. It's fine though, - # if it has a default value, or it is an '*args'-like - # parameter, left alone by the processing of positional - # arguments. - if (not partial and param.kind != _VAR_POSITIONAL and - param.default is _empty): - raise TypeError('missing a required argument: {arg!r}'. \ - format(arg=param_name)) from None - - else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - - arguments[param_name] = arg_val - - if kwargs: - if kwargs_param is not None: - # Process our '**kwargs'-like parameter - arguments[kwargs_param.name] = kwargs - else: - raise TypeError( - 'got an unexpected keyword argument {arg!r}'.format( - arg=next(iter(kwargs)))) - - return self._bound_arguments_cls(self, arguments) - - def bind(*args, **kwargs): - """Get a BoundArguments object, that maps the passed `args` - and `kwargs` to the function's signature. Raises `TypeError` - if the passed arguments can not be bound. - """ - return args[0]._bind(args[1:], kwargs) - - def bind_partial(*args, **kwargs): - """Get a BoundArguments object, that partially maps the - passed `args` and `kwargs` to the function's signature. - Raises `TypeError` if the passed arguments can not be bound. - """ - return args[0]._bind(args[1:], kwargs, partial=True) - - def __reduce__(self): - return (type(self), - (tuple(self._parameters.values()),), - {'_return_annotation': self._return_annotation}) - - def __setstate__(self, state): - self._return_annotation = state['_return_annotation'] - - def __repr__(self): - return '<{} {}>'.format(self.__class__.__name__, self) - - def __str__(self): - result = [] - render_pos_only_separator = False - render_kw_only_separator = True - for param in self.parameters.values(): - formatted = str(param) - - kind = param.kind - - if kind == _POSITIONAL_ONLY: - render_pos_only_separator = True - elif render_pos_only_separator: - # It's not a positional-only parameter, and the flag - # is set to 'True' (there were pos-only params before.) - result.append('/') - render_pos_only_separator = False - - if kind == _VAR_POSITIONAL: - # OK, we have an '*args'-like parameter, so we won't need - # a '*' to separate keyword-only arguments - render_kw_only_separator = False - elif kind == _KEYWORD_ONLY and render_kw_only_separator: - # We have a keyword-only parameter to render and we haven't - # rendered an '*args'-like parameter before, so add a '*' - # separator to the parameters list ("foo(arg1, *, arg2)" case) - result.append('*') - # This condition should be only triggered once, so - # reset the flag - render_kw_only_separator = False - - result.append(formatted) - - if render_pos_only_separator: - # There were only positional-only parameters, hence the - # flag was not reset to 'False' - result.append('/') - - rendered = '({})'.format(', '.join(result)) - - if self.return_annotation is not _empty: - anno = formatannotation(self.return_annotation) - rendered += ' -> {}'.format(anno) - - return rendered - - -def signature(obj, *, follow_wrapped=True): - """Get a signature object for the passed callable.""" - return Signature.from_callable(obj, follow_wrapped=follow_wrapped) - - -def _main(): - """ Logic for inspecting an object given at command line """ - import argparse - import importlib - - parser = argparse.ArgumentParser() - parser.add_argument( - 'object', - help="The object to be analysed. " - "It supports the 'module:qualname' syntax") - parser.add_argument( - '-d', '--details', action='store_true', - help='Display info about the module rather than its source code') - - args = parser.parse_args() - - target = args.object - mod_name, has_attrs, attrs = target.partition(":") - try: - obj = module = importlib.import_module(mod_name) - except Exception as exc: - msg = "Failed to import {} ({}: {})".format(mod_name, - type(exc).__name__, - exc) - print(msg, file=sys.stderr) - exit(2) - - if has_attrs: - parts = attrs.split(".") - obj = module - for part in parts: - obj = getattr(obj, part) - - if module.__name__ in sys.builtin_module_names: - print("Can't get info for builtin modules.", file=sys.stderr) - exit(1) - - if args.details: - print('Target: {}'.format(target)) - print('Origin: {}'.format(getsourcefile(module))) - print('Cached: {}'.format(module.__cached__)) - if obj is module: - print('Loader: {}'.format(repr(module.__loader__))) - if hasattr(module, '__path__'): - print('Submodule search path: {}'.format(module.__path__)) - else: - try: - __, lineno = findsource(obj) - except Exception: - pass - else: - print('Line: {}'.format(lineno)) - - print('\n') - else: - print(getsource(obj)) - - -if __name__ == "__main__": - _main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/io.py --- a/Lib/io.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/io.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,7 @@ At the top of the I/O hierarchy is the abstract base class IOBase. It defines the basic interface to a stream. Note, however, that there is no separation between reading and writing to streams; implementations are -allowed to raise an OSError if they do not support a given operation. +allowed to throw an IOError if they do not support a given operation. Extending IOBase is RawIOBase which deals simply with the reading and writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide @@ -19,7 +19,7 @@ Another IOBase subclass, TextIOBase, deals with the encoding and decoding of streams into text. TextIOWrapper, which extends it, is a buffered text interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO -is an in-memory stream for text. +is a in-memory stream for text. Argument names are not part of the specification, and only the arguments of open() are intended to be used as keyword arguments. @@ -34,6 +34,15 @@ """ # New I/O library conforming to PEP 3116. +# XXX edge cases when switching between reading/writing +# XXX need to support 1 meaning line-buffered +# XXX whenever an argument is None, use the default value +# XXX read/write ops should check readable/writable +# XXX buffered readinto should work with arbitrary buffer objects +# XXX use incremental encoder for text output, at least for UTF-16 and UTF-8-SIG +# XXX check writable, readable and seekable in appropriate places + + __author__ = ("Guido van Rossum , " "Mike Verdone , " "Mark Russell , " @@ -58,9 +67,6 @@ OpenWrapper = _io.open # for compatibility with _pyio -# Pretend this exception was created here. -UnsupportedOperation.__module__ = "io" - # for seek() SEEK_SET = 0 SEEK_CUR = 1 @@ -70,16 +76,16 @@ # Method descriptions and default implementations are inherited from the C # version however. class IOBase(_io._IOBase, metaclass=abc.ABCMeta): - __doc__ = _io._IOBase.__doc__ + pass class RawIOBase(_io._RawIOBase, IOBase): - __doc__ = _io._RawIOBase.__doc__ + pass class BufferedIOBase(_io._BufferedIOBase, IOBase): - __doc__ = _io._BufferedIOBase.__doc__ + pass class TextIOBase(_io._TextIOBase, IOBase): - __doc__ = _io._TextIOBase.__doc__ + pass RawIOBase.register(FileIO) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ipaddress.py --- a/Lib/ipaddress.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2290 +0,0 @@ -# Copyright 2007 Google Inc. -# Licensed to PSF under a Contributor Agreement. - -"""A fast, lightweight IPv4/IPv6 manipulation library in Python. - -This library is used to create/poke/manipulate IPv4 and IPv6 addresses -and networks. - -""" - -__version__ = '1.0' - - -import functools - -IPV4LENGTH = 32 -IPV6LENGTH = 128 - -class AddressValueError(ValueError): - """A Value Error related to the address.""" - - -class NetmaskValueError(ValueError): - """A Value Error related to the netmask.""" - - -def ip_address(address): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Address or IPv6Address object. - - Raises: - ValueError: if the *address* passed isn't either a v4 or a v6 - address - - """ - try: - return IPv4Address(address) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Address(address) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError('%r does not appear to be an IPv4 or IPv6 address' % - address) - - -def ip_network(address, strict=True): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP network. Either IPv4 or - IPv6 networks may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Network or IPv6Network object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. Or if the network has host bits set. - - """ - try: - return IPv4Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Network(address, strict) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError('%r does not appear to be an IPv4 or IPv6 network' % - address) - - -def ip_interface(address): - """Take an IP string/int and return an object of the correct type. - - Args: - address: A string or integer, the IP address. Either IPv4 or - IPv6 addresses may be supplied; integers less than 2**32 will - be considered to be IPv4 by default. - - Returns: - An IPv4Interface or IPv6Interface object. - - Raises: - ValueError: if the string passed isn't either a v4 or a v6 - address. - - Notes: - The IPv?Interface classes describe an Address on a particular - Network, so they're basically a combination of both the Address - and Network classes. - - """ - try: - return IPv4Interface(address) - except (AddressValueError, NetmaskValueError): - pass - - try: - return IPv6Interface(address) - except (AddressValueError, NetmaskValueError): - pass - - raise ValueError('%r does not appear to be an IPv4 or IPv6 interface' % - address) - - -def v4_int_to_packed(address): - """Represent an address as 4 packed bytes in network (big-endian) order. - - Args: - address: An integer representation of an IPv4 IP address. - - Returns: - The integer address packed as 4 bytes in network (big-endian) order. - - Raises: - ValueError: If the integer is negative or too large to be an - IPv4 IP address. - - """ - try: - return address.to_bytes(4, 'big') - except OverflowError: - raise ValueError("Address negative or too large for IPv4") - - -def v6_int_to_packed(address): - """Represent an address as 16 packed bytes in network (big-endian) order. - - Args: - address: An integer representation of an IPv6 IP address. - - Returns: - The integer address packed as 16 bytes in network (big-endian) order. - - """ - try: - return address.to_bytes(16, 'big') - except OverflowError: - raise ValueError("Address negative or too large for IPv6") - - -def _split_optional_netmask(address): - """Helper to split the netmask and raise AddressValueError if needed""" - addr = str(address).split('/') - if len(addr) > 2: - raise AddressValueError("Only one '/' permitted in %r" % address) - return addr - - -def _find_address_range(addresses): - """Find a sequence of sorted deduplicated IPv#Address. - - Args: - addresses: a list of IPv#Address objects. - - Yields: - A tuple containing the first and last IP addresses in the sequence. - - """ - it = iter(addresses) - first = last = next(it) - for ip in it: - if ip._ip != last._ip + 1: - yield first, last - first = ip - last = ip - yield first, last - - -def _count_righthand_zero_bits(number, bits): - """Count the number of zero bits on the right hand side. - - Args: - number: an integer. - bits: maximum number of bits to count. - - Returns: - The number of zero bits on the right hand side of the number. - - """ - if number == 0: - return bits - return min(bits, (~number & (number-1)).bit_length()) - - -def summarize_address_range(first, last): - """Summarize a network range given the first and last IP addresses. - - Example: - >>> list(summarize_address_range(IPv4Address('192.0.2.0'), - ... IPv4Address('192.0.2.130'))) - ... #doctest: +NORMALIZE_WHITESPACE - [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'), - IPv4Network('192.0.2.130/32')] - - Args: - first: the first IPv4Address or IPv6Address in the range. - last: the last IPv4Address or IPv6Address in the range. - - Returns: - An iterator of the summarized IPv(4|6) network objects. - - Raise: - TypeError: - If the first and last objects are not IP addresses. - If the first and last objects are not the same version. - ValueError: - If the last object is not greater than the first. - If the version of the first address is not 4 or 6. - - """ - if (not (isinstance(first, _BaseAddress) and - isinstance(last, _BaseAddress))): - raise TypeError('first and last must be IP addresses, not networks') - if first.version != last.version: - raise TypeError("%s and %s are not of the same version" % ( - first, last)) - if first > last: - raise ValueError('last IP address must be greater than first') - - if first.version == 4: - ip = IPv4Network - elif first.version == 6: - ip = IPv6Network - else: - raise ValueError('unknown IP version') - - ip_bits = first._max_prefixlen - first_int = first._ip - last_int = last._ip - while first_int <= last_int: - nbits = min(_count_righthand_zero_bits(first_int, ip_bits), - (last_int - first_int + 1).bit_length() - 1) - net = ip((first_int, ip_bits - nbits)) - yield net - first_int += 1 << nbits - if first_int - 1 == ip._ALL_ONES: - break - - -def _collapse_addresses_internal(addresses): - """Loops through the addresses, collapsing concurrent netblocks. - - Example: - - ip1 = IPv4Network('192.0.2.0/26') - ip2 = IPv4Network('192.0.2.64/26') - ip3 = IPv4Network('192.0.2.128/26') - ip4 = IPv4Network('192.0.2.192/26') - - _collapse_addresses_internal([ip1, ip2, ip3, ip4]) -> - [IPv4Network('192.0.2.0/24')] - - This shouldn't be called directly; it is called via - collapse_addresses([]). - - Args: - addresses: A list of IPv4Network's or IPv6Network's - - Returns: - A list of IPv4Network's or IPv6Network's depending on what we were - passed. - - """ - # First merge - to_merge = list(addresses) - subnets = {} - while to_merge: - net = to_merge.pop() - supernet = net.supernet() - existing = subnets.get(supernet) - if existing is None: - subnets[supernet] = net - elif existing != net: - # Merge consecutive subnets - del subnets[supernet] - to_merge.append(supernet) - # Then iterate over resulting networks, skipping subsumed subnets - last = None - for net in sorted(subnets.values()): - if last is not None: - # Since they are sorted, last.network_address <= net.network_address - # is a given. - if last.broadcast_address >= net.broadcast_address: - continue - yield net - last = net - - -def collapse_addresses(addresses): - """Collapse a list of IP objects. - - Example: - collapse_addresses([IPv4Network('192.0.2.0/25'), - IPv4Network('192.0.2.128/25')]) -> - [IPv4Network('192.0.2.0/24')] - - Args: - addresses: An iterator of IPv4Network or IPv6Network objects. - - Returns: - An iterator of the collapsed IPv(4|6)Network objects. - - Raises: - TypeError: If passed a list of mixed version objects. - - """ - addrs = [] - ips = [] - nets = [] - - # split IP addresses and networks - for ip in addresses: - if isinstance(ip, _BaseAddress): - if ips and ips[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - ip, ips[-1])) - ips.append(ip) - elif ip._prefixlen == ip._max_prefixlen: - if ips and ips[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - ip, ips[-1])) - try: - ips.append(ip.ip) - except AttributeError: - ips.append(ip.network_address) - else: - if nets and nets[-1]._version != ip._version: - raise TypeError("%s and %s are not of the same version" % ( - ip, nets[-1])) - nets.append(ip) - - # sort and dedup - ips = sorted(set(ips)) - - # find consecutive address ranges in the sorted sequence and summarize them - if ips: - for first, last in _find_address_range(ips): - addrs.extend(summarize_address_range(first, last)) - - return _collapse_addresses_internal(addrs + nets) - - -def get_mixed_type_key(obj): - """Return a key suitable for sorting between networks and addresses. - - Address and Network objects are not sortable by default; they're - fundamentally different so the expression - - IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') - - doesn't make any sense. There are some times however, where you may wish - to have ipaddress sort these for you anyway. If you need to do this, you - can use this function as the key= argument to sorted(). - - Args: - obj: either a Network or Address object. - Returns: - appropriate key. - - """ - if isinstance(obj, _BaseNetwork): - return obj._get_networks_key() - elif isinstance(obj, _BaseAddress): - return obj._get_address_key() - return NotImplemented - - -class _IPAddressBase: - - """The mother class.""" - - __slots__ = () - - @property - def exploded(self): - """Return the longhand version of the IP address as a string.""" - return self._explode_shorthand_ip_string() - - @property - def compressed(self): - """Return the shorthand version of the IP address as a string.""" - return str(self) - - @property - def reverse_pointer(self): - """The name of the reverse DNS pointer for the IP address, e.g.: - >>> ipaddress.ip_address("127.0.0.1").reverse_pointer - '1.0.0.127.in-addr.arpa' - >>> ipaddress.ip_address("2001:db8::1").reverse_pointer - '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa' - - """ - return self._reverse_pointer() - - @property - def version(self): - msg = '%200s has no version specified' % (type(self),) - raise NotImplementedError(msg) - - def _check_int_address(self, address): - if address < 0: - msg = "%d (< 0) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._version)) - if address > self._ALL_ONES: - msg = "%d (>= 2**%d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, self._max_prefixlen, - self._version)) - - def _check_packed_address(self, address, expected_len): - address_len = len(address) - if address_len != expected_len: - msg = "%r (len %d != %d) is not permitted as an IPv%d address" - raise AddressValueError(msg % (address, address_len, - expected_len, self._version)) - - @classmethod - def _ip_int_from_prefix(cls, prefixlen): - """Turn the prefix length into a bitwise netmask - - Args: - prefixlen: An integer, the prefix length. - - Returns: - An integer. - - """ - return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen) - - @classmethod - def _prefix_from_ip_int(cls, ip_int): - """Return prefix length from the bitwise netmask. - - Args: - ip_int: An integer, the netmask in expanded bitwise format - - Returns: - An integer, the prefix length. - - Raises: - ValueError: If the input intermingles zeroes & ones - """ - trailing_zeroes = _count_righthand_zero_bits(ip_int, - cls._max_prefixlen) - prefixlen = cls._max_prefixlen - trailing_zeroes - leading_ones = ip_int >> trailing_zeroes - all_ones = (1 << prefixlen) - 1 - if leading_ones != all_ones: - byteslen = cls._max_prefixlen // 8 - details = ip_int.to_bytes(byteslen, 'big') - msg = 'Netmask pattern %r mixes zeroes & ones' - raise ValueError(msg % details) - return prefixlen - - @classmethod - def _report_invalid_netmask(cls, netmask_str): - msg = '%r is not a valid netmask' % netmask_str - raise NetmaskValueError(msg) from None - - @classmethod - def _prefix_from_prefix_string(cls, prefixlen_str): - """Return prefix length from a numeric string - - Args: - prefixlen_str: The string to be converted - - Returns: - An integer, the prefix length. - - Raises: - NetmaskValueError: If the input is not a valid netmask - """ - # int allows a leading +/- as well as surrounding whitespace, - # so we ensure that isn't the case - if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str): - cls._report_invalid_netmask(prefixlen_str) - try: - prefixlen = int(prefixlen_str) - except ValueError: - cls._report_invalid_netmask(prefixlen_str) - if not (0 <= prefixlen <= cls._max_prefixlen): - cls._report_invalid_netmask(prefixlen_str) - return prefixlen - - @classmethod - def _prefix_from_ip_string(cls, ip_str): - """Turn a netmask/hostmask string into a prefix length - - Args: - ip_str: The netmask/hostmask to be converted - - Returns: - An integer, the prefix length. - - Raises: - NetmaskValueError: If the input is not a valid netmask/hostmask - """ - # Parse the netmask/hostmask like an IP address. - try: - ip_int = cls._ip_int_from_string(ip_str) - except AddressValueError: - cls._report_invalid_netmask(ip_str) - - # Try matching a netmask (this would be /1*0*/ as a bitwise regexp). - # Note that the two ambiguous cases (all-ones and all-zeroes) are - # treated as netmasks. - try: - return cls._prefix_from_ip_int(ip_int) - except ValueError: - pass - - # Invert the bits, and try matching a /0+1+/ hostmask instead. - ip_int ^= cls._ALL_ONES - try: - return cls._prefix_from_ip_int(ip_int) - except ValueError: - cls._report_invalid_netmask(ip_str) - - def __reduce__(self): - return self.__class__, (str(self),) - - -@functools.total_ordering -class _BaseAddress(_IPAddressBase): - - """A generic IP object. - - This IP class contains the version independent methods which are - used by single IP addresses. - """ - - __slots__ = () - - def __int__(self): - return self._ip - - def __eq__(self, other): - try: - return (self._ip == other._ip - and self._version == other._version) - except AttributeError: - return NotImplemented - - def __lt__(self, other): - if not isinstance(other, _BaseAddress): - return NotImplemented - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - self, other)) - if self._ip != other._ip: - return self._ip < other._ip - return False - - # Shorthand for Integer addition and subtraction. This is not - # meant to ever support addition/subtraction of addresses. - def __add__(self, other): - if not isinstance(other, int): - return NotImplemented - return self.__class__(int(self) + other) - - def __sub__(self, other): - if not isinstance(other, int): - return NotImplemented - return self.__class__(int(self) - other) - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, str(self)) - - def __str__(self): - return str(self._string_from_ip_int(self._ip)) - - def __hash__(self): - return hash(hex(int(self._ip))) - - def _get_address_key(self): - return (self._version, self) - - def __reduce__(self): - return self.__class__, (self._ip,) - - -@functools.total_ordering -class _BaseNetwork(_IPAddressBase): - - """A generic IP network object. - - This IP class contains the version independent methods which are - used by networks. - - """ - def __init__(self, address): - self._cache = {} - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, str(self)) - - def __str__(self): - return '%s/%d' % (self.network_address, self.prefixlen) - - def hosts(self): - """Generate Iterator over usable hosts in a network. - - This is like __iter__ except it doesn't return the network - or broadcast addresses. - - """ - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in range(network + 1, broadcast): - yield self._address_class(x) - - def __iter__(self): - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in range(network, broadcast + 1): - yield self._address_class(x) - - def __getitem__(self, n): - network = int(self.network_address) - broadcast = int(self.broadcast_address) - if n >= 0: - if network + n > broadcast: - raise IndexError - return self._address_class(network + n) - else: - n += 1 - if broadcast + n < network: - raise IndexError - return self._address_class(broadcast + n) - - def __lt__(self, other): - if not isinstance(other, _BaseNetwork): - return NotImplemented - if self._version != other._version: - raise TypeError('%s and %s are not of the same version' % ( - self, other)) - if self.network_address != other.network_address: - return self.network_address < other.network_address - if self.netmask != other.netmask: - return self.netmask < other.netmask - return False - - def __eq__(self, other): - try: - return (self._version == other._version and - self.network_address == other.network_address and - int(self.netmask) == int(other.netmask)) - except AttributeError: - return NotImplemented - - def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) - - def __contains__(self, other): - # always false if one is v4 and the other is v6. - if self._version != other._version: - return False - # dealing with another network. - if isinstance(other, _BaseNetwork): - return False - # dealing with another address - else: - # address - return (int(self.network_address) <= int(other._ip) <= - int(self.broadcast_address)) - - def overlaps(self, other): - """Tell if self is partly contained in other.""" - return self.network_address in other or ( - self.broadcast_address in other or ( - other.network_address in self or ( - other.broadcast_address in self))) - - @property - def broadcast_address(self): - x = self._cache.get('broadcast_address') - if x is None: - x = self._address_class(int(self.network_address) | - int(self.hostmask)) - self._cache['broadcast_address'] = x - return x - - @property - def hostmask(self): - x = self._cache.get('hostmask') - if x is None: - x = self._address_class(int(self.netmask) ^ self._ALL_ONES) - self._cache['hostmask'] = x - return x - - @property - def with_prefixlen(self): - return '%s/%d' % (self.network_address, self._prefixlen) - - @property - def with_netmask(self): - return '%s/%s' % (self.network_address, self.netmask) - - @property - def with_hostmask(self): - return '%s/%s' % (self.network_address, self.hostmask) - - @property - def num_addresses(self): - """Number of hosts in the current subnet.""" - return int(self.broadcast_address) - int(self.network_address) + 1 - - @property - def _address_class(self): - # Returning bare address objects (rather than interfaces) allows for - # more consistent behaviour across the network address, broadcast - # address and individual host addresses. - msg = '%200s has no associated address class' % (type(self),) - raise NotImplementedError(msg) - - @property - def prefixlen(self): - return self._prefixlen - - def address_exclude(self, other): - """Remove an address from a larger block. - - For example: - - addr1 = ip_network('192.0.2.0/28') - addr2 = ip_network('192.0.2.1/32') - addr1.address_exclude(addr2) = - [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'), - IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')] - - or IPv6: - - addr1 = ip_network('2001:db8::1/32') - addr2 = ip_network('2001:db8::1/128') - addr1.address_exclude(addr2) = - [ip_network('2001:db8::1/128'), - ip_network('2001:db8::2/127'), - ip_network('2001:db8::4/126'), - ip_network('2001:db8::8/125'), - ... - ip_network('2001:db8:8000::/33')] - - Args: - other: An IPv4Network or IPv6Network object of the same type. - - Returns: - An iterator of the IPv(4|6)Network objects which is self - minus other. - - Raises: - TypeError: If self and other are of differing address - versions, or if other is not a network object. - ValueError: If other is not completely contained by self. - - """ - if not self._version == other._version: - raise TypeError("%s and %s are not of the same version" % ( - self, other)) - - if not isinstance(other, _BaseNetwork): - raise TypeError("%s is not a network object" % other) - - if not (other.network_address >= self.network_address and - other.broadcast_address <= self.broadcast_address): - raise ValueError('%s not contained in %s' % (other, self)) - if other == self: - return - - # Make sure we're comparing the network of other. - other = other.__class__('%s/%s' % (other.network_address, - other.prefixlen)) - - s1, s2 = self.subnets() - while s1 != other and s2 != other: - if (other.network_address >= s1.network_address and - other.broadcast_address <= s1.broadcast_address): - yield s2 - s1, s2 = s1.subnets() - elif (other.network_address >= s2.network_address and - other.broadcast_address <= s2.broadcast_address): - yield s1 - s1, s2 = s2.subnets() - else: - # If we got here, there's a bug somewhere. - raise AssertionError('Error performing exclusion: ' - 's1: %s s2: %s other: %s' % - (s1, s2, other)) - if s1 == other: - yield s2 - elif s2 == other: - yield s1 - else: - # If we got here, there's a bug somewhere. - raise AssertionError('Error performing exclusion: ' - 's1: %s s2: %s other: %s' % - (s1, s2, other)) - - def compare_networks(self, other): - """Compare two IP objects. - - This is only concerned about the comparison of the integer - representation of the network addresses. This means that the - host bits aren't considered at all in this method. If you want - to compare host bits, you can easily enough do a - 'HostA._ip < HostB._ip' - - Args: - other: An IP object. - - Returns: - If the IP versions of self and other are the same, returns: - - -1 if self < other: - eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25') - IPv6Network('2001:db8::1000/124') < - IPv6Network('2001:db8::2000/124') - 0 if self == other - eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24') - IPv6Network('2001:db8::1000/124') == - IPv6Network('2001:db8::1000/124') - 1 if self > other - eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25') - IPv6Network('2001:db8::2000/124') > - IPv6Network('2001:db8::1000/124') - - Raises: - TypeError if the IP versions are different. - - """ - # does this need to raise a ValueError? - if self._version != other._version: - raise TypeError('%s and %s are not of the same type' % ( - self, other)) - # self._version == other._version below here: - if self.network_address < other.network_address: - return -1 - if self.network_address > other.network_address: - return 1 - # self.network_address == other.network_address below here: - if self.netmask < other.netmask: - return -1 - if self.netmask > other.netmask: - return 1 - return 0 - - def _get_networks_key(self): - """Network-only key function. - - Returns an object that identifies this address' network and - netmask. This function is a suitable "key" argument for sorted() - and list.sort(). - - """ - return (self._version, self.network_address, self.netmask) - - def subnets(self, prefixlen_diff=1, new_prefix=None): - """The subnets which join to make the current subnet. - - In the case that self contains only one IP - (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 - for IPv6), yield an iterator with just ourself. - - Args: - prefixlen_diff: An integer, the amount the prefix length - should be increased by. This should not be set if - new_prefix is also set. - new_prefix: The desired new prefix length. This must be a - larger number (smaller prefix) than the existing prefix. - This should not be set if prefixlen_diff is also set. - - Returns: - An iterator of IPv(4|6) objects. - - Raises: - ValueError: The prefixlen_diff is too small or too large. - OR - prefixlen_diff and new_prefix are both set or new_prefix - is a smaller number than the current prefix (smaller - number means a larger network) - - """ - if self._prefixlen == self._max_prefixlen: - yield self - return - - if new_prefix is not None: - if new_prefix < self._prefixlen: - raise ValueError('new prefix must be longer') - if prefixlen_diff != 1: - raise ValueError('cannot set prefixlen_diff and new_prefix') - prefixlen_diff = new_prefix - self._prefixlen - - if prefixlen_diff < 0: - raise ValueError('prefix length diff must be > 0') - new_prefixlen = self._prefixlen + prefixlen_diff - - if new_prefixlen > self._max_prefixlen: - raise ValueError( - 'prefix length diff %d is invalid for netblock %s' % ( - new_prefixlen, self)) - - start = int(self.network_address) - end = int(self.broadcast_address) - step = (int(self.hostmask) + 1) >> prefixlen_diff - for new_addr in range(start, end, step): - current = self.__class__((new_addr, new_prefixlen)) - yield current - - def supernet(self, prefixlen_diff=1, new_prefix=None): - """The supernet containing the current network. - - Args: - prefixlen_diff: An integer, the amount the prefix length of - the network should be decreased by. For example, given a - /24 network and a prefixlen_diff of 3, a supernet with a - /21 netmask is returned. - - Returns: - An IPv4 network object. - - Raises: - ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have - a negative prefix length. - OR - If prefixlen_diff and new_prefix are both set or new_prefix is a - larger number than the current prefix (larger number means a - smaller network) - - """ - if self._prefixlen == 0: - return self - - if new_prefix is not None: - if new_prefix > self._prefixlen: - raise ValueError('new prefix must be shorter') - if prefixlen_diff != 1: - raise ValueError('cannot set prefixlen_diff and new_prefix') - prefixlen_diff = self._prefixlen - new_prefix - - new_prefixlen = self.prefixlen - prefixlen_diff - if new_prefixlen < 0: - raise ValueError( - 'current prefixlen is %d, cannot have a prefixlen_diff of %d' % - (self.prefixlen, prefixlen_diff)) - return self.__class__(( - int(self.network_address) & (int(self.netmask) << prefixlen_diff), - new_prefixlen - )) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is a multicast address. - See RFC 2373 2.7 for details. - - """ - return (self.network_address.is_multicast and - self.broadcast_address.is_multicast) - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within one of the - reserved IPv6 Network ranges. - - """ - return (self.network_address.is_reserved and - self.broadcast_address.is_reserved) - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is reserved per RFC 4291. - - """ - return (self.network_address.is_link_local and - self.broadcast_address.is_link_local) - - @property - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry or iana-ipv6-special-registry. - - """ - return (self.network_address.is_private and - self.broadcast_address.is_private) - - @property - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, True if the address is not reserved per - iana-ipv4-special-registry or iana-ipv6-special-registry. - - """ - return not self.is_private - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 2373 2.5.2. - - """ - return (self.network_address.is_unspecified and - self.broadcast_address.is_unspecified) - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback address as defined in - RFC 2373 2.5.3. - - """ - return (self.network_address.is_loopback and - self.broadcast_address.is_loopback) - - -class _BaseV4: - - """Base IPv4 object. - - The following methods are used by IPv4 objects in both single IP - addresses and networks. - - """ - - __slots__ = () - _version = 4 - # Equivalent to 255.255.255.255 or 32 bits of 1's. - _ALL_ONES = (2**IPV4LENGTH) - 1 - _DECIMAL_DIGITS = frozenset('0123456789') - - # the valid octets for host and netmasks. only useful for IPv4. - _valid_mask_octets = frozenset({255, 254, 252, 248, 240, 224, 192, 128, 0}) - - _max_prefixlen = IPV4LENGTH - # There are only a handful of valid v4 netmasks, so we cache them all - # when constructed (see _make_netmask()). - _netmask_cache = {} - - def _explode_shorthand_ip_string(self): - return str(self) - - @classmethod - def _make_netmask(cls, arg): - """Make a (netmask, prefix_len) tuple from the given argument. - - Argument can be: - - an integer (the prefix length) - - a string representing the prefix length (e.g. "24") - - a string representing the prefix netmask (e.g. "255.255.255.0") - """ - if arg not in cls._netmask_cache: - if isinstance(arg, int): - prefixlen = arg - else: - try: - # Check for a netmask in prefix length form - prefixlen = cls._prefix_from_prefix_string(arg) - except NetmaskValueError: - # Check for a netmask or hostmask in dotted-quad form. - # This may raise NetmaskValueError. - prefixlen = cls._prefix_from_ip_string(arg) - netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen)) - cls._netmask_cache[arg] = netmask, prefixlen - return cls._netmask_cache[arg] - - @classmethod - def _ip_int_from_string(cls, ip_str): - """Turn the given IP string into an integer for comparison. - - Args: - ip_str: A string, the IP ip_str. - - Returns: - The IP ip_str as an integer. - - Raises: - AddressValueError: if ip_str isn't a valid IPv4 Address. - - """ - if not ip_str: - raise AddressValueError('Address cannot be empty') - - octets = ip_str.split('.') - if len(octets) != 4: - raise AddressValueError("Expected 4 octets in %r" % ip_str) - - try: - return int.from_bytes(map(cls._parse_octet, octets), 'big') - except ValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) from None - - @classmethod - def _parse_octet(cls, octet_str): - """Convert a decimal octet into an integer. - - Args: - octet_str: A string, the number to parse. - - Returns: - The octet as an integer. - - Raises: - ValueError: if the octet isn't strictly a decimal from [0..255]. - - """ - if not octet_str: - raise ValueError("Empty octet not permitted") - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not cls._DECIMAL_DIGITS.issuperset(octet_str): - msg = "Only decimal digits permitted in %r" - raise ValueError(msg % octet_str) - # We do the length check second, since the invalid character error - # is likely to be more informative for the user - if len(octet_str) > 3: - msg = "At most 3 characters permitted in %r" - raise ValueError(msg % octet_str) - # Convert to integer (we know digits are legal) - octet_int = int(octet_str, 10) - # Any octets that look like they *might* be written in octal, - # and which don't look exactly the same in both octal and - # decimal are rejected as ambiguous - if octet_int > 7 and octet_str[0] == '0': - msg = "Ambiguous (octal/decimal) value in %r not permitted" - raise ValueError(msg % octet_str) - if octet_int > 255: - raise ValueError("Octet %d (> 255) not permitted" % octet_int) - return octet_int - - @classmethod - def _string_from_ip_int(cls, ip_int): - """Turns a 32-bit integer into dotted decimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - The IP address as a string in dotted decimal notation. - - """ - return '.'.join(map(str, ip_int.to_bytes(4, 'big'))) - - def _is_valid_netmask(self, netmask): - """Verify that the netmask is valid. - - Args: - netmask: A string, either a prefix or dotted decimal - netmask. - - Returns: - A boolean, True if the prefix represents a valid IPv4 - netmask. - - """ - mask = netmask.split('.') - if len(mask) == 4: - try: - for x in mask: - if int(x) not in self._valid_mask_octets: - return False - except ValueError: - # Found something that isn't an integer or isn't valid - return False - for idx, y in enumerate(mask): - if idx > 0 and y > mask[idx - 1]: - return False - return True - try: - netmask = int(netmask) - except ValueError: - return False - return 0 <= netmask <= self._max_prefixlen - - def _is_hostmask(self, ip_str): - """Test if the IP string is a hostmask (rather than a netmask). - - Args: - ip_str: A string, the potential hostmask. - - Returns: - A boolean, True if the IP string is a hostmask. - - """ - bits = ip_str.split('.') - try: - parts = [x for x in map(int, bits) if x in self._valid_mask_octets] - except ValueError: - return False - if len(parts) != len(bits): - return False - if parts[0] < parts[-1]: - return True - return False - - def _reverse_pointer(self): - """Return the reverse DNS pointer name for the IPv4 address. - - This implements the method described in RFC1035 3.5. - - """ - reverse_octets = str(self).split('.')[::-1] - return '.'.join(reverse_octets) + '.in-addr.arpa' - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - -class IPv4Address(_BaseV4, _BaseAddress): - - """Represent and manipulate single IPv4 Addresses.""" - - __slots__ = ('_ip', '__weakref__') - - def __init__(self, address): - - """ - Args: - address: A string or integer representing the IP - - Additionally, an integer can be passed, so - IPv4Address('192.0.2.1') == IPv4Address(3221225985). - or, more generally - IPv4Address(int(IPv4Address('192.0.2.1'))) == - IPv4Address('192.0.2.1') - - Raises: - AddressValueError: If ipaddress isn't a valid IPv4 address. - - """ - # Efficient constructor from integer. - if isinstance(address, int): - self._check_int_address(address) - self._ip = address - return - - # Constructing from a packed address - if isinstance(address, bytes): - self._check_packed_address(address, 4) - self._ip = int.from_bytes(address, 'big') - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = str(address) - if '/' in addr_str: - raise AddressValueError("Unexpected '/' in %r" % address) - self._ip = self._ip_int_from_string(addr_str) - - @property - def packed(self): - """The binary representation of this address.""" - return v4_int_to_packed(self._ip) - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within the - reserved IPv4 Network range. - - """ - return self in self._constants._reserved_network - - @property - @functools.lru_cache() - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. - - """ - return any(self in net for net in self._constants._private_networks) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is multicast. - See RFC 3171 for details. - - """ - return self in self._constants._multicast_network - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 5735 3. - - """ - return self == self._constants._unspecified_address - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback per RFC 3330. - - """ - return self in self._constants._loopback_network - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is link-local per RFC 3927. - - """ - return self in self._constants._linklocal_network - - -class IPv4Interface(IPv4Address): - - def __init__(self, address): - if isinstance(address, (bytes, int)): - IPv4Address.__init__(self, address) - self.network = IPv4Network(self._ip) - self._prefixlen = self._max_prefixlen - return - - if isinstance(address, tuple): - IPv4Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - - self.network = IPv4Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return - - addr = _split_optional_netmask(address) - IPv4Address.__init__(self, addr[0]) - - self.network = IPv4Network(address, strict=False) - self._prefixlen = self.network._prefixlen - - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - - def __str__(self): - return '%s/%d' % (self._string_from_ip_int(self._ip), - self.network.prefixlen) - - def __eq__(self, other): - address_equal = IPv4Address.__eq__(self, other) - if not address_equal or address_equal is NotImplemented: - return address_equal - try: - return self.network == other.network - except AttributeError: - # An interface with an associated network is NOT the - # same as an unassociated address. That's why the hash - # takes the extra info into account. - return False - - def __lt__(self, other): - address_less = IPv4Address.__lt__(self, other) - if address_less is NotImplemented: - return NotImplemented - try: - return self.network < other.network - except AttributeError: - # We *do* allow addresses and interfaces to be sorted. The - # unassociated address is considered less than all interfaces. - return False - - def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) - - __reduce__ = _IPAddressBase.__reduce__ - - @property - def ip(self): - return IPv4Address(self._ip) - - @property - def with_prefixlen(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self._prefixlen) - - @property - def with_netmask(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self.netmask) - - @property - def with_hostmask(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self.hostmask) - - -class IPv4Network(_BaseV4, _BaseNetwork): - - """This class represents and manipulates 32-bit IPv4 network + addresses.. - - Attributes: [examples for IPv4Network('192.0.2.0/27')] - .network_address: IPv4Address('192.0.2.0') - .hostmask: IPv4Address('0.0.0.31') - .broadcast_address: IPv4Address('192.0.2.32') - .netmask: IPv4Address('255.255.255.224') - .prefixlen: 27 - - """ - # Class to use when creating address objects - _address_class = IPv4Address - - def __init__(self, address, strict=True): - - """Instantiate a new IPv4 network object. - - Args: - address: A string or integer representing the IP [& network]. - '192.0.2.0/24' - '192.0.2.0/255.255.255.0' - '192.0.0.2/0.0.0.255' - are all functionally the same in IPv4. Similarly, - '192.0.2.1' - '192.0.2.1/255.255.255.255' - '192.0.2.1/32' - are also functionally equivalent. That is to say, failing to - provide a subnetmask will create an object with a mask of /32. - - If the mask (portion after the / in the argument) is given in - dotted quad form, it is treated as a netmask if it starts with a - non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it - starts with a zero field (e.g. 0.255.255.255 == /8), with the - single exception of an all-zero mask which is treated as a - netmask == /0. If no mask is given, a default of /32 is used. - - Additionally, an integer can be passed, so - IPv4Network('192.0.2.1') == IPv4Network(3221225985) - or, more generally - IPv4Interface(int(IPv4Interface('192.0.2.1'))) == - IPv4Interface('192.0.2.1') - - Raises: - AddressValueError: If ipaddress isn't a valid IPv4 address. - NetmaskValueError: If the netmask isn't valid for - an IPv4 address. - ValueError: If strict is True and a network address is not - supplied. - - """ - _BaseNetwork.__init__(self, address) - - # Constructing from a packed address or integer - if isinstance(address, (int, bytes)): - self.network_address = IPv4Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - #fixme: address/network test here. - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - # We weren't given an address[1] - arg = self._max_prefixlen - self.network_address = IPv4Address(address[0]) - self.netmask, self._prefixlen = self._make_netmask(arg) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv4Address(packed & - int(self.netmask)) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv4Address(int(self.network_address) & int(self.netmask)) != - self.network_address): - raise ValueError('%s has host bits set' % self) - self.network_address = IPv4Address(int(self.network_address) & - int(self.netmask)) - - if self._prefixlen == (self._max_prefixlen - 1): - self.hosts = self.__iter__ - - @property - @functools.lru_cache() - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, True if the address is not reserved per - iana-ipv4-special-registry. - - """ - return (not (self.network_address in IPv4Network('100.64.0.0/10') and - self.broadcast_address in IPv4Network('100.64.0.0/10')) and - not self.is_private) - - -class _IPv4Constants: - _linklocal_network = IPv4Network('169.254.0.0/16') - - _loopback_network = IPv4Network('127.0.0.0/8') - - _multicast_network = IPv4Network('224.0.0.0/4') - - _private_networks = [ - IPv4Network('0.0.0.0/8'), - IPv4Network('10.0.0.0/8'), - IPv4Network('127.0.0.0/8'), - IPv4Network('169.254.0.0/16'), - IPv4Network('172.16.0.0/12'), - IPv4Network('192.0.0.0/29'), - IPv4Network('192.0.0.170/31'), - IPv4Network('192.0.2.0/24'), - IPv4Network('192.168.0.0/16'), - IPv4Network('198.18.0.0/15'), - IPv4Network('198.51.100.0/24'), - IPv4Network('203.0.113.0/24'), - IPv4Network('240.0.0.0/4'), - IPv4Network('255.255.255.255/32'), - ] - - _reserved_network = IPv4Network('240.0.0.0/4') - - _unspecified_address = IPv4Address('0.0.0.0') - - -IPv4Address._constants = _IPv4Constants - - -class _BaseV6: - - """Base IPv6 object. - - The following methods are used by IPv6 objects in both single IP - addresses and networks. - - """ - - __slots__ = () - _version = 6 - _ALL_ONES = (2**IPV6LENGTH) - 1 - _HEXTET_COUNT = 8 - _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') - _max_prefixlen = IPV6LENGTH - - # There are only a bunch of valid v6 netmasks, so we cache them all - # when constructed (see _make_netmask()). - _netmask_cache = {} - - @classmethod - def _make_netmask(cls, arg): - """Make a (netmask, prefix_len) tuple from the given argument. - - Argument can be: - - an integer (the prefix length) - - a string representing the prefix length (e.g. "24") - - a string representing the prefix netmask (e.g. "255.255.255.0") - """ - if arg not in cls._netmask_cache: - if isinstance(arg, int): - prefixlen = arg - else: - prefixlen = cls._prefix_from_prefix_string(arg) - netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen)) - cls._netmask_cache[arg] = netmask, prefixlen - return cls._netmask_cache[arg] - - @classmethod - def _ip_int_from_string(cls, ip_str): - """Turn an IPv6 ip_str into an integer. - - Args: - ip_str: A string, the IPv6 ip_str. - - Returns: - An int, the IPv6 address - - Raises: - AddressValueError: if ip_str isn't a valid IPv6 Address. - - """ - if not ip_str: - raise AddressValueError('Address cannot be empty') - - parts = ip_str.split(':') - - # An IPv6 address needs at least 2 colons (3 parts). - _min_parts = 3 - if len(parts) < _min_parts: - msg = "At least %d parts expected in %r" % (_min_parts, ip_str) - raise AddressValueError(msg) - - # If the address has an IPv4-style suffix, convert it to hexadecimal. - if '.' in parts[-1]: - try: - ipv4_int = IPv4Address(parts.pop())._ip - except AddressValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) from None - parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF)) - parts.append('%x' % (ipv4_int & 0xFFFF)) - - # An IPv6 address can't have more than 8 colons (9 parts). - # The extra colon comes from using the "::" notation for a single - # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 - if len(parts) > _max_parts: - msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) - raise AddressValueError(msg) - - # Disregarding the endpoints, find '::' with nothing in between. - # This indicates that a run of zeroes has been skipped. - skip_index = None - for i in range(1, len(parts) - 1): - if not parts[i]: - if skip_index is not None: - # Can't have more than one '::' - msg = "At most one '::' permitted in %r" % ip_str - raise AddressValueError(msg) - skip_index = i - - # parts_hi is the number of parts to copy from above/before the '::' - # parts_lo is the number of parts to copy from below/after the '::' - if skip_index is not None: - # If we found a '::', then check if it also covers the endpoints. - parts_hi = skip_index - parts_lo = len(parts) - skip_index - 1 - if not parts[0]: - parts_hi -= 1 - if parts_hi: - msg = "Leading ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # ^: requires ^:: - if not parts[-1]: - parts_lo -= 1 - if parts_lo: - msg = "Trailing ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # :$ requires ::$ - parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo) - if parts_skipped < 1: - msg = "Expected at most %d other parts with '::' in %r" - raise AddressValueError(msg % (cls._HEXTET_COUNT-1, ip_str)) - else: - # Otherwise, allocate the entire address to parts_hi. The - # endpoints could still be empty, but _parse_hextet() will check - # for that. - if len(parts) != cls._HEXTET_COUNT: - msg = "Exactly %d parts expected without '::' in %r" - raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str)) - if not parts[0]: - msg = "Leading ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # ^: requires ^:: - if not parts[-1]: - msg = "Trailing ':' only permitted as part of '::' in %r" - raise AddressValueError(msg % ip_str) # :$ requires ::$ - parts_hi = len(parts) - parts_lo = 0 - parts_skipped = 0 - - try: - # Now, parse the hextets into a 128-bit integer. - ip_int = 0 - for i in range(parts_hi): - ip_int <<= 16 - ip_int |= cls._parse_hextet(parts[i]) - ip_int <<= 16 * parts_skipped - for i in range(-parts_lo, 0): - ip_int <<= 16 - ip_int |= cls._parse_hextet(parts[i]) - return ip_int - except ValueError as exc: - raise AddressValueError("%s in %r" % (exc, ip_str)) from None - - @classmethod - def _parse_hextet(cls, hextet_str): - """Convert an IPv6 hextet string into an integer. - - Args: - hextet_str: A string, the number to parse. - - Returns: - The hextet as an integer. - - Raises: - ValueError: if the input isn't strictly a hex number from - [0..FFFF]. - - """ - # Whitelist the characters, since int() allows a lot of bizarre stuff. - if not cls._HEX_DIGITS.issuperset(hextet_str): - raise ValueError("Only hex digits permitted in %r" % hextet_str) - # We do the length check second, since the invalid character error - # is likely to be more informative for the user - if len(hextet_str) > 4: - msg = "At most 4 characters permitted in %r" - raise ValueError(msg % hextet_str) - # Length check means we can skip checking the integer value - return int(hextet_str, 16) - - @classmethod - def _compress_hextets(cls, hextets): - """Compresses a list of hextets. - - Compresses a list of strings, replacing the longest continuous - sequence of "0" in the list with "" and adding empty strings at - the beginning or at the end of the string such that subsequently - calling ":".join(hextets) will produce the compressed version of - the IPv6 address. - - Args: - hextets: A list of strings, the hextets to compress. - - Returns: - A list of strings. - - """ - best_doublecolon_start = -1 - best_doublecolon_len = 0 - doublecolon_start = -1 - doublecolon_len = 0 - for index, hextet in enumerate(hextets): - if hextet == '0': - doublecolon_len += 1 - if doublecolon_start == -1: - # Start of a sequence of zeros. - doublecolon_start = index - if doublecolon_len > best_doublecolon_len: - # This is the longest sequence of zeros so far. - best_doublecolon_len = doublecolon_len - best_doublecolon_start = doublecolon_start - else: - doublecolon_len = 0 - doublecolon_start = -1 - - if best_doublecolon_len > 1: - best_doublecolon_end = (best_doublecolon_start + - best_doublecolon_len) - # For zeros at the end of the address. - if best_doublecolon_end == len(hextets): - hextets += [''] - hextets[best_doublecolon_start:best_doublecolon_end] = [''] - # For zeros at the beginning of the address. - if best_doublecolon_start == 0: - hextets = [''] + hextets - - return hextets - - @classmethod - def _string_from_ip_int(cls, ip_int=None): - """Turns a 128-bit integer into hexadecimal notation. - - Args: - ip_int: An integer, the IP address. - - Returns: - A string, the hexadecimal representation of the address. - - Raises: - ValueError: The address is bigger than 128 bits of all ones. - - """ - if ip_int is None: - ip_int = int(cls._ip) - - if ip_int > cls._ALL_ONES: - raise ValueError('IPv6 address is too large') - - hex_str = '%032x' % ip_int - hextets = ['%x' % int(hex_str[x:x+4], 16) for x in range(0, 32, 4)] - - hextets = cls._compress_hextets(hextets) - return ':'.join(hextets) - - def _explode_shorthand_ip_string(self): - """Expand a shortened IPv6 address. - - Args: - ip_str: A string, the IPv6 address. - - Returns: - A string, the expanded IPv6 address. - - """ - if isinstance(self, IPv6Network): - ip_str = str(self.network_address) - elif isinstance(self, IPv6Interface): - ip_str = str(self.ip) - else: - ip_str = str(self) - - ip_int = self._ip_int_from_string(ip_str) - hex_str = '%032x' % ip_int - parts = [hex_str[x:x+4] for x in range(0, 32, 4)] - if isinstance(self, (_BaseNetwork, IPv6Interface)): - return '%s/%d' % (':'.join(parts), self._prefixlen) - return ':'.join(parts) - - def _reverse_pointer(self): - """Return the reverse DNS pointer name for the IPv6 address. - - This implements the method described in RFC3596 2.5. - - """ - reverse_chars = self.exploded[::-1].replace(':', '') - return '.'.join(reverse_chars) + '.ip6.arpa' - - @property - def max_prefixlen(self): - return self._max_prefixlen - - @property - def version(self): - return self._version - - -class IPv6Address(_BaseV6, _BaseAddress): - - """Represent and manipulate single IPv6 Addresses.""" - - __slots__ = ('_ip', '__weakref__') - - def __init__(self, address): - """Instantiate a new IPv6 address object. - - Args: - address: A string or integer representing the IP - - Additionally, an integer can be passed, so - IPv6Address('2001:db8::') == - IPv6Address(42540766411282592856903984951653826560) - or, more generally - IPv6Address(int(IPv6Address('2001:db8::'))) == - IPv6Address('2001:db8::') - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - - """ - # Efficient constructor from integer. - if isinstance(address, int): - self._check_int_address(address) - self._ip = address - return - - # Constructing from a packed address - if isinstance(address, bytes): - self._check_packed_address(address, 16) - self._ip = int.from_bytes(address, 'big') - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP string. - addr_str = str(address) - if '/' in addr_str: - raise AddressValueError("Unexpected '/' in %r" % address) - self._ip = self._ip_int_from_string(addr_str) - - @property - def packed(self): - """The binary representation of this address.""" - return v6_int_to_packed(self._ip) - - @property - def is_multicast(self): - """Test if the address is reserved for multicast use. - - Returns: - A boolean, True if the address is a multicast address. - See RFC 2373 2.7 for details. - - """ - return self in self._constants._multicast_network - - @property - def is_reserved(self): - """Test if the address is otherwise IETF reserved. - - Returns: - A boolean, True if the address is within one of the - reserved IPv6 Network ranges. - - """ - return any(self in x for x in self._constants._reserved_networks) - - @property - def is_link_local(self): - """Test if the address is reserved for link-local. - - Returns: - A boolean, True if the address is reserved per RFC 4291. - - """ - return self in self._constants._linklocal_network - - @property - def is_site_local(self): - """Test if the address is reserved for site-local. - - Note that the site-local address space has been deprecated by RFC 3879. - Use is_private to test if this address is in the space of unique local - addresses as defined by RFC 4193. - - Returns: - A boolean, True if the address is reserved per RFC 3513 2.5.6. - - """ - return self in self._constants._sitelocal_network - - @property - @functools.lru_cache() - def is_private(self): - """Test if this address is allocated for private networks. - - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry. - - """ - return any(self in net for net in self._constants._private_networks) - - @property - def is_global(self): - """Test if this address is allocated for public networks. - - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. - - """ - return not self.is_private - - @property - def is_unspecified(self): - """Test if the address is unspecified. - - Returns: - A boolean, True if this is the unspecified address as defined in - RFC 2373 2.5.2. - - """ - return self._ip == 0 - - @property - def is_loopback(self): - """Test if the address is a loopback address. - - Returns: - A boolean, True if the address is a loopback address as defined in - RFC 2373 2.5.3. - - """ - return self._ip == 1 - - @property - def ipv4_mapped(self): - """Return the IPv4 mapped address. - - Returns: - If the IPv6 address is a v4 mapped address, return the - IPv4 mapped address. Return None otherwise. - - """ - if (self._ip >> 32) != 0xFFFF: - return None - return IPv4Address(self._ip & 0xFFFFFFFF) - - @property - def teredo(self): - """Tuple of embedded teredo IPs. - - Returns: - Tuple of the (server, client) IPs or None if the address - doesn't appear to be a teredo address (doesn't start with - 2001::/32) - - """ - if (self._ip >> 96) != 0x20010000: - return None - return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF), - IPv4Address(~self._ip & 0xFFFFFFFF)) - - @property - def sixtofour(self): - """Return the IPv4 6to4 embedded address. - - Returns: - The IPv4 6to4-embedded address if present or None if the - address doesn't appear to contain a 6to4 embedded address. - - """ - if (self._ip >> 112) != 0x2002: - return None - return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) - - -class IPv6Interface(IPv6Address): - - def __init__(self, address): - if isinstance(address, (bytes, int)): - IPv6Address.__init__(self, address) - self.network = IPv6Network(self._ip) - self._prefixlen = self._max_prefixlen - return - if isinstance(address, tuple): - IPv6Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - self.network = IPv6Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return - - addr = _split_optional_netmask(address) - IPv6Address.__init__(self, addr[0]) - self.network = IPv6Network(address, strict=False) - self.netmask = self.network.netmask - self._prefixlen = self.network._prefixlen - self.hostmask = self.network.hostmask - - def __str__(self): - return '%s/%d' % (self._string_from_ip_int(self._ip), - self.network.prefixlen) - - def __eq__(self, other): - address_equal = IPv6Address.__eq__(self, other) - if not address_equal or address_equal is NotImplemented: - return address_equal - try: - return self.network == other.network - except AttributeError: - # An interface with an associated network is NOT the - # same as an unassociated address. That's why the hash - # takes the extra info into account. - return False - - def __lt__(self, other): - address_less = IPv6Address.__lt__(self, other) - if address_less is NotImplemented: - return NotImplemented - try: - return self.network < other.network - except AttributeError: - # We *do* allow addresses and interfaces to be sorted. The - # unassociated address is considered less than all interfaces. - return False - - def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) - - __reduce__ = _IPAddressBase.__reduce__ - - @property - def ip(self): - return IPv6Address(self._ip) - - @property - def with_prefixlen(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self._prefixlen) - - @property - def with_netmask(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self.netmask) - - @property - def with_hostmask(self): - return '%s/%s' % (self._string_from_ip_int(self._ip), - self.hostmask) - - @property - def is_unspecified(self): - return self._ip == 0 and self.network.is_unspecified - - @property - def is_loopback(self): - return self._ip == 1 and self.network.is_loopback - - -class IPv6Network(_BaseV6, _BaseNetwork): - - """This class represents and manipulates 128-bit IPv6 networks. - - Attributes: [examples for IPv6('2001:db8::1000/124')] - .network_address: IPv6Address('2001:db8::1000') - .hostmask: IPv6Address('::f') - .broadcast_address: IPv6Address('2001:db8::100f') - .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0') - .prefixlen: 124 - - """ - - # Class to use when creating address objects - _address_class = IPv6Address - - def __init__(self, address, strict=True): - """Instantiate a new IPv6 Network object. - - Args: - address: A string or integer representing the IPv6 network or the - IP and prefix/netmask. - '2001:db8::/128' - '2001:db8:0000:0000:0000:0000:0000:0000/128' - '2001:db8::' - are all functionally the same in IPv6. That is to say, - failing to provide a subnetmask will create an object with - a mask of /128. - - Additionally, an integer can be passed, so - IPv6Network('2001:db8::') == - IPv6Network(42540766411282592856903984951653826560) - or, more generally - IPv6Network(int(IPv6Network('2001:db8::'))) == - IPv6Network('2001:db8::') - - strict: A boolean. If true, ensure that we have been passed - A true network address, eg, 2001:db8::1000/124 and not an - IP address on a network, eg, 2001:db8::1/124. - - Raises: - AddressValueError: If address isn't a valid IPv6 address. - NetmaskValueError: If the netmask isn't valid for - an IPv6 address. - ValueError: If strict was True and a network address was not - supplied. - - """ - _BaseNetwork.__init__(self, address) - - # Efficient constructor from integer or packed address - if isinstance(address, (bytes, int)): - self.network_address = IPv6Address(address) - self.netmask, self._prefixlen = self._make_netmask(self._max_prefixlen) - return - - if isinstance(address, tuple): - if len(address) > 1: - arg = address[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - self.network_address = IPv6Address(address[0]) - packed = int(self.network_address) - if packed & int(self.netmask) != packed: - if strict: - raise ValueError('%s has host bits set' % self) - else: - self.network_address = IPv6Address(packed & - int(self.netmask)) - return - - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - addr = _split_optional_netmask(address) - - self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) - - if len(addr) == 2: - arg = addr[1] - else: - arg = self._max_prefixlen - self.netmask, self._prefixlen = self._make_netmask(arg) - - if strict: - if (IPv6Address(int(self.network_address) & int(self.netmask)) != - self.network_address): - raise ValueError('%s has host bits set' % self) - self.network_address = IPv6Address(int(self.network_address) & - int(self.netmask)) - - if self._prefixlen == (self._max_prefixlen - 1): - self.hosts = self.__iter__ - - def hosts(self): - """Generate Iterator over usable hosts in a network. - - This is like __iter__ except it doesn't return the - Subnet-Router anycast address. - - """ - network = int(self.network_address) - broadcast = int(self.broadcast_address) - for x in range(network + 1, broadcast + 1): - yield self._address_class(x) - - @property - def is_site_local(self): - """Test if the address is reserved for site-local. - - Note that the site-local address space has been deprecated by RFC 3879. - Use is_private to test if this address is in the space of unique local - addresses as defined by RFC 4193. - - Returns: - A boolean, True if the address is reserved per RFC 3513 2.5.6. - - """ - return (self.network_address.is_site_local and - self.broadcast_address.is_site_local) - - -class _IPv6Constants: - - _linklocal_network = IPv6Network('fe80::/10') - - _multicast_network = IPv6Network('ff00::/8') - - _private_networks = [ - IPv6Network('::1/128'), - IPv6Network('::/128'), - IPv6Network('::ffff:0:0/96'), - IPv6Network('100::/64'), - IPv6Network('2001::/23'), - IPv6Network('2001:2::/48'), - IPv6Network('2001:db8::/32'), - IPv6Network('2001:10::/28'), - IPv6Network('fc00::/7'), - IPv6Network('fe80::/10'), - ] - - _reserved_networks = [ - IPv6Network('::/8'), IPv6Network('100::/8'), - IPv6Network('200::/7'), IPv6Network('400::/6'), - IPv6Network('800::/5'), IPv6Network('1000::/4'), - IPv6Network('4000::/3'), IPv6Network('6000::/3'), - IPv6Network('8000::/3'), IPv6Network('A000::/3'), - IPv6Network('C000::/3'), IPv6Network('E000::/4'), - IPv6Network('F000::/5'), IPv6Network('F800::/6'), - IPv6Network('FE00::/9'), - ] - - _sitelocal_network = IPv6Network('fec0::/10') - - -IPv6Address._constants = _IPv6Constants diff -r 6db40a9955dc -r 0d413f60cc23 Lib/json/__init__.py --- a/Lib/json/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/json/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -3,8 +3,11 @@ interchange format. :mod:`json` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is derived from a -version of the externally maintained simplejson library. +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. Encoding basic Python object hierarchies:: @@ -36,7 +39,8 @@ Pretty printing:: >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + >>> print('\n'.join([l.rstrip() for l in s.splitlines()])) { "4": 5, "6": 7 @@ -93,17 +97,17 @@ "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + Expecting property name: line 1 column 2 (char 2) """ __version__ = '2.0.9' __all__ = [ 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'JSONDecoder', 'JSONEncoder', ] __author__ = 'Bob Ippolito ' -from .decoder import JSONDecoder, JSONDecodeError +from .decoder import JSONDecoder from .encoder import JSONEncoder _default_encoder = JSONEncoder( @@ -118,7 +122,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, - default=None, sort_keys=False, **kw): + default=None, **kw): """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). @@ -144,17 +148,13 @@ level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. - If *sort_keys* is ``True`` (default: ``False``), then the output of - dictionaries will be sorted by key. - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. @@ -164,7 +164,7 @@ if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and - default is None and not sort_keys and not kw): + default is None and not kw): iterable = _default_encoder.iterencode(obj) else: if cls is None: @@ -172,7 +172,7 @@ iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, - default=default, sort_keys=sort_keys, **kw).iterencode(obj) + default=default, **kw).iterencode(obj) # could accelerate with writelines in some versions of Python, at # a debuggability cost for chunk in iterable: @@ -181,10 +181,10 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, - default=None, sort_keys=False, **kw): + default=None, **kw): """Serialize ``obj`` to a JSON formatted ``str``. - If ``skipkeys`` is true then ``dict`` keys that are not basic types + If ``skipkeys`` is false then ``dict`` keys that are not basic types (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. @@ -206,17 +206,13 @@ level of 0 will only insert newlines. ``None`` is the most compact representation. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. - If *sort_keys* is ``True`` (default: ``False``), then the output of - dictionaries will be sorted by key. - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg; otherwise ``JSONEncoder`` is used. @@ -226,14 +222,14 @@ if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and - default is None and not sort_keys and not kw): + default is None and not kw): return _default_encoder.encode(obj) if cls is None: cls = JSONEncoder return cls( skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, default=default, sort_keys=sort_keys, + separators=separators, default=default, **kw).encode(obj) @@ -307,12 +303,6 @@ The ``encoding`` argument is ignored and deprecated. """ - if not isinstance(s, str): - raise TypeError('the JSON object must be str, not {!r}'.format( - s.__class__.__name__)) - if s.startswith(u'\ufeff'): - raise JSONDecodeError("Unexpected UTF-8 BOM (decode using utf-8-sig)", - s, 0) if (cls is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/json/decoder.py --- a/Lib/json/decoder.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/json/decoder.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,6 +1,9 @@ """Implementation of JSONDecoder """ +import binascii import re +import sys +import struct from json import scanner try: @@ -8,39 +11,46 @@ except ImportError: c_scanstring = None -__all__ = ['JSONDecoder', 'JSONDecodeError'] +__all__ = ['JSONDecoder'] FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL -NaN = float('nan') -PosInf = float('inf') -NegInf = float('-inf') +def _floatconstants(): + _BYTES = binascii.unhexlify(b'7FF80000000000007FF0000000000000') + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf +NaN, PosInf, NegInf = _floatconstants() -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - lineno: The line corresponding to pos - colno: The column corresponding to pos +def linecol(doc, pos): + if isinstance(doc, bytes): + newline = b'\n' + else: + newline = '\n' + lineno = doc.count(newline, 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex(newline, 0, pos) + return lineno, colno - """ - # Note that this exception is used from _json - def __init__(self, msg, doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - colno = pos - doc.rfind('\n', 0, pos) - errmsg = '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos) - ValueError.__init__(self, errmsg) - self.msg = msg - self.doc = doc - self.pos = pos - self.lineno = lineno - self.colno = colno - def __reduce__(self): - return self.__class__, (self.msg, self.doc, self.pos) +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _json + lineno, colno = linecol(doc, pos) + if end is None: + fmt = '{0}: line {1} column {2} (char {3})' + return fmt.format(msg, lineno, colno, pos) + #fmt = '%s: line %d column %d (char %d)' + #return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + #fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + #return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) _CONSTANTS = { @@ -56,16 +66,6 @@ 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': - try: - return int(esc, 16) - except ValueError: - pass - msg = "Invalid \\uXXXX escape" - raise JSONDecodeError(msg, s, pos) - def py_scanstring(s, end, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): """Scan the string s for a JSON string. End is the index of the @@ -82,7 +82,8 @@ while 1: chunk = _m(s, end) if chunk is None: - raise JSONDecodeError("Unterminated string starting at", s, begin) + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) end = chunk.end() content, terminator = chunk.groups() # Content is contains zero or more unescaped string characters @@ -96,31 +97,43 @@ if strict: #msg = "Invalid control character %r at" % (terminator,) msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) + raise ValueError(errmsg(msg, s, end)) else: _append(terminator) continue try: esc = s[end] except IndexError: - raise JSONDecodeError("Unterminated string starting at", s, begin) + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) # If not a unicode escape sequence, must be in the lookup table if esc != 'u': try: char = _b[esc] except KeyError: msg = "Invalid \\escape: {0!r}".format(esc) - raise JSONDecodeError(msg, s, end) + raise ValueError(errmsg(msg, s, end)) end += 1 else: - uni = _decode_uXXXX(s, end) - end += 5 - if 0xd800 <= uni <= 0xdbff and s[end:end + 2] == '\\u': - uni2 = _decode_uXXXX(s, end + 1) - if 0xdc00 <= uni2 <= 0xdfff: - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - end += 6 + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise ValueError(errmsg(msg, s, end)) + uni = int(esc, 16) + if 0xd800 <= uni <= 0xdbff: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise ValueError(errmsg(msg, s, end)) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise ValueError(errmsg(msg, s, end)) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 char = chr(uni) + + end = next_end _append(char) return ''.join(chunks), end @@ -153,14 +166,13 @@ if nextchar == '}': if object_pairs_hook is not None: result = object_pairs_hook(pairs) - return result, end + 1 + return result, end pairs = {} if object_hook is not None: pairs = object_hook(pairs) return pairs, end + 1 elif nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", s, end) + raise ValueError(errmsg("Expecting property name", s, end)) end += 1 while True: key, end = scanstring(s, end, strict) @@ -170,7 +182,7 @@ if s[end:end + 1] != ':': end = _w(s, end).end() if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting ':' delimiter", s, end) + raise ValueError(errmsg("Expecting : delimiter", s, end)) end += 1 try: @@ -183,8 +195,8 @@ try: value, end = scan_once(s, end) - except StopIteration as err: - raise JSONDecodeError("Expecting value", s, err.value) from None + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) pairs_append((key, value)) try: nextchar = s[end] @@ -198,13 +210,12 @@ if nextchar == '}': break elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': - raise JSONDecodeError( - "Expecting property name enclosed in double quotes", s, end - 1) + raise ValueError(errmsg("Expecting property name", s, end - 1)) if object_pairs_hook is not None: result = object_pairs_hook(pairs) return result, end @@ -227,8 +238,8 @@ while True: try: value, end = scan_once(s, end) - except StopIteration as err: - raise JSONDecodeError("Expecting value", s, err.value) from None + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) _append(value) nextchar = s[end:end + 1] if nextchar in _ws: @@ -238,7 +249,7 @@ if nextchar == ']': break elif nextchar != ',': - raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + raise ValueError(errmsg("Expecting , delimiter", s, end)) try: if s[end] in _ws: end += 1 @@ -339,7 +350,7 @@ obj, end = self.raw_decode(s, idx=_w(s, 0).end()) end = _w(s, end).end() if end != len(s): - raise JSONDecodeError("Extra data", s, end) + raise ValueError(errmsg("Extra data", s, end, len(s))) return obj def raw_decode(self, s, idx=0): @@ -353,6 +364,6 @@ """ try: obj, end = self.scan_once(s, idx) - except StopIteration as err: - raise JSONDecodeError("Expecting value", s, err.value) from None + except StopIteration: + raise ValueError("No JSON object could be decoded") return obj, end diff -r 6db40a9955dc -r 0d413f60cc23 Lib/json/encoder.py --- a/Lib/json/encoder.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/json/encoder.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,10 +7,6 @@ except ImportError: c_encode_basestring_ascii = None try: - from _json import encode_basestring as c_encode_basestring -except ImportError: - c_encode_basestring = None -try: from _json import make_encoder as c_make_encoder except ImportError: c_make_encoder = None @@ -31,10 +27,11 @@ ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) -INFINITY = float('inf') +# Assume this produces an infinity on all machines (probably not guaranteed) +INFINITY = float('1e66666') FLOAT_REPR = repr -def py_encode_basestring(s): +def encode_basestring(s): """Return a JSON representation of a Python string """ @@ -43,9 +40,6 @@ return '"' + ESCAPE.sub(replace, s) + '"' -encode_basestring = (c_encode_basestring or py_encode_basestring) - - def py_encode_basestring_ascii(s): """Return an ASCII-only JSON representation of a Python string @@ -134,10 +128,9 @@ indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. If specified, default is a function that gets called for objects that can't otherwise be serialized. It should return a JSON encodable @@ -153,8 +146,6 @@ self.indent = indent if separators is not None: self.item_separator, self.key_separator = separators - elif indent is not None: - self.item_separator = ',' if default is not None: self.default = default @@ -173,7 +164,6 @@ pass else: return list(iterable) - # Let the base class default method raise the TypeError return JSONEncoder.default(self, o) """ @@ -182,7 +172,6 @@ def encode(self, o): """Return a JSON string representation of a Python data structure. - >>> from json.encoder import JSONEncoder >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) '{"foo": ["bar", "baz"]}' @@ -306,13 +295,9 @@ elif value is False: yield buf + 'false' elif isinstance(value, int): - # Subclasses of int/float may override __str__, but we still - # want to encode them as integers/floats in JSON. One example - # within the standard library is IntEnum. - yield buf + str(int(value)) + yield buf + str(value) elif isinstance(value, float): - # see comment above for int - yield buf + _floatstr(float(value)) + yield buf + _floatstr(value) else: yield buf if isinstance(value, (list, tuple)): @@ -321,7 +306,8 @@ chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) - yield from chunks + for chunk in chunks: + yield chunk if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -358,8 +344,7 @@ # JavaScript is weakly typed for these, so it makes sense to # also allow them. Many encoders seem to do something like this. elif isinstance(key, float): - # see comment for int/float in _make_iterencode - key = _floatstr(float(key)) + key = _floatstr(key) elif key is True: key = 'true' elif key is False: @@ -367,8 +352,7 @@ elif key is None: key = 'null' elif isinstance(key, int): - # see comment for int/float in _make_iterencode - key = str(int(key)) + key = str(key) elif _skipkeys: continue else: @@ -388,11 +372,9 @@ elif value is False: yield 'false' elif isinstance(value, int): - # see comment for int/float in _make_iterencode - yield str(int(value)) + yield str(value) elif isinstance(value, float): - # see comment for int/float in _make_iterencode - yield _floatstr(float(value)) + yield _floatstr(value) else: if isinstance(value, (list, tuple)): chunks = _iterencode_list(value, _current_indent_level) @@ -400,7 +382,8 @@ chunks = _iterencode_dict(value, _current_indent_level) else: chunks = _iterencode(value, _current_indent_level) - yield from chunks + for chunk in chunks: + yield chunk if newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level @@ -418,15 +401,15 @@ elif o is False: yield 'false' elif isinstance(o, int): - # see comment for int/float in _make_iterencode - yield str(int(o)) + yield str(o) elif isinstance(o, float): - # see comment for int/float in _make_iterencode - yield _floatstr(float(o)) + yield _floatstr(o) elif isinstance(o, (list, tuple)): - yield from _iterencode_list(o, _current_indent_level) + for chunk in _iterencode_list(o, _current_indent_level): + yield chunk elif isinstance(o, dict): - yield from _iterencode_dict(o, _current_indent_level) + for chunk in _iterencode_dict(o, _current_indent_level): + yield chunk else: if markers is not None: markerid = id(o) @@ -434,7 +417,8 @@ raise ValueError("Circular reference detected") markers[markerid] = o o = _default(o) - yield from _iterencode(o, _current_indent_level) + for chunk in _iterencode(o, _current_indent_level): + yield chunk if markers is not None: del markers[markerid] return _iterencode diff -r 6db40a9955dc -r 0d413f60cc23 Lib/json/scanner.py --- a/Lib/json/scanner.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/json/scanner.py Mon Jan 25 17:05:13 2016 +0100 @@ -29,7 +29,7 @@ try: nextchar = string[idx] except IndexError: - raise StopIteration(idx) + raise StopIteration if nextchar == '"': return parse_string(string, idx + 1, strict) @@ -60,7 +60,7 @@ elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': return parse_constant('-Infinity'), idx + 9 else: - raise StopIteration(idx) + raise StopIteration def scan_once(string, idx): try: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/json/tool.py --- a/Lib/json/tool.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/json/tool.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,43 +7,30 @@ "json": "obj" } $ echo '{ 1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 3 (char 2) + Expecting property name: line 1 column 2 (char 2) """ -import argparse -import collections +import sys import json -import sys - def main(): - prog = 'python -m json.tool' - description = ('A simple command line interface for json module ' - 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(prog=prog, description=description) - parser.add_argument('infile', nargs='?', type=argparse.FileType(), - help='a JSON file to be validated or pretty-printed') - parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), - help='write the output of infile to outfile') - parser.add_argument('--sort-keys', action='store_true', default=False, - help='sort the output of dictionaries alphabetically by key') - options = parser.parse_args() - - infile = options.infile or sys.stdin - outfile = options.outfile or sys.stdout - sort_keys = options.sort_keys - with infile: - try: - if sort_keys: - obj = json.load(infile) - else: - obj = json.load(infile, - object_pairs_hook=collections.OrderedDict) - except ValueError as e: - raise SystemExit(e) - with outfile: - json.dump(obj, outfile, sort_keys=sort_keys, indent=4) - outfile.write('\n') + if len(sys.argv) == 1: + infile = sys.stdin + outfile = sys.stdout + elif len(sys.argv) == 2: + infile = open(sys.argv[1], 'rb') + outfile = sys.stdout + elif len(sys.argv) == 3: + infile = open(sys.argv[1], 'rb') + outfile = open(sys.argv[2], 'wb') + else: + raise SystemExit(sys.argv[0] + " [infile [outfile]]") + try: + obj = json.load(infile) + except ValueError as e: + raise SystemExit(e) + json.dump(obj, outfile, sort_keys=True, indent=4) + outfile.write('\n') if __name__ == '__main__': diff -r 6db40a9955dc -r 0d413f60cc23 Lib/keyword.py --- a/Lib/keyword.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/keyword.py Mon Jan 25 17:05:13 2016 +0100 @@ -60,12 +60,6 @@ if len(args) > 1: optfile = args[1] else: optfile = "Lib/keyword.py" - # load the output skeleton from the target, taking care to preserve its - # newline convention. - with open(optfile, newline='') as fp: - format = fp.readlines() - nl = format[0][len(format[0].strip()):] if format else '\n' - # scan the source file for keywords with open(iptfile) as fp: strprog = re.compile('"([^"]+)"') @@ -74,21 +68,26 @@ if '{1, "' in line: match = strprog.search(line) if match: - lines.append(" '" + match.group(1) + "'," + nl) + lines.append(" '" + match.group(1) + "',\n") lines.sort() - # insert the lines of keywords into the skeleton + # load the output skeleton from the target + with open(optfile) as fp: + format = fp.readlines() + + # insert the lines of keywords try: - start = format.index("#--start keywords--" + nl) + 1 - end = format.index("#--end keywords--" + nl) + start = format.index("#--start keywords--\n") + 1 + end = format.index("#--end keywords--\n") format[start:end] = lines except ValueError: sys.stderr.write("target does not contain format markers\n") sys.exit(1) # write the output file - with open(optfile, 'w', newline='') as fp: - fp.writelines(format) + fp = open(optfile, 'w') + fp.write(''.join(format)) + fp.close() if __name__ == "__main__": main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/Grammar.txt --- a/Lib/lib2to3/Grammar.txt Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/Grammar.txt Mon Jan 25 17:05:13 2016 +0100 @@ -33,8 +33,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) -async_funcdef: ASYNC funcdef +decorated: decorators (classdef | funcdef) funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' typedargslist: ((tfpdef ['=' test] ',')* @@ -57,7 +56,7 @@ expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | +augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal assignments, additional restrictions enforced by the interpreter print_stmt: 'print' ( [ test (',' test)* [','] ] | @@ -83,8 +82,7 @@ exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] while_stmt: 'while' test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] @@ -121,9 +119,9 @@ and_expr: shift_expr ('&' shift_expr)* shift_expr: arith_expr (('<<'|'>>') arith_expr)* arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* +term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power -power: [AWAIT] atom trailer* ['**' factor] +power: atom trailer* ['**' factor] atom: ('(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' | '{' [dictsetmaker] '}' | @@ -144,7 +142,7 @@ classdef: 'class' NAME ['(' [arglist] ')'] ':' suite arglist: (argument ',')* (argument [','] - |'*' test (',' argument)* [',' '**' test] + |'*' test (',' argument)* [',' '**' test] |'**' test) argument: test [comp_for] | test '=' test # Really [keyword '='] test @@ -157,5 +155,4 @@ # not used in grammar, but may appear in "node" passed from Parser to Compiler encoding_decl: NAME -yield_expr: 'yield' [yield_arg] -yield_arg: 'from' test | testlist +yield_expr: 'yield' [testlist] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/btm_utils.py --- a/Lib/lib2to3/btm_utils.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/btm_utils.py Mon Jan 25 17:05:13 2016 +0100 @@ -96,7 +96,8 @@ def leaves(self): "Generator that returns the leaves of the tree" for child in self.children: - yield from child.leaves() + for x in child.leaves(): + yield x if not self.children: yield self @@ -276,6 +277,7 @@ sub-iterables""" for x in sequence: if isinstance(x, (list, tuple)): - yield from rec_test(x, test_func) + for y in rec_test(x, test_func): + yield y else: yield test_func(x) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixer_util.py --- a/Lib/lib2to3/fixer_util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixer_util.py Mon Jan 25 17:05:13 2016 +0100 @@ -129,29 +129,6 @@ imp = Node(syms.import_from, children) return imp -def ImportAndCall(node, results, names): - """Returns an import statement and calls a method - of the module: - - import module - module.name()""" - obj = results["obj"].clone() - if obj.type == syms.arglist: - newarglist = obj.clone() - else: - newarglist = Node(syms.arglist, [obj.clone()]) - after = results["after"] - if after: - after = [n.clone() for n in after] - new = Node(syms.power, - Attr(Name(names[0]), Name(names[1])) + - [Node(syms.trailer, - [results["lpar"].clone(), - newarglist, - results["rpar"].clone()])] + after) - new.prefix = node.prefix - return new - ########################################################### ### Determine whether a node represents a given literal @@ -187,8 +164,8 @@ return Node(syms.atom, [LParen(), node, RParen()]) -consuming_calls = {"sorted", "list", "set", "any", "all", "tuple", "sum", - "min", "max", "enumerate"} +consuming_calls = set(["sorted", "list", "set", "any", "all", "tuple", "sum", + "min", "max"]) def attr_chain(obj, attr): """Follow an attribute chain. @@ -215,14 +192,14 @@ p1 = """ power< ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' | - 'any' | 'all' | 'enumerate' | (any* trailer< '.' 'join' >) ) + 'any' | 'all' | (any* trailer< '.' 'join' >) ) trailer< '(' node=any ')' > any* > """ p2 = """ power< - ( 'sorted' | 'enumerate' ) + 'sorted' trailer< '(' arglist ')' > any* > @@ -230,14 +207,14 @@ pats_built = False def in_special_context(node): """ Returns true if node is in an environment where all that is required - of it is being iterable (ie, it doesn't matter if it returns a list - or an iterator). + of it is being itterable (ie, it doesn't matter if it returns a list + or an itterator). See test_map_nochange in test_fixers.py for some examples and tests. """ global p0, p1, p2, pats_built if not pats_built: + p1 = patcomp.compile_pattern(p1) p0 = patcomp.compile_pattern(p0) - p1 = patcomp.compile_pattern(p1) p2 = patcomp.compile_pattern(p2) pats_built = True patterns = [p0, p1, p2] @@ -297,9 +274,9 @@ """Find the top level namespace.""" # Scamper up to the top level namespace while node.type != syms.file_input: + assert node.parent, "Tree is insane! root found before "\ + "file_input node was found." node = node.parent - if not node: - raise ValueError("root found before file_input node was found.") return node def does_tree_import(package, name, node): @@ -359,7 +336,7 @@ root.insert_child(insert_pos, Node(syms.simple_stmt, children)) -_def_syms = {syms.classdef, syms.funcdef} +_def_syms = set([syms.classdef, syms.funcdef]) def find_binding(name, node, package=None): """ Returns the node which binds variable name, otherwise None. If optional argument package is supplied, only imports will @@ -402,7 +379,7 @@ return ret return None -_block_syms = {syms.funcdef, syms.classdef, syms.trailer} +_block_syms = set([syms.funcdef, syms.classdef, syms.trailer]) def _find(name, node): nodes = [node] while nodes: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_asserts.py --- a/Lib/lib2to3/fixes/fix_asserts.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,34 +0,0 @@ -"""Fixer that replaces deprecated unittest method names.""" - -# Author: Ezio Melotti - -from ..fixer_base import BaseFix -from ..fixer_util import Name - -NAMES = dict( - assert_="assertTrue", - assertEquals="assertEqual", - assertNotEquals="assertNotEqual", - assertAlmostEquals="assertAlmostEqual", - assertNotAlmostEquals="assertNotAlmostEqual", - assertRegexpMatches="assertRegex", - assertRaisesRegexp="assertRaisesRegex", - failUnlessEqual="assertEqual", - failIfEqual="assertNotEqual", - failUnlessAlmostEqual="assertAlmostEqual", - failIfAlmostEqual="assertNotAlmostEqual", - failUnless="assertTrue", - failUnlessRaises="assertRaises", - failIf="assertFalse", -) - - -class FixAsserts(BaseFix): - - PATTERN = """ - power< any+ trailer< '.' meth=(%s)> any* > - """ % '|'.join(map(repr, NAMES)) - - def transform(self, node, results): - name = results["meth"][0] - name.replace(Name(NAMES[str(name)], prefix=name.prefix)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_dict.py --- a/Lib/lib2to3/fixes/fix_dict.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_dict.py Mon Jan 25 17:05:13 2016 +0100 @@ -36,7 +36,7 @@ from .. import fixer_util -iter_exempt = fixer_util.consuming_calls | {"iter"} +iter_exempt = fixer_util.consuming_calls | set(["iter"]) class FixDict(fixer_base.BaseFix): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_exitfunc.py --- a/Lib/lib2to3/fixes/fix_exitfunc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_exitfunc.py Mon Jan 25 17:05:13 2016 +0100 @@ -35,7 +35,7 @@ self.sys_import = None def transform(self, node, results): - # First, find the sys import. We'll just hope it's global scope. + # First, find a the sys import. We'll just hope it's global scope. if "sys_import" in results: if self.sys_import is None: self.sys_import = results["sys_import"] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_import.py --- a/Lib/lib2to3/fixes/fix_import.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_import.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,7 +32,7 @@ elif node.type == syms.dotted_as_names: pending.extend(node.children[::-2]) else: - raise AssertionError("unknown node type") + raise AssertionError("unkown node type") class FixImport(fixer_base.BaseFix): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_input.py --- a/Lib/lib2to3/fixes/fix_input.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_input.py Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ """ def transform(self, node, results): - # If we're already wrapped in an eval() call, we're done. + # If we're already wrapped in a eval() call, we're done. if context.match(node.parent.parent): return diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_intern.py --- a/Lib/lib2to3/fixes/fix_intern.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_intern.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,8 +6,9 @@ intern(s) -> sys.intern(s)""" # Local imports +from .. import pytree from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import +from ..fixer_util import Name, Attr, touch_import class FixIntern(fixer_base.BaseFix): @@ -25,7 +26,21 @@ """ def transform(self, node, results): - names = ('sys', 'intern') - new = ImportAndCall(node, results, names) + syms = self.syms + obj = results["obj"].clone() + if obj.type == syms.arglist: + newarglist = obj.clone() + else: + newarglist = pytree.Node(syms.arglist, [obj.clone()]) + after = results["after"] + if after: + after = [n.clone() for n in after] + new = pytree.Node(syms.power, + Attr(Name("sys"), Name("intern")) + + [pytree.Node(syms.trailer, + [results["lpar"].clone(), + newarglist, + results["rpar"].clone()])] + after) + new.prefix = node.prefix touch_import(None, 'sys', node) return new diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_itertools.py --- a/Lib/lib2to3/fixes/fix_itertools.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_itertools.py Mon Jan 25 17:05:13 2016 +0100 @@ -34,8 +34,8 @@ # Remove the 'itertools' prefix = it.prefix it.remove() - # Replace the node which contains ('.', 'function') with the - # function (to be consistent with the second part of the pattern) + # Replace the node wich contains ('.', 'function') with the + # function (to be consistant with the second part of the pattern) dot.remove() func.parent.replace(func) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_metaclass.py --- a/Lib/lib2to3/fixes/fix_metaclass.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_metaclass.py Mon Jan 25 17:05:13 2016 +0100 @@ -71,7 +71,7 @@ def fixup_simple_stmt(parent, i, stmt_node): """ if there is a semi-colon all the parts count as part of the same simple_stmt. We just want the __metaclass__ part so we move - everything after the semi-colon into its own simple_stmt node + everything efter the semi-colon into its own simple_stmt node """ for semi_ind, node in enumerate(stmt_node.children): if node.type == token.SEMI: # *sigh* @@ -114,7 +114,7 @@ left_node = expr_node.children[0] if isinstance(left_node, Leaf) and \ left_node.value == '__metaclass__': - # We found an assignment to __metaclass__. + # We found a assignment to __metaclass__. fixup_simple_stmt(node, i, simple_node) remove_trailing_newline(simple_node) yield (node, i, simple_node) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_reload.py --- a/Lib/lib2to3/fixes/fix_reload.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -"""Fixer for reload(). - -reload(s) -> imp.reload(s)""" - -# Local imports -from .. import fixer_base -from ..fixer_util import ImportAndCall, touch_import - - -class FixReload(fixer_base.BaseFix): - BM_compatible = True - order = "pre" - - PATTERN = """ - power< 'reload' - trailer< lpar='(' - ( not(arglist | argument) any ','> ) - rpar=')' > - after=any* - > - """ - - def transform(self, node, results): - names = ('imp', 'reload') - new = ImportAndCall(node, results, names) - touch_import(None, 'imp', node) - return new diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_types.py --- a/Lib/lib2to3/fixes/fix_types.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_types.py Mon Jan 25 17:05:13 2016 +0100 @@ -42,7 +42,7 @@ 'NotImplementedType' : 'type(NotImplemented)', 'SliceType' : 'slice', 'StringType': 'bytes', # XXX ? - 'StringTypes' : '(str,)', # XXX ? + 'StringTypes' : 'str', # XXX ? 'TupleType': 'tuple', 'TypeType' : 'type', 'UnicodeType': 'str', diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/fixes/fix_unicode.py --- a/Lib/lib2to3/fixes/fix_unicode.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/fixes/fix_unicode.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,42 +1,25 @@ -r"""Fixer for unicode. - -* Changes unicode to str and unichr to chr. - -* If "...\u..." is not unicode literal change it into "...\\u...". - -* Change u"..." into "...". +"""Fixer that changes unicode to str, unichr to chr, and u"..." into "...". """ +import re from ..pgen2 import token from .. import fixer_base _mapping = {"unichr" : "chr", "unicode" : "str"} +_literal_re = re.compile(r"[uU][rR]?[\'\"]") class FixUnicode(fixer_base.BaseFix): BM_compatible = True PATTERN = "STRING | 'unicode' | 'unichr'" - def start_tree(self, tree, filename): - super(FixUnicode, self).start_tree(tree, filename) - self.unicode_literals = 'unicode_literals' in tree.future_features - def transform(self, node, results): if node.type == token.NAME: new = node.clone() new.value = _mapping[node.value] return new elif node.type == token.STRING: - val = node.value - if not self.unicode_literals and val[0] in '\'"' and '\\' in val: - val = r'\\'.join([ - v.replace('\\u', r'\\u').replace('\\U', r'\\U') - for v in val.split(r'\\') - ]) - if val[0] in 'uU': - val = val[1:] - if val == node.value: - return node - new = node.clone() - new.value = val - return new + if _literal_re.match(node.value): + new = node.clone() + new.value = new.value[1:] + return new diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/main.py --- a/Lib/lib2to3/main.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/main.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,7 +2,7 @@ Main program for 2to3. """ -from __future__ import with_statement, print_function +from __future__ import with_statement import sys import os @@ -90,11 +90,11 @@ if os.path.lexists(backup): try: os.remove(backup) - except OSError as err: + except os.error as err: self.log_message("Can't remove backup %s", backup) try: os.rename(filename, backup) - except OSError as err: + except os.error as err: self.log_message("Can't rename %s to %s", filename, backup) # Actually write the new file write = super(StdoutRefactoringTool, self).write_file diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/patcomp.py --- a/Lib/lib2to3/patcomp.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/patcomp.py Mon Jan 25 17:05:13 2016 +0100 @@ -32,7 +32,7 @@ def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" - skip = {token.NEWLINE, token.INDENT, token.DEDENT} + skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) tokens = tokenize.generate_tokens(io.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pgen2/conv.py --- a/Lib/lib2to3/pgen2/conv.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pgen2/conv.py Mon Jan 25 17:05:13 2016 +0100 @@ -60,7 +60,7 @@ """ try: f = open(filename) - except OSError as err: + except IOError as err: print("Can't open %s: %s" % (filename, err)) return False self.symbol2number = {} @@ -111,7 +111,7 @@ """ try: f = open(filename) - except OSError as err: + except IOError as err: print("Can't open %s: %s" % (filename, err)) return False # The code below essentially uses f's iterator-ness! diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pgen2/driver.py --- a/Lib/lib2to3/pgen2/driver.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pgen2/driver.py Mon Jan 25 17:05:13 2016 +0100 @@ -123,7 +123,7 @@ logger.info("Writing grammar tables to %s", gp) try: g.dump(gp) - except OSError as e: + except IOError as e: logger.info("Writing failed:"+str(e)) else: g = grammar.Grammar() @@ -138,20 +138,3 @@ if not os.path.exists(b): return True return os.path.getmtime(a) >= os.path.getmtime(b) - - -def main(*args): - """Main program, when run as a script: produce grammar pickle files. - - Calls load_grammar for each argument, a path to a grammar text file. - """ - if not args: - args = sys.argv[1:] - logging.basicConfig(level=logging.INFO, stream=sys.stdout, - format='%(message)s') - for gt in args: - load_grammar(gt, save=True, force=True) - return True - -if __name__ == "__main__": - sys.exit(int(not main())) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pgen2/grammar.py --- a/Lib/lib2to3/pgen2/grammar.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pgen2/grammar.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,7 +20,7 @@ class Grammar(object): - """Pgen parsing tables conversion class. + """Pgen parsing tables tables conversion class. Once initialized, this class supplies the grammar tables for the parsing engine implemented by parse.py. The parsing engine @@ -45,7 +45,7 @@ these two are each other's inverse. states -- a list of DFAs, where each DFA is a list of - states, each state is a list of arcs, and each + states, each state is is a list of arcs, and each arc is a (i, j) pair where i is a label and j is a state number. The DFA number is the index into this list. (This name is slightly confusing.) @@ -86,13 +86,15 @@ def dump(self, filename): """Dump the grammar tables to a pickle file.""" - with open(filename, "wb") as f: - pickle.dump(self.__dict__, f, 2) + f = open(filename, "wb") + pickle.dump(self.__dict__, f, 2) + f.close() def load(self, filename): """Load the grammar tables from a pickle file.""" - with open(filename, "rb") as f: - d = pickle.load(f) + f = open(filename, "rb") + d = pickle.load(f) + f.close() self.__dict__.update(d) def copy(self): @@ -149,7 +151,6 @@ { LBRACE } RBRACE @ AT -@= ATEQUAL == EQEQUAL != NOTEQUAL <> NOTEQUAL diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pgen2/token.py --- a/Lib/lib2to3/pgen2/token.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pgen2/token.py Mon Jan 25 17:05:13 2016 +0100 @@ -57,15 +57,12 @@ DOUBLESLASH = 48 DOUBLESLASHEQUAL = 49 AT = 50 -ATEQUAL = 51 -OP = 52 -COMMENT = 53 -NL = 54 -RARROW = 55 -AWAIT = 56 -ASYNC = 57 -ERRORTOKEN = 58 -N_TOKENS = 59 +OP = 51 +COMMENT = 52 +NL = 53 +RARROW = 54 +ERRORTOKEN = 55 +N_TOKENS = 56 NT_OFFSET = 256 #--end constants-- diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pgen2/tokenize.py --- a/Lib/lib2to3/pgen2/tokenize.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pgen2/tokenize.py Mon Jan 25 17:05:13 2016 +0100 @@ -84,7 +84,7 @@ # recognized as two instances of =). Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", r"//=?", r"->", - r"[+\-*/%&@|^=<>]=?", + r"[+\-*/%&|^=<>]=?", r"~") Bracket = '[][(){}]' @@ -220,7 +220,7 @@ for tok in iterable: toknum, tokval = tok[:2] - if toknum in (NAME, NUMBER, ASYNC, AWAIT): + if toknum in (NAME, NUMBER): tokval += ' ' if toknum == INDENT: @@ -236,8 +236,7 @@ startline = False toks_append(tokval) -cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII) -blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) +cookie_re = re.compile("coding[:=]\s*([-\w.]+)") def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -253,7 +252,7 @@ def detect_encoding(readline): """ The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argument, readline, + be used to decode a Python source file. It requires one argment, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used @@ -282,10 +281,11 @@ line_string = line.decode('ascii') except UnicodeDecodeError: return None - match = cookie_re.match(line_string) - if not match: + + matches = cookie_re.findall(line_string) + if not matches: return None - encoding = _get_normal_name(match.group(1)) + encoding = _get_normal_name(matches[0]) try: codec = lookup(encoding) except LookupError: @@ -310,8 +310,6 @@ encoding = find_cookie(first) if encoding: return encoding, [first] - if not blank_re.match(first): - return default, [first] second = read_or_stop() if not second: @@ -346,7 +344,7 @@ def generate_tokens(readline): """ - The generate_tokens() generator requires one argument, readline, which + The generate_tokens() generator requires one argment, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as a string. Alternately, readline @@ -366,12 +364,6 @@ contline = None indents = [0] - # 'stashed' and 'async_*' are used for async/await parsing - stashed = None - async_def = False - async_def_indent = 0 - async_def_nl = False - while 1: # loop over lines in stream try: line = readline() @@ -412,10 +404,6 @@ pos = pos + 1 if pos == max: break - if stashed: - yield stashed - stashed = None - if line[pos] in '#\r\n': # skip comments or blank lines if line[pos] == '#': comment_token = line[pos:].rstrip('\r\n') @@ -438,19 +426,8 @@ "unindent does not match any outer indentation level", ("", lnum, pos, line)) indents = indents[:-1] - - if async_def and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - yield (DEDENT, '', (lnum, pos), (lnum, pos), line) - if async_def and async_def_nl and async_def_indent >= indents[-1]: - async_def = False - async_def_nl = False - async_def_indent = 0 - else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (lnum, 0)) @@ -470,18 +447,9 @@ newline = NEWLINE if parenlev > 0: newline = NL - elif async_def: - async_def_nl = True - if stashed: - yield stashed - stashed = None yield (newline, token, spos, epos, line) - elif initial == '#': assert not token.endswith("\n") - if stashed: - yield stashed - stashed = None yield (COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] @@ -489,9 +457,6 @@ if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] - if stashed: - yield stashed - stashed = None yield (STRING, token, spos, (lnum, pos), line) else: strstart = (lnum, start) # multiple lines @@ -509,63 +474,22 @@ contline = line break else: # ordinary string - if stashed: - yield stashed - stashed = None yield (STRING, token, spos, epos, line) elif initial in namechars: # ordinary name - if token in ('async', 'await'): - if async_def: - yield (ASYNC if token == 'async' else AWAIT, - token, spos, epos, line) - continue - - tok = (NAME, token, spos, epos, line) - if token == 'async' and not stashed: - stashed = tok - continue - - if token == 'def': - if (stashed - and stashed[0] == NAME - and stashed[1] == 'async'): - - async_def = True - async_def_indent = indents[-1] - - yield (ASYNC, stashed[1], - stashed[2], stashed[3], - stashed[4]) - stashed = None - - if stashed: - yield stashed - stashed = None - - yield tok + yield (NAME, token, spos, epos, line) elif initial == '\\': # continued stmt # This yield is new; needed for better idempotency: - if stashed: - yield stashed - stashed = None yield (NL, token, spos, (lnum, pos), line) continued = 1 else: if initial in '([{': parenlev = parenlev + 1 elif initial in ')]}': parenlev = parenlev - 1 - if stashed: - yield stashed - stashed = None yield (OP, token, spos, epos, line) else: yield (ERRORTOKEN, line[pos], (lnum, pos), (lnum, pos+1), line) pos = pos + 1 - if stashed: - yield stashed - stashed = None - for indent in indents[1:]: # pop remaining indent levels yield (DEDENT, '', (lnum, 0), (lnum, 0), '') yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/pytree.py --- a/Lib/lib2to3/pytree.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/pytree.py Mon Jan 25 17:05:13 2016 +0100 @@ -64,6 +64,16 @@ __hash__ = None # For Py3 compatibility. + def __ne__(self, other): + """ + Compare two nodes for inequality. + + This calls the method _eq(). + """ + if self.__class__ is not other.__class__: + return NotImplemented + return not self._eq(other) + def _eq(self, other): """ Compare two nodes for equality. @@ -184,7 +194,8 @@ def leaves(self): for child in self.children: - yield from child.leaves() + for x in child.leaves(): + yield x def depth(self): if self.parent is None: @@ -263,14 +274,16 @@ def post_order(self): """Return a post-order iterator for the tree.""" for child in self.children: - yield from child.post_order() + for node in child.post_order(): + yield node yield self def pre_order(self): """Return a pre-order iterator for the tree.""" yield self for child in self.children: - yield from child.pre_order() + for node in child.pre_order(): + yield node def _prefix_getter(self): """ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/refactor.py --- a/Lib/lib2to3/refactor.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/refactor.py Mon Jan 25 17:05:13 2016 +0100 @@ -57,7 +57,7 @@ # Always return leafs if pat.type is None: raise _EveryNode - return {pat.type} + return set([pat.type]) if isinstance(pat, pytree.NegatedPattern): if pat.content: @@ -133,7 +133,7 @@ def advance(): tok = next(gen) return tok[0], tok[1] - ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) + ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT)) features = set() try: while True: @@ -255,7 +255,7 @@ fixer = fix_class(self.options, self.fixer_log) if fixer.explicit and self.explicit is not True and \ fix_mod_path not in self.explicit: - self.log_message("Skipping optional fixer: %s", fix_name) + self.log_message("Skipping implicit fixer: %s", fix_name) continue self.log_debug("Adding transformation: %s", fix_name) @@ -326,7 +326,7 @@ """ try: f = open(filename, "rb") - except OSError as err: + except IOError as err: self.log_error("Can't open %s: %s", filename, err) return None, None try: @@ -445,7 +445,7 @@ try: find_root(node) - except ValueError: + except AssertionError: # this node has been cut off from a # previous transformation ; skip continue @@ -534,12 +534,12 @@ """ try: f = _open_with_encoding(filename, "w", encoding=encoding) - except OSError as err: + except os.error as err: self.log_error("Can't create %s: %s", filename, err) return try: f.write(_to_system_newlines(new_text)) - except OSError as err: + except os.error as err: self.log_error("Can't write %s: %s", filename, err) finally: f.close() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/__init__.py --- a/Lib/lib2to3/tests/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,24 @@ +"""Make tests/ into a package. This allows us to "import tests" and +have tests.all_tests be a TestSuite representing all test cases +from all test_*.py files in tests/.""" # Author: Collin Winter import os +import os.path import unittest +import types -from test.support import load_package_tests +from . import support -def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) +all_tests = unittest.TestSuite() + +tests_dir = os.path.join(os.path.dirname(__file__), '..', 'tests') +tests = [t[0:-3] for t in os.listdir(tests_dir) + if t.startswith('test_') and t.endswith('.py')] + +loader = unittest.TestLoader() + +for t in tests: + __import__("",globals(),locals(),[t],level=1) + mod = globals()[t] + all_tests.addTests(loader.loadTestsFromModule(mod)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/__main__.py --- a/Lib/lib2to3/tests/__main__.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,4 +0,0 @@ -from . import load_tests -import unittest - -unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/data/false_encoding.py --- a/Lib/lib2to3/tests/data/false_encoding.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,2 +0,0 @@ -#!/usr/bin/env python -print '#coding=0' diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/pytree_idempotency.py --- a/Lib/lib2to3/tests/pytree_idempotency.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/pytree_idempotency.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,8 +4,6 @@ """Main program for testing the infrastructure.""" -from __future__ import print_function - __author__ = "Guido van Rossum " # Support imports (need to be imported first) @@ -55,7 +53,7 @@ for dir in sys.path: try: names = os.listdir(dir) - except OSError: + except os.error: continue print("Scanning", dir, "...", file=sys.stderr) for name in names: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_all_fixers.py --- a/Lib/lib2to3/tests/test_all_fixers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_all_fixers.py Mon Jan 25 17:05:13 2016 +0100 @@ -7,14 +7,12 @@ # Python imports import unittest -import test.support # Local imports from lib2to3 import refactor from . import support -@test.support.requires_resource('cpu') class Test_all(support.TestCase): def setUp(self): @@ -23,6 +21,3 @@ def test_all_project_files(self): for filepath in support.all_project_files(): self.refactor.refactor_file(filepath) - -if __name__ == '__main__': - unittest.main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_fixers.py --- a/Lib/lib2to3/tests/test_fixers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_fixers.py Mon Jan 25 17:05:13 2016 +0100 @@ -41,7 +41,7 @@ def warns(self, before, after, message, unchanged=False): tree = self._check(before, after) - self.assertIn(message, "".join(self.fixer_log)) + self.assertTrue(message in "".join(self.fixer_log)) if not unchanged: self.assertTrue(tree.was_changed) @@ -282,65 +282,6 @@ b = """f(*args, **kwds)""" self.check(a, b) -class Test_reload(FixerTestCase): - fixer = "reload" - - def test(self): - b = """reload(a)""" - a = """import imp\nimp.reload(a)""" - self.check(b, a) - - def test_comment(self): - b = """reload( a ) # comment""" - a = """import imp\nimp.reload( a ) # comment""" - self.check(b, a) - - # PEP 8 comments - b = """reload( a ) # comment""" - a = """import imp\nimp.reload( a ) # comment""" - self.check(b, a) - - def test_space(self): - b = """reload( a )""" - a = """import imp\nimp.reload( a )""" - self.check(b, a) - - b = """reload( a)""" - a = """import imp\nimp.reload( a)""" - self.check(b, a) - - b = """reload(a )""" - a = """import imp\nimp.reload(a )""" - self.check(b, a) - - def test_unchanged(self): - s = """reload(a=1)""" - self.unchanged(s) - - s = """reload(f, g)""" - self.unchanged(s) - - s = """reload(f, *h)""" - self.unchanged(s) - - s = """reload(f, *h, **i)""" - self.unchanged(s) - - s = """reload(f, **i)""" - self.unchanged(s) - - s = """reload(*h, **i)""" - self.unchanged(s) - - s = """reload(*h)""" - self.unchanged(s) - - s = """reload(**i)""" - self.unchanged(s) - - s = """reload()""" - self.unchanged(s) - class Test_intern(FixerTestCase): fixer = "intern" @@ -1464,27 +1405,27 @@ a = "d.values()" self.check(b, a) - def test_28(self): + def test_14(self): b = "[i for i in d.viewkeys()]" a = "[i for i in d.keys()]" self.check(b, a) - def test_29(self): + def test_15(self): b = "(i for i in d.viewkeys())" a = "(i for i in d.keys())" self.check(b, a) - def test_30(self): + def test_17(self): b = "iter(d.viewkeys())" a = "iter(d.keys())" self.check(b, a) - def test_31(self): + def test_18(self): b = "list(d.viewkeys())" a = "list(d.keys())" self.check(b, a) - def test_32(self): + def test_19(self): b = "sorted(d.viewkeys())" a = "sorted(d.keys())" self.check(b, a) @@ -2883,43 +2824,6 @@ a = """R'''x''' """ self.check(b, a) - def test_native_literal_escape_u(self): - b = r"""'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\\u20ac\\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_bytes_literal_escape_u(self): - b = r"""b'\\\u20ac\U0001d121\\u20ac'""" - a = r"""b'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""br'\\\u20ac\U0001d121\\u20ac'""" - a = r"""br'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_unicode_literal_escape_u(self): - b = r"""u'\\\u20ac\U0001d121\\u20ac'""" - a = r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = r"""ur'\\\u20ac\U0001d121\\u20ac'""" - a = r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - def test_native_unicode_literal_escape_u(self): - f = 'from __future__ import unicode_literals\n' - b = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - - b = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - a = f + r"""r'\\\u20ac\U0001d121\\u20ac'""" - self.check(b, a) - class Test_callable(FixerTestCase): fixer = "callable" @@ -3077,10 +2981,6 @@ self.unchanged(a) a = """sorted(filter(f, 'abc'), key=blah)[0]""" self.unchanged(a) - a = """enumerate(filter(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(filter(f, 'abc'), start=1)""" - self.unchanged(a) a = """for i in filter(f, 'abc'): pass""" self.unchanged(a) a = """[x for x in filter(f, 'abc')]""" @@ -3189,10 +3089,6 @@ self.unchanged(a) a = """sorted(map(f, 'abc'), key=blah)[0]""" self.unchanged(a) - a = """enumerate(map(f, 'abc'))""" - self.unchanged(a) - a = """enumerate(map(f, 'abc'), start=1)""" - self.unchanged(a) a = """for i in map(f, 'abc'): pass""" self.unchanged(a) a = """[x for x in map(f, 'abc')]""" @@ -3256,10 +3152,6 @@ self.unchanged(a) a = """sorted(zip(a, b), key=blah)[0]""" self.unchanged(a) - a = """enumerate(zip(a, b))""" - self.unchanged(a) - a = """enumerate(zip(a, b), start=1)""" - self.unchanged(a) a = """for i in zip(a, b): pass""" self.unchanged(a) a = """[x for x in zip(a, b)]""" @@ -3322,10 +3214,6 @@ a = """type(None)""" self.check(b, a) - b = "types.StringTypes" - a = "(str,)" - self.check(b, a) - class Test_idioms(FixerTestCase): fixer = "idioms" @@ -4639,53 +4527,3 @@ def test_unchanged(self): s = """f(sys.exitfunc)""" self.unchanged(s) - - -class Test_asserts(FixerTestCase): - - fixer = "asserts" - - def test_deprecated_names(self): - tests = [ - ('self.assert_(True)', 'self.assertTrue(True)'), - ('self.assertEquals(2, 2)', 'self.assertEqual(2, 2)'), - ('self.assertNotEquals(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.assertAlmostEquals(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.assertNotAlmostEquals(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnlessEqual(2, 2)', 'self.assertEqual(2, 2)'), - ('self.failIfEqual(2, 3)', 'self.assertNotEqual(2, 3)'), - ('self.failUnlessAlmostEqual(2, 3)', 'self.assertAlmostEqual(2, 3)'), - ('self.failIfAlmostEqual(2, 8)', 'self.assertNotAlmostEqual(2, 8)'), - ('self.failUnless(True)', 'self.assertTrue(True)'), - ('self.failUnlessRaises(foo)', 'self.assertRaises(foo)'), - ('self.failIf(False)', 'self.assertFalse(False)'), - ] - for b, a in tests: - self.check(b, a) - - def test_variants(self): - b = 'eq = self.assertEquals' - a = 'eq = self.assertEqual' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail")' - a = 'self.assertEqual(2, 3, msg="fail")' - self.check(b, a) - b = 'self.assertEquals(2, 3, msg="fail") # foo' - a = 'self.assertEqual(2, 3, msg="fail") # foo' - self.check(b, a) - b = 'self.assertEquals (2, 3)' - a = 'self.assertEqual (2, 3)' - self.check(b, a) - b = ' self.assertEquals (2, 3)' - a = ' self.assertEqual (2, 3)' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion): explode()' - a = 'with self.assertRaises(Explosion): explode()' - self.check(b, a) - b = 'with self.failUnlessRaises(Explosion) as cm: explode()' - a = 'with self.assertRaises(Explosion) as cm: explode()' - self.check(b, a) - - def test_unchanged(self): - self.unchanged('self.assertEqualsOnSaturday') - self.unchanged('self.assertEqualsOnSaturday(3, 5)') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_main.py --- a/Lib/lib2to3/tests/test_main.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_main.py Mon Jan 25 17:05:13 2016 +0100 @@ -49,9 +49,9 @@ ret = self.run_2to3_capture(["-"], input_stream, out_enc, err) self.assertEqual(ret, 0) output = out.getvalue().decode("ascii") - self.assertIn("-print 'nothing'", output) - self.assertIn("WARNING: couldn't encode 's diff for " - "your terminal", err.getvalue()) + self.assertTrue("-print 'nothing'" in output) + self.assertTrue("WARNING: couldn't encode 's diff for " + "your terminal" in err.getvalue()) def setup_test_source_trees(self): """Setup a test source tree and output destination tree.""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_parser.py --- a/Lib/lib2to3/tests/test_parser.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_parser.py Mon Jan 25 17:05:13 2016 +0100 @@ -48,71 +48,6 @@ raise AssertionError("Syntax shouldn't have been valid") -class TestMatrixMultiplication(GrammarTest): - def test_matrix_multiplication_operator(self): - self.validate("a @ b") - self.validate("a @= b") - - -class TestYieldFrom(GrammarTest): - def test_yield_from(self): - self.validate("yield from x") - self.validate("(yield from x) + y") - self.invalid_syntax("yield from") - - -class TestAsyncAwait(GrammarTest): - def test_await_expr(self): - self.validate("""async def foo(): - await x - """) - - self.validate("""async def foo(): - - def foo(): pass - - def foo(): pass - - await x - """) - - self.validate("""async def foo(): return await a""") - - self.validate("""def foo(): - def foo(): pass - async def foo(): await x - """) - - self.invalid_syntax("await x") - self.invalid_syntax("""def foo(): - await x""") - - self.invalid_syntax("""def foo(): - def foo(): pass - async def foo(): pass - await x - """) - - def test_async_var(self): - self.validate("""async = 1""") - self.validate("""await = 1""") - self.validate("""def async(): pass""") - - def test_async_with(self): - self.validate("""async def foo(): - async for a in b: pass""") - - self.invalid_syntax("""def foo(): - async for a in b: pass""") - - def test_async_for(self): - self.validate("""async def foo(): - async with a: pass""") - - self.invalid_syntax("""def foo(): - async with a: pass""") - - class TestRaiseChanges(GrammarTest): def test_2x_style_1(self): self.validate("raise") @@ -142,7 +77,7 @@ self.invalid_syntax("raise E from") -# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef +# Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): def test_1(self): self.validate("""def f(x) -> list: pass""") @@ -233,8 +168,8 @@ for filepath in support.all_project_files(): with open(filepath, "rb") as fp: encoding = tokenize.detect_encoding(fp.readline)[0] - self.assertIsNotNone(encoding, - "can't detect encoding for %s" % filepath) + self.assertTrue(encoding is not None, + "can't detect encoding for %s" % filepath) with open(filepath, "r", encoding=encoding) as fp: source = fp.read() try: diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_pytree.py --- a/Lib/lib2to3/tests/test_pytree.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_pytree.py Mon Jan 25 17:05:13 2016 +0100 @@ -143,12 +143,12 @@ l3 = pytree.Leaf(100, "bar") n1 = pytree.Node(1000, [l1, l2, l3]) self.assertEqual(n1.children, [l1, l2, l3]) - self.assertIsInstance(n1.children, list) + self.assertTrue(isinstance(n1.children, list)) self.assertFalse(n1.was_changed) l2new = pytree.Leaf(100, "-") l2.replace(l2new) self.assertEqual(n1.children, [l1, l2new, l3]) - self.assertIsInstance(n1.children, list) + self.assertTrue(isinstance(n1.children, list)) self.assertTrue(n1.was_changed) def test_replace_with_list(self): @@ -159,7 +159,7 @@ l2.replace([pytree.Leaf(100, "*"), pytree.Leaf(100, "*")]) self.assertEqual(str(n1), "foo**bar") - self.assertIsInstance(n1.children, list) + self.assertTrue(isinstance(n1.children, list)) def test_leaves(self): l1 = pytree.Leaf(100, "foo") @@ -330,7 +330,7 @@ n2 = pytree.Node(1000, []) p1 = pytree.Node(1000, [n1, n2]) - self.assertIs(n1.next_sibling, n2) + self.assertTrue(n1.next_sibling is n2) self.assertEqual(n2.next_sibling, None) self.assertEqual(p1.next_sibling, None) @@ -339,7 +339,7 @@ l2 = pytree.Leaf(100, "b") p1 = pytree.Node(1000, [l1, l2]) - self.assertIs(l1.next_sibling, l2) + self.assertTrue(l1.next_sibling is l2) self.assertEqual(l2.next_sibling, None) self.assertEqual(p1.next_sibling, None) @@ -348,7 +348,7 @@ n2 = pytree.Node(1000, []) p1 = pytree.Node(1000, [n1, n2]) - self.assertIs(n2.prev_sibling, n1) + self.assertTrue(n2.prev_sibling is n1) self.assertEqual(n1.prev_sibling, None) self.assertEqual(p1.prev_sibling, None) @@ -357,7 +357,7 @@ l2 = pytree.Leaf(100, "b") p1 = pytree.Node(1000, [l1, l2]) - self.assertIs(l2.prev_sibling, l1) + self.assertTrue(l2.prev_sibling is l1) self.assertEqual(l1.prev_sibling, None) self.assertEqual(p1.prev_sibling, None) @@ -430,7 +430,7 @@ r = {} self.assertTrue(pw.match_seq([l1, l3], r)) self.assertEqual(r, {"pl": l3, "pw": [l1, l3]}) - self.assertIs(r["pl"], l3) + self.assertTrue(r["pl"] is l3) r = {} def test_generate_matches(self): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lib2to3/tests/test_refactor.py --- a/Lib/lib2to3/tests/test_refactor.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lib2to3/tests/test_refactor.py Mon Jan 25 17:05:13 2016 +0100 @@ -49,9 +49,9 @@ def test_print_function_option(self): rt = self.rt({"print_function" : True}) - self.assertIs(rt.grammar, pygram.python_grammar_no_print_statement) - self.assertIs(rt.driver.grammar, - pygram.python_grammar_no_print_statement) + self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement) + self.assertTrue(rt.driver.grammar is + pygram.python_grammar_no_print_statement) def test_write_unchanged_files_option(self): rt = self.rt() @@ -271,10 +271,6 @@ fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) - def test_false_file_encoding(self): - fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") - data = self.check_file_refactoring(fn) - def test_bom(self): fn = os.path.join(TEST_DATA_DIR, "bom.py") data = self.check_file_refactoring(fn) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/linecache.py --- a/Lib/linecache.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/linecache.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,10 @@ -"""Cache lines from Python source files. +"""Cache lines from files. This is intended to read lines from modules imported -- hence if a filename is not found, it will look down the module search path for a file by that name. """ -import functools import sys import os import tokenize @@ -22,9 +21,7 @@ # The cache -# The cache. Maps filenames to either a thunk which will provide source code, -# or a tuple (size, mtime, lines, fullname) once loaded. -cache = {} +cache = {} # The cache def clearcache(): @@ -35,19 +32,13 @@ def getlines(filename, module_globals=None): - """Get the lines for a Python source file from the cache. + """Get the lines for a file from the cache. Update the cache if it doesn't contain an entry for this file already.""" if filename in cache: - entry = cache[filename] - if len(entry) != 1: - return cache[filename][2] - - try: + return cache[filename][2] + else: return updatecache(filename, module_globals) - except MemoryError: - clearcache() - return [] def checkcache(filename=None): @@ -63,16 +54,12 @@ return for filename in filenames: - entry = cache[filename] - if len(entry) == 1: - # lazy cache entry, leave it lazy. - continue - size, mtime, lines, fullname = entry + size, mtime, lines, fullname = cache[filename] if mtime is None: continue # no-op for files loaded via a __loader__ try: stat = os.stat(fullname) - except OSError: + except os.error: del cache[filename] continue if size != stat.st_size or mtime != stat.st_mtime: @@ -85,8 +72,7 @@ and return an empty list.""" if filename in cache: - if len(cache[filename]) != 1: - del cache[filename] + del cache[filename] if not filename or (filename.startswith('<') and filename.endswith('>')): return [] @@ -96,23 +82,27 @@ except OSError: basename = filename - # Realise a lazy loader based lookup if there is one - # otherwise try to lookup right now. - if lazycache(filename, module_globals): - try: - data = cache[filename][0]() - except (ImportError, OSError): - pass - else: - if data is None: - # No luck, the PEP302 loader cannot find the source - # for this module. - return [] - cache[filename] = ( - len(data), None, - [line+'\n' for line in data.splitlines()], fullname - ) - return cache[filename][2] + # Try for a __loader__, if available + if module_globals and '__loader__' in module_globals: + name = module_globals.get('__name__') + loader = module_globals['__loader__'] + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + try: + data = get_source(name) + except (ImportError, IOError): + pass + else: + if data is None: + # No luck, the PEP302 loader cannot find the source + # for this module. + return [] + cache[filename] = ( + len(data), None, + [line+'\n' for line in data.splitlines()], fullname + ) + return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. @@ -128,50 +118,17 @@ try: stat = os.stat(fullname) break - except OSError: + except os.error: pass else: return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() - except OSError: + except IOError: return [] if lines and not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines - - -def lazycache(filename, module_globals): - """Seed the cache for filename with module_globals. - - The module loader will be asked for the source only when getlines is - called, not immediately. - - If there is an entry in the cache already, it is not altered. - - :return: True if a lazy load is registered in the cache, - otherwise False. To register such a load a module loader with a - get_source method must be found, the filename must be a cachable - filename, and the filename must not be already cached. - """ - if filename in cache: - if len(cache[filename]) == 1: - return True - else: - return False - if not filename or (filename.startswith('<') and filename.endswith('>')): - return False - # Try for a __loader__, if available - if module_globals and '__loader__' in module_globals: - name = module_globals.get('__name__') - loader = module_globals['__loader__'] - get_source = getattr(loader, 'get_source', None) - - if name and get_source: - get_lines = functools.partial(get_source, name) - cache[filename] = (get_lines,) - return True - return False diff -r 6db40a9955dc -r 0d413f60cc23 Lib/locale.py --- a/Lib/locale.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/locale.py Mon Jan 25 17:05:13 2016 +0100 @@ -301,29 +301,22 @@ """Convert float to integer, taking the locale into account.""" return format("%.12g", val) -def delocalize(string): - "Parses a string as a normalized number according to the locale settings." - - conv = localeconv() - +def atof(string, func=float): + "Parses a string as a float according to the locale settings." #First, get rid of the grouping - ts = conv['thousands_sep'] + ts = localeconv()['thousands_sep'] if ts: string = string.replace(ts, '') - #next, replace the decimal point with a dot - dd = conv['decimal_point'] + dd = localeconv()['decimal_point'] if dd: string = string.replace(dd, '.') - return string + #finally, parse the string + return func(string) -def atof(string, func=float): - "Parses a string as a float according to the locale settings." - return func(delocalize(string)) - -def atoi(string): +def atoi(str): "Converts a string to an integer according to the locale settings." - return int(delocalize(string)) + return atof(str, int) def _test(): setlocale(LC_ALL, "") @@ -343,40 +336,6 @@ # overridden below) _setlocale = setlocale -def _replace_encoding(code, encoding): - if '.' in code: - langname = code[:code.index('.')] - else: - langname = code - # Convert the encoding to a C lib compatible encoding string - norm_encoding = encodings.normalize_encoding(encoding) - #print('norm encoding: %r' % norm_encoding) - norm_encoding = encodings.aliases.aliases.get(norm_encoding.lower(), - norm_encoding) - #print('aliased encoding: %r' % norm_encoding) - encoding = norm_encoding - norm_encoding = norm_encoding.lower() - if norm_encoding in locale_encoding_alias: - encoding = locale_encoding_alias[norm_encoding] - else: - norm_encoding = norm_encoding.replace('_', '') - norm_encoding = norm_encoding.replace('-', '') - if norm_encoding in locale_encoding_alias: - encoding = locale_encoding_alias[norm_encoding] - #print('found encoding %r' % encoding) - return langname + '.' + encoding - -def _append_modifier(code, modifier): - if modifier == 'euro': - if '.' not in code: - return code + '.ISO8859-15' - _, _, encoding = code.partition('.') - if encoding in ('ISO8859-15', 'UTF-8'): - return code - if encoding == 'ISO8859-1': - return _replace_encoding(code, 'ISO8859-15') - return code + '@' + modifier - def normalize(localename): """ Returns a normalized locale code for the given locale @@ -393,72 +352,55 @@ does. """ - # Normalize the locale name and extract the encoding and modifier - code = localename.lower() - if ':' in code: + # Normalize the locale name and extract the encoding + fullname = localename.lower() + if ':' in fullname: # ':' is sometimes used as encoding delimiter. - code = code.replace(':', '.') - if '@' in code: - code, modifier = code.split('@', 1) + fullname = fullname.replace(':', '.') + if '.' in fullname: + langname, encoding = fullname.split('.')[:2] + fullname = langname + '.' + encoding else: - modifier = '' - if '.' in code: - langname, encoding = code.split('.')[:2] - else: - langname = code + langname = fullname encoding = '' - # First lookup: fullname (possibly with encoding and modifier) - lang_enc = langname - if encoding: - norm_encoding = encoding.replace('-', '') - norm_encoding = norm_encoding.replace('_', '') - lang_enc += '.' + norm_encoding - lookup_name = lang_enc - if modifier: - lookup_name += '@' + modifier + # First lookup: fullname (possibly with encoding) + norm_encoding = encoding.replace('-', '') + norm_encoding = norm_encoding.replace('_', '') + lookup_name = langname + '.' + encoding code = locale_alias.get(lookup_name, None) if code is not None: return code - #print('first lookup failed') + #print 'first lookup failed' - if modifier: - # Second try: fullname without modifier (possibly with encoding) - code = locale_alias.get(lang_enc, None) - if code is not None: - #print('lookup without modifier succeeded') - if '@' not in code: - return _append_modifier(code, modifier) - if code.split('@', 1)[1].lower() == modifier: - return code - #print('second lookup failed') + # Second try: langname (without encoding) + code = locale_alias.get(langname, None) + if code is not None: + #print 'langname lookup succeeded' + if '.' in code: + langname, defenc = code.split('.') + else: + langname = code + defenc = '' + if encoding: + # Convert the encoding to a C lib compatible encoding string + norm_encoding = encodings.normalize_encoding(encoding) + #print 'norm encoding: %r' % norm_encoding + norm_encoding = encodings.aliases.aliases.get(norm_encoding, + norm_encoding) + #print 'aliased encoding: %r' % norm_encoding + encoding = locale_encoding_alias.get(norm_encoding, + norm_encoding) + else: + encoding = defenc + #print 'found encoding %r' % encoding + if encoding: + return langname + '.' + encoding + else: + return langname - if encoding: - # Third try: langname (without encoding, possibly with modifier) - lookup_name = langname - if modifier: - lookup_name += '@' + modifier - code = locale_alias.get(lookup_name, None) - if code is not None: - #print('lookup without encoding succeeded') - if '@' not in code: - return _replace_encoding(code, encoding) - code, modifier = code.split('@', 1) - return _replace_encoding(code, encoding) + '@' + modifier - - if modifier: - # Fourth try: langname (without encoding and modifier) - code = locale_alias.get(langname, None) - if code is not None: - #print('lookup without modifier and encoding succeeded') - if '@' not in code: - code = _replace_encoding(code, encoding) - return _append_modifier(code, modifier) - code, defmod = code.split('@', 1) - if defmod.lower() == modifier: - return _replace_encoding(code, encoding) + '@' + defmod - - return localename + else: + return localename def _parse_localename(localename): @@ -477,7 +419,7 @@ code = normalize(localename) if '@' in code: # Deal with locale modifiers - code, modifier = code.split('@', 1) + code, modifier = code.split('@') if modifier == 'euro' and '.' not in code: # Assume Latin-9 for @euro locales. This is bogus, # since some systems may use other encodings for these @@ -612,8 +554,8 @@ # On Win32, this will return the ANSI code page def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using.""" - import _bootlocale - return _bootlocale.getpreferredencoding(False) + import _locale + return _locale._getdefaultlocale()[1] else: # On Unix, if CODESET is available, use that. try: @@ -632,16 +574,27 @@ def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using, according to the system configuration.""" - import _bootlocale if do_setlocale: oldloc = setlocale(LC_CTYPE) try: setlocale(LC_CTYPE, "") except Error: pass - result = _bootlocale.getpreferredencoding(False) - if do_setlocale: + result = nl_langinfo(CODESET) + if not result and sys.platform == 'darwin': + # nl_langinfo can return an empty string + # when the setting has an invalid value. + # Default to UTF-8 in that case because + # UTF-8 is the default charset on OSX and + # returning nothing will crash the + # interpreter. + result = 'UTF-8' setlocale(LC_CTYPE, oldloc) + else: + result = nl_langinfo(CODESET) + if not result and sys.platform == 'darwin': + # See above for explanation + result = 'UTF-8' return result @@ -669,14 +622,6 @@ 'jis': 'JIS7', 'jis7': 'JIS7', 'ajec': 'eucJP', - 'koi8c': 'KOI8-C', - 'microsoftcp1251': 'CP1251', - 'microsoftcp1255': 'CP1255', - 'microsoftcp1256': 'CP1256', - '88591': 'ISO8859-1', - '88592': 'ISO8859-2', - '88595': 'ISO8859-5', - '885915': 'ISO8859-15', # Mappings from Python codec names to C lib encoding names 'ascii': 'ISO8859-1', @@ -703,21 +648,11 @@ 'euc_kr': 'eucKR', 'utf_8': 'UTF-8', 'koi8_r': 'KOI8-R', - 'koi8_t': 'KOI8-T', 'koi8_u': 'KOI8-U', - 'kz1048': 'RK1048', - 'cp1251': 'CP1251', - 'cp1255': 'CP1255', - 'cp1256': 'CP1256', - # XXX This list is still incomplete. If you know more # mappings, please file a bug report. Thanks. } -for k, v in sorted(locale_encoding_alias.items()): - k = k.replace('_', '') - locale_encoding_alias.setdefault(k, v) - # # The locale_alias table maps lowercase alias names to C locale names # (case-sensitive). Encodings are always separated from the locale @@ -808,281 +743,458 @@ # updated 'sr_yu.utf8@cyrillic' -> 'sr_CS.UTF-8' to 'sr_RS.UTF-8' # updated 'sr_yu@cyrillic' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' # -# SS 2013-12-20: -# Updated alias mapping to most recent locale.alias file -# from X.org distribution using makelocalealias.py. -# -# These are the differences compared to the old mapping (Python 3.3.3 -# and older): -# -# updated 'a3' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' -# updated 'a3_az' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' -# updated 'a3_az.koi8c' -> 'a3_AZ.KOI8-C' to 'az_AZ.KOI8-C' -# updated 'cs_cs.iso88592' -> 'cs_CS.ISO8859-2' to 'cs_CZ.ISO8859-2' -# updated 'hebrew' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' -# updated 'hebrew.iso88598' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' -# updated 'sd' -> 'sd_IN@devanagari.UTF-8' to 'sd_IN.UTF-8' -# updated 'sr@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' -# updated 'sr_cs' -> 'sr_RS.UTF-8' to 'sr_CS.UTF-8' -# updated 'sr_cs.utf8@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' -# updated 'sr_cs@latn' -> 'sr_RS.UTF-8@latin' to 'sr_CS.UTF-8@latin' -# -# SS 2014-10-01: -# Updated alias mapping with glibc 2.19 supported locales. locale_alias = { - 'a3': 'az_AZ.KOI8-C', - 'a3_az': 'az_AZ.KOI8-C', - 'a3_az.koic': 'az_AZ.KOI8-C', - 'aa_dj': 'aa_DJ.ISO8859-1', - 'aa_er': 'aa_ER.UTF-8', - 'aa_et': 'aa_ET.UTF-8', + 'a3': 'a3_AZ.KOI8-C', + 'a3_az': 'a3_AZ.KOI8-C', + 'a3_az.koi8c': 'a3_AZ.KOI8-C', 'af': 'af_ZA.ISO8859-1', 'af_za': 'af_ZA.ISO8859-1', + 'af_za.iso88591': 'af_ZA.ISO8859-1', 'am': 'am_ET.UTF-8', 'am_et': 'am_ET.UTF-8', 'american': 'en_US.ISO8859-1', - 'an_es': 'an_ES.ISO8859-15', + 'american.iso88591': 'en_US.ISO8859-1', 'ar': 'ar_AA.ISO8859-6', 'ar_aa': 'ar_AA.ISO8859-6', + 'ar_aa.iso88596': 'ar_AA.ISO8859-6', 'ar_ae': 'ar_AE.ISO8859-6', + 'ar_ae.iso88596': 'ar_AE.ISO8859-6', 'ar_bh': 'ar_BH.ISO8859-6', + 'ar_bh.iso88596': 'ar_BH.ISO8859-6', 'ar_dz': 'ar_DZ.ISO8859-6', + 'ar_dz.iso88596': 'ar_DZ.ISO8859-6', 'ar_eg': 'ar_EG.ISO8859-6', - 'ar_in': 'ar_IN.UTF-8', + 'ar_eg.iso88596': 'ar_EG.ISO8859-6', 'ar_iq': 'ar_IQ.ISO8859-6', + 'ar_iq.iso88596': 'ar_IQ.ISO8859-6', 'ar_jo': 'ar_JO.ISO8859-6', + 'ar_jo.iso88596': 'ar_JO.ISO8859-6', 'ar_kw': 'ar_KW.ISO8859-6', + 'ar_kw.iso88596': 'ar_KW.ISO8859-6', 'ar_lb': 'ar_LB.ISO8859-6', + 'ar_lb.iso88596': 'ar_LB.ISO8859-6', 'ar_ly': 'ar_LY.ISO8859-6', + 'ar_ly.iso88596': 'ar_LY.ISO8859-6', 'ar_ma': 'ar_MA.ISO8859-6', + 'ar_ma.iso88596': 'ar_MA.ISO8859-6', 'ar_om': 'ar_OM.ISO8859-6', + 'ar_om.iso88596': 'ar_OM.ISO8859-6', 'ar_qa': 'ar_QA.ISO8859-6', + 'ar_qa.iso88596': 'ar_QA.ISO8859-6', 'ar_sa': 'ar_SA.ISO8859-6', + 'ar_sa.iso88596': 'ar_SA.ISO8859-6', 'ar_sd': 'ar_SD.ISO8859-6', + 'ar_sd.iso88596': 'ar_SD.ISO8859-6', 'ar_sy': 'ar_SY.ISO8859-6', + 'ar_sy.iso88596': 'ar_SY.ISO8859-6', 'ar_tn': 'ar_TN.ISO8859-6', + 'ar_tn.iso88596': 'ar_TN.ISO8859-6', 'ar_ye': 'ar_YE.ISO8859-6', + 'ar_ye.iso88596': 'ar_YE.ISO8859-6', 'arabic': 'ar_AA.ISO8859-6', + 'arabic.iso88596': 'ar_AA.ISO8859-6', 'as': 'as_IN.UTF-8', - 'as_in': 'as_IN.UTF-8', - 'ast_es': 'ast_ES.ISO8859-15', - 'ayc_pe': 'ayc_PE.UTF-8', 'az': 'az_AZ.ISO8859-9E', 'az_az': 'az_AZ.ISO8859-9E', 'az_az.iso88599e': 'az_AZ.ISO8859-9E', 'be': 'be_BY.CP1251', 'be@latin': 'be_BY.UTF-8@latin', - 'be_bg.utf8': 'bg_BG.UTF-8', 'be_by': 'be_BY.CP1251', + 'be_by.cp1251': 'be_BY.CP1251', + 'be_by.microsoftcp1251': 'be_BY.CP1251', + 'be_by.utf8@latin': 'be_BY.UTF-8@latin', 'be_by@latin': 'be_BY.UTF-8@latin', - 'bem_zm': 'bem_ZM.UTF-8', - 'ber_dz': 'ber_DZ.UTF-8', - 'ber_ma': 'ber_MA.UTF-8', 'bg': 'bg_BG.CP1251', 'bg_bg': 'bg_BG.CP1251', - 'bho_in': 'bho_IN.UTF-8', - 'bn_bd': 'bn_BD.UTF-8', + 'bg_bg.cp1251': 'bg_BG.CP1251', + 'bg_bg.iso88595': 'bg_BG.ISO8859-5', + 'bg_bg.koi8r': 'bg_BG.KOI8-R', + 'bg_bg.microsoftcp1251': 'bg_BG.CP1251', 'bn_in': 'bn_IN.UTF-8', - 'bo_cn': 'bo_CN.UTF-8', - 'bo_in': 'bo_IN.UTF-8', 'bokmal': 'nb_NO.ISO8859-1', 'bokm\xe5l': 'nb_NO.ISO8859-1', 'br': 'br_FR.ISO8859-1', 'br_fr': 'br_FR.ISO8859-1', - 'brx_in': 'brx_IN.UTF-8', + 'br_fr.iso88591': 'br_FR.ISO8859-1', + 'br_fr.iso885914': 'br_FR.ISO8859-14', + 'br_fr.iso885915': 'br_FR.ISO8859-15', + 'br_fr.iso885915@euro': 'br_FR.ISO8859-15', + 'br_fr.utf8@euro': 'br_FR.UTF-8', + 'br_fr@euro': 'br_FR.ISO8859-15', 'bs': 'bs_BA.ISO8859-2', 'bs_ba': 'bs_BA.ISO8859-2', + 'bs_ba.iso88592': 'bs_BA.ISO8859-2', 'bulgarian': 'bg_BG.CP1251', - 'byn_er': 'byn_ER.UTF-8', 'c': 'C', 'c-french': 'fr_CA.ISO8859-1', - 'c.ascii': 'C', + 'c-french.iso88591': 'fr_CA.ISO8859-1', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', - 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', 'ca_ad': 'ca_AD.ISO8859-1', + 'ca_ad.iso88591': 'ca_AD.ISO8859-1', + 'ca_ad.iso885915': 'ca_AD.ISO8859-15', + 'ca_ad.iso885915@euro': 'ca_AD.ISO8859-15', + 'ca_ad.utf8@euro': 'ca_AD.UTF-8', + 'ca_ad@euro': 'ca_AD.ISO8859-15', 'ca_es': 'ca_ES.ISO8859-1', - 'ca_es@valencia': 'ca_ES.ISO8859-15@valencia', + 'ca_es.iso88591': 'ca_ES.ISO8859-1', + 'ca_es.iso885915': 'ca_ES.ISO8859-15', + 'ca_es.iso885915@euro': 'ca_ES.ISO8859-15', + 'ca_es.utf8@euro': 'ca_ES.UTF-8', + 'ca_es@euro': 'ca_ES.ISO8859-15', 'ca_fr': 'ca_FR.ISO8859-1', + 'ca_fr.iso88591': 'ca_FR.ISO8859-1', + 'ca_fr.iso885915': 'ca_FR.ISO8859-15', + 'ca_fr.iso885915@euro': 'ca_FR.ISO8859-15', + 'ca_fr.utf8@euro': 'ca_FR.UTF-8', + 'ca_fr@euro': 'ca_FR.ISO8859-15', 'ca_it': 'ca_IT.ISO8859-1', + 'ca_it.iso88591': 'ca_IT.ISO8859-1', + 'ca_it.iso885915': 'ca_IT.ISO8859-15', + 'ca_it.iso885915@euro': 'ca_IT.ISO8859-15', + 'ca_it.utf8@euro': 'ca_IT.UTF-8', + 'ca_it@euro': 'ca_IT.ISO8859-15', 'catalan': 'ca_ES.ISO8859-1', 'cextend': 'en_US.ISO8859-1', + 'cextend.en': 'en_US.ISO8859-1', 'chinese-s': 'zh_CN.eucCN', 'chinese-t': 'zh_TW.eucTW', - 'crh_ua': 'crh_UA.UTF-8', 'croatian': 'hr_HR.ISO8859-2', 'cs': 'cs_CZ.ISO8859-2', 'cs_cs': 'cs_CZ.ISO8859-2', + 'cs_cs.iso88592': 'cs_CS.ISO8859-2', 'cs_cz': 'cs_CZ.ISO8859-2', - 'csb_pl': 'csb_PL.UTF-8', - 'cv_ru': 'cv_RU.UTF-8', + 'cs_cz.iso88592': 'cs_CZ.ISO8859-2', 'cy': 'cy_GB.ISO8859-1', 'cy_gb': 'cy_GB.ISO8859-1', + 'cy_gb.iso88591': 'cy_GB.ISO8859-1', + 'cy_gb.iso885914': 'cy_GB.ISO8859-14', + 'cy_gb.iso885915': 'cy_GB.ISO8859-15', + 'cy_gb@euro': 'cy_GB.ISO8859-15', 'cz': 'cs_CZ.ISO8859-2', 'cz_cz': 'cs_CZ.ISO8859-2', 'czech': 'cs_CZ.ISO8859-2', 'da': 'da_DK.ISO8859-1', + 'da.iso885915': 'da_DK.ISO8859-15', 'da_dk': 'da_DK.ISO8859-1', + 'da_dk.88591': 'da_DK.ISO8859-1', + 'da_dk.885915': 'da_DK.ISO8859-15', + 'da_dk.iso88591': 'da_DK.ISO8859-1', + 'da_dk.iso885915': 'da_DK.ISO8859-15', + 'da_dk@euro': 'da_DK.ISO8859-15', 'danish': 'da_DK.ISO8859-1', + 'danish.iso88591': 'da_DK.ISO8859-1', 'dansk': 'da_DK.ISO8859-1', 'de': 'de_DE.ISO8859-1', + 'de.iso885915': 'de_DE.ISO8859-15', 'de_at': 'de_AT.ISO8859-1', + 'de_at.iso88591': 'de_AT.ISO8859-1', + 'de_at.iso885915': 'de_AT.ISO8859-15', + 'de_at.iso885915@euro': 'de_AT.ISO8859-15', + 'de_at.utf8@euro': 'de_AT.UTF-8', + 'de_at@euro': 'de_AT.ISO8859-15', 'de_be': 'de_BE.ISO8859-1', + 'de_be.iso88591': 'de_BE.ISO8859-1', + 'de_be.iso885915': 'de_BE.ISO8859-15', + 'de_be.iso885915@euro': 'de_BE.ISO8859-15', + 'de_be.utf8@euro': 'de_BE.UTF-8', + 'de_be@euro': 'de_BE.ISO8859-15', 'de_ch': 'de_CH.ISO8859-1', + 'de_ch.iso88591': 'de_CH.ISO8859-1', + 'de_ch.iso885915': 'de_CH.ISO8859-15', + 'de_ch@euro': 'de_CH.ISO8859-15', 'de_de': 'de_DE.ISO8859-1', - 'de_li.utf8': 'de_LI.UTF-8', + 'de_de.88591': 'de_DE.ISO8859-1', + 'de_de.885915': 'de_DE.ISO8859-15', + 'de_de.885915@euro': 'de_DE.ISO8859-15', + 'de_de.iso88591': 'de_DE.ISO8859-1', + 'de_de.iso885915': 'de_DE.ISO8859-15', + 'de_de.iso885915@euro': 'de_DE.ISO8859-15', + 'de_de.utf8@euro': 'de_DE.UTF-8', + 'de_de@euro': 'de_DE.ISO8859-15', 'de_lu': 'de_LU.ISO8859-1', + 'de_lu.iso88591': 'de_LU.ISO8859-1', + 'de_lu.iso885915': 'de_LU.ISO8859-15', + 'de_lu.iso885915@euro': 'de_LU.ISO8859-15', + 'de_lu.utf8@euro': 'de_LU.UTF-8', + 'de_lu@euro': 'de_LU.ISO8859-15', 'deutsch': 'de_DE.ISO8859-1', - 'doi_in': 'doi_IN.UTF-8', 'dutch': 'nl_NL.ISO8859-1', 'dutch.iso88591': 'nl_BE.ISO8859-1', - 'dv_mv': 'dv_MV.UTF-8', - 'dz_bt': 'dz_BT.UTF-8', 'ee': 'ee_EE.ISO8859-4', 'ee_ee': 'ee_EE.ISO8859-4', + 'ee_ee.iso88594': 'ee_EE.ISO8859-4', 'eesti': 'et_EE.ISO8859-1', 'el': 'el_GR.ISO8859-7', - 'el_cy': 'el_CY.ISO8859-7', 'el_gr': 'el_GR.ISO8859-7', + 'el_gr.iso88597': 'el_GR.ISO8859-7', 'el_gr@euro': 'el_GR.ISO8859-15', 'en': 'en_US.ISO8859-1', - 'en_ag': 'en_AG.UTF-8', + 'en.iso88591': 'en_US.ISO8859-1', 'en_au': 'en_AU.ISO8859-1', + 'en_au.iso88591': 'en_AU.ISO8859-1', 'en_be': 'en_BE.ISO8859-1', + 'en_be@euro': 'en_BE.ISO8859-15', 'en_bw': 'en_BW.ISO8859-1', + 'en_bw.iso88591': 'en_BW.ISO8859-1', 'en_ca': 'en_CA.ISO8859-1', - 'en_dk': 'en_DK.ISO8859-1', - 'en_dl.utf8': 'en_DL.UTF-8', + 'en_ca.iso88591': 'en_CA.ISO8859-1', 'en_gb': 'en_GB.ISO8859-1', + 'en_gb.88591': 'en_GB.ISO8859-1', + 'en_gb.iso88591': 'en_GB.ISO8859-1', + 'en_gb.iso885915': 'en_GB.ISO8859-15', + 'en_gb@euro': 'en_GB.ISO8859-15', 'en_hk': 'en_HK.ISO8859-1', + 'en_hk.iso88591': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', + 'en_ie.iso88591': 'en_IE.ISO8859-1', + 'en_ie.iso885915': 'en_IE.ISO8859-15', + 'en_ie.iso885915@euro': 'en_IE.ISO8859-15', + 'en_ie.utf8@euro': 'en_IE.UTF-8', + 'en_ie@euro': 'en_IE.ISO8859-15', 'en_in': 'en_IN.ISO8859-1', - 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', + 'en_nz.iso88591': 'en_NZ.ISO8859-1', 'en_ph': 'en_PH.ISO8859-1', + 'en_ph.iso88591': 'en_PH.ISO8859-1', 'en_sg': 'en_SG.ISO8859-1', + 'en_sg.iso88591': 'en_SG.ISO8859-1', 'en_uk': 'en_GB.ISO8859-1', 'en_us': 'en_US.ISO8859-1', + 'en_us.88591': 'en_US.ISO8859-1', + 'en_us.885915': 'en_US.ISO8859-15', + 'en_us.iso88591': 'en_US.ISO8859-1', + 'en_us.iso885915': 'en_US.ISO8859-15', + 'en_us.iso885915@euro': 'en_US.ISO8859-15', + 'en_us@euro': 'en_US.ISO8859-15', 'en_us@euro@euro': 'en_US.ISO8859-15', 'en_za': 'en_ZA.ISO8859-1', - 'en_zm': 'en_ZM.UTF-8', + 'en_za.88591': 'en_ZA.ISO8859-1', + 'en_za.iso88591': 'en_ZA.ISO8859-1', + 'en_za.iso885915': 'en_ZA.ISO8859-15', + 'en_za@euro': 'en_ZA.ISO8859-15', 'en_zw': 'en_ZW.ISO8859-1', - 'en_zw.utf8': 'en_ZS.UTF-8', + 'en_zw.iso88591': 'en_ZW.ISO8859-1', 'eng_gb': 'en_GB.ISO8859-1', + 'eng_gb.8859': 'en_GB.ISO8859-1', 'english': 'en_EN.ISO8859-1', + 'english.iso88591': 'en_EN.ISO8859-1', 'english_uk': 'en_GB.ISO8859-1', + 'english_uk.8859': 'en_GB.ISO8859-1', 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', 'english_us': 'en_US.ISO8859-1', + 'english_us.8859': 'en_US.ISO8859-1', + 'english_us.ascii': 'en_US.ISO8859-1', 'eo': 'eo_XX.ISO8859-3', - 'eo.utf8': 'eo.UTF-8', 'eo_eo': 'eo_EO.ISO8859-3', - 'eo_us.utf8': 'eo_US.UTF-8', + 'eo_eo.iso88593': 'eo_EO.ISO8859-3', 'eo_xx': 'eo_XX.ISO8859-3', + 'eo_xx.iso88593': 'eo_XX.ISO8859-3', 'es': 'es_ES.ISO8859-1', 'es_ar': 'es_AR.ISO8859-1', + 'es_ar.iso88591': 'es_AR.ISO8859-1', 'es_bo': 'es_BO.ISO8859-1', + 'es_bo.iso88591': 'es_BO.ISO8859-1', 'es_cl': 'es_CL.ISO8859-1', + 'es_cl.iso88591': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', + 'es_co.iso88591': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', - 'es_cu': 'es_CU.UTF-8', + 'es_cr.iso88591': 'es_CR.ISO8859-1', 'es_do': 'es_DO.ISO8859-1', + 'es_do.iso88591': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', + 'es_ec.iso88591': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', + 'es_es.88591': 'es_ES.ISO8859-1', + 'es_es.iso88591': 'es_ES.ISO8859-1', + 'es_es.iso885915': 'es_ES.ISO8859-15', + 'es_es.iso885915@euro': 'es_ES.ISO8859-15', + 'es_es.utf8@euro': 'es_ES.UTF-8', + 'es_es@euro': 'es_ES.ISO8859-15', 'es_gt': 'es_GT.ISO8859-1', + 'es_gt.iso88591': 'es_GT.ISO8859-1', 'es_hn': 'es_HN.ISO8859-1', + 'es_hn.iso88591': 'es_HN.ISO8859-1', 'es_mx': 'es_MX.ISO8859-1', + 'es_mx.iso88591': 'es_MX.ISO8859-1', 'es_ni': 'es_NI.ISO8859-1', + 'es_ni.iso88591': 'es_NI.ISO8859-1', 'es_pa': 'es_PA.ISO8859-1', + 'es_pa.iso88591': 'es_PA.ISO8859-1', + 'es_pa.iso885915': 'es_PA.ISO8859-15', + 'es_pa@euro': 'es_PA.ISO8859-15', 'es_pe': 'es_PE.ISO8859-1', + 'es_pe.iso88591': 'es_PE.ISO8859-1', + 'es_pe.iso885915': 'es_PE.ISO8859-15', + 'es_pe@euro': 'es_PE.ISO8859-15', 'es_pr': 'es_PR.ISO8859-1', + 'es_pr.iso88591': 'es_PR.ISO8859-1', 'es_py': 'es_PY.ISO8859-1', + 'es_py.iso88591': 'es_PY.ISO8859-1', + 'es_py.iso885915': 'es_PY.ISO8859-15', + 'es_py@euro': 'es_PY.ISO8859-15', 'es_sv': 'es_SV.ISO8859-1', + 'es_sv.iso88591': 'es_SV.ISO8859-1', + 'es_sv.iso885915': 'es_SV.ISO8859-15', + 'es_sv@euro': 'es_SV.ISO8859-15', 'es_us': 'es_US.ISO8859-1', + 'es_us.iso88591': 'es_US.ISO8859-1', 'es_uy': 'es_UY.ISO8859-1', + 'es_uy.iso88591': 'es_UY.ISO8859-1', + 'es_uy.iso885915': 'es_UY.ISO8859-15', + 'es_uy@euro': 'es_UY.ISO8859-15', 'es_ve': 'es_VE.ISO8859-1', + 'es_ve.iso88591': 'es_VE.ISO8859-1', + 'es_ve.iso885915': 'es_VE.ISO8859-15', + 'es_ve@euro': 'es_VE.ISO8859-15', 'estonian': 'et_EE.ISO8859-1', 'et': 'et_EE.ISO8859-15', 'et_ee': 'et_EE.ISO8859-15', + 'et_ee.iso88591': 'et_EE.ISO8859-1', + 'et_ee.iso885913': 'et_EE.ISO8859-13', + 'et_ee.iso885915': 'et_EE.ISO8859-15', + 'et_ee.iso88594': 'et_EE.ISO8859-4', + 'et_ee@euro': 'et_EE.ISO8859-15', 'eu': 'eu_ES.ISO8859-1', 'eu_es': 'eu_ES.ISO8859-1', - 'eu_fr': 'eu_FR.ISO8859-1', + 'eu_es.iso88591': 'eu_ES.ISO8859-1', + 'eu_es.iso885915': 'eu_ES.ISO8859-15', + 'eu_es.iso885915@euro': 'eu_ES.ISO8859-15', + 'eu_es.utf8@euro': 'eu_ES.UTF-8', + 'eu_es@euro': 'eu_ES.ISO8859-15', 'fa': 'fa_IR.UTF-8', 'fa_ir': 'fa_IR.UTF-8', 'fa_ir.isiri3342': 'fa_IR.ISIRI-3342', - 'ff_sn': 'ff_SN.UTF-8', 'fi': 'fi_FI.ISO8859-15', + 'fi.iso885915': 'fi_FI.ISO8859-15', 'fi_fi': 'fi_FI.ISO8859-15', - 'fil_ph': 'fil_PH.UTF-8', + 'fi_fi.88591': 'fi_FI.ISO8859-1', + 'fi_fi.iso88591': 'fi_FI.ISO8859-1', + 'fi_fi.iso885915': 'fi_FI.ISO8859-15', + 'fi_fi.iso885915@euro': 'fi_FI.ISO8859-15', + 'fi_fi.utf8@euro': 'fi_FI.UTF-8', + 'fi_fi@euro': 'fi_FI.ISO8859-15', 'finnish': 'fi_FI.ISO8859-1', + 'finnish.iso88591': 'fi_FI.ISO8859-1', 'fo': 'fo_FO.ISO8859-1', 'fo_fo': 'fo_FO.ISO8859-1', + 'fo_fo.iso88591': 'fo_FO.ISO8859-1', + 'fo_fo.iso885915': 'fo_FO.ISO8859-15', + 'fo_fo@euro': 'fo_FO.ISO8859-15', 'fr': 'fr_FR.ISO8859-1', + 'fr.iso885915': 'fr_FR.ISO8859-15', 'fr_be': 'fr_BE.ISO8859-1', + 'fr_be.88591': 'fr_BE.ISO8859-1', + 'fr_be.iso88591': 'fr_BE.ISO8859-1', + 'fr_be.iso885915': 'fr_BE.ISO8859-15', + 'fr_be.iso885915@euro': 'fr_BE.ISO8859-15', + 'fr_be.utf8@euro': 'fr_BE.UTF-8', + 'fr_be@euro': 'fr_BE.ISO8859-15', 'fr_ca': 'fr_CA.ISO8859-1', + 'fr_ca.88591': 'fr_CA.ISO8859-1', + 'fr_ca.iso88591': 'fr_CA.ISO8859-1', + 'fr_ca.iso885915': 'fr_CA.ISO8859-15', + 'fr_ca@euro': 'fr_CA.ISO8859-15', 'fr_ch': 'fr_CH.ISO8859-1', + 'fr_ch.88591': 'fr_CH.ISO8859-1', + 'fr_ch.iso88591': 'fr_CH.ISO8859-1', + 'fr_ch.iso885915': 'fr_CH.ISO8859-15', + 'fr_ch@euro': 'fr_CH.ISO8859-15', 'fr_fr': 'fr_FR.ISO8859-1', + 'fr_fr.88591': 'fr_FR.ISO8859-1', + 'fr_fr.iso88591': 'fr_FR.ISO8859-1', + 'fr_fr.iso885915': 'fr_FR.ISO8859-15', + 'fr_fr.iso885915@euro': 'fr_FR.ISO8859-15', + 'fr_fr.utf8@euro': 'fr_FR.UTF-8', + 'fr_fr@euro': 'fr_FR.ISO8859-15', 'fr_lu': 'fr_LU.ISO8859-1', + 'fr_lu.88591': 'fr_LU.ISO8859-1', + 'fr_lu.iso88591': 'fr_LU.ISO8859-1', + 'fr_lu.iso885915': 'fr_LU.ISO8859-15', + 'fr_lu.iso885915@euro': 'fr_LU.ISO8859-15', + 'fr_lu.utf8@euro': 'fr_LU.UTF-8', + 'fr_lu@euro': 'fr_LU.ISO8859-15', 'fran\xe7ais': 'fr_FR.ISO8859-1', 'fre_fr': 'fr_FR.ISO8859-1', + 'fre_fr.8859': 'fr_FR.ISO8859-1', 'french': 'fr_FR.ISO8859-1', 'french.iso88591': 'fr_CH.ISO8859-1', 'french_france': 'fr_FR.ISO8859-1', - 'fur_it': 'fur_IT.UTF-8', - 'fy_de': 'fy_DE.UTF-8', - 'fy_nl': 'fy_NL.UTF-8', + 'french_france.8859': 'fr_FR.ISO8859-1', 'ga': 'ga_IE.ISO8859-1', 'ga_ie': 'ga_IE.ISO8859-1', + 'ga_ie.iso88591': 'ga_IE.ISO8859-1', + 'ga_ie.iso885914': 'ga_IE.ISO8859-14', + 'ga_ie.iso885915': 'ga_IE.ISO8859-15', + 'ga_ie.iso885915@euro': 'ga_IE.ISO8859-15', + 'ga_ie.utf8@euro': 'ga_IE.UTF-8', + 'ga_ie@euro': 'ga_IE.ISO8859-15', 'galego': 'gl_ES.ISO8859-1', 'galician': 'gl_ES.ISO8859-1', 'gd': 'gd_GB.ISO8859-1', 'gd_gb': 'gd_GB.ISO8859-1', + 'gd_gb.iso88591': 'gd_GB.ISO8859-1', + 'gd_gb.iso885914': 'gd_GB.ISO8859-14', + 'gd_gb.iso885915': 'gd_GB.ISO8859-15', + 'gd_gb@euro': 'gd_GB.ISO8859-15', 'ger_de': 'de_DE.ISO8859-1', + 'ger_de.8859': 'de_DE.ISO8859-1', 'german': 'de_DE.ISO8859-1', 'german.iso88591': 'de_CH.ISO8859-1', 'german_germany': 'de_DE.ISO8859-1', - 'gez_er': 'gez_ER.UTF-8', - 'gez_et': 'gez_ET.UTF-8', + 'german_germany.8859': 'de_DE.ISO8859-1', 'gl': 'gl_ES.ISO8859-1', 'gl_es': 'gl_ES.ISO8859-1', + 'gl_es.iso88591': 'gl_ES.ISO8859-1', + 'gl_es.iso885915': 'gl_ES.ISO8859-15', + 'gl_es.iso885915@euro': 'gl_ES.ISO8859-15', + 'gl_es.utf8@euro': 'gl_ES.UTF-8', + 'gl_es@euro': 'gl_ES.ISO8859-15', 'greek': 'el_GR.ISO8859-7', + 'greek.iso88597': 'el_GR.ISO8859-7', 'gu_in': 'gu_IN.UTF-8', 'gv': 'gv_GB.ISO8859-1', 'gv_gb': 'gv_GB.ISO8859-1', - 'ha_ng': 'ha_NG.UTF-8', + 'gv_gb.iso88591': 'gv_GB.ISO8859-1', + 'gv_gb.iso885914': 'gv_GB.ISO8859-14', + 'gv_gb.iso885915': 'gv_GB.ISO8859-15', + 'gv_gb@euro': 'gv_GB.ISO8859-15', 'he': 'he_IL.ISO8859-8', 'he_il': 'he_IL.ISO8859-8', - 'hebrew': 'he_IL.ISO8859-8', + 'he_il.cp1255': 'he_IL.CP1255', + 'he_il.iso88598': 'he_IL.ISO8859-8', + 'he_il.microsoftcp1255': 'he_IL.CP1255', + 'hebrew': 'iw_IL.ISO8859-8', + 'hebrew.iso88598': 'iw_IL.ISO8859-8', 'hi': 'hi_IN.ISCII-DEV', 'hi_in': 'hi_IN.ISCII-DEV', 'hi_in.isciidev': 'hi_IN.ISCII-DEV', 'hne': 'hne_IN.UTF-8', - 'hne_in': 'hne_IN.UTF-8', 'hr': 'hr_HR.ISO8859-2', 'hr_hr': 'hr_HR.ISO8859-2', + 'hr_hr.iso88592': 'hr_HR.ISO8859-2', 'hrvatski': 'hr_HR.ISO8859-2', - 'hsb_de': 'hsb_DE.ISO8859-2', - 'ht_ht': 'ht_HT.UTF-8', 'hu': 'hu_HU.ISO8859-2', 'hu_hu': 'hu_HU.ISO8859-2', + 'hu_hu.iso88592': 'hu_HU.ISO8859-2', 'hungarian': 'hu_HU.ISO8859-2', - 'hy_am': 'hy_AM.UTF-8', - 'hy_am.armscii8': 'hy_AM.ARMSCII_8', - 'ia': 'ia.UTF-8', - 'ia_fr': 'ia_FR.UTF-8', 'icelandic': 'is_IS.ISO8859-1', + 'icelandic.iso88591': 'is_IS.ISO8859-1', 'id': 'id_ID.ISO8859-1', 'id_id': 'id_ID.ISO8859-1', - 'ig_ng': 'ig_NG.UTF-8', - 'ik_ca': 'ik_CA.UTF-8', 'in': 'id_ID.ISO8859-1', 'in_id': 'id_ID.ISO8859-1', 'is': 'is_IS.ISO8859-1', 'is_is': 'is_IS.ISO8859-1', + 'is_is.iso88591': 'is_IS.ISO8859-1', + 'is_is.iso885915': 'is_IS.ISO8859-15', + 'is_is@euro': 'is_IS.ISO8859-15', 'iso-8859-1': 'en_US.ISO8859-1', 'iso-8859-15': 'en_US.ISO8859-15', 'iso8859-1': 'en_US.ISO8859-1', @@ -1090,55 +1202,76 @@ 'iso_8859_1': 'en_US.ISO8859-1', 'iso_8859_15': 'en_US.ISO8859-15', 'it': 'it_IT.ISO8859-1', + 'it.iso885915': 'it_IT.ISO8859-15', 'it_ch': 'it_CH.ISO8859-1', + 'it_ch.iso88591': 'it_CH.ISO8859-1', + 'it_ch.iso885915': 'it_CH.ISO8859-15', + 'it_ch@euro': 'it_CH.ISO8859-15', 'it_it': 'it_IT.ISO8859-1', + 'it_it.88591': 'it_IT.ISO8859-1', + 'it_it.iso88591': 'it_IT.ISO8859-1', + 'it_it.iso885915': 'it_IT.ISO8859-15', + 'it_it.iso885915@euro': 'it_IT.ISO8859-15', + 'it_it.utf8@euro': 'it_IT.UTF-8', + 'it_it@euro': 'it_IT.ISO8859-15', 'italian': 'it_IT.ISO8859-1', + 'italian.iso88591': 'it_IT.ISO8859-1', 'iu': 'iu_CA.NUNACOM-8', 'iu_ca': 'iu_CA.NUNACOM-8', 'iu_ca.nunacom8': 'iu_CA.NUNACOM-8', 'iw': 'he_IL.ISO8859-8', 'iw_il': 'he_IL.ISO8859-8', - 'iw_il.utf8': 'iw_IL.UTF-8', + 'iw_il.iso88598': 'he_IL.ISO8859-8', 'ja': 'ja_JP.eucJP', + 'ja.jis': 'ja_JP.JIS7', + 'ja.sjis': 'ja_JP.SJIS', 'ja_jp': 'ja_JP.eucJP', + 'ja_jp.ajec': 'ja_JP.eucJP', 'ja_jp.euc': 'ja_JP.eucJP', + 'ja_jp.eucjp': 'ja_JP.eucJP', + 'ja_jp.iso-2022-jp': 'ja_JP.JIS7', + 'ja_jp.iso2022jp': 'ja_JP.JIS7', + 'ja_jp.jis': 'ja_JP.JIS7', + 'ja_jp.jis7': 'ja_JP.JIS7', 'ja_jp.mscode': 'ja_JP.SJIS', 'ja_jp.pck': 'ja_JP.SJIS', + 'ja_jp.sjis': 'ja_JP.SJIS', + 'ja_jp.ujis': 'ja_JP.eucJP', 'japan': 'ja_JP.eucJP', 'japanese': 'ja_JP.eucJP', 'japanese-euc': 'ja_JP.eucJP', 'japanese.euc': 'ja_JP.eucJP', + 'japanese.sjis': 'ja_JP.SJIS', 'jp_jp': 'ja_JP.eucJP', 'ka': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge.georgianacademy': 'ka_GE.GEORGIAN-ACADEMY', 'ka_ge.georgianps': 'ka_GE.GEORGIAN-PS', 'ka_ge.georgianrs': 'ka_GE.GEORGIAN-ACADEMY', - 'kk_kz': 'kk_KZ.RK1048', 'kl': 'kl_GL.ISO8859-1', 'kl_gl': 'kl_GL.ISO8859-1', + 'kl_gl.iso88591': 'kl_GL.ISO8859-1', + 'kl_gl.iso885915': 'kl_GL.ISO8859-15', + 'kl_gl@euro': 'kl_GL.ISO8859-15', 'km_kh': 'km_KH.UTF-8', 'kn': 'kn_IN.UTF-8', 'kn_in': 'kn_IN.UTF-8', 'ko': 'ko_KR.eucKR', 'ko_kr': 'ko_KR.eucKR', 'ko_kr.euc': 'ko_KR.eucKR', - 'kok_in': 'kok_IN.UTF-8', + 'ko_kr.euckr': 'ko_KR.eucKR', 'korean': 'ko_KR.eucKR', 'korean.euc': 'ko_KR.eucKR', 'ks': 'ks_IN.UTF-8', - 'ks_in': 'ks_IN.UTF-8', - 'ks_in@devanagari.utf8': 'ks_IN.UTF-8@devanagari', - 'ku_tr': 'ku_TR.ISO8859-9', + 'ks_in@devanagari': 'ks_IN@devanagari.UTF-8', 'kw': 'kw_GB.ISO8859-1', 'kw_gb': 'kw_GB.ISO8859-1', + 'kw_gb.iso88591': 'kw_GB.ISO8859-1', + 'kw_gb.iso885914': 'kw_GB.ISO8859-14', + 'kw_gb.iso885915': 'kw_GB.ISO8859-15', + 'kw_gb@euro': 'kw_GB.ISO8859-15', 'ky': 'ky_KG.UTF-8', 'ky_kg': 'ky_KG.UTF-8', - 'lb_lu': 'lb_LU.UTF-8', - 'lg_ug': 'lg_UG.ISO8859-10', - 'li_be': 'li_BE.UTF-8', - 'li_nl': 'li_NL.UTF-8', - 'lij_it': 'lij_IT.UTF-8', 'lithuanian': 'lt_LT.ISO8859-13', 'lo': 'lo_LA.MULELAO-1', 'lo_la': 'lo_LA.MULELAO-1', @@ -1147,102 +1280,150 @@ 'lo_la.mulelao1': 'lo_LA.MULELAO-1', 'lt': 'lt_LT.ISO8859-13', 'lt_lt': 'lt_LT.ISO8859-13', + 'lt_lt.iso885913': 'lt_LT.ISO8859-13', + 'lt_lt.iso88594': 'lt_LT.ISO8859-4', 'lv': 'lv_LV.ISO8859-13', 'lv_lv': 'lv_LV.ISO8859-13', - 'mag_in': 'mag_IN.UTF-8', + 'lv_lv.iso885913': 'lv_LV.ISO8859-13', + 'lv_lv.iso88594': 'lv_LV.ISO8859-4', 'mai': 'mai_IN.UTF-8', - 'mai_in': 'mai_IN.UTF-8', - 'mg_mg': 'mg_MG.ISO8859-15', - 'mhr_ru': 'mhr_RU.UTF-8', 'mi': 'mi_NZ.ISO8859-1', 'mi_nz': 'mi_NZ.ISO8859-1', + 'mi_nz.iso88591': 'mi_NZ.ISO8859-1', 'mk': 'mk_MK.ISO8859-5', 'mk_mk': 'mk_MK.ISO8859-5', + 'mk_mk.cp1251': 'mk_MK.CP1251', + 'mk_mk.iso88595': 'mk_MK.ISO8859-5', + 'mk_mk.microsoftcp1251': 'mk_MK.CP1251', 'ml': 'ml_IN.UTF-8', - 'ml_in': 'ml_IN.UTF-8', - 'mn_mn': 'mn_MN.UTF-8', - 'mni_in': 'mni_IN.UTF-8', 'mr': 'mr_IN.UTF-8', 'mr_in': 'mr_IN.UTF-8', 'ms': 'ms_MY.ISO8859-1', 'ms_my': 'ms_MY.ISO8859-1', + 'ms_my.iso88591': 'ms_MY.ISO8859-1', 'mt': 'mt_MT.ISO8859-3', 'mt_mt': 'mt_MT.ISO8859-3', - 'my_mm': 'my_MM.UTF-8', - 'nan_tw@latin': 'nan_TW.UTF-8@latin', + 'mt_mt.iso88593': 'mt_MT.ISO8859-3', 'nb': 'nb_NO.ISO8859-1', 'nb_no': 'nb_NO.ISO8859-1', - 'nds_de': 'nds_DE.UTF-8', - 'nds_nl': 'nds_NL.UTF-8', - 'ne_np': 'ne_NP.UTF-8', - 'nhn_mx': 'nhn_MX.UTF-8', - 'niu_nu': 'niu_NU.UTF-8', - 'niu_nz': 'niu_NZ.UTF-8', + 'nb_no.88591': 'nb_NO.ISO8859-1', + 'nb_no.iso88591': 'nb_NO.ISO8859-1', + 'nb_no.iso885915': 'nb_NO.ISO8859-15', + 'nb_no@euro': 'nb_NO.ISO8859-15', 'nl': 'nl_NL.ISO8859-1', - 'nl_aw': 'nl_AW.UTF-8', + 'nl.iso885915': 'nl_NL.ISO8859-15', 'nl_be': 'nl_BE.ISO8859-1', + 'nl_be.88591': 'nl_BE.ISO8859-1', + 'nl_be.iso88591': 'nl_BE.ISO8859-1', + 'nl_be.iso885915': 'nl_BE.ISO8859-15', + 'nl_be.iso885915@euro': 'nl_BE.ISO8859-15', + 'nl_be.utf8@euro': 'nl_BE.UTF-8', + 'nl_be@euro': 'nl_BE.ISO8859-15', 'nl_nl': 'nl_NL.ISO8859-1', + 'nl_nl.88591': 'nl_NL.ISO8859-1', + 'nl_nl.iso88591': 'nl_NL.ISO8859-1', + 'nl_nl.iso885915': 'nl_NL.ISO8859-15', + 'nl_nl.iso885915@euro': 'nl_NL.ISO8859-15', + 'nl_nl.utf8@euro': 'nl_NL.UTF-8', + 'nl_nl@euro': 'nl_NL.ISO8859-15', 'nn': 'nn_NO.ISO8859-1', 'nn_no': 'nn_NO.ISO8859-1', + 'nn_no.88591': 'nn_NO.ISO8859-1', + 'nn_no.iso88591': 'nn_NO.ISO8859-1', + 'nn_no.iso885915': 'nn_NO.ISO8859-15', + 'nn_no@euro': 'nn_NO.ISO8859-15', 'no': 'no_NO.ISO8859-1', 'no@nynorsk': 'ny_NO.ISO8859-1', 'no_no': 'no_NO.ISO8859-1', + 'no_no.88591': 'no_NO.ISO8859-1', + 'no_no.iso88591': 'no_NO.ISO8859-1', + 'no_no.iso885915': 'no_NO.ISO8859-15', 'no_no.iso88591@bokmal': 'no_NO.ISO8859-1', 'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1', + 'no_no@euro': 'no_NO.ISO8859-15', 'norwegian': 'no_NO.ISO8859-1', + 'norwegian.iso88591': 'no_NO.ISO8859-1', 'nr': 'nr_ZA.ISO8859-1', 'nr_za': 'nr_ZA.ISO8859-1', + 'nr_za.iso88591': 'nr_ZA.ISO8859-1', 'nso': 'nso_ZA.ISO8859-15', 'nso_za': 'nso_ZA.ISO8859-15', + 'nso_za.iso885915': 'nso_ZA.ISO8859-15', 'ny': 'ny_NO.ISO8859-1', 'ny_no': 'ny_NO.ISO8859-1', + 'ny_no.88591': 'ny_NO.ISO8859-1', + 'ny_no.iso88591': 'ny_NO.ISO8859-1', + 'ny_no.iso885915': 'ny_NO.ISO8859-15', + 'ny_no@euro': 'ny_NO.ISO8859-15', 'nynorsk': 'nn_NO.ISO8859-1', 'oc': 'oc_FR.ISO8859-1', 'oc_fr': 'oc_FR.ISO8859-1', - 'om_et': 'om_ET.UTF-8', - 'om_ke': 'om_KE.ISO8859-1', + 'oc_fr.iso88591': 'oc_FR.ISO8859-1', + 'oc_fr.iso885915': 'oc_FR.ISO8859-15', + 'oc_fr@euro': 'oc_FR.ISO8859-15', 'or': 'or_IN.UTF-8', - 'or_in': 'or_IN.UTF-8', - 'os_ru': 'os_RU.UTF-8', 'pa': 'pa_IN.UTF-8', 'pa_in': 'pa_IN.UTF-8', - 'pa_pk': 'pa_PK.UTF-8', - 'pap_an': 'pap_AN.UTF-8', 'pd': 'pd_US.ISO8859-1', 'pd_de': 'pd_DE.ISO8859-1', + 'pd_de.iso88591': 'pd_DE.ISO8859-1', + 'pd_de.iso885915': 'pd_DE.ISO8859-15', + 'pd_de@euro': 'pd_DE.ISO8859-15', 'pd_us': 'pd_US.ISO8859-1', + 'pd_us.iso88591': 'pd_US.ISO8859-1', + 'pd_us.iso885915': 'pd_US.ISO8859-15', + 'pd_us@euro': 'pd_US.ISO8859-15', 'ph': 'ph_PH.ISO8859-1', 'ph_ph': 'ph_PH.ISO8859-1', + 'ph_ph.iso88591': 'ph_PH.ISO8859-1', 'pl': 'pl_PL.ISO8859-2', 'pl_pl': 'pl_PL.ISO8859-2', + 'pl_pl.iso88592': 'pl_PL.ISO8859-2', 'polish': 'pl_PL.ISO8859-2', 'portuguese': 'pt_PT.ISO8859-1', + 'portuguese.iso88591': 'pt_PT.ISO8859-1', 'portuguese_brazil': 'pt_BR.ISO8859-1', + 'portuguese_brazil.8859': 'pt_BR.ISO8859-1', 'posix': 'C', 'posix-utf2': 'C', 'pp': 'pp_AN.ISO8859-1', 'pp_an': 'pp_AN.ISO8859-1', - 'ps_af': 'ps_AF.UTF-8', + 'pp_an.iso88591': 'pp_AN.ISO8859-1', 'pt': 'pt_PT.ISO8859-1', + 'pt.iso885915': 'pt_PT.ISO8859-15', 'pt_br': 'pt_BR.ISO8859-1', + 'pt_br.88591': 'pt_BR.ISO8859-1', + 'pt_br.iso88591': 'pt_BR.ISO8859-1', + 'pt_br.iso885915': 'pt_BR.ISO8859-15', + 'pt_br@euro': 'pt_BR.ISO8859-15', 'pt_pt': 'pt_PT.ISO8859-1', + 'pt_pt.88591': 'pt_PT.ISO8859-1', + 'pt_pt.iso88591': 'pt_PT.ISO8859-1', + 'pt_pt.iso885915': 'pt_PT.ISO8859-15', + 'pt_pt.iso885915@euro': 'pt_PT.ISO8859-15', + 'pt_pt.utf8@euro': 'pt_PT.UTF-8', + 'pt_pt@euro': 'pt_PT.ISO8859-15', 'ro': 'ro_RO.ISO8859-2', 'ro_ro': 'ro_RO.ISO8859-2', + 'ro_ro.iso88592': 'ro_RO.ISO8859-2', 'romanian': 'ro_RO.ISO8859-2', 'ru': 'ru_RU.UTF-8', + 'ru.koi8r': 'ru_RU.KOI8-R', 'ru_ru': 'ru_RU.UTF-8', + 'ru_ru.cp1251': 'ru_RU.CP1251', + 'ru_ru.iso88595': 'ru_RU.ISO8859-5', + 'ru_ru.koi8r': 'ru_RU.KOI8-R', + 'ru_ru.microsoftcp1251': 'ru_RU.CP1251', 'ru_ua': 'ru_UA.KOI8-U', + 'ru_ua.cp1251': 'ru_UA.CP1251', + 'ru_ua.koi8u': 'ru_UA.KOI8-U', + 'ru_ua.microsoftcp1251': 'ru_UA.CP1251', 'rumanian': 'ro_RO.ISO8859-2', 'russian': 'ru_RU.ISO8859-5', 'rw': 'rw_RW.ISO8859-1', 'rw_rw': 'rw_RW.ISO8859-1', - 'sa_in': 'sa_IN.UTF-8', - 'sat_in': 'sat_IN.UTF-8', - 'sc_it': 'sc_IT.UTF-8', - 'sd': 'sd_IN.UTF-8', - 'sd_in': 'sd_IN.UTF-8', - 'sd_in@devanagari.utf8': 'sd_IN.UTF-8@devanagari', - 'sd_pk': 'sd_PK.UTF-8', + 'rw_rw.iso88591': 'rw_RW.ISO8859-1', + 'sd': 'sd_IN@devanagari.UTF-8', 'se_no': 'se_NO.UTF-8', 'serbocroatian': 'sr_RS.UTF-8@latin', 'sh': 'sr_RS.UTF-8@latin', @@ -1251,38 +1432,42 @@ 'sh_hr.iso88592': 'hr_HR.ISO8859-2', 'sh_sp': 'sr_CS.ISO8859-2', 'sh_yu': 'sr_RS.UTF-8@latin', - 'shs_ca': 'shs_CA.UTF-8', 'si': 'si_LK.UTF-8', 'si_lk': 'si_LK.UTF-8', - 'sid_et': 'sid_ET.UTF-8', 'sinhala': 'si_LK.UTF-8', 'sk': 'sk_SK.ISO8859-2', 'sk_sk': 'sk_SK.ISO8859-2', + 'sk_sk.iso88592': 'sk_SK.ISO8859-2', 'sl': 'sl_SI.ISO8859-2', 'sl_cs': 'sl_CS.ISO8859-2', 'sl_si': 'sl_SI.ISO8859-2', + 'sl_si.iso88592': 'sl_SI.ISO8859-2', 'slovak': 'sk_SK.ISO8859-2', 'slovene': 'sl_SI.ISO8859-2', 'slovenian': 'sl_SI.ISO8859-2', - 'so_dj': 'so_DJ.ISO8859-1', - 'so_et': 'so_ET.UTF-8', - 'so_ke': 'so_KE.ISO8859-1', - 'so_so': 'so_SO.ISO8859-1', 'sp': 'sr_CS.ISO8859-5', 'sp_yu': 'sr_CS.ISO8859-5', 'spanish': 'es_ES.ISO8859-1', + 'spanish.iso88591': 'es_ES.ISO8859-1', 'spanish_spain': 'es_ES.ISO8859-1', + 'spanish_spain.8859': 'es_ES.ISO8859-1', 'sq': 'sq_AL.ISO8859-2', 'sq_al': 'sq_AL.ISO8859-2', - 'sq_mk': 'sq_MK.UTF-8', + 'sq_al.iso88592': 'sq_AL.ISO8859-2', 'sr': 'sr_RS.UTF-8', 'sr@cyrillic': 'sr_RS.UTF-8', - 'sr@latn': 'sr_CS.UTF-8@latin', - 'sr_cs': 'sr_CS.UTF-8', + 'sr@latin': 'sr_RS.UTF-8@latin', + 'sr@latn': 'sr_RS.UTF-8@latin', + 'sr_cs': 'sr_RS.UTF-8', + 'sr_cs.iso88592': 'sr_CS.ISO8859-2', 'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2', - 'sr_cs@latn': 'sr_CS.UTF-8@latin', + 'sr_cs.iso88595': 'sr_CS.ISO8859-5', + 'sr_cs.utf8@latn': 'sr_RS.UTF-8@latin', + 'sr_cs@latn': 'sr_RS.UTF-8@latin', 'sr_me': 'sr_ME.UTF-8', 'sr_rs': 'sr_RS.UTF-8', + 'sr_rs.utf8@latn': 'sr_RS.UTF-8@latin', + 'sr_rs@latin': 'sr_RS.UTF-8@latin', 'sr_rs@latn': 'sr_RS.UTF-8@latin', 'sr_sp': 'sr_CS.ISO8859-2', 'sr_yu': 'sr_RS.UTF-8@latin', @@ -1291,64 +1476,78 @@ 'sr_yu.iso88595': 'sr_CS.ISO8859-5', 'sr_yu.iso88595@cyrillic': 'sr_CS.ISO8859-5', 'sr_yu.microsoftcp1251@cyrillic': 'sr_CS.CP1251', - 'sr_yu.utf8': 'sr_RS.UTF-8', 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8', 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', 'ss_za': 'ss_ZA.ISO8859-1', + 'ss_za.iso88591': 'ss_ZA.ISO8859-1', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', + 'st_za.iso88591': 'st_ZA.ISO8859-1', 'sv': 'sv_SE.ISO8859-1', + 'sv.iso885915': 'sv_SE.ISO8859-15', 'sv_fi': 'sv_FI.ISO8859-1', + 'sv_fi.iso88591': 'sv_FI.ISO8859-1', + 'sv_fi.iso885915': 'sv_FI.ISO8859-15', + 'sv_fi.iso885915@euro': 'sv_FI.ISO8859-15', + 'sv_fi.utf8@euro': 'sv_FI.UTF-8', + 'sv_fi@euro': 'sv_FI.ISO8859-15', 'sv_se': 'sv_SE.ISO8859-1', - 'sw_ke': 'sw_KE.UTF-8', - 'sw_tz': 'sw_TZ.UTF-8', + 'sv_se.88591': 'sv_SE.ISO8859-1', + 'sv_se.iso88591': 'sv_SE.ISO8859-1', + 'sv_se.iso885915': 'sv_SE.ISO8859-15', + 'sv_se@euro': 'sv_SE.ISO8859-15', 'swedish': 'sv_SE.ISO8859-1', - 'szl_pl': 'szl_PL.UTF-8', + 'swedish.iso88591': 'sv_SE.ISO8859-1', 'ta': 'ta_IN.TSCII-0', 'ta_in': 'ta_IN.TSCII-0', 'ta_in.tscii': 'ta_IN.TSCII-0', 'ta_in.tscii0': 'ta_IN.TSCII-0', - 'ta_lk': 'ta_LK.UTF-8', 'te': 'te_IN.UTF-8', - 'te_in': 'te_IN.UTF-8', 'tg': 'tg_TJ.KOI8-C', 'tg_tj': 'tg_TJ.KOI8-C', + 'tg_tj.koi8c': 'tg_TJ.KOI8-C', 'th': 'th_TH.ISO8859-11', 'th_th': 'th_TH.ISO8859-11', + 'th_th.iso885911': 'th_TH.ISO8859-11', 'th_th.tactis': 'th_TH.TIS620', 'th_th.tis620': 'th_TH.TIS620', 'thai': 'th_TH.ISO8859-11', - 'ti_er': 'ti_ER.UTF-8', - 'ti_et': 'ti_ET.UTF-8', - 'tig_er': 'tig_ER.UTF-8', - 'tk_tm': 'tk_TM.UTF-8', 'tl': 'tl_PH.ISO8859-1', 'tl_ph': 'tl_PH.ISO8859-1', + 'tl_ph.iso88591': 'tl_PH.ISO8859-1', 'tn': 'tn_ZA.ISO8859-15', 'tn_za': 'tn_ZA.ISO8859-15', + 'tn_za.iso885915': 'tn_ZA.ISO8859-15', 'tr': 'tr_TR.ISO8859-9', - 'tr_cy': 'tr_CY.ISO8859-9', 'tr_tr': 'tr_TR.ISO8859-9', + 'tr_tr.iso88599': 'tr_TR.ISO8859-9', 'ts': 'ts_ZA.ISO8859-1', 'ts_za': 'ts_ZA.ISO8859-1', + 'ts_za.iso88591': 'ts_ZA.ISO8859-1', 'tt': 'tt_RU.TATAR-CYR', 'tt_ru': 'tt_RU.TATAR-CYR', + 'tt_ru.koi8c': 'tt_RU.KOI8-C', 'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR', - 'tt_ru@iqtelif': 'tt_RU.UTF-8@iqtelif', 'turkish': 'tr_TR.ISO8859-9', - 'ug_cn': 'ug_CN.UTF-8', + 'turkish.iso88599': 'tr_TR.ISO8859-9', 'uk': 'uk_UA.KOI8-U', 'uk_ua': 'uk_UA.KOI8-U', + 'uk_ua.cp1251': 'uk_UA.CP1251', + 'uk_ua.iso88595': 'uk_UA.ISO8859-5', + 'uk_ua.koi8u': 'uk_UA.KOI8-U', + 'uk_ua.microsoftcp1251': 'uk_UA.CP1251', 'univ': 'en_US.utf', 'universal': 'en_US.utf', 'universal.utf8@ucs4': 'en_US.UTF-8', - 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', - 'ur_in': 'ur_IN.UTF-8', 'ur_pk': 'ur_PK.CP1256', + 'ur_pk.cp1256': 'ur_PK.CP1256', + 'ur_pk.microsoftcp1256': 'ur_PK.CP1256', 'uz': 'uz_UZ.UTF-8', 'uz_uz': 'uz_UZ.UTF-8', + 'uz_uz.iso88591': 'uz_UZ.ISO8859-1', + 'uz_uz.utf8@cyrillic': 'uz_UZ.UTF-8', 'uz_uz@cyrillic': 'uz_UZ.UTF-8', 've': 've_ZA.UTF-8', 've_za': 've_ZA.UTF-8', @@ -1360,28 +1559,35 @@ 'vi_vn.viscii111': 'vi_VN.VISCII', 'wa': 'wa_BE.ISO8859-1', 'wa_be': 'wa_BE.ISO8859-1', - 'wae_ch': 'wae_CH.UTF-8', - 'wal_et': 'wal_ET.UTF-8', - 'wo_sn': 'wo_SN.UTF-8', + 'wa_be.iso88591': 'wa_BE.ISO8859-1', + 'wa_be.iso885915': 'wa_BE.ISO8859-15', + 'wa_be.iso885915@euro': 'wa_BE.ISO8859-15', + 'wa_be@euro': 'wa_BE.ISO8859-15', 'xh': 'xh_ZA.ISO8859-1', 'xh_za': 'xh_ZA.ISO8859-1', + 'xh_za.iso88591': 'xh_ZA.ISO8859-1', 'yi': 'yi_US.CP1255', 'yi_us': 'yi_US.CP1255', - 'yo_ng': 'yo_NG.UTF-8', - 'yue_hk': 'yue_HK.UTF-8', + 'yi_us.cp1255': 'yi_US.CP1255', + 'yi_us.microsoftcp1255': 'yi_US.CP1255', 'zh': 'zh_CN.eucCN', 'zh_cn': 'zh_CN.gb2312', 'zh_cn.big5': 'zh_TW.big5', 'zh_cn.euc': 'zh_CN.eucCN', + 'zh_cn.gb18030': 'zh_CN.gb18030', + 'zh_cn.gb2312': 'zh_CN.gb2312', + 'zh_cn.gbk': 'zh_CN.gbk', 'zh_hk': 'zh_HK.big5hkscs', + 'zh_hk.big5': 'zh_HK.big5', 'zh_hk.big5hk': 'zh_HK.big5hkscs', - 'zh_sg': 'zh_SG.GB2312', - 'zh_sg.gbk': 'zh_SG.GBK', + 'zh_hk.big5hkscs': 'zh_HK.big5hkscs', 'zh_tw': 'zh_TW.big5', + 'zh_tw.big5': 'zh_TW.big5', 'zh_tw.euc': 'zh_TW.eucTW', 'zh_tw.euctw': 'zh_TW.eucTW', 'zu': 'zu_ZA.ISO8859-1', 'zu_za': 'zu_ZA.ISO8859-1', + 'zu_za.iso88591': 'zu_ZA.ISO8859-1', } # diff -r 6db40a9955dc -r 0d413f60cc23 Lib/logging/__init__.py --- a/Lib/logging/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/logging/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -# Copyright 2001-2015 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,13 +18,12 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2015 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ -import sys, os, time, io, traceback, warnings, weakref, collections - +import sys, os, time, io, traceback, warnings, weakref from string import Template __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', @@ -33,9 +32,8 @@ 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', - 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', - 'lastResort', 'raiseExceptions'] + 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', + 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] try: import threading @@ -44,7 +42,6 @@ __author__ = "Vinay Sajip " __status__ = "production" -# The following module attributes are no longer updated. __version__ = "0.5.1.2" __date__ = "07 February 2010" @@ -53,6 +50,34 @@ #--------------------------------------------------------------------------- # +# _srcfile is used when walking the stack to check when we've got the first +# caller stack frame. +# +if hasattr(sys, 'frozen'): #support for py2exe + _srcfile = "logging%s__init__%s" % (os.sep, __file__[-4:]) +else: + _srcfile = __file__ +_srcfile = os.path.normcase(_srcfile) + + +if hasattr(sys, '_getframe'): + currentframe = lambda: sys._getframe(3) +else: #pragma: no cover + def currentframe(): + """Return the frame object for the caller's stack frame.""" + try: + raise Exception + except: + return sys.exc_info()[2].tb_frame.f_back + +# _srcfile is only used in conjunction with sys._getframe(). +# To provide compatibility with older versions of Python, set _srcfile +# to None if _getframe() is not available; this value will prevent +# findCaller() from being called. +#if not hasattr(sys, "_getframe"): +# _srcfile = None + +# #_startTime is used as the base when calculating the relative time of events # _startTime = time.time() @@ -98,22 +123,20 @@ DEBUG = 10 NOTSET = 0 -_levelToName = { - CRITICAL: 'CRITICAL', - ERROR: 'ERROR', - WARNING: 'WARNING', - INFO: 'INFO', - DEBUG: 'DEBUG', - NOTSET: 'NOTSET', -} -_nameToLevel = { - 'CRITICAL': CRITICAL, - 'ERROR': ERROR, - 'WARN': WARNING, - 'WARNING': WARNING, - 'INFO': INFO, - 'DEBUG': DEBUG, - 'NOTSET': NOTSET, +_levelNames = { + CRITICAL : 'CRITICAL', + ERROR : 'ERROR', + WARNING : 'WARNING', + INFO : 'INFO', + DEBUG : 'DEBUG', + NOTSET : 'NOTSET', + 'CRITICAL' : CRITICAL, + 'ERROR' : ERROR, + 'WARN' : WARNING, + 'WARNING' : WARNING, + 'INFO' : INFO, + 'DEBUG' : DEBUG, + 'NOTSET' : NOTSET, } def getLevelName(level): @@ -130,8 +153,7 @@ Otherwise, the string "Level %s" % level is returned. """ - # See Issue #22386 for the reason for this convoluted expression - return _levelToName.get(level, _nameToLevel.get(level, ("Level %s" % level))) + return _levelNames.get(level, ("Level %s" % level)) def addLevelName(level, levelName): """ @@ -141,52 +163,18 @@ """ _acquireLock() try: #unlikely to cause an exception, but you never know... - _levelToName[level] = levelName - _nameToLevel[levelName] = level + _levelNames[level] = levelName + _levelNames[levelName] = level finally: _releaseLock() -if hasattr(sys, '_getframe'): - currentframe = lambda: sys._getframe(3) -else: #pragma: no cover - def currentframe(): - """Return the frame object for the caller's stack frame.""" - try: - raise Exception - except Exception: - return sys.exc_info()[2].tb_frame.f_back - -# -# _srcfile is used when walking the stack to check when we've got the first -# caller stack frame, by skipping frames whose filename is that of this -# module's source. It therefore should contain the filename of this module's -# source file. -# -# Ordinarily we would use __file__ for this, but frozen modules don't always -# have __file__ set, for some reason (see Issue #21736). Thus, we get the -# filename from a handy code object from a function defined in this module. -# (There's no particular reason for picking addLevelName.) -# - -_srcfile = os.path.normcase(addLevelName.__code__.co_filename) - -# _srcfile is only used in conjunction with sys._getframe(). -# To provide compatibility with older versions of Python, set _srcfile -# to None if _getframe() is not available; this value will prevent -# findCaller() from being called. You can also do this if you want to avoid -# the overhead of fetching caller information, even when _getframe() is -# available. -#if not hasattr(sys, '_getframe'): -# _srcfile = None - - def _checkLevel(level): if isinstance(level, int): rv = level elif str(level) == level: - if level not in _nameToLevel: + if level not in _levelNames: raise ValueError("Unknown level: %r" % level) - rv = _nameToLevel[level] + rv = _levelNames[level] else: raise TypeError("Level not an integer or a valid string: %r" % level) return rv @@ -262,13 +250,7 @@ # 'Value is %d' instead of 'Value is 0'. # For the use case of passing a dictionary, this should not be a # problem. - # Issue #21172: a request was made to relax the isinstance check - # to hasattr(args[0], '__getitem__'). However, the docs on string - # formatting still seem to suggest a mapping object is required. - # Thus, while not removing the isinstance check, it does now look - # for collections.Mapping rather than, as before, dict. - if (args and len(args) == 1 and isinstance(args[0], collections.Mapping) - and args[0]): + if args and len(args) == 1 and isinstance(args[0], dict) and args[0]: args = args[0] self.args = args self.levelname = getLevelName(level) @@ -317,8 +299,6 @@ return ''%(self.name, self.levelno, self.pathname, self.lineno, self.msg) - __repr__ = __str__ - def getMessage(self): """ Return the message for this LogRecord. @@ -408,12 +388,10 @@ def format(self, record): return self._tpl.substitute(**record.__dict__) -BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" - _STYLES = { - '%': (PercentStyle, BASIC_FORMAT), - '{': (StrFormatStyle, '{levelname}:{name}:{message}'), - '$': (StringTemplateStyle, '${levelname}:${name}:${message}'), + '%': PercentStyle, + '{': StrFormatStyle, + '$': StringTemplateStyle } class Formatter(object): @@ -478,7 +456,7 @@ if style not in _STYLES: raise ValueError('Style must be one of: %s' % ','.join( _STYLES.keys())) - self._style = _STYLES[style][0](fmt) + self._style = _STYLES[style](fmt) self._fmt = self._style._fmt self.datefmt = datefmt @@ -728,17 +706,15 @@ Remove a handler reference from the internal cleanup list. """ # This function can be called during module teardown, when globals are - # set to None. It can also be called from another thread. So we need to - # pre-emptively grab the necessary globals and check if they're None, - # to prevent race conditions and failures during interpreter shutdown. - acquire, release, handlers = _acquireLock, _releaseLock, _handlerList - if acquire and release and handlers: - acquire() + # set to None. If _acquireLock is None, assume this is the case and do + # nothing. + if _acquireLock is not None: + _acquireLock() try: - if wr in handlers: - handlers.remove(wr) + if wr in _handlerList: + _handlerList.remove(wr) finally: - release() + _releaseLock() def _addHandlerRef(handler): """ @@ -903,37 +879,16 @@ The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 - t, v, tb = sys.exc_info() + ei = sys.exc_info() try: - sys.stderr.write('--- Logging error ---\n') - traceback.print_exception(t, v, tb, None, sys.stderr) - sys.stderr.write('Call stack:\n') - # Walk the stack frame up until we're out of logging, - # so as to print the calling context. - frame = tb.tb_frame - while (frame and os.path.dirname(frame.f_code.co_filename) == - __path__[0]): - frame = frame.f_back - if frame: - traceback.print_stack(frame, file=sys.stderr) - else: - # couldn't find the right stack frame, for some reason - sys.stderr.write('Logged from file %s, line %s\n' % ( - record.filename, record.lineno)) - # Issue 18671: output logging message and arguments - try: - sys.stderr.write('Message: %r\n' - 'Arguments: %s\n' % (record.msg, - record.args)) - except Exception: - sys.stderr.write('Unable to print the message and arguments' - ' - possible formatting error.\nUse the' - ' traceback above to help find the error.\n' - ) - except OSError: #pragma: no cover + traceback.print_exception(ei[0], ei[1], ei[2], + None, sys.stderr) + sys.stderr.write('Logged from file %s, line %s\n' % ( + record.filename, record.lineno)) + except IOError: #pragma: no cover pass # see issue 5971 finally: - del t, v, tb + del ei class StreamHandler(Handler): """ @@ -983,7 +938,9 @@ stream.write(msg) stream.write(self.terminator) self.flush() - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) class FileHandler(StreamHandler): @@ -999,7 +956,6 @@ self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding - self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. @@ -1014,19 +970,12 @@ """ self.acquire() try: - try: - if self.stream: - try: - self.flush() - finally: - stream = self.stream - self.stream = None - if hasattr(stream, "close"): - stream.close() - finally: - # Issue #19523: call unconditionally to - # prevent a handler leak when delay is set + if self.stream: + self.flush() + if hasattr(self.stream, "close"): + self.stream.close() StreamHandler.close(self) + self.stream = None finally: self.release() @@ -1094,6 +1043,7 @@ # # Determine which class to use when instantiating loggers. # +_loggerClass = None def setLoggerClass(klass): """ @@ -1112,6 +1062,7 @@ """ Return the class to be used when instantiating a logger. """ + return _loggerClass class Manager(object): @@ -1308,11 +1259,12 @@ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) - def exception(self, msg, *args, exc_info=True, **kwargs): + def exception(self, msg, *args, **kwargs): """ Convenience method for logging an ERROR with exception information. """ - self.error(msg, *args, exc_info=exc_info, **kwargs) + kwargs['exc_info'] = True + self.error(msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ @@ -1397,7 +1349,7 @@ """ sinfo = None if _srcfile: - #IronPython doesn't track Python frames, so findCaller raises an + #IronPython doesn't track Python frames, so findCaller throws an #exception on some versions of IronPython. We trap it here so that #IronPython can use logging. try: @@ -1407,9 +1359,7 @@ else: # pragma: no cover fn, lno, func = "(unknown file)", 0, "(unknown function)" if exc_info: - if isinstance(exc_info, BaseException): - exc_info = (type(exc_info), exc_info, exc_info.__traceback__) - elif not isinstance(exc_info, tuple): + if not isinstance(exc_info, tuple): exc_info = sys.exc_info() record = self.makeRecord(self.name, level, fn, lno, msg, args, exc_info, func, extra, sinfo) @@ -1619,11 +1569,12 @@ """ self.log(ERROR, msg, *args, **kwargs) - def exception(self, msg, *args, exc_info=True, **kwargs): + def exception(self, msg, *args, **kwargs): """ Delegate an exception call to the underlying logger. """ - self.log(ERROR, msg, *args, exc_info=exc_info, **kwargs) + kwargs["exc_info"] = True + self.log(ERROR, msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ @@ -1674,6 +1625,8 @@ # Configuration classes and functions #--------------------------------------------------------------------------- +BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" + def basicConfig(**kwargs): """ Do basic configuration for the logging system. @@ -1729,7 +1682,7 @@ _acquireLock() try: if len(root.handlers) == 0: - handlers = kwargs.pop("handlers", None) + handlers = kwargs.get("handlers") if handlers is None: if "stream" in kwargs and "filename" in kwargs: raise ValueError("'stream' and 'filename' should not be " @@ -1739,31 +1692,25 @@ raise ValueError("'stream' or 'filename' should not be " "specified together with 'handlers'") if handlers is None: - filename = kwargs.pop("filename", None) - mode = kwargs.pop("filemode", 'a') + filename = kwargs.get("filename") if filename: + mode = kwargs.get("filemode", 'a') h = FileHandler(filename, mode) else: - stream = kwargs.pop("stream", None) + stream = kwargs.get("stream") h = StreamHandler(stream) handlers = [h] - dfs = kwargs.pop("datefmt", None) - style = kwargs.pop("style", '%') - if style not in _STYLES: - raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) - fs = kwargs.pop("format", _STYLES[style][1]) + fs = kwargs.get("format", BASIC_FORMAT) + dfs = kwargs.get("datefmt", None) + style = kwargs.get("style", '%') fmt = Formatter(fs, dfs, style) for h in handlers: if h.formatter is None: h.setFormatter(fmt) root.addHandler(h) - level = kwargs.pop("level", None) + level = kwargs.get("level") if level is not None: root.setLevel(level) - if kwargs: - keys = ', '.join(kwargs.keys()) - raise ValueError('Unrecognised argument(s): %s' % keys) finally: _releaseLock() @@ -1805,13 +1752,14 @@ basicConfig() root.error(msg, *args, **kwargs) -def exception(msg, *args, exc_info=True, **kwargs): +def exception(msg, *args, **kwargs): """ Log a message with severity 'ERROR' on the root logger, with exception information. If the logger has no handlers, basicConfig() is called to add a console handler with a pre-defined format. """ - error(msg, *args, exc_info=exc_info, **kwargs) + kwargs['exc_info'] = True + error(msg, *args, **kwargs) def warning(msg, *args, **kwargs): """ @@ -1881,7 +1829,7 @@ h.acquire() h.flush() h.close() - except (OSError, ValueError): + except (IOError, ValueError): # Ignore errors which might be caused # because handlers have been closed but # references to them are still around at @@ -1889,7 +1837,7 @@ pass finally: h.release() - except: # ignore everything, as we're shutting down + except: if raiseExceptions: raise #else, swallow diff -r 6db40a9955dc -r 0d413f60cc23 Lib/logging/config.py --- a/Lib/logging/config.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/logging/config.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2010 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,19 +19,13 @@ is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2010 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ -import errno +import sys, logging, logging.handlers, socket, struct, traceback, re import io -import logging -import logging.handlers -import re -import struct -import sys -import traceback try: import _thread as thread @@ -44,7 +38,10 @@ DEFAULT_LOGGING_CONFIG_PORT = 9030 -RESET_ERROR = errno.ECONNRESET +if sys.platform == "win32": + RESET_ERROR = 10054 #WSAECONNRESET +else: + RESET_ERROR = 104 #ECONNRESET # # The following code implements a socket listener for on-the-fly @@ -64,14 +61,11 @@ """ import configparser - if isinstance(fname, configparser.RawConfigParser): - cp = fname + cp = configparser.ConfigParser(defaults) + if hasattr(fname, 'readline'): + cp.read_file(fname) else: - cp = configparser.ConfigParser(defaults) - if hasattr(fname, 'readline'): - cp.read_file(fname) - else: - cp.read(fname) + cp.read(fname) formatters = _create_formatters(cp) @@ -116,12 +110,11 @@ sectname = "formatter_%s" % form fs = cp.get(sectname, "format", raw=True, fallback=None) dfs = cp.get(sectname, "datefmt", raw=True, fallback=None) - stl = cp.get(sectname, "style", raw=True, fallback='%') c = logging.Formatter class_name = cp[sectname].get("class") if class_name: c = _resolve(class_name) - f = c(fs, dfs, stl) + f = c(fs, dfs) formatters[form] = f return formatters @@ -148,7 +141,7 @@ h = klass(*args) if "level" in section: level = section["level"] - h.setLevel(level) + h.setLevel(logging._levelNames[level]) if len(fmt): h.setFormatter(formatters[fmt]) if issubclass(klass, logging.handlers.MemoryHandler): @@ -179,8 +172,8 @@ logger.level = logging.NOTSET logger.handlers = [] logger.propagate = True - else: - logger.disabled = disable_existing + elif disable_existing: + logger.disabled = True def _install_loggers(cp, handlers, disable_existing): """Create and install loggers""" @@ -195,7 +188,7 @@ log = root if "level" in section: level = section["level"] - log.setLevel(level) + log.setLevel(logging._levelNames[level]) for h in root.handlers[:]: root.removeHandler(h) hlist = section["handlers"] @@ -241,7 +234,7 @@ existing.remove(qn) if "level" in section: level = section["level"] - logger.setLevel(level) + logger.setLevel(logging._levelNames[level]) for h in logger.handlers[:]: logger.removeHandler(h) logger.propagate = propagate @@ -278,30 +271,6 @@ return True -class ConvertingMixin(object): - """For ConvertingXXX's, this mixin class provides common functions""" - - def convert_with_key(self, key, value, replace=True): - result = self.configurator.convert(value) - #If the converted value is different, save for next time - if value is not result: - if replace: - self[key] = result - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): - result.parent = self - result.key = key - return result - - def convert(self, value): - result = self.configurator.convert(value) - if value is not result: - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): - result.parent = self - return result - - # The ConvertingXXX classes are wrappers around standard Python containers, # and they serve to convert any suitable values in the container. The # conversion converts base dicts, lists and tuples to their wrapped @@ -311,37 +280,77 @@ # Each wrapper should have a configurator attribute holding the actual # configurator to use for conversion. -class ConvertingDict(dict, ConvertingMixin): +class ConvertingDict(dict): """A converting dictionary wrapper.""" def __getitem__(self, key): value = dict.__getitem__(self, key) - return self.convert_with_key(key, value) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result def get(self, key, default=None): value = dict.get(self, key, default) - return self.convert_with_key(key, value) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result def pop(self, key, default=None): value = dict.pop(self, key, default) - return self.convert_with_key(key, value, replace=False) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result -class ConvertingList(list, ConvertingMixin): +class ConvertingList(list): """A converting list wrapper.""" def __getitem__(self, key): value = list.__getitem__(self, key) - return self.convert_with_key(key, value) + result = self.configurator.convert(value) + #If the converted value is different, save for next time + if value is not result: + self[key] = result + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result def pop(self, idx=-1): value = list.pop(self, idx) - return self.convert(value) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + return result -class ConvertingTuple(tuple, ConvertingMixin): +class ConvertingTuple(tuple): """A converting tuple wrapper.""" def __getitem__(self, key): value = tuple.__getitem__(self, key) - # Can't replace a tuple entry. - return self.convert_with_key(key, value, replace=False) + result = self.configurator.convert(value) + if value is not result: + if type(result) in (ConvertingDict, ConvertingList, + ConvertingTuple): + result.parent = self + result.key = key + return result class BaseConfigurator(object): """ @@ -552,29 +561,14 @@ # As handlers can refer to other handlers, sort the keys # to allow a deterministic order of configuration handlers = config.get('handlers', EMPTY_DICT) - deferred = [] for name in sorted(handlers): try: handler = self.configure_handler(handlers[name]) handler.name = name handlers[name] = handler except Exception as e: - if 'target not configured yet' in str(e): - deferred.append(name) - else: - raise ValueError('Unable to configure handler ' - '%r: %s' % (name, e)) - - # Now do any that were deferred - for name in deferred: - try: - handler = self.configure_handler(handlers[name]) - handler.name = name - handlers[name] = handler - except Exception as e: raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) - # Next, do loggers - they refer to handlers and filters #we don't want to lose the existing loggers, @@ -660,13 +654,7 @@ else: fmt = config.get('format', None) dfmt = config.get('datefmt', None) - style = config.get('style', '%') - cname = config.get('class', None) - if not cname: - c = logging.Formatter - else: - c = _resolve(cname) - result = c(fmt, dfmt, style) + result = logging.Formatter(fmt, dfmt) return result def configure_filter(self, config): @@ -688,7 +676,6 @@ def configure_handler(self, config): """Configure a handler from a dictionary.""" - config_copy = dict(config) # for restoring in case of error formatter = config.pop('formatter', None) if formatter: try: @@ -704,17 +691,12 @@ c = self.resolve(c) factory = c else: - cname = config.pop('class') - klass = self.resolve(cname) + klass = self.resolve(config.pop('class')) #Special case for handler which refers to another handler if issubclass(klass, logging.handlers.MemoryHandler) and\ 'target' in config: try: - th = self.config['handlers'][config['target']] - if not isinstance(th, logging.Handler): - config.update(config_copy) # restore for deferred cfg - raise TypeError('target not configured yet') - config['target'] = th + config['target'] = self.config['handlers'][config['target']] except Exception as e: raise ValueError('Unable to set target handler ' '%r: %s' % (config['target'], e)) @@ -725,7 +707,6 @@ 'address' in config: config['address'] = self.as_tuple(config['address']) factory = klass - props = config.pop('.', None) kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) try: result = factory(**kwargs) @@ -744,9 +725,6 @@ result.setLevel(logging._checkLevel(level)) if filters: self.add_filters(result, filters) - if props: - for name, value in props.items(): - setattr(result, name, value) return result def add_handlers(self, logger, handlers): @@ -795,7 +773,7 @@ dictConfigClass(config).configure() -def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None): +def listen(port=DEFAULT_LOGGING_CONFIG_PORT): """ Start up a socket server on the specified port, and listen for new configurations. @@ -804,15 +782,6 @@ Returns a Thread object on which you can call start() to start the server, and which you can join() when appropriate. To stop the server, call stopListening(). - - Use the ``verify`` argument to verify any bytes received across the wire - from a client. If specified, it should be a callable which receives a - single argument - the bytes of configuration data received across the - network - and it should return either ``None``, to indicate that the - passed in bytes could not be verified and should be discarded, or a - byte string which is then passed to the configuration machinery as - normal. Note that you can return transformed bytes, e.g. by decrypting - the bytes passed in. """ if not thread: #pragma: no cover raise NotImplementedError("listen() needs threading to work") @@ -840,28 +809,31 @@ chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + conn.recv(slen - len(chunk)) - if self.server.verify is not None: - chunk = self.server.verify(chunk) - if chunk is not None: # verified, can process - chunk = chunk.decode("utf-8") + chunk = chunk.decode("utf-8") + try: + import json + d =json.loads(chunk) + assert isinstance(d, dict) + dictConfig(d) + except: + #Apply new configuration. + + file = io.StringIO(chunk) try: - import json - d =json.loads(chunk) - assert isinstance(d, dict) - dictConfig(d) - except Exception: - #Apply new configuration. - - file = io.StringIO(chunk) - try: - fileConfig(file) - except Exception: - traceback.print_exc() + fileConfig(file) + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: + traceback.print_exc() if self.server.ready: self.server.ready.set() - except OSError as e: - if e.errno != RESET_ERROR: + except socket.error as e: + if not isinstance(e.args, tuple): raise + else: + errcode = e.args[0] + if errcode != RESET_ERROR: + raise class ConfigSocketReceiver(ThreadingTCPServer): """ @@ -871,14 +843,13 @@ allow_reuse_address = 1 def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, - handler=None, ready=None, verify=None): + handler=None, ready=None): ThreadingTCPServer.__init__(self, (host, port), handler) logging._acquireLock() self.abort = 0 logging._releaseLock() self.timeout = 1 self.ready = ready - self.verify = verify def serve_until_stopped(self): import select @@ -896,18 +867,16 @@ class Server(threading.Thread): - def __init__(self, rcvr, hdlr, port, verify): + def __init__(self, rcvr, hdlr, port): super(Server, self).__init__() self.rcvr = rcvr self.hdlr = hdlr self.port = port - self.verify = verify self.ready = threading.Event() def run(self): server = self.rcvr(port=self.port, handler=self.hdlr, - ready=self.ready, - verify=self.verify) + ready=self.ready) if self.port == 0: self.port = server.server_address[1] self.ready.set() @@ -917,7 +886,7 @@ logging._releaseLock() server.serve_until_stopped() - return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify) + return Server(ConfigSocketReceiver, ConfigStreamHandler, port) def stopListening(): """ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/logging/handlers.py --- a/Lib/logging/handlers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/logging/handlers.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,4 +1,4 @@ -# Copyright 2001-2015 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,12 +18,13 @@ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2015 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ import logging, socket, os, pickle, struct, time, re +from codecs import BOM_UTF8 from stat import ST_DEV, ST_INO, ST_MTIME import queue try: @@ -71,7 +72,9 @@ if self.shouldRollover(record): self.doRollover() logging.FileHandler.emit(self, record) - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) def rotation_filename(self, default_name): @@ -108,9 +111,7 @@ what the source is rotated to, e.g. 'test.log.1'. """ if not callable(self.rotator): - # Issue 18940: A file may not have been created if delay is True. - if os.path.exists(source): - os.rename(source, dest) + os.rename(source, dest) else: self.rotator(source, dest) @@ -171,8 +172,8 @@ if os.path.exists(dfn): os.remove(dfn) self.rotate(self.baseFilename, dfn) - if not self.delay: - self.stream = self._open() + self.mode = 'w' + self.stream = self._open() def shouldRollover(self, record): """ @@ -198,12 +199,11 @@ If backupCount is > 0, when rollover is done, no more than backupCount files are kept - the oldest ones are deleted. """ - def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None): + def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) self.when = when.upper() self.backupCount = backupCount self.utc = utc - self.atTime = atTime # Calculate the real rollover interval, which is just the number of # seconds between rollovers. Also set the filename suffix used when # a rollover occurs. Current 'when' events supported: @@ -273,22 +273,9 @@ currentHour = t[3] currentMinute = t[4] currentSecond = t[5] - currentDay = t[6] - # r is the number of seconds left between now and the next rotation - if self.atTime is None: - rotate_ts = _MIDNIGHT - else: - rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 + - self.atTime.second) - - r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 + - currentSecond) - if r < 0: - # Rotate time is before the current time (for example when - # self.rotateAt is 13:45 and it now 14:15), rotation is - # tomorrow. - r += _MIDNIGHT - currentDay = (currentDay + 1) % 7 + # r is the number of seconds left between now and midnight + r = _MIDNIGHT - ((currentHour * 60 + currentMinute) * 60 + + currentSecond) result = currentTime + r # If we are rolling over on a certain day, add in the number of days until # the next rollover, but offset by 1 since we just calculated the time @@ -306,7 +293,7 @@ # This is because the above time calculation takes us to midnight on this # day, i.e. the start of the next day. if self.when.startswith('W'): - day = currentDay # 0 is Monday + day = t[6] # 0 is Monday if day != self.dayOfWeek: if day < self.dayOfWeek: daysToWait = self.dayOfWeek - day @@ -394,8 +381,8 @@ if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) - if not self.delay: - self.stream = self._open() + self.mode = 'w' + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval @@ -432,53 +419,35 @@ """ def __init__(self, filename, mode='a', encoding=None, delay=False): logging.FileHandler.__init__(self, filename, mode, encoding, delay) - self.dev, self.ino = -1, -1 - self._statstream() - - def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] - - def reopenIfNeeded(self): - """ - Reopen log file if needed. - - Checks if the underlying file has changed, and if it - has, close the old stream and reopen the file to get the - current stream. - """ - # Reduce the chance of race conditions by stat'ing by path only - # once and then fstat'ing our new fd if we opened a new log stream. - # See issue #14632: Thanks to John Mulligan for the problem report - # and patch. - try: - # stat the file by path, checking for existence - sres = os.stat(self.baseFilename) - except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + if not os.path.exists(self.baseFilename): + self.dev, self.ino = -1, -1 + else: + stat = os.stat(self.baseFilename) + self.dev, self.ino = stat[ST_DEV], stat[ST_INO] def emit(self, record): """ Emit a record. - If underlying file has changed, reopen the file before emitting the - record to it. + First check if the underlying file has changed, and if it + has, close the old stream and reopen the file to get the + current stream. """ - self.reopenIfNeeded() + if not os.path.exists(self.baseFilename): + stat = None + changed = True + else: + stat = os.stat(self.baseFilename) + changed = (stat[ST_DEV] != self.dev) or (stat[ST_INO] != self.ino) + if changed and self.stream is not None: + self.stream.flush() + self.stream.close() + self.stream = self._open() + if stat is None: + stat = os.stat(self.baseFilename) + self.dev, self.ino = stat[ST_DEV], stat[ST_INO] logging.FileHandler.emit(self, record) - class SocketHandler(logging.Handler): """ A handler class which writes logging records, in pickle format, to @@ -503,10 +472,6 @@ logging.Handler.__init__(self) self.host = host self.port = port - if port is None: - self.address = host - else: - self.address = (host, port) self.sock = None self.closeOnError = False self.retryTime = None @@ -522,17 +487,15 @@ A factory method which allows subclasses to define the precise type of socket they want. """ - if self.port is not None: - result = socket.create_connection(self.address, timeout=timeout) - else: - result = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - result.settimeout(timeout) - try: - result.connect(self.address) - except OSError: - result.close() # Issue 19182 - raise - return result + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if hasattr(s, 'settimeout'): + s.settimeout(timeout) + try: + s.connect((self.host, self.port)) + return s + except socket.error: + s.close() + raise def createSocket(self): """ @@ -552,7 +515,7 @@ try: self.sock = self.makeSocket() self.retryTime = None # next time, no delay before trying - except OSError: + except socket.error: #Creation failed, so set the retry time and return. if self.retryTime is None: self.retryPeriod = self.retryStart @@ -576,8 +539,16 @@ #but are still unable to connect. if self.sock: try: - self.sock.sendall(s) - except OSError: #pragma: no cover + if hasattr(self.sock, "sendall"): + self.sock.sendall(s) + else: #pragma: no cover + sentsofar = 0 + left = len(s) + while left > 0: + sent = self.sock.send(s[sentsofar:]) + sentsofar = sentsofar + sent + left = left - sent + except socket.error: #pragma: no cover self.sock.close() self.sock = None # so we can call createSocket next time @@ -588,18 +559,11 @@ """ ei = record.exc_info if ei: - # just to get traceback text into record.exc_text ... - dummy = self.format(record) - # See issue #14436: If msg or args are objects, they may not be - # available on the receiving end. So we convert the msg % args - # to a string, save it as msg and zap the args. - d = dict(record.__dict__) - d['msg'] = record.getMessage() - d['args'] = None - d['exc_info'] = None - # Issue #25685: delete 'message' if present: redundant with 'msg' - d.pop('message', None) - s = pickle.dumps(d, 1) + dummy = self.format(record) # just to get traceback text into record.exc_text + record.exc_info = None # to avoid Unpickleable error + s = pickle.dumps(record.__dict__, 1) + if ei: + record.exc_info = ei # for next handler slen = struct.pack(">L", len(s)) return slen + s @@ -629,7 +593,9 @@ try: s = self.makePickle(record) self.send(s) - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) def close(self): @@ -638,10 +604,9 @@ """ self.acquire() try: - sock = self.sock - if sock: + if self.sock: + self.sock.close() self.sock = None - sock.close() logging.Handler.close(self) finally: self.release() @@ -669,11 +634,7 @@ The factory method of SocketHandler is here overridden to create a UDP socket (SOCK_DGRAM). """ - if self.port is None: - family = socket.AF_UNIX - else: - family = socket.AF_INET - s = socket.socket(family, socket.SOCK_DGRAM) + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) return s def send(self, s): @@ -686,7 +647,7 @@ """ if self.sock is None: self.createSocket() - self.sock.sendto(s, self.address) + self.sock.sendto(s, (self.host, self.port)) class SysLogHandler(logging.Handler): """ @@ -792,17 +753,13 @@ } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), - facility=LOG_USER, socktype=None): + facility=LOG_USER, socktype=socket.SOCK_DGRAM): """ Initialize a handler. If address is specified as a string, a UNIX socket is used. To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be used. - If facility is not specified, LOG_USER is used. If socktype is - specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific - socket type will be used. For Unix sockets, you can also specify a - socktype of None, in which case socket.SOCK_DGRAM will be used, falling - back to socket.SOCK_STREAM. + If facility is not specified, LOG_USER is used. """ logging.Handler.__init__(self) @@ -815,37 +772,20 @@ self._connect_unixsocket(address) else: self.unixsocket = False - if socktype is None: - socktype = socket.SOCK_DGRAM self.socket = socket.socket(socket.AF_INET, socktype) if socktype == socket.SOCK_STREAM: self.socket.connect(address) - self.socktype = socktype self.formatter = None def _connect_unixsocket(self, address): - use_socktype = self.socktype - if use_socktype is None: - use_socktype = socket.SOCK_DGRAM - self.socket = socket.socket(socket.AF_UNIX, use_socktype) + self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + # syslog may require either DGRAM or STREAM sockets try: self.socket.connect(address) - # it worked, so set self.socktype to the used type - self.socktype = use_socktype - except OSError: + except socket.error: self.socket.close() - if self.socktype is not None: - # user didn't specify falling back, so fail - raise - use_socktype = socket.SOCK_STREAM - self.socket = socket.socket(socket.AF_UNIX, use_socktype) - try: - self.socket.connect(address) - # it worked, so set self.socktype to the used type - self.socktype = use_socktype - except OSError: - self.socket.close() - raise + self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.socket.connect(address) def encodePriority(self, facility, priority): """ @@ -891,33 +831,35 @@ The record is formatted, and then sent to the syslog server. If exception information is present, it is NOT sent to the server. """ + msg = self.format(record) + if self.ident: + msg = self.ident + msg + if self.append_nul: + msg += '\000' + """ + We need to convert record level to lowercase, maybe this will + change in the future. + """ + prio = '<%d>' % self.encodePriority(self.facility, + self.mapPriority(record.levelname)) + prio = prio.encode('utf-8') + # Message is a string. Convert to bytes as required by RFC 5424 + msg = msg.encode('utf-8') + msg = prio + BOM_UTF8 + msg try: - msg = self.format(record) - if self.ident: - msg = self.ident + msg - if self.append_nul: - msg += '\000' - - # We need to convert record level to lowercase, maybe this will - # change in the future. - prio = '<%d>' % self.encodePriority(self.facility, - self.mapPriority(record.levelname)) - prio = prio.encode('utf-8') - # Message is a string. Convert to bytes as required by RFC 5424 - msg = msg.encode('utf-8') - msg = prio + msg if self.unixsocket: try: self.socket.send(msg) - except OSError: - self.socket.close() + except socket.error: self._connect_unixsocket(self.address) self.socket.send(msg) elif self.socktype == socket.SOCK_DGRAM: self.socket.sendto(msg, self.address) else: self.socket.sendall(msg) - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) class SMTPHandler(logging.Handler): @@ -925,7 +867,7 @@ A handler class which sends an SMTP email for each logging event. """ def __init__(self, mailhost, fromaddr, toaddrs, subject, - credentials=None, secure=None, timeout=5.0): + credentials=None, secure=None, timeout=1.0): """ Initialize the handler. @@ -943,11 +885,11 @@ default is one second). """ logging.Handler.__init__(self) - if isinstance(mailhost, (list, tuple)): + if isinstance(mailhost, tuple): self.mailhost, self.mailport = mailhost else: self.mailhost, self.mailport = mailhost, None - if isinstance(credentials, (list, tuple)): + if isinstance(credentials, tuple): self.username, self.password = credentials else: self.username = None @@ -976,28 +918,28 @@ """ try: import smtplib - from email.message import EmailMessage - import email.utils - + from email.utils import formatdate port = self.mailport if not port: port = smtplib.SMTP_PORT smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout) - msg = EmailMessage() - msg['From'] = self.fromaddr - msg['To'] = ','.join(self.toaddrs) - msg['Subject'] = self.getSubject(record) - msg['Date'] = email.utils.localtime() - msg.set_content(self.format(record)) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + ",".join(self.toaddrs), + self.getSubject(record), + formatdate(), msg) if self.username: if self.secure is not None: smtp.ehlo() smtp.starttls(*self.secure) smtp.ehlo() smtp.login(self.username, self.password) - smtp.send_message(msg) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) smtp.quit() - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) class NTEventLogHandler(logging.Handler): @@ -1082,7 +1024,9 @@ type = self.getEventType(record) msg = self.format(record) self._welu.ReportEvent(self.appname, id, cat, type, [msg]) - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) def close(self): @@ -1103,8 +1047,7 @@ A class which sends records to a Web server, using either GET or POST semantics. """ - def __init__(self, host, url, method="GET", secure=False, credentials=None, - context=None): + def __init__(self, host, url, method="GET", secure=False, credentials=None): """ Initialize the instance with the host, the request URL, and the method ("GET" or "POST") @@ -1113,15 +1056,11 @@ method = method.upper() if method not in ["GET", "POST"]: raise ValueError("method must be GET or POST") - if not secure and context is not None: - raise ValueError("context parameter only makes sense " - "with secure=True") self.host = host self.url = url self.method = method self.secure = secure self.credentials = credentials - self.context = context def mapLogRecord(self, record): """ @@ -1141,7 +1080,7 @@ import http.client, urllib.parse host = self.host if self.secure: - h = http.client.HTTPSConnection(host, context=self.context) + h = http.client.HTTPSConnection(host) else: h = http.client.HTTPConnection(host) url = self.url @@ -1172,7 +1111,9 @@ if self.method == "POST": h.send(data.encode('utf-8')) h.getresponse() #can't do anything with the result - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) class BufferingHandler(logging.Handler): @@ -1227,10 +1168,8 @@ This version just flushes and chains to the parent class' close(). """ - try: - self.flush() - finally: - logging.Handler.close(self) + self.flush() + logging.Handler.close(self) class MemoryHandler(BufferingHandler): """ @@ -1284,15 +1223,13 @@ """ Flush, set the target to None and lose the buffer. """ + self.flush() + self.acquire() try: - self.flush() + self.target = None + BufferingHandler.close(self) finally: - self.acquire() - try: - self.target = None - BufferingHandler.close(self) - finally: - self.release() + self.release() class QueueHandler(logging.Handler): @@ -1356,7 +1293,9 @@ """ try: self.enqueue(self.prepare(record)) - except Exception: + except (KeyboardInterrupt, SystemExit): #pragma: no cover + raise + except: self.handleError(record) if threading: @@ -1368,7 +1307,7 @@ """ _sentinel = None - def __init__(self, queue, *handlers, respect_handler_level=False): + def __init__(self, queue, *handlers): """ Initialise an instance with the specified queue and handlers. @@ -1377,7 +1316,6 @@ self.handlers = handlers self._stop = threading.Event() self._thread = None - self.respect_handler_level = respect_handler_level def dequeue(self, block): """ @@ -1418,12 +1356,7 @@ """ record = self.prepare(record) for handler in self.handlers: - if not self.respect_handler_level: - process = True - else: - process = record.levelno >= handler.level - if process: - handler.handle(record) + handler.handle(record) def _monitor(self): """ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/lzma.py --- a/Lib/lzma.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/lzma.py Mon Jan 25 17:05:13 2016 +0100 @@ -18,23 +18,22 @@ "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME", "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError", - "open", "compress", "decompress", "is_check_supported", + "compress", "decompress", "check_is_supported", ] -import builtins import io from _lzma import * -from _lzma import _encode_filter_properties, _decode_filter_properties -import _compression _MODE_CLOSED = 0 _MODE_READ = 1 -# Value 2 no longer used +_MODE_READ_EOF = 2 _MODE_WRITE = 3 +_BUFFER_SIZE = 8192 -class LZMAFile(_compression.BaseStream): + +class LZMAFile(io.BufferedIOBase): """A file object providing transparent LZMA (de)compression. @@ -46,16 +45,16 @@ """ def __init__(self, filename=None, mode="r", *, - format=None, check=-1, preset=None, filters=None): - """Open an LZMA-compressed file in binary mode. + fileobj=None, format=None, check=-1, + preset=None, filters=None): + """Open an LZMA-compressed file. - filename can be either an actual file name (given as a str or - bytes object), in which case the named file is opened, or it can - be an existing file object to read from or write to. + If filename is given, open the named file. Otherwise, operate on + the file object given by fileobj. Exactly one of these two + parameters should be provided. - mode can be "r" for reading (default), "w" for (over)writing, - "x" for creating exclusively, or "a" for appending. These can - equivalently be given as "rb", "wb", "xb" and "ab" respectively. + mode can be "r" for reading (default), "w" for (over)writing, or + "a" for appending. format specifies the container format to use for the file. If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the @@ -91,8 +90,10 @@ self._fp = None self._closefp = False self._mode = _MODE_CLOSED + self._pos = 0 + self._size = -1 - if mode in ("r", "rb"): + if mode == "r": if check != -1: raise ValueError("Cannot specify an integrity check " "when opening a file for reading") @@ -102,32 +103,31 @@ if format is None: format = FORMAT_AUTO mode_code = _MODE_READ - elif mode in ("w", "wb", "a", "ab", "x", "xb"): + # Save the args to pass to the LZMADecompressor initializer. + # If the file contains multiple compressed streams, each + # stream will need a separate decompressor object. + self._init_args = {"format":format, "filters":filters} + self._decompressor = LZMADecompressor(**self._init_args) + self._buffer = None + elif mode in ("w", "a"): if format is None: format = FORMAT_XZ mode_code = _MODE_WRITE self._compressor = LZMACompressor(format=format, check=check, preset=preset, filters=filters) - self._pos = 0 else: raise ValueError("Invalid mode: {!r}".format(mode)) - if isinstance(filename, (str, bytes)): - if "b" not in mode: - mode += "b" - self._fp = builtins.open(filename, mode) + if filename is not None and fileobj is None: + mode += "b" + self._fp = open(filename, mode) self._closefp = True self._mode = mode_code - elif hasattr(filename, "read") or hasattr(filename, "write"): - self._fp = filename + elif fileobj is not None and filename is None: + self._fp = fileobj self._mode = mode_code else: - raise TypeError("filename must be a str or bytes object, or a file") - - if self._mode == _MODE_READ: - raw = _compression.DecompressReader(self._fp, LZMADecompressor, - trailing_error=LZMAError, format=format, filters=filters) - self._buffer = io.BufferedReader(raw) + raise ValueError("Must give exactly one of filename and fileobj") def close(self): """Flush and close the file. @@ -138,8 +138,8 @@ if self._mode == _MODE_CLOSED: return try: - if self._mode == _MODE_READ: - self._buffer.close() + if self._mode in (_MODE_READ, _MODE_READ_EOF): + self._decompressor = None self._buffer = None elif self._mode == _MODE_WRITE: self._fp.write(self._compressor.flush()) @@ -165,18 +165,96 @@ def seekable(self): """Return whether the file supports seeking.""" - return self.readable() and self._buffer.seekable() + return self.readable() and self._fp.seekable() def readable(self): """Return whether the file was opened for reading.""" self._check_not_closed() - return self._mode == _MODE_READ + return self._mode in (_MODE_READ, _MODE_READ_EOF) def writable(self): """Return whether the file was opened for writing.""" self._check_not_closed() return self._mode == _MODE_WRITE + # Mode-checking helper functions. + + def _check_not_closed(self): + if self.closed: + raise ValueError("I/O operation on closed file") + + def _check_can_read(self): + if not self.readable(): + raise io.UnsupportedOperation("File not open for reading") + + def _check_can_write(self): + if not self.writable(): + raise io.UnsupportedOperation("File not open for writing") + + def _check_can_seek(self): + if not self.readable(): + raise io.UnsupportedOperation("Seeking is only supported " + "on files open for reading") + if not self._fp.seekable(): + raise io.UnsupportedOperation("The underlying file object " + "does not support seeking") + + # Fill the readahead buffer if it is empty. Returns False on EOF. + def _fill_buffer(self): + if self._buffer: + return True + + if self._decompressor.unused_data: + rawblock = self._decompressor.unused_data + else: + rawblock = self._fp.read(_BUFFER_SIZE) + + if not rawblock: + if self._decompressor.eof: + self._mode = _MODE_READ_EOF + self._size = self._pos + return False + else: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + + # Continue to next stream. + if self._decompressor.eof: + self._decompressor = LZMADecompressor(**self._init_args) + + self._buffer = self._decompressor.decompress(rawblock) + return True + + # Read data until EOF. + # If return_data is false, consume the data without returning it. + def _read_all(self, return_data=True): + blocks = [] + while self._fill_buffer(): + if return_data: + blocks.append(self._buffer) + self._pos += len(self._buffer) + self._buffer = None + if return_data: + return b"".join(blocks) + + # Read a block of up to n bytes. + # If return_data is false, consume the data without returning it. + def _read_block(self, n, return_data=True): + blocks = [] + while n > 0 and self._fill_buffer(): + if n < len(self._buffer): + data = self._buffer[:n] + self._buffer = self._buffer[n:] + else: + data = self._buffer + self._buffer = None + if return_data: + blocks.append(data) + self._pos += len(data) + n -= len(data) + if return_data: + return b"".join(blocks) + def peek(self, size=-1): """Return buffered data without advancing the file position. @@ -184,9 +262,9 @@ The exact number of bytes returned is unspecified. """ self._check_can_read() - # Relies on the undocumented fact that BufferedReader.peek() always - # returns at least one byte (except at EOF) - return self._buffer.peek(size) + if self._mode == _MODE_READ_EOF or not self._fill_buffer(): + return b"" + return self._buffer def read(self, size=-1): """Read up to size uncompressed bytes from the file. @@ -195,29 +273,31 @@ Returns b"" if the file is already at EOF. """ self._check_can_read() - return self._buffer.read(size) + if self._mode == _MODE_READ_EOF or size == 0: + return b"" + elif size < 0: + return self._read_all() + else: + return self._read_block(size) def read1(self, size=-1): - """Read up to size uncompressed bytes, while trying to avoid - making multiple reads from the underlying stream. Reads up to a - buffer's worth of data if size is negative. + """Read up to size uncompressed bytes with at most one read + from the underlying stream. Returns b"" if the file is at EOF. """ self._check_can_read() - if size < 0: - size = io.DEFAULT_BUFFER_SIZE - return self._buffer.read1(size) - - def readline(self, size=-1): - """Read a line of uncompressed bytes from the file. - - The terminating newline (if present) is retained. If size is - non-negative, no more than size bytes will be read (in which - case the line may be incomplete). Returns b'' if already at EOF. - """ - self._check_can_read() - return self._buffer.readline(size) + if (size == 0 or self._mode == _MODE_READ_EOF or + not self._fill_buffer()): + return b"" + if 0 < size < len(self._buffer): + data = self._buffer[:size] + self._buffer = self._buffer[size:] + else: + data = self._buffer + self._buffer = None + self._pos += len(data) + return data def write(self, data): """Write a bytes object to the file. @@ -232,7 +312,15 @@ self._pos += len(data) return len(data) - def seek(self, offset, whence=io.SEEK_SET): + # Rewind the file to the beginning of the data stream. + def _rewind(self): + self._fp.seek(0, 0) + self._mode = _MODE_READ + self._pos = 0 + self._decompressor = LZMADecompressor(**self._init_args) + self._buffer = None + + def seek(self, offset, whence=0): """Change the file position. The new position is specified by offset, relative to the @@ -244,74 +332,49 @@ Returns the new file position. - Note that seeking is emulated, so depending on the parameters, + Note that seeking is emulated, sp depending on the parameters, this operation may be extremely slow. """ self._check_can_seek() - return self._buffer.seek(offset, whence) + + # Recalculate offset as an absolute file position. + if whence == 0: + pass + elif whence == 1: + offset = self._pos + offset + elif whence == 2: + # Seeking relative to EOF - we need to know the file's size. + if self._size < 0: + self._read_all(return_data=False) + offset = self._size + offset + else: + raise ValueError("Invalid value for whence: {}".format(whence)) + + # Make it so that offset is the number of bytes to skip forward. + if offset < self._pos: + self._rewind() + else: + offset -= self._pos + + # Read and discard data until we reach the desired position. + if self._mode != _MODE_READ_EOF: + self._read_block(offset, return_data=False) + + return self._pos def tell(self): """Return the current file position.""" self._check_not_closed() - if self._mode == _MODE_READ: - return self._buffer.tell() return self._pos -def open(filename, mode="rb", *, - format=None, check=-1, preset=None, filters=None, - encoding=None, errors=None, newline=None): - """Open an LZMA-compressed file in binary or text mode. - - filename can be either an actual file name (given as a str or bytes - object), in which case the named file is opened, or it can be an - existing file object to read from or write to. - - The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb", - "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text - mode. - - The format, check, preset and filters arguments specify the - compression settings, as for LZMACompressor, LZMADecompressor and - LZMAFile. - - For binary mode, this function is equivalent to the LZMAFile - constructor: LZMAFile(filename, mode, ...). In this case, the - encoding, errors and newline arguments must not be provided. - - For text mode, a LZMAFile object is created, and wrapped in an - io.TextIOWrapper instance with the specified encoding, error - handling behavior, and line ending(s). - - """ - if "t" in mode: - if "b" in mode: - raise ValueError("Invalid mode: %r" % (mode,)) - else: - if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") - if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") - if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") - - lz_mode = mode.replace("t", "") - binary_file = LZMAFile(filename, lz_mode, format=format, check=check, - preset=preset, filters=filters) - - if "t" in mode: - return io.TextIOWrapper(binary_file, encoding, errors, newline) - else: - return binary_file - - def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None): """Compress a block of data. Refer to LZMACompressor's docstring for a description of the optional arguments *format*, *check*, *preset* and *filters*. - For incremental compression, use an LZMACompressor instead. + For incremental compression, use an LZMACompressor object instead. """ comp = LZMACompressor(format, check, preset, filters) return comp.compress(data) + comp.flush() @@ -323,23 +386,16 @@ Refer to LZMADecompressor's docstring for a description of the optional arguments *format*, *check* and *filters*. - For incremental decompression, use an LZMADecompressor instead. + For incremental decompression, use a LZMADecompressor object instead. """ results = [] while True: decomp = LZMADecompressor(format, memlimit, filters) - try: - res = decomp.decompress(data) - except LZMAError: - if results: - break # Leftover data is not a valid LZMA/XZ stream; ignore it. - else: - raise # Error on the first iteration; bail out. - results.append(res) + results.append(decomp.decompress(data)) if not decomp.eof: raise LZMAError("Compressed data ended before the " "end-of-stream marker was reached") + if not decomp.unused_data: + return b"".join(results) + # There is unused data left over. Proceed to next stream. data = decomp.unused_data - if not data: - break - return b"".join(results) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/macpath.py --- a/Lib/macpath.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/macpath.py Mon Jan 25 17:05:13 2016 +0100 @@ -50,26 +50,20 @@ def join(s, *p): - try: - colon = _get_colon(s) - path = s - if not p: - path[:0] + colon #23780: Ensure compatible data type even if p is null. - for t in p: - if (not path) or isabs(t): - path = t - continue - if t[:1] == colon: - t = t[1:] - if colon not in path: - path = colon + path - if path[-1:] != colon: - path = path + colon - path = path + t - return path - except (TypeError, AttributeError, BytesWarning): - genericpath._check_arg_types('join', s, *p) - raise + colon = _get_colon(s) + path = s + for t in p: + if (not s) or isabs(t): + path = t + continue + if t[:1] == colon: + t = t[1:] + if colon not in path: + path = colon + path + if path[-1:] != colon: + path = path + colon + path = path + t + return path def split(s): @@ -133,7 +127,7 @@ try: st = os.lstat(path) - except OSError: + except os.error: return False return True diff -r 6db40a9955dc -r 0d413f60cc23 Lib/macurl2path.py --- a/Lib/macurl2path.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/macurl2path.py Mon Jan 25 17:05:13 2016 +0100 @@ -75,3 +75,23 @@ def _pncomp2url(component): # We want to quote slashes return urllib.parse.quote(component[:31], safe='') + +def test(): + for url in ["index.html", + "bar/index.html", + "/foo/bar/index.html", + "/foo/bar/", + "/"]: + print('%r -> %r' % (url, url2pathname(url))) + for path in ["drive:", + "drive:dir:", + "drive:dir:file", + "drive:file", + "file", + ":file", + ":dir:", + ":dir:file"]: + print('%r -> %r' % (path, pathname2url(path))) + +if __name__ == '__main__': + test() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/mailbox.py --- a/Lib/mailbox.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/mailbox.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,3 +1,5 @@ +#! /usr/bin/env python3 + """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes.""" # Notes for authors of new mailbox subclasses: @@ -6,6 +8,7 @@ # or returning from a flush() method. See functions _sync_flush() and # _sync_close(). +import sys import os import time import calendar @@ -19,6 +22,9 @@ import io import contextlib try: + if sys.platform == 'os2emx': + # OS/2 EMX fcntl() not adequate + raise ImportError import fcntl except ImportError: fcntl = None @@ -103,7 +109,7 @@ def itervalues(self): """Return an iterator over all messages.""" - for key in self.iterkeys(): + for key in self.keys(): try: value = self[key] except KeyError: @@ -119,7 +125,7 @@ def iteritems(self): """Return an iterator over (key, message) tuples.""" - for key in self.iterkeys(): + for key in self.keys(): try: value = self[key] except KeyError: @@ -154,7 +160,7 @@ def popitem(self): """Delete an arbitrary (key, message) pair and return it.""" - for key in self.iterkeys(): + for key in self.keys(): return (key, self.pop(key)) # This is only run once. else: raise KeyError('No messages in mailbox') @@ -162,7 +168,7 @@ def update(self, arg=None): """Change the messages that correspond to certain keys.""" if hasattr(arg, 'iteritems'): - source = arg.iteritems() + source = arg.items() elif hasattr(arg, 'items'): source = arg.items() else: @@ -202,9 +208,6 @@ raise ValueError("String input must be ASCII-only; " "use bytes or a Message instead") - # Whether each message must end in a newline - _append_newline = False - def _dump_message(self, message, target, mangle_from_=False): # This assumes the target file is open in binary mode. """Dump message contents to target file.""" @@ -216,9 +219,6 @@ data = buffer.read() data = data.replace(b'\n', linesep) target.write(data) - if self._append_newline and not data.endswith(linesep): - # Make sure the message ends with a newline - target.write(linesep) elif isinstance(message, (str, bytes, io.StringIO)): if isinstance(message, io.StringIO): warnings.warn("Use of StringIO input is deprecated, " @@ -230,15 +230,11 @@ message = message.replace(b'\nFrom ', b'\n>From ') message = message.replace(b'\n', linesep) target.write(message) - if self._append_newline and not message.endswith(linesep): - # Make sure the message ends with a newline - target.write(linesep) elif hasattr(message, 'read'): if hasattr(message, 'buffer'): warnings.warn("Use of text mode files is deprecated, " "use a binary mode file instead", DeprecationWarning, 3) message = message.buffer - lastline = None while True: line = message.readline() # Universal newline support. @@ -252,10 +248,6 @@ line = b'>From ' + line[5:] line = line.replace(b'\n', linesep) target.write(line) - lastline = line - if self._append_newline and lastline and not lastline.endswith(linesep): - # Make sure the message ends with a newline - target.write(linesep) else: raise TypeError('Invalid message type: %s' % type(message)) @@ -305,12 +297,6 @@ suffix = '' uniq = os.path.basename(tmp_file.name).split(self.colon)[0] dest = os.path.join(self._path, subdir, uniq + suffix) - if isinstance(message, MaildirMessage): - os.utime(tmp_file.name, - (os.path.getatime(tmp_file.name), message.get_date())) - # No file modification should be done after the file is moved to its - # final position in order to prevent race conditions with changes - # from other programs try: if hasattr(os, 'link'): os.link(tmp_file.name, dest) @@ -324,6 +310,8 @@ % dest) else: raise + if isinstance(message, MaildirMessage): + os.utime(dest, (os.path.getatime(dest), message.get_date())) return uniq def remove(self, key): @@ -335,8 +323,11 @@ # This overrides an inapplicable implementation in the superclass. try: self.remove(key) - except (KeyError, FileNotFoundError): + except KeyError: pass + except OSError as e: + if e.errno != errno.ENOENT: + raise def __setitem__(self, key, message): """Replace the keyed message; raise KeyError if it doesn't exist.""" @@ -355,24 +346,23 @@ else: suffix = '' self.discard(key) - tmp_path = os.path.join(self._path, temp_subpath) new_path = os.path.join(self._path, subdir, key + suffix) + os.rename(os.path.join(self._path, temp_subpath), new_path) if isinstance(message, MaildirMessage): - os.utime(tmp_path, - (os.path.getatime(tmp_path), message.get_date())) - # No file modification should be done after the file is moved to its - # final position in order to prevent race conditions with changes - # from other programs - os.rename(tmp_path, new_path) + os.utime(new_path, (os.path.getatime(new_path), + message.get_date())) def get_message(self, key): """Return a Message representation or raise a KeyError.""" subpath = self._lookup(key) - with open(os.path.join(self._path, subpath), 'rb') as f: + f = open(os.path.join(self._path, subpath), 'rb') + try: if self._factory: msg = self._factory(f) else: msg = MaildirMessage(f) + finally: + f.close() subdir, name = os.path.split(subpath) msg.set_subdir(subdir) if self.colon in name: @@ -382,8 +372,11 @@ def get_bytes(self, key): """Return a bytes representation or raise a KeyError.""" - with open(os.path.join(self._path, self._lookup(key)), 'rb') as f: + f = open(os.path.join(self._path, self._lookup(key)), 'rb') + try: return f.read().replace(linesep, b'\n') + finally: + f.close() def get_file(self, key): """Return a file-like representation or raise a KeyError.""" @@ -495,12 +488,16 @@ path = os.path.join(self._path, 'tmp', uniq) try: os.stat(path) - except FileNotFoundError: - Maildir._count += 1 - try: - return _create_carefully(path) - except FileExistsError: - pass + except OSError as e: + if e.errno == errno.ENOENT: + Maildir._count += 1 + try: + return _create_carefully(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + else: + raise # Fall through to here if stat succeeded or open raised EEXIST. raise ExternalClashError('Name clash prevented file creation: %s' % @@ -559,7 +556,7 @@ def next(self): """Return the next message in a one-time iteration.""" if not hasattr(self, '_onetime_keys'): - self._onetime_keys = self.iterkeys() + self._onetime_keys = iter(self.keys()) while True: try: return self[next(self._onetime_keys)] @@ -577,7 +574,7 @@ Mailbox.__init__(self, path, factory, create) try: f = open(self._path, 'rb+') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: if create: f = open(self._path, 'wb+') @@ -590,19 +587,16 @@ self._file = f self._toc = None self._next_key = 0 - self._pending = False # No changes require rewriting the file. - self._pending_sync = False # No need to sync the file + self._pending = False # No changes require rewriting the file. self._locked = False - self._file_length = None # Used to record mailbox size + self._file_length = None # Used to record mailbox size def add(self, message): """Add message and return assigned key.""" self._lookup() self._toc[self._next_key] = self._append_message(message) self._next_key += 1 - # _append_message appends the message to the mailbox file. We - # don't need a full rewrite + rename, sync is enough. - self._pending_sync = True + self._pending = True return self._next_key - 1 def remove(self, key): @@ -620,7 +614,8 @@ def iterkeys(self): """Return an iterator over keys.""" self._lookup() - yield from self._toc.keys() + for key in self._toc.keys(): + yield key def __contains__(self, key): """Return True if the keyed message exists, False otherwise.""" @@ -647,11 +642,6 @@ def flush(self): """Write any pending changes to disk.""" if not self._pending: - if self._pending_sync: - # Messages have only been added, so syncing the file - # is enough. - _sync_flush(self._file) - self._pending_sync = False return # In order to be writing anything out at all, self._toc must @@ -685,7 +675,6 @@ new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) - self._file_length = new_file.tell() except: new_file.close() os.remove(new_file.name) @@ -693,18 +682,18 @@ _sync_close(new_file) # self._file is about to get replaced, so no need to sync. self._file.close() - # Make sure the new file's mode is the same as the old file's - mode = os.stat(self._path).st_mode - os.chmod(new_file.name, mode) try: os.rename(new_file.name, self._path) - except FileExistsError: - os.remove(self._path) - os.rename(new_file.name, self._path) + except OSError as e: + if e.errno == errno.EEXIST or \ + (os.name == 'os2' and e.errno == errno.EACCES): + os.remove(self._path) + os.rename(new_file.name, self._path) + else: + raise self._file = open(self._path, 'rb+') self._toc = new_toc self._pending = False - self._pending_sync = False if self._locked: _lock_file(self._file, dotlock=False) @@ -722,14 +711,10 @@ def close(self): """Flush and close the mailbox.""" - try: - self.flush() - finally: - try: - if self._locked: - self.unlock() - finally: - self._file.close() # Sync has been done by self.flush() above. + self.flush() + if self._locked: + self.unlock() + self._file.close() # Sync has been done by self.flush() above. def _lookup(self, key=None): """Return (start, stop) or raise KeyError.""" @@ -745,12 +730,6 @@ """Append message to mailbox and return (start, stop) offsets.""" self._file.seek(0, 2) before = self._file.tell() - if len(self._toc) == 0 and not self._pending: - # This is the first message, and the _pre_mailbox_hook - # hasn't yet been called. If self._pending is True, - # messages have been removed, so _pre_mailbox_hook must - # have been called already. - self._pre_mailbox_hook(self._file) try: self._pre_message_hook(self._file) offsets = self._install_message(message) @@ -835,48 +814,30 @@ _mangle_from_ = True - # All messages must end in a newline character, and - # _post_message_hooks outputs an empty line between messages. - _append_newline = True - def __init__(self, path, factory=None, create=True): """Initialize an mbox mailbox.""" self._message_factory = mboxMessage _mboxMMDF.__init__(self, path, factory, create) - def _post_message_hook(self, f): - """Called after writing each message to file f.""" - f.write(linesep) + def _pre_message_hook(self, f): + """Called before writing each message to file f.""" + if f.tell() != 0: + f.write(linesep) def _generate_toc(self): """Generate key-to-(start, stop) table of contents.""" starts, stops = [], [] - last_was_empty = False self._file.seek(0) while True: line_pos = self._file.tell() line = self._file.readline() if line.startswith(b'From '): if len(stops) < len(starts): - if last_was_empty: - stops.append(line_pos - len(linesep)) - else: - # The last line before the "From " line wasn't - # blank, but we consider it a start of a - # message anyway. - stops.append(line_pos) + stops.append(line_pos - len(linesep)) starts.append(line_pos) - last_was_empty = False elif not line: - if last_was_empty: - stops.append(line_pos - len(linesep)) - else: - stops.append(line_pos) + stops.append(line_pos) break - elif line == linesep: - last_was_empty = True - else: - last_was_empty = False self._toc = dict(enumerate(zip(starts, stops))) self._next_key = len(self._toc) self._file_length = self._file.tell() @@ -981,7 +942,7 @@ path = os.path.join(self._path, str(key)) try: f = open(path, 'rb+') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: @@ -995,7 +956,7 @@ path = os.path.join(self._path, str(key)) try: f = open(path, 'rb+') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: @@ -1021,12 +982,12 @@ f = open(os.path.join(self._path, str(key)), 'rb+') else: f = open(os.path.join(self._path, str(key)), 'rb') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise - with f: + try: if self._locked: _lock_file(f) try: @@ -1034,6 +995,8 @@ finally: if self._locked: _unlock_file(f) + finally: + f.close() for name, key_list in self.get_sequences().items(): if key in key_list: msg.add_sequence(name) @@ -1046,12 +1009,12 @@ f = open(os.path.join(self._path, str(key)), 'rb+') else: f = open(os.path.join(self._path, str(key)), 'rb') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: raise - with f: + try: if self._locked: _lock_file(f) try: @@ -1059,12 +1022,14 @@ finally: if self._locked: _unlock_file(f) + finally: + f.close() def get_file(self, key): """Return a file-like representation or raise a KeyError.""" try: f = open(os.path.join(self._path, str(key)), 'rb') - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: raise KeyError('No message with key: %s' % key) else: @@ -1082,7 +1047,7 @@ def __len__(self): """Return a count of messages in the mailbox.""" - return len(list(self.iterkeys())) + return len(list(self.keys())) def lock(self): """Lock the mailbox.""" @@ -1196,7 +1161,7 @@ sequences = self.get_sequences() prev = 0 changes = [] - for key in self.iterkeys(): + for key in self.keys(): if key - 1 != prev: changes.append((key, prev + 1)) if hasattr(os, 'link'): @@ -1234,8 +1199,8 @@ class Babyl(_singlefileMailbox): """An Rmail-style Babyl mailbox.""" - _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered', - 'forwarded', 'edited', 'resent'}) + _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered', + 'forwarded', 'edited', 'resent')) def __init__(self, path, factory=None, create=True): """Initialize a Babyl mailbox.""" @@ -1456,24 +1421,17 @@ line = line[:-1] + b'\n' self._file.write(line.replace(b'\n', linesep)) if line == b'\n' or not line: + self._file.write(b'*** EOOH ***' + linesep) if first_pass: first_pass = False - self._file.write(b'*** EOOH ***' + linesep) message.seek(original_pos) else: break while True: - line = message.readline() - if not line: + buffer = message.read(4096) # Buffer size is arbitrary. + if not buffer: break - # Universal newline support. - if line.endswith(b'\r\n'): - line = line[:-2] + linesep - elif line.endswith(b'\r'): - line = line[:-1] + linesep - elif line.endswith(b'\n'): - line = line[:-1] + linesep - self._file.write(line) + self._file.write(buffer.replace(b'\n', linesep)) else: raise TypeError('Invalid message type: %s' % type(message)) stop = self._file.tell() @@ -1504,10 +1462,9 @@ def _become_message(self, message): """Assume the non-format-specific state of message.""" - type_specific = getattr(message, '_type_specific_attributes', []) - for name in message.__dict__: - if name not in type_specific: - self.__dict__[name] = message.__dict__[name] + for name in ('_headers', '_unixfrom', '_payload', '_charset', + 'preamble', 'epilogue', 'defects', '_default_type'): + self.__dict__[name] = message.__dict__[name] def _explain_to(self, message): """Copy format-specific state to message insofar as possible.""" @@ -1520,8 +1477,6 @@ class MaildirMessage(Message): """Message with Maildir-specific properties.""" - _type_specific_attributes = ['_subdir', '_info', '_date'] - def __init__(self, message=None): """Initialize a MaildirMessage instance.""" self._subdir = 'new' @@ -1629,8 +1584,6 @@ class _mboxMMDFMessage(Message): """Message with mbox- or MMDF-specific properties.""" - _type_specific_attributes = ['_from'] - def __init__(self, message=None): """Initialize an mboxMMDFMessage instance.""" self.set_from('MAILER-DAEMON', True) @@ -1746,8 +1699,6 @@ class MHMessage(Message): """Message with MH-specific properties.""" - _type_specific_attributes = ['_sequences'] - def __init__(self, message=None): """Initialize an MHMessage instance.""" self._sequences = [] @@ -1818,8 +1769,6 @@ class BabylMessage(Message): """Message with Babyl-specific properties.""" - _type_specific_attributes = ['_labels', '_visible'] - def __init__(self, message=None): """Initialize an BabylMessage instance.""" self._labels = [] @@ -1953,7 +1902,7 @@ while True: line = self.readline() if not line: - return + raise StopIteration yield line def tell(self): @@ -1970,11 +1919,9 @@ def close(self): """Close the file.""" if hasattr(self, '_file'): - try: - if hasattr(self._file, 'close'): - self._file.close() - finally: - del self._file + if hasattr(self._file, 'close'): + self._file.close() + del self._file def _read(self, size, read_method): """Read size bytes using read_method.""" @@ -1986,7 +1933,7 @@ return result def __enter__(self): - """Context management protocol support.""" + """Context manager protocol support.""" return self def __exit__(self, *exc): @@ -2059,7 +2006,7 @@ if fcntl: try: fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as e: + except IOError as e: if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): raise ExternalClashError('lockf: lock unavailable: %s' % f.name) @@ -2069,7 +2016,7 @@ try: pre_lock = _create_temporary(f.name + '.lock') pre_lock.close() - except OSError as e: + except IOError as e: if e.errno in (errno.EACCES, errno.EROFS): return # Without write access, just skip dotlocking. else: @@ -2082,10 +2029,14 @@ else: os.rename(pre_lock.name, f.name + '.lock') dotlock_done = True - except FileExistsError: - os.remove(pre_lock.name) - raise ExternalClashError('dot lock unavailable: %s' % - f.name) + except OSError as e: + if e.errno == errno.EEXIST or \ + (os.name == 'os2' and e.errno == errno.EACCES): + os.remove(pre_lock.name) + raise ExternalClashError('dot lock unavailable: %s' % + f.name) + else: + raise except: if fcntl: fcntl.lockf(f, fcntl.LOCK_UN) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/mailcap.py --- a/Lib/mailcap.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/mailcap.py Mon Jan 25 17:05:13 2016 +0100 @@ -20,10 +20,10 @@ for mailcap in listmailcapfiles(): try: fp = open(mailcap, 'r') - except OSError: + except IOError: continue - with fp: - morecaps = readmailcapfile(fp) + morecaps = readmailcapfile(fp) + fp.close() for key, value in morecaps.items(): if not key in caps: caps[key] = value diff -r 6db40a9955dc -r 0d413f60cc23 Lib/mimetypes.py --- a/Lib/mimetypes.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/mimetypes.py Mon Jan 25 17:05:13 2016 +0100 @@ -246,25 +246,22 @@ except EnvironmentError: break else: - if '\0' not in ctype: - yield ctype + yield ctype i += 1 - with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, '') as hkcr: - for subkeyname in enum_types(hkcr): + with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, + r'MIME\Database\Content Type') as mimedb: + for ctype in enum_types(mimedb): try: - with _winreg.OpenKey(hkcr, subkeyname) as subkey: - # Only check file extensions - if not subkeyname.startswith("."): - continue - # raises EnvironmentError if no 'Content Type' value - mimetype, datatype = _winreg.QueryValueEx( - subkey, 'Content Type') - if datatype != _winreg.REG_SZ: - continue - self.add_type(mimetype, subkeyname, strict) + with _winreg.OpenKey(mimedb, ctype) as key: + suffix, datatype = _winreg.QueryValueEx(key, + 'Extension') except EnvironmentError: continue + if datatype != _winreg.REG_SZ: + continue + self.add_type(ctype, suffix, strict) + def guess_type(url, strict=True): """Guess the type of a file based on its URL. @@ -362,12 +359,11 @@ def read_mime_types(file): try: f = open(file) - except OSError: + except IOError: return None - with f: - db = MimeTypes() - db.readfp(f, True) - return db.types_map[True] + db = MimeTypes() + db.readfp(f, True) + return db.types_map[True] def _default_mime_types(): @@ -382,14 +378,12 @@ '.taz': '.tar.gz', '.tz': '.tar.gz', '.tbz2': '.tar.bz2', - '.txz': '.tar.xz', } encodings_map = { '.gz': 'gzip', '.Z': 'compress', '.bz2': 'bzip2', - '.xz': 'xz', } # Before adding new types, make sure they are either registered with IANA, @@ -435,7 +429,7 @@ '.jpe' : 'image/jpeg', '.jpeg' : 'image/jpeg', '.jpg' : 'image/jpeg', - '.js' : 'application/javascript', + '.js' : 'application/x-javascript', '.ksh' : 'text/plain', '.latex' : 'application/x-latex', '.m1v' : 'video/mpeg', diff -r 6db40a9955dc -r 0d413f60cc23 Lib/modulefinder.py --- a/Lib/modulefinder.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/modulefinder.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,17 +1,12 @@ """Find modules used by a script, using introspection.""" import dis -import importlib._bootstrap_external -import importlib.machinery +import imp import marshal import os import sys import types import struct -import warnings -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - import imp # XXX Clean up once str8's cstor matches bytes. LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) @@ -223,17 +218,16 @@ if not m.__path__: return modules = {} - # 'suffixes' used to be a list hardcoded to [".py", ".pyc"]. + # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. # But we must also collect Python extension modules - although # we cannot separate normal dlls from Python extensions. suffixes = [] - suffixes += importlib.machinery.EXTENSION_SUFFIXES[:] - suffixes += importlib.machinery.SOURCE_SUFFIXES[:] - suffixes += importlib.machinery.BYTECODE_SUFFIXES[:] + for triple in imp.get_suffixes(): + suffixes.append(triple[0]) for dir in m.__path__: try: names = os.listdir(dir) - except OSError: + except os.error: self.msg(2, "can't list directory", dir) continue for name in names: @@ -288,12 +282,11 @@ if type == imp.PY_SOURCE: co = compile(fp.read()+'\n', pathname, 'exec') elif type == imp.PY_COMPILED: - try: - marshal_data = importlib._bootstrap_external._validate_bytecode_header(fp.read()) - except ImportError as exc: - self.msgout(2, "raise ImportError: " + str(exc), pathname) - raise - co = marshal.loads(marshal_data) + if fp.read(4) != imp.get_magic(): + self.msgout(2, "raise ImportError: Bad magic number", pathname) + raise ImportError("Bad magic number in %s" % pathname) + fp.read(4) + co = marshal.load(fp) else: co = None m = self.add_module(fqname) @@ -337,6 +330,30 @@ fullname = name + "." + sub self._add_badmodule(fullname, caller) + def scan_opcodes(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Version for Python 2.4 and older + code = co.co_code + names = co.co_names + consts = co.co_consts + while code: + c = code[0] + if c in STORE_OPS: + oparg, = unpack('= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + def scan_opcodes_25(self, co, unpack = struct.unpack): # Scan the code, and yield 'interesting' opcode combinations @@ -368,7 +385,10 @@ def scan_code(self, co, m): code = co.co_code - scanner = self.scan_opcodes_25 + if sys.version_info >= (2, 5): + scanner = self.scan_opcodes_25 + else: + scanner = self.scan_opcodes for what, args in scanner(co): if what == "store": name, = args @@ -487,7 +507,7 @@ # Print modules that may be missing, but then again, maybe not... if maybe: print() - print("Submodules that appear to be missing, but could also be", end=' ') + print("Submodules thay appear to be missing, but could also be", end=' ') print("global names in the parent package:") for name in maybe: mods = sorted(self.badmodules[name].keys()) @@ -568,12 +588,11 @@ if isinstance(consts[i], type(co)): consts[i] = self.replace_paths_in_code(consts[i]) - return types.CodeType(co.co_argcount, co.co_kwonlyargcount, - co.co_nlocals, co.co_stacksize, co.co_flags, - co.co_code, tuple(consts), co.co_names, - co.co_varnames, new_filename, co.co_name, - co.co_firstlineno, co.co_lnotab, co.co_freevars, - co.co_cellvars) + return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, + co.co_flags, co.co_code, tuple(consts), co.co_names, + co.co_varnames, new_filename, co.co_name, + co.co_firstlineno, co.co_lnotab, + co.co_freevars, co.co_cellvars) def test(): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/msilib/__init__.py --- a/Lib/msilib/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/msilib/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,11 +1,7 @@ # Copyright (C) 2005 Martin v. Löwis # Licensed to PSF under a Contributor Agreement. from _msi import * -import fnmatch -import os -import re -import string -import sys +import os, string, re, sys AMD64 = "AMD64" in sys.version Itanium = "Itanium" in sys.version @@ -329,7 +325,7 @@ def add_file(self, file, src=None, version=None, language=None): """Add a file to the current component of the directory, starting a new one - if there is no current component. By default, the file name in the source + one if there is no current component. By default, the file name in the source and the file table will be identical. If the src file is specified, it is interpreted relative to the current directory. Optionally, a version and a language can be specified for the entry in the File table.""" @@ -365,7 +361,7 @@ # [(logical, 0, filehash.IntegerData(1), # filehash.IntegerData(2), filehash.IntegerData(3), # filehash.IntegerData(4))]) - # Automatically remove .pyc files on uninstall (2) + # Automatically remove .pyc/.pyo files on uninstall (2) # XXX: adding so many RemoveFile entries makes installer unbelievably # slow. So instead, we have to use wildcard remove entries if file.endswith(".py"): @@ -379,22 +375,17 @@ def glob(self, pattern, exclude = None): """Add a list of files to the current component as specified in the glob pattern. Individual files can be excluded in the exclude list.""" - try: - files = os.listdir(self.absolute) - except OSError: - return [] - if pattern[:1] != '.': - files = (f for f in files if f[0] != '.') - files = fnmatch.filter(files, pattern) + files = glob.glob1(self.absolute, pattern) for f in files: if exclude and f in exclude: continue self.add_file(f) return files def remove_pyc(self): - "Remove .pyc files on uninstall" + "Remove .pyc/.pyo files on uninstall" add_data(self.db, "RemoveFile", - [(self.component+"c", self.component, "*.pyc", self.logical, 2)]) + [(self.component+"c", self.component, "*.pyc", self.logical, 2), + (self.component+"o", self.component, "*.pyo", self.logical, 2)]) class Binary: def __init__(self, fname): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/msilib/schema.py --- a/Lib/msilib/schema.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/msilib/schema.py Mon Jan 25 17:05:13 2016 +0100 @@ -733,7 +733,7 @@ ('CustomAction','Source','Y',None, None, None, None, 'CustomSource',None, 'The table reference of the source of the code.',), ('CustomAction','Target','Y',None, None, None, None, 'Formatted',None, 'Excecution parameter, depends on the type of custom action',), ('DrLocator','Signature_','N',None, None, None, None, 'Identifier',None, 'The Signature_ represents a unique file signature and is also the foreign key in the Signature table.',), -('DrLocator','Path','Y',None, None, None, None, 'AnyPath',None, 'The path on the user system. This is either a subpath below the value of the Parent or a full path. The path may contain properties enclosed within [ ] that will be expanded.',), +('DrLocator','Path','Y',None, None, None, None, 'AnyPath',None, 'The path on the user system. This is a either a subpath below the value of the Parent or a full path. The path may contain properties enclosed within [ ] that will be expanded.',), ('DrLocator','Depth','Y',0,32767,None, None, None, None, 'The depth below the path to which the Signature_ is recursively searched. If absent, the depth is assumed to be 0.',), ('DrLocator','Parent','Y',None, None, None, None, 'Identifier',None, 'The parent file signature. It is also a foreign key in the Signature table. If null and the Path column does not expand to a full path, then all the fixed drives of the user system are searched using the Path.',), ('DuplicateFile','File_','N',None, None, 'File',1,'Identifier',None, 'Foreign key referencing the source file to be duplicated.',), diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/__init__.py --- a/Lib/multiprocessing/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -8,31 +8,276 @@ # subpackage 'multiprocessing.dummy' has the same API but is a simple # wrapper for 'threading'. # +# Try calling `multiprocessing.doc.main()` to read the html +# documentation in a webbrowser. +# +# # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # -import sys -from . import context +__version__ = '0.70a1' + +__all__ = [ + 'Process', 'current_process', 'active_children', 'freeze_support', + 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger', + 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError', + 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', + 'Event', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool', 'Value', 'Array', + 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING', + ] + +__author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)' # -# Copy stuff from default context +# Imports # -globals().update((name, getattr(context._default_context, name)) - for name in context._default_context.__all__) -__all__ = context._default_context.__all__ +import os +import sys + +from multiprocessing.process import Process, current_process, active_children +from multiprocessing.util import SUBDEBUG, SUBWARNING # -# XXX These should not really be documented or public. +# Exceptions # -SUBDEBUG = 5 -SUBWARNING = 25 +class ProcessError(Exception): + pass + +class BufferTooShort(ProcessError): + pass + +class TimeoutError(ProcessError): + pass + +class AuthenticationError(ProcessError): + pass + +import _multiprocessing # -# Alias for main module -- will be reset by bootstrapping child processes +# Definitions not depending on native semaphores # -if '__main__' in sys.modules: - sys.modules['__mp_main__'] = sys.modules['__main__'] +def Manager(): + ''' + Returns a manager associated with a running server process + + The managers methods such as `Lock()`, `Condition()` and `Queue()` + can be used to create shared objects. + ''' + from multiprocessing.managers import SyncManager + m = SyncManager() + m.start() + return m + +def Pipe(duplex=True): + ''' + Returns two connection object connected by a pipe + ''' + from multiprocessing.connection import Pipe + return Pipe(duplex) + +def cpu_count(): + ''' + Returns the number of CPUs in the system + ''' + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif 'bsd' in sys.platform or sys.platform == 'darwin': + comm = '/sbin/sysctl -n hw.ncpu' + if sys.platform == 'darwin': + comm = '/usr' + comm + try: + with os.popen(comm) as p: + num = int(p.read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + + if num >= 1: + return num + else: + raise NotImplementedError('cannot determine number of cpus') + +def freeze_support(): + ''' + Check whether this is a fake forked process in a frozen executable. + If so then run code specified by commandline and exit. + ''' + if sys.platform == 'win32' and getattr(sys, 'frozen', False): + from multiprocessing.forking import freeze_support + freeze_support() + +def get_logger(): + ''' + Return package logger -- if it does not already exist then it is created + ''' + from multiprocessing.util import get_logger + return get_logger() + +def log_to_stderr(level=None): + ''' + Turn on logging and add a handler which prints to stderr + ''' + from multiprocessing.util import log_to_stderr + return log_to_stderr(level) + +def allow_connection_pickling(): + ''' + Install support for sending connections and sockets between processes + ''' + from multiprocessing import reduction + +# +# Definitions depending on native semaphores +# + +def Lock(): + ''' + Returns a non-recursive lock object + ''' + from multiprocessing.synchronize import Lock + return Lock() + +def RLock(): + ''' + Returns a recursive lock object + ''' + from multiprocessing.synchronize import RLock + return RLock() + +def Condition(lock=None): + ''' + Returns a condition object + ''' + from multiprocessing.synchronize import Condition + return Condition(lock) + +def Semaphore(value=1): + ''' + Returns a semaphore object + ''' + from multiprocessing.synchronize import Semaphore + return Semaphore(value) + +def BoundedSemaphore(value=1): + ''' + Returns a bounded semaphore object + ''' + from multiprocessing.synchronize import BoundedSemaphore + return BoundedSemaphore(value) + +def Event(): + ''' + Returns an event object + ''' + from multiprocessing.synchronize import Event + return Event() + +def Queue(maxsize=0): + ''' + Returns a queue object + ''' + from multiprocessing.queues import Queue + return Queue(maxsize) + +def JoinableQueue(maxsize=0): + ''' + Returns a queue object + ''' + from multiprocessing.queues import JoinableQueue + return JoinableQueue(maxsize) + +def SimpleQueue(): + ''' + Returns a queue object + ''' + from multiprocessing.queues import SimpleQueue + return SimpleQueue() + +def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None): + ''' + Returns a process pool object + ''' + from multiprocessing.pool import Pool + return Pool(processes, initializer, initargs, maxtasksperchild) + +def RawValue(typecode_or_type, *args): + ''' + Returns a shared object + ''' + from multiprocessing.sharedctypes import RawValue + return RawValue(typecode_or_type, *args) + +def RawArray(typecode_or_type, size_or_initializer): + ''' + Returns a shared array + ''' + from multiprocessing.sharedctypes import RawArray + return RawArray(typecode_or_type, size_or_initializer) + +def Value(typecode_or_type, *args, **kwds): + ''' + Returns a synchronized shared object + ''' + from multiprocessing.sharedctypes import Value + return Value(typecode_or_type, *args, **kwds) + +def Array(typecode_or_type, size_or_initializer, **kwds): + ''' + Returns a synchronized shared array + ''' + from multiprocessing.sharedctypes import Array + return Array(typecode_or_type, size_or_initializer, **kwds) + +# +# +# + +if sys.platform == 'win32': + + def set_executable(executable): + ''' + Sets the path to a python.exe or pythonw.exe binary used to run + child processes on Windows instead of sys.executable. + Useful for people embedding Python. + ''' + from multiprocessing.forking import set_executable + set_executable(executable) + + __all__ += ['set_executable'] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/connection.py --- a/Lib/multiprocessing/connection.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/connection.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,32 @@ # multiprocessing/connection.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] @@ -12,27 +37,26 @@ import io import os import sys +import pickle +import select import socket import struct +import errno import time import tempfile import itertools import _multiprocessing - -from . import reduction -from . import util - -from . import AuthenticationError, BufferTooShort -from .reduction import ForkingPickler - +from multiprocessing import current_process, AuthenticationError, BufferTooShort +from multiprocessing.util import ( + get_temp_dir, Finalize, sub_debug, debug, _eintr_retry) try: - import _winapi - from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE + from _multiprocessing import win32 + from _subprocess import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE except ImportError: if sys.platform == 'win32': raise - _winapi = None + win32 = None # # @@ -73,24 +97,13 @@ if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + return tempfile.mktemp(prefix='listener-', dir=get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % - (os.getpid(), next(_mmap_counter)), dir="") + (os.getpid(), next(_mmap_counter))) else: raise ValueError('unrecognized family') -def _validate_family(family): - ''' - Checks if the family is valid for the current environment. - ''' - if sys.platform != 'win32' and family == 'AF_PIPE': - raise ValueError('Family %s is not recognized.' % family) - - if sys.platform == 'win32' and family == 'AF_UNIX': - # double check - if not hasattr(socket, family): - raise ValueError('Family %s is not recognized.' % family) def address_type(address): ''' @@ -133,22 +146,22 @@ def _check_closed(self): if self._handle is None: - raise OSError("handle is closed") + raise IOError("handle is closed") def _check_readable(self): if not self._readable: - raise OSError("connection is write-only") + raise IOError("connection is write-only") def _check_writable(self): if not self._writable: - raise OSError("connection is read-only") + raise IOError("connection is read-only") def _bad_message_length(self): if self._writable: self._readable = False else: self.close() - raise OSError("bad message length") + raise IOError("bad message length") @property def closed(self): @@ -203,7 +216,8 @@ """Send a (picklable) object""" self._check_closed() self._check_writable() - self._send_bytes(ForkingPickler.dumps(obj)) + buf = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + self._send_bytes(memoryview(buf)) def recv_bytes(self, maxlength=None): """ @@ -220,7 +234,7 @@ def recv_bytes_into(self, buf, offset=0): """ - Receive bytes data into a writeable bytes-like object. + Receive bytes data into a writeable buffer-like object. Return the number of bytes read. """ self._check_closed() @@ -248,7 +262,7 @@ self._check_closed() self._check_readable() buf = self._recv_bytes() - return ForkingPickler.loads(buf.getbuffer()) + return pickle.loads(buf.getbuffer()) def poll(self, timeout=0.0): """Whether there is any input available to be read""" @@ -256,14 +270,8 @@ self._check_readable() return self._poll(timeout) - def __enter__(self): - return self - def __exit__(self, exc_type, exc_value, exc_tb): - self.close() - - -if _winapi: +if win32: class PipeConnection(_ConnectionBase): """ @@ -273,14 +281,14 @@ """ _got_empty_message = False - def _close(self, _CloseHandle=_winapi.CloseHandle): + def _close(self, _CloseHandle=win32.CloseHandle): _CloseHandle(self._handle) def _send_bytes(self, buf): - ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) + ov, err = win32.WriteFile(self._handle, buf, overlapped=True) try: - if err == _winapi.ERROR_IO_PENDING: - waitres = _winapi.WaitForMultipleObjects( + if err == win32.ERROR_IO_PENDING: + waitres = win32.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: @@ -298,11 +306,11 @@ else: bsize = 128 if maxsize is None else min(maxsize, 128) try: - ov, err = _winapi.ReadFile(self._handle, bsize, - overlapped=True) + ov, err = win32.ReadFile(self._handle, bsize, + overlapped=True) try: - if err == _winapi.ERROR_IO_PENDING: - waitres = _winapi.WaitForMultipleObjects( + if err == win32.ERROR_IO_PENDING: + waitres = win32.WaitForMultipleObjects( [ov.event], False, INFINITE) assert waitres == WAIT_OBJECT_0 except: @@ -314,10 +322,10 @@ f = io.BytesIO() f.write(ov.getbuffer()) return f - elif err == _winapi.ERROR_MORE_DATA: + elif err == win32.ERROR_MORE_DATA: return self._get_more_data(ov, maxsize) - except OSError as e: - if e.winerror == _winapi.ERROR_BROKEN_PIPE: + except IOError as e: + if e.winerror == win32.ERROR_BROKEN_PIPE: raise EOFError else: raise @@ -325,19 +333,21 @@ def _poll(self, timeout): if (self._got_empty_message or - _winapi.PeekNamedPipe(self._handle)[0] != 0): + win32.PeekNamedPipe(self._handle)[0] != 0): return True + if timeout < 0: + timeout = None return bool(wait([self], timeout)) def _get_more_data(self, ov, maxsize): buf = ov.getbuffer() f = io.BytesIO() f.write(buf) - left = _winapi.PeekNamedPipe(self._handle)[1] + left = win32.PeekNamedPipe(self._handle)[1] assert left > 0 if maxsize is not None and len(buf) + left > maxsize: self._bad_message_length() - ov, err = _winapi.ReadFile(self._handle, left, overlapped=True) + ov, err = win32.ReadFile(self._handle, left, overlapped=True) rbytes, err = ov.GetOverlappedResult(True) assert err == 0 assert rbytes == left @@ -351,11 +361,11 @@ a socket handle (Windows). """ - if _winapi: - def _close(self, _close=_multiprocessing.closesocket): + if win32: + def _close(self, _close=win32.closesocket): _close(self._handle) - _write = _multiprocessing.send - _read = _multiprocessing.recv + _write = win32.send + _read = win32.recv else: def _close(self, _close=os.close): _close(self._handle) @@ -382,26 +392,19 @@ if remaining == size: raise EOFError else: - raise OSError("got end of file during message") + raise IOError("got end of file during message") buf.write(chunk) remaining -= n return buf def _send_bytes(self, buf): + # For wire compatibility with 3.2 and lower n = len(buf) - # For wire compatibility with 3.2 and lower - header = struct.pack("!i", n) - if n > 16384: - # The payload is large so Nagle's algorithm won't be triggered - # and we'd better avoid the cost of concatenation. - self._send(header) + self._send(struct.pack("!i", n)) + # The condition is necessary to avoid "broken pipe" errors + # when sending a 0-length buffer if the other end closed the pipe. + if n > 0: self._send(buf) - else: - # Issue #20540: concatenate before sending, to avoid delays due - # to Nagle's algorithm on a TCP socket. - # Also note we want to avoid sending a 0-length buffer separately, - # to avoid "broken pipe" errors if the other end closed the pipe. - self._send(header + buf) def _recv_bytes(self, maxsize=None): buf = self._recv(4) @@ -411,7 +414,9 @@ return self._recv(size) def _poll(self, timeout): - r = wait([self], timeout) + if timeout < 0.0: + timeout = None + r = wait([self._handle], timeout) return bool(r) @@ -431,7 +436,6 @@ or default_family address = address or arbitrary_address(family) - _validate_family(family) if family == 'AF_PIPE': self._listener = PipeListener(address, backlog) else: @@ -448,8 +452,6 @@ Returns a `Connection` object. ''' - if self._listener is None: - raise OSError('listener is closed') c = self._listener.accept() if self._authkey: deliver_challenge(c, self._authkey) @@ -460,27 +462,17 @@ ''' Close the bound socket or named pipe of `self`. ''' - listener = self._listener - if listener is not None: - self._listener = None - listener.close() + return self._listener.close() address = property(lambda self: self._listener._address) last_accepted = property(lambda self: self._listener._last_accepted) - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_tb): - self.close() - def Client(address, family=None, authkey=None): ''' Returns a connection to the address of a `Listener` ''' family = family or address_type(address) - _validate_family(family) if family == 'AF_PIPE': c = PipeClient(address) else: @@ -504,8 +496,6 @@ ''' if duplex: s1, s2 = socket.socketpair() - s1.setblocking(True) - s2.setblocking(True) c1 = Connection(s1.detach()) c2 = Connection(s2.detach()) else: @@ -523,32 +513,30 @@ ''' address = arbitrary_address('AF_PIPE') if duplex: - openmode = _winapi.PIPE_ACCESS_DUPLEX - access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE + openmode = win32.PIPE_ACCESS_DUPLEX + access = win32.GENERIC_READ | win32.GENERIC_WRITE obsize, ibsize = BUFSIZE, BUFSIZE else: - openmode = _winapi.PIPE_ACCESS_INBOUND - access = _winapi.GENERIC_WRITE + openmode = win32.PIPE_ACCESS_INBOUND + access = win32.GENERIC_WRITE obsize, ibsize = 0, BUFSIZE - h1 = _winapi.CreateNamedPipe( - address, openmode | _winapi.FILE_FLAG_OVERLAPPED | - _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, - _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | - _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, - # default security descriptor: the handle cannot be inherited - _winapi.NULL + h1 = win32.CreateNamedPipe( + address, openmode | win32.FILE_FLAG_OVERLAPPED | + win32.FILE_FLAG_FIRST_PIPE_INSTANCE, + win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE | + win32.PIPE_WAIT, + 1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL ) - h2 = _winapi.CreateFile( - address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, - _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL + h2 = win32.CreateFile( + address, access, 0, win32.NULL, win32.OPEN_EXISTING, + win32.FILE_FLAG_OVERLAPPED, win32.NULL ) - _winapi.SetNamedPipeHandleState( - h2, _winapi.PIPE_READMODE_MESSAGE, None, None + win32.SetNamedPipeHandleState( + h2, win32.PIPE_READMODE_MESSAGE, None, None ) - overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True) + overlapped = win32.ConnectNamedPipe(h1, overlapped=True) _, err = overlapped.GetOverlappedResult(True) assert err == 0 @@ -572,7 +560,6 @@ if os.name == 'posix': self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self._socket.setblocking(True) self._socket.bind(address) self._socket.listen(backlog) self._address = self._socket.getsockname() @@ -583,7 +570,7 @@ self._last_accepted = None if family == 'AF_UNIX': - self._unlink = util.Finalize( + self._unlink = Finalize( self, os.unlink, args=(address,), exitpriority=0 ) else: @@ -591,17 +578,15 @@ def accept(self): s, self._last_accepted = self._socket.accept() - s.setblocking(True) - return Connection(s.detach()) + fd = duplicate(s.fileno()) + conn = Connection(fd) + s.close() + return conn def close(self): - try: - self._socket.close() - finally: - unlink = self._unlink - if unlink is not None: - self._unlink = None - unlink() + self._socket.close() + if self._unlink is not None: + self._unlink() def SocketClient(address): @@ -610,9 +595,10 @@ ''' family = address_type(address) with socket.socket( getattr(socket, family) ) as s: - s.setblocking(True) s.connect(address) - return Connection(s.detach()) + fd = duplicate(s.fileno()) + conn = Connection(fd) + return conn # # Definitions for connections based on named pipes @@ -629,52 +615,44 @@ self._handle_queue = [self._new_handle(first=True)] self._last_accepted = None - util.sub_debug('listener created with address=%r', self._address) - self.close = util.Finalize( + sub_debug('listener created with address=%r', self._address) + self.close = Finalize( self, PipeListener._finalize_pipe_listener, args=(self._handle_queue, self._address), exitpriority=0 ) def _new_handle(self, first=False): - flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED + flags = win32.PIPE_ACCESS_DUPLEX | win32.FILE_FLAG_OVERLAPPED if first: - flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE - return _winapi.CreateNamedPipe( + flags |= win32.FILE_FLAG_FIRST_PIPE_INSTANCE + return win32.CreateNamedPipe( self._address, flags, - _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | - _winapi.PIPE_WAIT, - _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, - _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL + win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE | + win32.PIPE_WAIT, + win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, + win32.NMPWAIT_WAIT_FOREVER, win32.NULL ) def accept(self): self._handle_queue.append(self._new_handle()) handle = self._handle_queue.pop(0) + ov = win32.ConnectNamedPipe(handle, overlapped=True) try: - ov = _winapi.ConnectNamedPipe(handle, overlapped=True) - except OSError as e: - if e.winerror != _winapi.ERROR_NO_DATA: - raise - # ERROR_NO_DATA can occur if a client has already connected, - # written data and then disconnected -- see Issue 14725. - else: - try: - res = _winapi.WaitForMultipleObjects( - [ov.event], False, INFINITE) - except: - ov.cancel() - _winapi.CloseHandle(handle) - raise - finally: - _, err = ov.GetOverlappedResult(True) - assert err == 0 + res = win32.WaitForMultipleObjects([ov.event], False, INFINITE) + except: + ov.cancel() + win32.CloseHandle(handle) + raise + finally: + _, err = ov.GetOverlappedResult(True) + assert err == 0 return PipeConnection(handle) @staticmethod def _finalize_pipe_listener(queue, address): - util.sub_debug('closing listener with address=%r', address) + sub_debug('closing listener with address=%r', address) for handle in queue: - _winapi.CloseHandle(handle) + close(handle) def PipeClient(address): ''' @@ -683,23 +661,23 @@ t = _init_timeout() while 1: try: - _winapi.WaitNamedPipe(address, 1000) - h = _winapi.CreateFile( - address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, - 0, _winapi.NULL, _winapi.OPEN_EXISTING, - _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL + win32.WaitNamedPipe(address, 1000) + h = win32.CreateFile( + address, win32.GENERIC_READ | win32.GENERIC_WRITE, + 0, win32.NULL, win32.OPEN_EXISTING, + win32.FILE_FLAG_OVERLAPPED, win32.NULL ) - except OSError as e: - if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT, - _winapi.ERROR_PIPE_BUSY) or _check_timeout(t): + except WindowsError as e: + if e.winerror not in (win32.ERROR_SEM_TIMEOUT, + win32.ERROR_PIPE_BUSY) or _check_timeout(t): raise else: break else: raise - _winapi.SetNamedPipeHandleState( - h, _winapi.PIPE_READMODE_MESSAGE, None, None + win32.SetNamedPipeHandleState( + h, win32.PIPE_READMODE_MESSAGE, None, None ) return PipeConnection(h) @@ -718,7 +696,7 @@ assert isinstance(authkey, bytes) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message).digest() response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) @@ -732,7 +710,7 @@ message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message).digest() connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message if response != WELCOME: @@ -788,7 +766,7 @@ L = list(handles) ready = [] while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + res = win32.WaitForMultipleObjects(L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): @@ -802,7 +780,7 @@ timeout = 0 return ready - _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED} + _ready_errors = {win32.ERROR_BROKEN_PIPE, win32.ERROR_NETNAME_DELETED} def wait(object_list, timeout=None): ''' @@ -832,27 +810,18 @@ else: # start an overlapped read of length zero try: - ov, err = _winapi.ReadFile(fileno(), 0, True) + ov, err = win32.ReadFile(fileno(), 0, True) except OSError as e: - ov, err = None, e.winerror + err = e.winerror if err not in _ready_errors: raise - if err == _winapi.ERROR_IO_PENDING: + if err == win32.ERROR_IO_PENDING: ov_list.append(ov) waithandle_to_obj[ov.event] = o else: # If o.fileno() is an overlapped pipe handle and # err == 0 then there is a zero length message - # in the pipe, but it HAS NOT been consumed... - if ov and sys.getwindowsversion()[:2] >= (6, 2): - # ... except on Windows 8 and later, where - # the message HAS been consumed. - try: - _, err = ov.GetOverlappedResult(False) - except OSError as e: - err = e.winerror - if not err and hasattr(o, '_got_empty_message'): - o._got_empty_message = True + # in the pipe, but it HAS NOT been consumed. ready_objects.add(o) timeout = 0 @@ -870,7 +839,7 @@ err = e.winerror if err not in _ready_errors: raise - if err != _winapi.ERROR_OPERATION_ABORTED: + if err != win32.ERROR_OPERATION_ABORTED: o = waithandle_to_obj[ov.event] ready_objects.add(o) if err == 0: @@ -884,70 +853,26 @@ else: - import selectors - - # poll/select have the advantage of not requiring any extra file - # descriptor, contrarily to epoll/kqueue (also, they require a single - # syscall). - if hasattr(selectors, 'PollSelector'): - _WaitSelector = selectors.PollSelector - else: - _WaitSelector = selectors.SelectSelector - def wait(object_list, timeout=None): ''' Wait till an object in object_list is ready/readable. Returns list of those objects in object_list which are ready/readable. ''' - with _WaitSelector() as selector: - for obj in object_list: - selector.register(obj, selectors.EVENT_READ) + if timeout is not None: + if timeout <= 0: + return select.select(object_list, [], [], 0)[0] + else: + deadline = time.time() + timeout + while True: + try: + return select.select(object_list, [], [], timeout)[0] + except OSError as e: + if e.errno != errno.EINTR: + raise + if timeout is not None: + timeout = deadline - time.time() - if timeout is not None: - deadline = time.time() + timeout - while True: - ready = selector.select(timeout) - if ready: - return [key.fileobj for (key, events) in ready] - else: - if timeout is not None: - timeout = deadline - time.time() - if timeout < 0: - return ready - -# -# Make connection and socket objects sharable if possible -# - -if sys.platform == 'win32': - def reduce_connection(conn): - handle = conn.fileno() - with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s: - from . import resource_sharer - ds = resource_sharer.DupSocket(s) - return rebuild_connection, (ds, conn.readable, conn.writable) - def rebuild_connection(ds, readable, writable): - sock = ds.detach() - return Connection(sock.detach(), readable, writable) - reduction.register(Connection, reduce_connection) - - def reduce_pipe_connection(conn): - access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) | - (_winapi.FILE_GENERIC_WRITE if conn.writable else 0)) - dh = reduction.DupHandle(conn.fileno(), access) - return rebuild_pipe_connection, (dh, conn.readable, conn.writable) - def rebuild_pipe_connection(dh, readable, writable): - handle = dh.detach() - return PipeConnection(handle, readable, writable) - reduction.register(PipeConnection, reduce_pipe_connection) - -else: - def reduce_connection(conn): - df = reduction.DupFd(conn.fileno()) - return rebuild_connection, (df, conn.readable, conn.writable) - def rebuild_connection(df, readable, writable): - fd = df.detach() - return Connection(fd, readable, writable) - reduction.register(Connection, reduce_connection) +# Late import because of circular import +from multiprocessing.forking import duplicate, close diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/context.py --- a/Lib/multiprocessing/context.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,348 +0,0 @@ -import os -import sys -import threading - -from . import process - -__all__ = [] # things are copied from here to __init__.py - -# -# Exceptions -# - -class ProcessError(Exception): - pass - -class BufferTooShort(ProcessError): - pass - -class TimeoutError(ProcessError): - pass - -class AuthenticationError(ProcessError): - pass - -# -# Base type for contexts -# - -class BaseContext(object): - - ProcessError = ProcessError - BufferTooShort = BufferTooShort - TimeoutError = TimeoutError - AuthenticationError = AuthenticationError - - current_process = staticmethod(process.current_process) - active_children = staticmethod(process.active_children) - - def cpu_count(self): - '''Returns the number of CPUs in the system''' - num = os.cpu_count() - if num is None: - raise NotImplementedError('cannot determine number of cpus') - else: - return num - - def Manager(self): - '''Returns a manager associated with a running server process - - The managers methods such as `Lock()`, `Condition()` and `Queue()` - can be used to create shared objects. - ''' - from .managers import SyncManager - m = SyncManager(ctx=self.get_context()) - m.start() - return m - - def Pipe(self, duplex=True): - '''Returns two connection object connected by a pipe''' - from .connection import Pipe - return Pipe(duplex) - - def Lock(self): - '''Returns a non-recursive lock object''' - from .synchronize import Lock - return Lock(ctx=self.get_context()) - - def RLock(self): - '''Returns a recursive lock object''' - from .synchronize import RLock - return RLock(ctx=self.get_context()) - - def Condition(self, lock=None): - '''Returns a condition object''' - from .synchronize import Condition - return Condition(lock, ctx=self.get_context()) - - def Semaphore(self, value=1): - '''Returns a semaphore object''' - from .synchronize import Semaphore - return Semaphore(value, ctx=self.get_context()) - - def BoundedSemaphore(self, value=1): - '''Returns a bounded semaphore object''' - from .synchronize import BoundedSemaphore - return BoundedSemaphore(value, ctx=self.get_context()) - - def Event(self): - '''Returns an event object''' - from .synchronize import Event - return Event(ctx=self.get_context()) - - def Barrier(self, parties, action=None, timeout=None): - '''Returns a barrier object''' - from .synchronize import Barrier - return Barrier(parties, action, timeout, ctx=self.get_context()) - - def Queue(self, maxsize=0): - '''Returns a queue object''' - from .queues import Queue - return Queue(maxsize, ctx=self.get_context()) - - def JoinableQueue(self, maxsize=0): - '''Returns a queue object''' - from .queues import JoinableQueue - return JoinableQueue(maxsize, ctx=self.get_context()) - - def SimpleQueue(self): - '''Returns a queue object''' - from .queues import SimpleQueue - return SimpleQueue(ctx=self.get_context()) - - def Pool(self, processes=None, initializer=None, initargs=(), - maxtasksperchild=None): - '''Returns a process pool object''' - from .pool import Pool - return Pool(processes, initializer, initargs, maxtasksperchild, - context=self.get_context()) - - def RawValue(self, typecode_or_type, *args): - '''Returns a shared object''' - from .sharedctypes import RawValue - return RawValue(typecode_or_type, *args) - - def RawArray(self, typecode_or_type, size_or_initializer): - '''Returns a shared array''' - from .sharedctypes import RawArray - return RawArray(typecode_or_type, size_or_initializer) - - def Value(self, typecode_or_type, *args, lock=True): - '''Returns a synchronized shared object''' - from .sharedctypes import Value - return Value(typecode_or_type, *args, lock=lock, - ctx=self.get_context()) - - def Array(self, typecode_or_type, size_or_initializer, *, lock=True): - '''Returns a synchronized shared array''' - from .sharedctypes import Array - return Array(typecode_or_type, size_or_initializer, lock=lock, - ctx=self.get_context()) - - def freeze_support(self): - '''Check whether this is a fake forked process in a frozen executable. - If so then run code specified by commandline and exit. - ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): - from .spawn import freeze_support - freeze_support() - - def get_logger(self): - '''Return package logger -- if it does not already exist then - it is created. - ''' - from .util import get_logger - return get_logger() - - def log_to_stderr(self, level=None): - '''Turn on logging and add a handler which prints to stderr''' - from .util import log_to_stderr - return log_to_stderr(level) - - def allow_connection_pickling(self): - '''Install support for sending connections and sockets - between processes - ''' - # This is undocumented. In previous versions of multiprocessing - # its only effect was to make socket objects inheritable on Windows. - from . import connection - - def set_executable(self, executable): - '''Sets the path to a python.exe or pythonw.exe binary used to run - child processes instead of sys.executable when using the 'spawn' - start method. Useful for people embedding Python. - ''' - from .spawn import set_executable - set_executable(executable) - - def set_forkserver_preload(self, module_names): - '''Set list of module names to try to load in forkserver process. - This is really just a hint. - ''' - from .forkserver import set_forkserver_preload - set_forkserver_preload(module_names) - - def get_context(self, method=None): - if method is None: - return self - try: - ctx = _concrete_contexts[method] - except KeyError: - raise ValueError('cannot find context for %r' % method) - ctx._check_available() - return ctx - - def get_start_method(self, allow_none=False): - return self._name - - def set_start_method(self, method=None): - raise ValueError('cannot set start method of concrete context') - - def _check_available(self): - pass - -# -# Type of default context -- underlying context can be set at most once -# - -class Process(process.BaseProcess): - _start_method = None - @staticmethod - def _Popen(process_obj): - return _default_context.get_context().Process._Popen(process_obj) - -class DefaultContext(BaseContext): - Process = Process - - def __init__(self, context): - self._default_context = context - self._actual_context = None - - def get_context(self, method=None): - if method is None: - if self._actual_context is None: - self._actual_context = self._default_context - return self._actual_context - else: - return super().get_context(method) - - def set_start_method(self, method, force=False): - if self._actual_context is not None and not force: - raise RuntimeError('context has already been set') - if method is None and force: - self._actual_context = None - return - self._actual_context = self.get_context(method) - - def get_start_method(self, allow_none=False): - if self._actual_context is None: - if allow_none: - return None - self._actual_context = self._default_context - return self._actual_context._name - - def get_all_start_methods(self): - if sys.platform == 'win32': - return ['spawn'] - else: - from . import reduction - if reduction.HAVE_SEND_HANDLE: - return ['fork', 'spawn', 'forkserver'] - else: - return ['fork', 'spawn'] - -DefaultContext.__all__ = list(x for x in dir(DefaultContext) if x[0] != '_') - -# -# Context types for fixed start method -# - -if sys.platform != 'win32': - - class ForkProcess(process.BaseProcess): - _start_method = 'fork' - @staticmethod - def _Popen(process_obj): - from .popen_fork import Popen - return Popen(process_obj) - - class SpawnProcess(process.BaseProcess): - _start_method = 'spawn' - @staticmethod - def _Popen(process_obj): - from .popen_spawn_posix import Popen - return Popen(process_obj) - - class ForkServerProcess(process.BaseProcess): - _start_method = 'forkserver' - @staticmethod - def _Popen(process_obj): - from .popen_forkserver import Popen - return Popen(process_obj) - - class ForkContext(BaseContext): - _name = 'fork' - Process = ForkProcess - - class SpawnContext(BaseContext): - _name = 'spawn' - Process = SpawnProcess - - class ForkServerContext(BaseContext): - _name = 'forkserver' - Process = ForkServerProcess - def _check_available(self): - from . import reduction - if not reduction.HAVE_SEND_HANDLE: - raise ValueError('forkserver start method not available') - - _concrete_contexts = { - 'fork': ForkContext(), - 'spawn': SpawnContext(), - 'forkserver': ForkServerContext(), - } - _default_context = DefaultContext(_concrete_contexts['fork']) - -else: - - class SpawnProcess(process.BaseProcess): - _start_method = 'spawn' - @staticmethod - def _Popen(process_obj): - from .popen_spawn_win32 import Popen - return Popen(process_obj) - - class SpawnContext(BaseContext): - _name = 'spawn' - Process = SpawnProcess - - _concrete_contexts = { - 'spawn': SpawnContext(), - } - _default_context = DefaultContext(_concrete_contexts['spawn']) - -# -# Force the start method -# - -def _force_start_method(method): - _default_context._actual_context = _concrete_contexts[method] - -# -# Check that the current thread is spawning a child process -# - -_tls = threading.local() - -def get_spawning_popen(): - return getattr(_tls, 'spawning_popen', None) - -def set_spawning_popen(popen): - _tls.spawning_popen = popen - -def assert_spawning(obj): - if get_spawning_popen() is None: - raise RuntimeError( - '%s objects should only be shared between processes' - ' through inheritance' % type(obj).__name__ - ) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/dummy/__init__.py --- a/Lib/multiprocessing/dummy/__init__.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/dummy/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,13 +4,38 @@ # multiprocessing/dummy/__init__.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Process', 'current_process', 'active_children', 'freeze_support', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', - 'Event', 'Barrier', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' + 'Event', 'Queue', 'Manager', 'Pipe', 'Pool', 'JoinableQueue' ] # @@ -22,9 +47,9 @@ import weakref import array -from .connection import Pipe +from multiprocessing.dummy.connection import Pipe from threading import Lock, RLock, Semaphore, BoundedSemaphore -from threading import Event, Condition, Barrier +from threading import Event, Condition from queue import Queue # @@ -43,8 +68,7 @@ def start(self): assert self._parent is current_process() self._start_called = True - if hasattr(self._parent, '_children'): - self._parent._children[self] = None + self._parent._children[self] = None threading.Thread.start(self) @property @@ -86,7 +110,7 @@ if not name.startswith('_'): temp.append('%s=%r' % (name, value)) temp.sort() - return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) + return 'Namespace(%s)' % str.join(', ', temp) dict = dict list = list @@ -104,7 +128,7 @@ self._value = value value = property(_get, _set) def __repr__(self): - return '<%s(%r, %r)>'%(type(self).__name__,self._typecode,self._value) + return '<%r(%r, %r)>'%(type(self).__name__,self._typecode,self._value) def Manager(): return sys.modules[__name__] @@ -113,7 +137,7 @@ pass def Pool(processes=None, initializer=None, initargs=()): - from ..pool import ThreadPool + from multiprocessing.pool import ThreadPool return ThreadPool(processes, initializer, initargs) JoinableQueue = Queue diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/dummy/connection.py --- a/Lib/multiprocessing/dummy/connection.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/dummy/connection.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,32 @@ # multiprocessing/dummy/connection.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'Client', 'Listener', 'Pipe' ] @@ -28,12 +53,6 @@ address = property(lambda self: self._backlog_queue) - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_tb): - self.close() - def Client(address): _in, _out = Queue(), Queue() @@ -59,15 +78,10 @@ return True if timeout <= 0.0: return False - with self._in.not_empty: - self._in.not_empty.wait(timeout) + self._in.not_empty.acquire() + self._in.not_empty.wait(timeout) + self._in.not_empty.release() return self._in.qsize() > 0 def close(self): pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, exc_tb): - self.close() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/forking.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/multiprocessing/forking.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,514 @@ +# +# Module for starting a process object using os.fork() or CreateProcess() +# +# multiprocessing/forking.py +# +# Copyright (c) 2006-2008, R Oudkerk +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +import os +import sys +import signal + +from multiprocessing import util, process + +__all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler'] + +# +# Check that the current thread is spawning a child process +# + +def assert_spawning(self): + if not Popen.thread_is_spawning(): + raise RuntimeError( + '%s objects should only be shared between processes' + ' through inheritance' % type(self).__name__ + ) + +# +# Try making some callable types picklable +# + +from pickle import Pickler +from copyreg import dispatch_table + +class ForkingPickler(Pickler): + _extra_reducers = {} + def __init__(self, *args): + Pickler.__init__(self, *args) + self.dispatch_table = dispatch_table.copy() + self.dispatch_table.update(self._extra_reducers) + @classmethod + def register(cls, type, reduce): + cls._extra_reducers[type] = reduce + +def _reduce_method(m): + if m.__self__ is None: + return getattr, (m.__class__, m.__func__.__name__) + else: + return getattr, (m.__self__, m.__func__.__name__) +class _C: + def f(self): + pass +ForkingPickler.register(type(_C().f), _reduce_method) + + +def _reduce_method_descriptor(m): + return getattr, (m.__objclass__, m.__name__) +ForkingPickler.register(type(list.append), _reduce_method_descriptor) +ForkingPickler.register(type(int.__add__), _reduce_method_descriptor) + +try: + from functools import partial +except ImportError: + pass +else: + def _reduce_partial(p): + return _rebuild_partial, (p.func, p.args, p.keywords or {}) + def _rebuild_partial(func, args, keywords): + return partial(func, *args, **keywords) + ForkingPickler.register(partial, _reduce_partial) + +# +# Unix +# + +if sys.platform != 'win32': + import select + + exit = os._exit + duplicate = os.dup + close = os.close + _select = util._eintr_retry(select.select) + + # + # We define a Popen class similar to the one from subprocess, but + # whose constructor takes a process object as its argument. + # + + class Popen(object): + + def __init__(self, process_obj): + sys.stdout.flush() + sys.stderr.flush() + self.returncode = None + + r, w = os.pipe() + self.sentinel = r + + self.pid = os.fork() + if self.pid == 0: + os.close(r) + if 'random' in sys.modules: + import random + random.seed() + code = process_obj._bootstrap() + os._exit(code) + + # `w` will be closed when the child exits, at which point `r` + # will become ready for reading (using e.g. select()). + os.close(w) + util.Finalize(self, os.close, (r,)) + + def poll(self, flag=os.WNOHANG): + if self.returncode is None: + try: + pid, sts = os.waitpid(self.pid, flag) + except os.error: + # Child process not yet created. See #1731717 + # e.errno == errno.ECHILD == 10 + return None + if pid == self.pid: + if os.WIFSIGNALED(sts): + self.returncode = -os.WTERMSIG(sts) + else: + assert os.WIFEXITED(sts) + self.returncode = os.WEXITSTATUS(sts) + return self.returncode + + def wait(self, timeout=None): + if self.returncode is None: + if timeout is not None: + r = _select([self.sentinel], [], [], timeout)[0] + if not r: + return None + # This shouldn't block if select() returned successfully. + return self.poll(os.WNOHANG if timeout == 0.0 else 0) + return self.returncode + + def terminate(self): + if self.returncode is None: + try: + os.kill(self.pid, signal.SIGTERM) + except OSError: + if self.wait(timeout=0.1) is None: + raise + + @staticmethod + def thread_is_spawning(): + return False + +# +# Windows +# + +else: + import _thread + import msvcrt + import _subprocess + + from pickle import load, HIGHEST_PROTOCOL + from _multiprocessing import win32 + + def dump(obj, file, protocol=None): + ForkingPickler(file, protocol).dump(obj) + + # + # + # + + TERMINATE = 0x10000 + WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) + WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + + exit = win32.ExitProcess + close = win32.CloseHandle + + # + # _python_exe is the assumed path to the python executable. + # People embedding Python want to modify it. + # + + if WINSERVICE: + _python_exe = os.path.join(sys.exec_prefix, 'python.exe') + else: + _python_exe = sys.executable + + def set_executable(exe): + global _python_exe + _python_exe = exe + + # + # + # + + def duplicate(handle, target_process=None, inheritable=False): + if target_process is None: + target_process = _subprocess.GetCurrentProcess() + return _subprocess.DuplicateHandle( + _subprocess.GetCurrentProcess(), handle, target_process, + 0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS + ).Detach() + + # + # We define a Popen class similar to the one from subprocess, but + # whose constructor takes a process object as its argument. + # + + class Popen(object): + ''' + Start a subprocess to run the code of a process object + ''' + _tls = _thread._local() + + def __init__(self, process_obj): + # create pipe for communication with child + rfd, wfd = os.pipe() + + # get handle for read end of the pipe and make it inheritable + rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True) + os.close(rfd) + + # start process + cmd = get_command_line() + [rhandle] + cmd = ' '.join('"%s"' % x for x in cmd) + hp, ht, pid, tid = _subprocess.CreateProcess( + _python_exe, cmd, None, None, 1, 0, None, None, None + ) + ht.Close() + close(rhandle) + + # set attributes of self + self.pid = pid + self.returncode = None + self._handle = hp + self.sentinel = int(hp) + + # send information to child + prep_data = get_preparation_data(process_obj._name) + to_child = os.fdopen(wfd, 'wb') + Popen._tls.process_handle = int(hp) + try: + dump(prep_data, to_child, HIGHEST_PROTOCOL) + dump(process_obj, to_child, HIGHEST_PROTOCOL) + finally: + del Popen._tls.process_handle + to_child.close() + + @staticmethod + def thread_is_spawning(): + return getattr(Popen._tls, 'process_handle', None) is not None + + @staticmethod + def duplicate_for_child(handle): + return duplicate(handle, Popen._tls.process_handle) + + def wait(self, timeout=None): + if self.returncode is None: + if timeout is None: + msecs = _subprocess.INFINITE + else: + msecs = max(0, int(timeout * 1000 + 0.5)) + + res = _subprocess.WaitForSingleObject(int(self._handle), msecs) + if res == _subprocess.WAIT_OBJECT_0: + code = _subprocess.GetExitCodeProcess(self._handle) + if code == TERMINATE: + code = -signal.SIGTERM + self.returncode = code + + return self.returncode + + def poll(self): + return self.wait(timeout=0) + + def terminate(self): + if self.returncode is None: + try: + _subprocess.TerminateProcess(int(self._handle), TERMINATE) + except WindowsError: + if self.wait(timeout=0.1) is None: + raise + + # + # + # + + def is_forking(argv): + ''' + Return whether commandline indicates we are forking + ''' + if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': + assert len(argv) == 3 + return True + else: + return False + + + def freeze_support(): + ''' + Run code for process object if this in not the main process + ''' + if is_forking(sys.argv): + main() + sys.exit() + + + def get_command_line(): + ''' + Returns prefix of command line used for spawning a child process + ''' + if process.current_process()._identity==() and is_forking(sys.argv): + raise RuntimeError(''' + Attempt to start a new process before the current process + has finished its bootstrapping phase. + + This probably means that you are on Windows and you have + forgotten to use the proper idiom in the main module: + + if __name__ == '__main__': + freeze_support() + ... + + The "freeze_support()" line can be omitted if the program + is not going to be frozen to produce a Windows executable.''') + + if getattr(sys, 'frozen', False): + return [sys.executable, '--multiprocessing-fork'] + else: + prog = 'from multiprocessing.forking import main; main()' + return [_python_exe, '-c', prog, '--multiprocessing-fork'] + + + def main(): + ''' + Run code specifed by data received over pipe + ''' + assert is_forking(sys.argv) + + handle = int(sys.argv[-1]) + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + from_parent = os.fdopen(fd, 'rb') + + process.current_process()._inheriting = True + preparation_data = load(from_parent) + prepare(preparation_data) + self = load(from_parent) + process.current_process()._inheriting = False + + from_parent.close() + + exitcode = self._bootstrap() + exit(exitcode) + + + def get_preparation_data(name): + ''' + Return info about parent needed by child to unpickle process object + ''' + from .util import _logger, _log_to_stderr + + d = dict( + name=name, + sys_path=sys.path, + sys_argv=sys.argv, + log_to_stderr=_log_to_stderr, + orig_dir=process.ORIGINAL_DIR, + authkey=process.current_process().authkey, + ) + + if _logger is not None: + d['log_level'] = _logger.getEffectiveLevel() + + if not WINEXE and not WINSERVICE: + main_path = getattr(sys.modules['__main__'], '__file__', None) + if not main_path and sys.argv[0] not in ('', '-c'): + main_path = sys.argv[0] + if main_path is not None: + if not os.path.isabs(main_path) and \ + process.ORIGINAL_DIR is not None: + main_path = os.path.join(process.ORIGINAL_DIR, main_path) + d['main_path'] = os.path.normpath(main_path) + + return d + + # + # Make (Pipe)Connection picklable + # + + # Late import because of circular import + from .connection import Connection, PipeConnection + + def reduce_connection(conn): + if not Popen.thread_is_spawning(): + raise RuntimeError( + 'By default %s objects can only be shared between processes\n' + 'using inheritance' % type(conn).__name__ + ) + return type(conn), (Popen.duplicate_for_child(conn.fileno()), + conn.readable, conn.writable) + + ForkingPickler.register(Connection, reduce_connection) + ForkingPickler.register(PipeConnection, reduce_connection) + +# +# Prepare current process +# + +old_main_modules = [] + +def prepare(data): + ''' + Try to get current process ready to unpickle process object + ''' + old_main_modules.append(sys.modules['__main__']) + + if 'name' in data: + process.current_process().name = data['name'] + + if 'authkey' in data: + process.current_process()._authkey = data['authkey'] + + if 'log_to_stderr' in data and data['log_to_stderr']: + util.log_to_stderr() + + if 'log_level' in data: + util.get_logger().setLevel(data['log_level']) + + if 'sys_path' in data: + sys.path = data['sys_path'] + + if 'sys_argv' in data: + sys.argv = data['sys_argv'] + + if 'dir' in data: + os.chdir(data['dir']) + + if 'orig_dir' in data: + process.ORIGINAL_DIR = data['orig_dir'] + + if 'main_path' in data: + # XXX (ncoghlan): The following code makes several bogus + # assumptions regarding the relationship between __file__ + # and a module's real name. See PEP 302 and issue #10845 + main_path = data['main_path'] + main_name = os.path.splitext(os.path.basename(main_path))[0] + if main_name == '__init__': + main_name = os.path.basename(os.path.dirname(main_path)) + + if main_name == '__main__': + main_module = sys.modules['__main__'] + main_module.__file__ = main_path + elif main_name != 'ipython': + # Main modules not actually called __main__.py may + # contain additional code that should still be executed + import imp + + if main_path is None: + dirs = None + elif os.path.basename(main_path).startswith('__init__.py'): + dirs = [os.path.dirname(os.path.dirname(main_path))] + else: + dirs = [os.path.dirname(main_path)] + + assert main_name not in sys.modules, main_name + file, path_name, etc = imp.find_module(main_name, dirs) + try: + # We would like to do "imp.load_module('__main__', ...)" + # here. However, that would cause 'if __name__ == + # "__main__"' clauses to be executed. + main_module = imp.load_module( + '__parents_main__', file, path_name, etc + ) + finally: + if file: + file.close() + + sys.modules['__main__'] = main_module + main_module.__name__ = '__main__' + + # Try to make the potentially picklable objects in + # sys.modules['__main__'] realize they are in the main + # module -- somewhat ugly. + for obj in list(main_module.__dict__.values()): + try: + if obj.__module__ == '__parents_main__': + obj.__module__ = '__main__' + except Exception: + pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/forkserver.py --- a/Lib/multiprocessing/forkserver.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,253 +0,0 @@ -import errno -import os -import selectors -import signal -import socket -import struct -import sys -import threading - -from . import connection -from . import process -from . import reduction -from . import semaphore_tracker -from . import spawn -from . import util - -__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process', - 'set_forkserver_preload'] - -# -# -# - -MAXFDS_TO_SEND = 256 -UNSIGNED_STRUCT = struct.Struct('Q') # large enough for pid_t - -# -# Forkserver class -# - -class ForkServer(object): - - def __init__(self): - self._forkserver_address = None - self._forkserver_alive_fd = None - self._inherited_fds = None - self._lock = threading.Lock() - self._preload_modules = ['__main__'] - - def set_forkserver_preload(self, modules_names): - '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): - raise TypeError('module_names must be a list of strings') - self._preload_modules = modules_names - - def get_inherited_fds(self): - '''Return list of fds inherited from parent process. - - This returns None if the current process was not started by fork - server. - ''' - return self._inherited_fds - - def connect_to_new_process(self, fds): - '''Request forkserver to create a child process. - - Returns a pair of fds (status_r, data_w). The calling process can read - the child process's pid and (eventually) its returncode from status_r. - The calling process should write to data_w the pickled preparation and - process data. - ''' - self.ensure_running() - if len(fds) + 4 >= MAXFDS_TO_SEND: - raise ValueError('too many fds') - with socket.socket(socket.AF_UNIX) as client: - client.connect(self._forkserver_address) - parent_r, child_w = os.pipe() - child_r, parent_w = os.pipe() - allfds = [child_r, child_w, self._forkserver_alive_fd, - semaphore_tracker.getfd()] - allfds += fds - try: - reduction.sendfds(client, allfds) - return parent_r, parent_w - except: - os.close(parent_r) - os.close(parent_w) - raise - finally: - os.close(child_r) - os.close(child_w) - - def ensure_running(self): - '''Make sure that a fork server is running. - - This can be called from any process. Note that usually a child - process will just reuse the forkserver started by its parent, so - ensure_running() will do nothing. - ''' - with self._lock: - semaphore_tracker.ensure_running() - if self._forkserver_alive_fd is not None: - return - - cmd = ('from multiprocessing.forkserver import main; ' + - 'main(%d, %d, %r, **%r)') - - if self._preload_modules: - desired_keys = {'main_path', 'sys_path'} - data = spawn.get_preparation_data('ignore') - data = dict((x,y) for (x,y) in data.items() - if x in desired_keys) - else: - data = {} - - with socket.socket(socket.AF_UNIX) as listener: - address = connection.arbitrary_address('AF_UNIX') - listener.bind(address) - os.chmod(address, 0o600) - listener.listen() - - # all client processes own the write end of the "alive" pipe; - # when they all terminate the read end becomes ready. - alive_r, alive_w = os.pipe() - try: - fds_to_pass = [listener.fileno(), alive_r] - cmd %= (listener.fileno(), alive_r, self._preload_modules, - data) - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd] - pid = util.spawnv_passfds(exe, args, fds_to_pass) - except: - os.close(alive_w) - raise - finally: - os.close(alive_r) - self._forkserver_address = address - self._forkserver_alive_fd = alive_w - -# -# -# - -def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): - '''Run forkserver.''' - if preload: - if '__main__' in preload and main_path is not None: - process.current_process()._inheriting = True - try: - spawn.import_main_path(main_path) - finally: - del process.current_process()._inheriting - for modname in preload: - try: - __import__(modname) - except ImportError: - pass - - # close sys.stdin - if sys.stdin is not None: - try: - sys.stdin.close() - sys.stdin = open(os.devnull) - except (OSError, ValueError): - pass - - # ignoring SIGCHLD means no need to reap zombie processes - handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) - with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \ - selectors.DefaultSelector() as selector: - _forkserver._forkserver_address = listener.getsockname() - - selector.register(listener, selectors.EVENT_READ) - selector.register(alive_r, selectors.EVENT_READ) - - while True: - try: - while True: - rfds = [key.fileobj for (key, events) in selector.select()] - if rfds: - break - - if alive_r in rfds: - # EOF because no more client processes left - assert os.read(alive_r, 1) == b'' - raise SystemExit - - assert listener in rfds - with listener.accept()[0] as s: - code = 1 - if os.fork() == 0: - try: - _serve_one(s, listener, alive_r, handler) - except Exception: - sys.excepthook(*sys.exc_info()) - sys.stderr.flush() - finally: - os._exit(code) - - except OSError as e: - if e.errno != errno.ECONNABORTED: - raise - -def _serve_one(s, listener, alive_r, handler): - # close unnecessary stuff and reset SIGCHLD handler - listener.close() - os.close(alive_r) - signal.signal(signal.SIGCHLD, handler) - - # receive fds from parent process - fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) - s.close() - assert len(fds) <= MAXFDS_TO_SEND - (child_r, child_w, _forkserver._forkserver_alive_fd, - stfd, *_forkserver._inherited_fds) = fds - semaphore_tracker._semaphore_tracker._fd = stfd - - # send pid to client processes - write_unsigned(child_w, os.getpid()) - - # reseed random number generator - if 'random' in sys.modules: - import random - random.seed() - - # run process object received over pipe - code = spawn._main(child_r) - - # write the exit code to the pipe - write_unsigned(child_w, code) - -# -# Read and write unsigned numbers -# - -def read_unsigned(fd): - data = b'' - length = UNSIGNED_STRUCT.size - while len(data) < length: - s = os.read(fd, length - len(data)) - if not s: - raise EOFError('unexpected EOF') - data += s - return UNSIGNED_STRUCT.unpack(data)[0] - -def write_unsigned(fd, n): - msg = UNSIGNED_STRUCT.pack(n) - while msg: - nbytes = os.write(fd, msg) - if nbytes == 0: - raise RuntimeError('should not get here') - msg = msg[nbytes:] - -# -# -# - -_forkserver = ForkServer() -ensure_running = _forkserver.ensure_running -get_inherited_fds = _forkserver.get_inherited_fds -connect_to_new_process = _forkserver.connect_to_new_process -set_forkserver_preload = _forkserver.set_forkserver_preload diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/heap.py --- a/Lib/multiprocessing/heap.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/heap.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,93 +4,83 @@ # multiprocessing/heap.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import bisect import mmap import os import sys -import tempfile import threading +import itertools -from . import context -from . import reduction -from . import util +import _multiprocessing +from multiprocessing.util import Finalize, info +from multiprocessing.forking import assert_spawning __all__ = ['BufferWrapper'] # -# Inheritable class which wraps an mmap, and from which blocks can be allocated +# Inheirtable class which wraps an mmap, and from which blocks can be allocated # if sys.platform == 'win32': - import _winapi + from _multiprocessing import win32 class Arena(object): - _rand = tempfile._RandomNameSequence() + _counter = itertools.count() def __init__(self, size): self.size = size - for i in range(100): - name = 'pym-%d-%s' % (os.getpid(), next(self._rand)) - buf = mmap.mmap(-1, size, tagname=name) - if _winapi.GetLastError() == 0: - break - # We have reopened a preexisting mmap. - buf.close() - else: - raise FileExistsError('Cannot find name for new mmap') - self.name = name - self.buffer = buf + self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter)) + self.buffer = mmap.mmap(-1, self.size, tagname=self.name) + assert win32.GetLastError() == 0, 'tagname already in use' self._state = (self.size, self.name) def __getstate__(self): - context.assert_spawning(self) + assert_spawning(self) return self._state def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) - # XXX Temporarily preventing buildbot failures while determining - # XXX the correct long-term fix. See issue 23060 - #assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS + assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS else: class Arena(object): - def __init__(self, size, fd=-1): + def __init__(self, size): + self.buffer = mmap.mmap(-1, size) self.size = size - self.fd = fd - if fd == -1: - self.fd, name = tempfile.mkstemp( - prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir()) - os.unlink(name) - util.Finalize(self, os.close, (self.fd,)) - with open(self.fd, 'wb', closefd=False) as f: - bs = 1024 * 1024 - if size >= bs: - zeros = b'\0' * bs - for _ in range(size // bs): - f.write(zeros) - del zeros - f.write(b'\0' * (size % bs)) - assert f.tell() == size - self.buffer = mmap.mmap(self.fd, self.size) - - def reduce_arena(a): - if a.fd == -1: - raise ValueError('Arena is unpicklable because ' - 'forking was enabled when it was created') - return rebuild_arena, (a.size, reduction.DupFd(a.fd)) - - def rebuild_arena(size, dupfd): - return Arena(size, dupfd.detach()) - - reduction.register(Arena, reduce_arena) + self.name = None # # Class allowing allocation of chunks of memory from arenas @@ -125,7 +115,7 @@ if i == len(self._lengths): length = self._roundup(max(self._size, size), mmap.PAGESIZE) self._size *= 2 - util.info('allocating a new mmap of length %d', length) + info('allocating a new mmap of length %d', length) arena = Arena(length) self._arenas.append(arena) return (arena, 0, length) @@ -225,8 +215,9 @@ assert 0 <= size < sys.maxsize if os.getpid() != self._lastpid: self.__init__() # reinitialize after fork - with self._lock: - self._free_pending_blocks() + self._lock.acquire() + self._free_pending_blocks() + try: size = self._roundup(max(size,1), self._alignment) (arena, start, stop) = self._malloc(size) new_stop = start + size @@ -235,9 +226,11 @@ block = (arena, start, new_stop) self._allocated_blocks.add(block) return block + finally: + self._lock.release() # -# Class representing a chunk of an mmap -- can be inherited by child process +# Class representing a chunk of an mmap -- can be inherited # class BufferWrapper(object): @@ -248,8 +241,13 @@ assert 0 <= size < sys.maxsize block = BufferWrapper._heap.malloc(size) self._state = (block, size) - util.Finalize(self, BufferWrapper._heap.free, args=(block,)) + Finalize(self, BufferWrapper._heap.free, args=(block,)) - def create_memoryview(self): + def get_address(self): (arena, start, stop), size = self._state - return memoryview(arena.buffer)[start:start+size] + address, length = _multiprocessing.address_of_buffer(arena.buffer) + assert size <= length + return address + start + + def get_size(self): + return self._state[1] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/managers.py --- a/Lib/multiprocessing/managers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/managers.py Mon Jan 25 17:05:13 2016 +0100 @@ -5,7 +5,32 @@ # multiprocessing/managers.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ] @@ -19,16 +44,10 @@ import array import queue -from time import time as _time from traceback import format_exc - -from . import connection -from . import context -from . import pool -from . import process -from . import reduction -from . import util -from . import get_context +from multiprocessing import Process, current_process, active_children, Pool, util, connection +from multiprocessing.process import AuthenticationString +from multiprocessing.forking import exit, Popen, ForkingPickler # # Register some things for pickling @@ -36,14 +55,16 @@ def reduce_array(a): return array.array, (a.typecode, a.tobytes()) -reduction.register(array.array, reduce_array) +ForkingPickler.register(array.array, reduce_array) view_types = [type(getattr({}, name)()) for name in ('items','keys','values')] if view_types[0] is not list: # only needed in Py3.0 def rebuild_as_list(obj): return list, (list(obj),) for view_type in view_types: - reduction.register(view_type, rebuild_as_list) + ForkingPickler.register(view_type, rebuild_as_list) + import copyreg + copyreg.pickle(view_type, rebuild_as_list) # # Type for identifying shared objects @@ -65,8 +86,8 @@ (self.typeid, self.address, self.id) = state def __repr__(self): - return '%s(typeid=%r, address=%r, id=%r)' % \ - (self.__class__.__name__, self.typeid, self.address, self.id) + return 'Token(typeid=%r, address=%r, id=%r)' % \ + (self.typeid, self.address, self.id) # # Function for communication with a manager's server process @@ -133,7 +154,7 @@ def __init__(self, registry, address, authkey, serializer): assert isinstance(authkey, bytes) self.registry = registry - self.authkey = process.AuthenticationString(authkey) + self.authkey = AuthenticationString(authkey) Listener, Client = listener_client[serializer] # do authentication later @@ -143,38 +164,28 @@ self.id_to_obj = {'0': (None, ())} self.id_to_refcount = {} self.mutex = threading.RLock() + self.stop = 0 def serve_forever(self): ''' Run the server forever ''' - self.stop_event = threading.Event() - process.current_process()._manager_server = self + current_process()._manager_server = self try: - accepter = threading.Thread(target=self.accepter) - accepter.daemon = True - accepter.start() try: - while not self.stop_event.is_set(): - self.stop_event.wait(1) + while 1: + try: + c = self.listener.accept() + except (OSError, IOError): + continue + t = threading.Thread(target=self.handle_request, args=(c,)) + t.daemon = True + t.start() except (KeyboardInterrupt, SystemExit): pass finally: - if sys.stdout != sys.__stdout__: - util.debug('resetting stdout, stderr') - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - sys.exit(0) - - def accepter(self): - while True: - try: - c = self.listener.accept() - except OSError: - continue - t = threading.Thread(target=self.handle_request, args=(c,)) - t.daemon = True - t.start() + self.stop = 999 + self.listener.close() def handle_request(self, c): ''' @@ -221,7 +232,7 @@ send = conn.send id_to_obj = self.id_to_obj - while not self.stop_event.is_set(): + while not self.stop: try: methodname = obj = None @@ -306,7 +317,8 @@ ''' Return some info --- useful to spot problems with refcounting ''' - with self.mutex: + self.mutex.acquire() + try: result = [] keys = list(self.id_to_obj.keys()) keys.sort() @@ -316,6 +328,8 @@ (ident, self.id_to_refcount[ident], str(self.id_to_obj[ident][0])[:75])) return '\n'.join(result) + finally: + self.mutex.release() def number_of_objects(self, c): ''' @@ -328,19 +342,39 @@ Shutdown this process ''' try: - util.debug('manager received shutdown message') - c.send(('#RETURN', None)) - except: - import traceback - traceback.print_exc() + try: + util.debug('manager received shutdown message') + c.send(('#RETURN', None)) + + if sys.stdout != sys.__stdout__: + util.debug('resetting stdout, stderr') + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + util._run_finalizers(0) + + for p in active_children(): + util.debug('terminating a child process of manager') + p.terminate() + + for p in active_children(): + util.debug('terminating a child process of manager') + p.join() + + util._run_finalizers() + util.info('manager exiting with exitcode 0') + except: + import traceback + traceback.print_exc() finally: - self.stop_event.set() + exit(0) def create(self, c, typeid, *args, **kwds): ''' Create a new shared object and return its id ''' - with self.mutex: + self.mutex.acquire() + try: callable, exposed, method_to_typeid, proxytype = \ self.registry[typeid] @@ -370,6 +404,8 @@ # has been created. self.incref(c, ident) return ident, tuple(exposed) + finally: + self.mutex.release() def get_methods(self, c, token): ''' @@ -386,16 +422,22 @@ self.serve_client(c) def incref(self, c, ident): - with self.mutex: + self.mutex.acquire() + try: self.id_to_refcount[ident] += 1 + finally: + self.mutex.release() def decref(self, c, ident): - with self.mutex: + self.mutex.acquire() + try: assert self.id_to_refcount[ident] >= 1 self.id_to_refcount[ident] -= 1 if self.id_to_refcount[ident] == 0: del self.id_to_obj[ident], self.id_to_refcount[ident] util.debug('disposing of obj with id %r', ident) + finally: + self.mutex.release() # # Class to represent state of a manager @@ -427,17 +469,19 @@ _registry = {} _Server = Server - def __init__(self, address=None, authkey=None, serializer='pickle', - ctx=None): + def __init__(self, address=None, authkey=None, serializer='pickle'): if authkey is None: - authkey = process.current_process().authkey + authkey = current_process().authkey self._address = address # XXX not final address if eg ('', 0) - self._authkey = process.AuthenticationString(authkey) + self._authkey = AuthenticationString(authkey) self._state = State() self._state.value = State.INITIAL self._serializer = serializer self._Listener, self._Client = listener_client[serializer] - self._ctx = ctx or get_context() + + def __reduce__(self): + return type(self).from_address, \ + (self._address, self._authkey, self._serializer) def get_server(self): ''' @@ -469,7 +513,7 @@ reader, writer = connection.Pipe(duplex=False) # spawn process which runs a server - self._process = self._ctx.Process( + self._process = Process( target=type(self)._run_server, args=(self._registry, self._address, self._authkey, self._serializer, writer, initializer, initargs), @@ -528,10 +572,7 @@ ''' Join the manager process (if it has been spawned) ''' - if self._process is not None: - self._process.join(timeout) - if not self._process.is_alive(): - self._process = None + self._process.join(timeout) def _debug_info(self): ''' @@ -554,9 +595,6 @@ conn.close() def __enter__(self): - if self._state.value == State.INITIAL: - self.start() - assert self._state.value == State.STARTED return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -578,7 +616,7 @@ except Exception: pass - process.join(timeout=1.0) + process.join(timeout=0.2) if process.is_alive(): util.info('manager still alive') if hasattr(process, 'terminate'): @@ -659,11 +697,14 @@ def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True): - with BaseProxy._mutex: + BaseProxy._mutex.acquire() + try: tls_idset = BaseProxy._address_to_local.get(token.address, None) if tls_idset is None: tls_idset = util.ForkAwareLocal(), ProcessLocalSet() BaseProxy._address_to_local[token.address] = tls_idset + finally: + BaseProxy._mutex.release() # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address @@ -681,11 +722,11 @@ self._Client = listener_client[serializer][1] if authkey is not None: - self._authkey = process.AuthenticationString(authkey) + self._authkey = AuthenticationString(authkey) elif self._manager is not None: self._authkey = self._manager._authkey else: - self._authkey = process.current_process().authkey + self._authkey = current_process().authkey if incref: self._incref() @@ -694,7 +735,7 @@ def _connect(self): util.debug('making connection to manager') - name = process.current_process().name + name = current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name conn = self._Client(self._token.address, authkey=self._authkey) @@ -721,7 +762,6 @@ elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] - token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed @@ -788,7 +828,7 @@ def __reduce__(self): kwds = {} - if context.get_spawning_popen() is not None: + if Popen.thread_is_spawning(): kwds['authkey'] = self._authkey if getattr(self, '_isauto', False): @@ -803,8 +843,8 @@ return self._getvalue() def __repr__(self): - return '<%s object, typeid %r at %#x>' % \ - (type(self).__name__, self._token.typeid, id(self)) + return '<%s object, typeid %r at %s>' % \ + (type(self).__name__, self._token.typeid, '0x%x' % id(self)) def __str__(self): ''' @@ -825,14 +865,14 @@ If possible the shared object is returned, or otherwise a proxy for it. ''' - server = getattr(process.current_process(), '_manager_server', None) + server = getattr(current_process(), '_manager_server', None) if server and server.address == token.address: return server.id_to_obj[token.id][0] else: incref = ( kwds.pop('incref', True) and - not getattr(process.current_process(), '_inheriting', False) + not getattr(current_process(), '_inheriting', False) ) return func(token, serializer, incref=incref, **kwds) @@ -879,7 +919,7 @@ if authkey is None and manager is not None: authkey = manager._authkey if authkey is None: - authkey = process.current_process().authkey + authkey = current_process().authkey ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed) proxy = ProxyType(token, serializer, manager=manager, authkey=authkey, @@ -901,7 +941,7 @@ if not name.startswith('_'): temp.append('%s=%r' % (name, value)) temp.sort() - return '%s(%s)' % (self.__class__.__name__, ', '.join(temp)) + return 'Namespace(%s)' % str.join(', ', temp) class Value(object): def __init__(self, typecode, value, lock=True): @@ -938,9 +978,8 @@ class AcquirerProxy(BaseProxy): _exposed_ = ('acquire', 'release') - def acquire(self, blocking=True, timeout=None): - args = (blocking,) if timeout is None else (blocking, timeout) - return self._callmethod('acquire', args) + def acquire(self, blocking=True): + return self._callmethod('acquire', (blocking,)) def release(self): return self._callmethod('release') def __enter__(self): @@ -957,24 +996,6 @@ return self._callmethod('notify') def notify_all(self): return self._callmethod('notify_all') - def wait_for(self, predicate, timeout=None): - result = predicate() - if result: - return result - if timeout is not None: - endtime = _time() + timeout - else: - endtime = None - waittime = None - while not result: - if endtime is not None: - waittime = endtime - _time() - if waittime <= 0: - break - self.wait(waittime) - result = predicate() - return result - class EventProxy(BaseProxy): _exposed_ = ('is_set', 'set', 'clear', 'wait') @@ -987,26 +1008,6 @@ def wait(self, timeout=None): return self._callmethod('wait', (timeout,)) - -class BarrierProxy(BaseProxy): - _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset') - def wait(self, timeout=None): - return self._callmethod('wait', (timeout,)) - def abort(self): - return self._callmethod('abort') - def reset(self): - return self._callmethod('reset') - @property - def parties(self): - return self._callmethod('__getattribute__', ('parties',)) - @property - def n_waiting(self): - return self._callmethod('__getattribute__', ('n_waiting',)) - @property - def broken(self): - return self._callmethod('__getattribute__', ('broken',)) - - class NamespaceProxy(BaseProxy): _exposed_ = ('__getattribute__', '__setattr__', '__delattr__') def __getattr__(self, key): @@ -1036,11 +1037,12 @@ BaseListProxy = MakeProxyType('BaseListProxy', ( - '__add__', '__contains__', '__delitem__', '__getitem__', '__len__', - '__mul__', '__reversed__', '__rmul__', '__setitem__', + '__add__', '__contains__', '__delitem__', '__delslice__', + '__getitem__', '__getslice__', '__len__', '__mul__', + '__reversed__', '__rmul__', '__setitem__', '__setslice__', 'append', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort', '__imul__' - )) + )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 class ListProxy(BaseListProxy): def __iadd__(self, value): self._callmethod('extend', (value,)) @@ -1058,26 +1060,21 @@ ArrayProxy = MakeProxyType('ArrayProxy', ( - '__len__', '__getitem__', '__setitem__' + '__len__', '__getitem__', '__setitem__', '__getslice__', '__setslice__' + )) # XXX __getslice__ and __setslice__ unneeded in Py3.0 + + +PoolProxy = MakeProxyType('PoolProxy', ( + 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join', + 'map', 'map_async', 'starmap', 'starmap_async', 'terminate' )) - - -BasePoolProxy = MakeProxyType('PoolProxy', ( - 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join', - 'map', 'map_async', 'starmap', 'starmap_async', 'terminate', - )) -BasePoolProxy._method_to_typeid_ = { +PoolProxy._method_to_typeid_ = { 'apply_async': 'AsyncResult', 'map_async': 'AsyncResult', 'starmap_async': 'AsyncResult', 'imap': 'Iterator', 'imap_unordered': 'Iterator' } -class PoolProxy(BasePoolProxy): - def __enter__(self): - return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.terminate() # # Definition of SyncManager @@ -1103,8 +1100,7 @@ SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore, AcquirerProxy) SyncManager.register('Condition', threading.Condition, ConditionProxy) -SyncManager.register('Barrier', threading.Barrier, BarrierProxy) -SyncManager.register('Pool', pool.Pool, PoolProxy) +SyncManager.register('Pool', Pool, PoolProxy) SyncManager.register('list', list, ListProxy) SyncManager.register('dict', dict, DictProxy) SyncManager.register('Value', Value, ValueProxy) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/pool.py --- a/Lib/multiprocessing/pool.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/pool.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,35 @@ # multiprocessing/pool.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # -__all__ = ['Pool', 'ThreadPool'] +__all__ = ['Pool'] # # Imports @@ -17,14 +42,10 @@ import queue import itertools import collections -import os import time -import traceback -# If threading is available then ThreadPool should be provided. Therefore -# we avoid top-level imports which are liable to fail on some systems. -from . import util -from . import get_context, TimeoutError +from multiprocessing import Process, cpu_count, TimeoutError +from multiprocessing.util import Finalize, debug # # Constants representing the state of a pool @@ -47,29 +68,6 @@ return list(itertools.starmap(args[0], args[1])) # -# Hack to embed stringification of remote traceback in local traceback -# - -class RemoteTraceback(Exception): - def __init__(self, tb): - self.tb = tb - def __str__(self): - return self.tb - -class ExceptionWithTraceback: - def __init__(self, exc, tb): - tb = traceback.format_exception(type(exc), exc, tb) - tb = ''.join(tb) - self.exc = exc - self.tb = '\n"""\n%s"""' % tb - def __reduce__(self): - return rebuild_exc, (self.exc, self.tb) - -def rebuild_exc(exc, tb): - exc.__cause__ = RemoteTraceback(tb) - return exc - -# # Code run by worker processes # @@ -87,11 +85,10 @@ self.exc) def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self) + return "" % str(self) -def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, - wrap_exception=False): +def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None): assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) put = outqueue.put get = inqueue.get @@ -106,30 +103,28 @@ while maxtasks is None or (maxtasks and completed < maxtasks): try: task = get() - except (EOFError, OSError): - util.debug('worker got EOFError or OSError -- exiting') + except (EOFError, IOError): + debug('worker got EOFError or IOError -- exiting') break if task is None: - util.debug('worker got sentinel -- exiting') + debug('worker got sentinel -- exiting') break job, i, func, args, kwds = task try: result = (True, func(*args, **kwds)) except Exception as e: - if wrap_exception: - e = ExceptionWithTraceback(e, e.__traceback__) result = (False, e) try: put((job, i, result)) except Exception as e: wrapped = MaybeEncodingError(e, result[1]) - util.debug("Possible encoding error while sending result: %s" % ( + debug("Possible encoding error while sending result: %s" % ( wrapped)) put((job, i, (False, wrapped))) completed += 1 - util.debug('worker exiting after %d tasks' % completed) + debug('worker exiting after %d tasks' % completed) # # Class representing a process pool @@ -139,14 +134,10 @@ ''' Class which supports an async version of applying functions to arguments. ''' - _wrap_exception = True - - def Process(self, *args, **kwds): - return self._ctx.Process(*args, **kwds) + Process = Process def __init__(self, processes=None, initializer=None, initargs=(), - maxtasksperchild=None, context=None): - self._ctx = context or get_context() + maxtasksperchild=None): self._setup_queues() self._taskqueue = queue.Queue() self._cache = {} @@ -156,7 +147,10 @@ self._initargs = initargs if processes is None: - processes = os.cpu_count() or 1 + try: + processes = cpu_count() + except NotImplementedError: + processes = 1 if processes < 1: raise ValueError("Number of processes must be at least 1") @@ -178,8 +172,7 @@ self._task_handler = threading.Thread( target=Pool._handle_tasks, - args=(self._taskqueue, self._quick_put, self._outqueue, - self._pool, self._cache) + args=(self._taskqueue, self._quick_put, self._outqueue, self._pool) ) self._task_handler.daemon = True self._task_handler._state = RUN @@ -193,7 +186,7 @@ self._result_handler._state = RUN self._result_handler.start() - self._terminate = util.Finalize( + self._terminate = Finalize( self, self._terminate_pool, args=(self._taskqueue, self._inqueue, self._outqueue, self._pool, self._worker_handler, self._task_handler, @@ -210,7 +203,7 @@ worker = self._pool[i] if worker.exitcode is not None: # worker exited - util.debug('cleaning up worker %d' % i) + debug('cleaning up worker %d' % i) worker.join() cleaned = True del self._pool[i] @@ -224,14 +217,13 @@ w = self.Process(target=worker, args=(self._inqueue, self._outqueue, self._initializer, - self._initargs, self._maxtasksperchild, - self._wrap_exception) + self._initargs, self._maxtasksperchild) ) self._pool.append(w) w.name = w.name.replace('Process', 'PoolWorker') w.daemon = True w.start() - util.debug('added worker') + debug('added worker') def _maintain_pool(self): """Clean up any exited workers and start replacements for them. @@ -240,8 +232,9 @@ self._repopulate_pool() def _setup_queues(self): - self._inqueue = self._ctx.SimpleQueue() - self._outqueue = self._ctx.SimpleQueue() + from .queues import SimpleQueue + self._inqueue = SimpleQueue() + self._outqueue = SimpleQueue() self._quick_put = self._inqueue._writer.send self._quick_get = self._outqueue._reader.recv @@ -257,6 +250,7 @@ Apply `func` to each element in `iterable`, collecting the results in a list that is returned. ''' + assert self._state == RUN return self._map_async(func, iterable, mapstar, chunksize).get() def starmap(self, func, iterable, chunksize=None): @@ -265,6 +259,7 @@ be iterables as well and will be unpacked as arguments. Hence `func` and (a, b) becomes func(a, b). ''' + assert self._state == RUN return self._map_async(func, iterable, starmapstar, chunksize).get() def starmap_async(self, func, iterable, chunksize=None, callback=None, @@ -272,6 +267,7 @@ ''' Asynchronous version of `starmap()` method. ''' + assert self._state == RUN return self._map_async(func, iterable, starmapstar, chunksize, callback, error_callback) @@ -279,8 +275,7 @@ ''' Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. ''' - if self._state != RUN: - raise ValueError("Pool not running") + assert self._state == RUN if chunksize == 1: result = IMapIterator(self._cache) self._taskqueue.put((((result._job, i, func, (x,), {}) @@ -298,8 +293,7 @@ ''' Like `imap()` method but ordering of results is arbitrary. ''' - if self._state != RUN: - raise ValueError("Pool not running") + assert self._state == RUN if chunksize == 1: result = IMapUnorderedIterator(self._cache) self._taskqueue.put((((result._job, i, func, (x,), {}) @@ -318,8 +312,7 @@ ''' Asynchronous version of `apply()` method. ''' - if self._state != RUN: - raise ValueError("Pool not running") + assert self._state == RUN result = ApplyResult(self._cache, callback, error_callback) self._taskqueue.put(([(result._job, None, func, args, kwds)], None)) return result @@ -329,16 +322,14 @@ ''' Asynchronous version of `map()` method. ''' - return self._map_async(func, iterable, mapstar, chunksize, callback, - error_callback) + assert self._state == RUN + return self._map_async(func, iterable, mapstar, chunksize) def _map_async(self, func, iterable, mapper, chunksize=None, callback=None, error_callback=None): ''' Helper function to implement map, starmap and their async counterparts. ''' - if self._state != RUN: - raise ValueError("Pool not running") if not hasattr(iterable, '__len__'): iterable = list(iterable) @@ -367,58 +358,46 @@ time.sleep(0.1) # send sentinel to stop workers pool._taskqueue.put(None) - util.debug('worker handler exiting') + debug('worker handler exiting') @staticmethod - def _handle_tasks(taskqueue, put, outqueue, pool, cache): + def _handle_tasks(taskqueue, put, outqueue, pool): thread = threading.current_thread() for taskseq, set_length in iter(taskqueue.get, None): - task = None i = -1 - try: - for i, task in enumerate(taskseq): - if thread._state: - util.debug('task handler found thread._state != RUN') - break - try: - put(task) - except Exception as e: - job, ind = task[:2] - try: - cache[job]._set(ind, (False, e)) - except KeyError: - pass - else: - if set_length: - util.debug('doing set_length()') - set_length(i+1) - continue - break - except Exception as ex: - job, ind = task[:2] if task else (0, 0) - if job in cache: - cache[job]._set(ind + 1, (False, ex)) + for i, task in enumerate(taskseq): + if thread._state: + debug('task handler found thread._state != RUN') + break + try: + put(task) + except IOError: + debug('could not put task on queue') + break + else: if set_length: - util.debug('doing set_length()') + debug('doing set_length()') set_length(i+1) + continue + break else: - util.debug('task handler got sentinel') + debug('task handler got sentinel') try: # tell result handler to finish when cache is empty - util.debug('task handler sending sentinel to result handler') + debug('task handler sending sentinel to result handler') outqueue.put(None) # tell workers there is no more work - util.debug('task handler sending sentinel to workers') + debug('task handler sending sentinel to workers') for p in pool: put(None) - except OSError: - util.debug('task handler got OSError when sending sentinels') + except IOError: + debug('task handler got IOError when sending sentinels') - util.debug('task handler exiting') + debug('task handler exiting') @staticmethod def _handle_results(outqueue, get, cache): @@ -427,17 +406,17 @@ while 1: try: task = get() - except (OSError, EOFError): - util.debug('result handler got EOFError/OSError -- exiting') + except (IOError, EOFError): + debug('result handler got EOFError/IOError -- exiting') return if thread._state: assert thread._state == TERMINATE - util.debug('result handler found thread._state=TERMINATE') + debug('result handler found thread._state=TERMINATE') break if task is None: - util.debug('result handler got sentinel') + debug('result handler got sentinel') break job, i, obj = task @@ -449,12 +428,12 @@ while cache and thread._state != TERMINATE: try: task = get() - except (OSError, EOFError): - util.debug('result handler got EOFError/OSError -- exiting') + except (IOError, EOFError): + debug('result handler got EOFError/IOError -- exiting') return if task is None: - util.debug('result handler ignoring extra sentinel') + debug('result handler ignoring extra sentinel') continue job, i, obj = task try: @@ -463,7 +442,7 @@ pass if hasattr(outqueue, '_reader'): - util.debug('ensuring that outqueue is not full') + debug('ensuring that outqueue is not full') # If we don't make room available in outqueue then # attempts to add the sentinel (None) to outqueue may # block. There is guaranteed to be no more than 2 sentinels. @@ -472,10 +451,10 @@ if not outqueue._reader.poll(): break get() - except (OSError, EOFError): + except (IOError, EOFError): pass - util.debug('result handler exiting: len(cache)=%s, thread._state=%s', + debug('result handler exiting: len(cache)=%s, thread._state=%s', len(cache), thread._state) @staticmethod @@ -493,19 +472,19 @@ ) def close(self): - util.debug('closing pool') + debug('closing pool') if self._state == RUN: self._state = CLOSE self._worker_handler._state = CLOSE def terminate(self): - util.debug('terminating pool') + debug('terminating pool') self._state = TERMINATE self._worker_handler._state = TERMINATE self._terminate() def join(self): - util.debug('joining pool') + debug('joining pool') assert self._state in (CLOSE, TERMINATE) self._worker_handler.join() self._task_handler.join() @@ -516,7 +495,7 @@ @staticmethod def _help_stuff_finish(inqueue, task_handler, size): # task_handler may be blocked trying to put items on inqueue - util.debug('removing tasks from inqueue until task handler finished') + debug('removing tasks from inqueue until task handler finished') inqueue._rlock.acquire() while task_handler.is_alive() and inqueue._reader.poll(): inqueue._reader.recv() @@ -526,12 +505,12 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, worker_handler, task_handler, result_handler, cache): # this is guaranteed to only be called once - util.debug('finalizing pool') + debug('finalizing pool') worker_handler._state = TERMINATE task_handler._state = TERMINATE - util.debug('helping task handler/workers to finish') + debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) assert result_handler.is_alive() or len(cache) == 0 @@ -541,39 +520,30 @@ # We must wait for the worker handler to exit before terminating # workers because we don't want workers to be restarted behind our back. - util.debug('joining worker handler') - if threading.current_thread() is not worker_handler: - worker_handler.join() + debug('joining worker handler') + worker_handler.join() # Terminate workers which haven't already finished. if pool and hasattr(pool[0], 'terminate'): - util.debug('terminating workers') + debug('terminating workers') for p in pool: if p.exitcode is None: p.terminate() - util.debug('joining task handler') - if threading.current_thread() is not task_handler: - task_handler.join() + debug('joining task handler') + task_handler.join() - util.debug('joining result handler') - if threading.current_thread() is not result_handler: - result_handler.join() + debug('joining result handler') + result_handler.join() if pool and hasattr(pool[0], 'terminate'): - util.debug('joining pool workers') + debug('joining pool workers') for p in pool: if p.is_alive(): # worker has not yet exited - util.debug('cleaning up worker %d' % p.pid) + debug('cleaning up worker %d' % p.pid) p.join() - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.terminate() - # # Class whose instances are returned by `Pool.apply_async()` # @@ -581,26 +551,32 @@ class ApplyResult(object): def __init__(self, cache, callback, error_callback): - self._event = threading.Event() + self._cond = threading.Condition(threading.Lock()) self._job = next(job_counter) self._cache = cache + self._ready = False self._callback = callback self._error_callback = error_callback cache[self._job] = self def ready(self): - return self._event.is_set() + return self._ready def successful(self): - assert self.ready() + assert self._ready return self._success def wait(self, timeout=None): - self._event.wait(timeout) + self._cond.acquire() + try: + if not self._ready: + self._cond.wait(timeout) + finally: + self._cond.release() def get(self, timeout=None): self.wait(timeout) - if not self.ready(): + if not self._ready: raise TimeoutError if self._success: return self._value @@ -613,11 +589,14 @@ self._callback(self._value) if self._error_callback and not self._success: self._error_callback(self._value) - self._event.set() + self._cond.acquire() + try: + self._ready = True + self._cond.notify() + finally: + self._cond.release() del self._cache[self._job] -AsyncResult = ApplyResult # create alias -- see #17805 - # # Class whose instances are returned by `Pool.map_async()` # @@ -632,8 +611,7 @@ self._chunksize = chunksize if chunksize <= 0: self._number_left = 0 - self._event.set() - del cache[self._job] + self._ready = True else: self._number_left = length//chunksize + bool(length % chunksize) @@ -646,14 +624,24 @@ if self._callback: self._callback(self._value) del self._cache[self._job] - self._event.set() + self._cond.acquire() + try: + self._ready = True + self._cond.notify() + finally: + self._cond.release() else: self._success = False self._value = result if self._error_callback: self._error_callback(self._value) del self._cache[self._job] - self._event.set() + self._cond.acquire() + try: + self._ready = True + self._cond.notify() + finally: + self._cond.release() # # Class whose instances are returned by `Pool.imap()` @@ -675,7 +663,8 @@ return self def next(self, timeout=None): - with self._cond: + self._cond.acquire() + try: try: item = self._items.popleft() except IndexError: @@ -688,6 +677,8 @@ if self._index == self._length: raise StopIteration raise TimeoutError + finally: + self._cond.release() success, value = item if success: @@ -697,7 +688,8 @@ __next__ = next # XXX def _set(self, i, obj): - with self._cond: + self._cond.acquire() + try: if self._index == i: self._items.append(obj) self._index += 1 @@ -711,13 +703,18 @@ if self._index == self._length: del self._cache[self._job] + finally: + self._cond.release() def _set_length(self, length): - with self._cond: + self._cond.acquire() + try: self._length = length if self._index == self._length: self._cond.notify() del self._cache[self._job] + finally: + self._cond.release() # # Class whose instances are returned by `Pool.imap_unordered()` @@ -726,24 +723,23 @@ class IMapUnorderedIterator(IMapIterator): def _set(self, i, obj): - with self._cond: + self._cond.acquire() + try: self._items.append(obj) self._index += 1 self._cond.notify() if self._index == self._length: del self._cache[self._job] + finally: + self._cond.release() # # # class ThreadPool(Pool): - _wrap_exception = False - @staticmethod - def Process(*args, **kwds): - from .dummy import Process - return Process(*args, **kwds) + from .dummy import Process def __init__(self, processes=None, initializer=None, initargs=()): Pool.__init__(self, processes, initializer, initargs) @@ -757,7 +753,10 @@ @staticmethod def _help_stuff_finish(inqueue, task_handler, size): # put sentinels at head of inqueue to make workers finish - with inqueue.not_empty: + inqueue.not_empty.acquire() + try: inqueue.queue.clear() inqueue.queue.extend([None] * size) inqueue.not_empty.notify_all() + finally: + inqueue.not_empty.release() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/popen_fork.py --- a/Lib/multiprocessing/popen_fork.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,80 +0,0 @@ -import os -import sys -import signal - -from . import util - -__all__ = ['Popen'] - -# -# Start child process using fork -# - -class Popen(object): - method = 'fork' - - def __init__(self, process_obj): - sys.stdout.flush() - sys.stderr.flush() - self.returncode = None - self._launch(process_obj) - - def duplicate_for_child(self, fd): - return fd - - def poll(self, flag=os.WNOHANG): - if self.returncode is None: - while True: - try: - pid, sts = os.waitpid(self.pid, flag) - except OSError as e: - # Child process not yet created. See #1731717 - # e.errno == errno.ECHILD == 10 - return None - else: - break - if pid == self.pid: - if os.WIFSIGNALED(sts): - self.returncode = -os.WTERMSIG(sts) - else: - assert os.WIFEXITED(sts) - self.returncode = os.WEXITSTATUS(sts) - return self.returncode - - def wait(self, timeout=None): - if self.returncode is None: - if timeout is not None: - from multiprocessing.connection import wait - if not wait([self.sentinel], timeout): - return None - # This shouldn't block if wait() returned successfully. - return self.poll(os.WNOHANG if timeout == 0.0 else 0) - return self.returncode - - def terminate(self): - if self.returncode is None: - try: - os.kill(self.pid, signal.SIGTERM) - except ProcessLookupError: - pass - except OSError: - if self.wait(timeout=0.1) is None: - raise - - def _launch(self, process_obj): - code = 1 - parent_r, child_w = os.pipe() - self.pid = os.fork() - if self.pid == 0: - try: - os.close(parent_r) - if 'random' in sys.modules: - import random - random.seed() - code = process_obj._bootstrap() - finally: - os._exit(code) - else: - os.close(child_w) - util.Finalize(self, os.close, (parent_r,)) - self.sentinel = parent_r diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/popen_forkserver.py --- a/Lib/multiprocessing/popen_forkserver.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,69 +0,0 @@ -import io -import os - -from . import reduction -if not reduction.HAVE_SEND_HANDLE: - raise ImportError('No support for sending fds between processes') -from . import context -from . import forkserver -from . import popen_fork -from . import spawn -from . import util - - -__all__ = ['Popen'] - -# -# Wrapper for an fd used while launching a process -# - -class _DupFd(object): - def __init__(self, ind): - self.ind = ind - def detach(self): - return forkserver.get_inherited_fds()[self.ind] - -# -# Start child process using a server process -# - -class Popen(popen_fork.Popen): - method = 'forkserver' - DupFd = _DupFd - - def __init__(self, process_obj): - self._fds = [] - super().__init__(process_obj) - - def duplicate_for_child(self, fd): - self._fds.append(fd) - return len(self._fds) - 1 - - def _launch(self, process_obj): - prep_data = spawn.get_preparation_data(process_obj._name) - buf = io.BytesIO() - context.set_spawning_popen(self) - try: - reduction.dump(prep_data, buf) - reduction.dump(process_obj, buf) - finally: - context.set_spawning_popen(None) - - self.sentinel, w = forkserver.connect_to_new_process(self._fds) - util.Finalize(self, os.close, (self.sentinel,)) - with open(w, 'wb', closefd=True) as f: - f.write(buf.getbuffer()) - self.pid = forkserver.read_unsigned(self.sentinel) - - def poll(self, flag=os.WNOHANG): - if self.returncode is None: - from multiprocessing.connection import wait - timeout = 0 if flag == os.WNOHANG else None - if not wait([self.sentinel], timeout): - return None - try: - self.returncode = forkserver.read_unsigned(self.sentinel) - except (OSError, EOFError): - # The process ended abnormally perhaps because of a signal - self.returncode = 255 - return self.returncode diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/popen_spawn_posix.py --- a/Lib/multiprocessing/popen_spawn_posix.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,69 +0,0 @@ -import io -import os - -from . import context -from . import popen_fork -from . import reduction -from . import spawn -from . import util - -__all__ = ['Popen'] - - -# -# Wrapper for an fd used while launching a process -# - -class _DupFd(object): - def __init__(self, fd): - self.fd = fd - def detach(self): - return self.fd - -# -# Start child process using a fresh interpreter -# - -class Popen(popen_fork.Popen): - method = 'spawn' - DupFd = _DupFd - - def __init__(self, process_obj): - self._fds = [] - super().__init__(process_obj) - - def duplicate_for_child(self, fd): - self._fds.append(fd) - return fd - - def _launch(self, process_obj): - from . import semaphore_tracker - tracker_fd = semaphore_tracker.getfd() - self._fds.append(tracker_fd) - prep_data = spawn.get_preparation_data(process_obj._name) - fp = io.BytesIO() - context.set_spawning_popen(self) - try: - reduction.dump(prep_data, fp) - reduction.dump(process_obj, fp) - finally: - context.set_spawning_popen(None) - - parent_r = child_w = child_r = parent_w = None - try: - parent_r, child_w = os.pipe() - child_r, parent_w = os.pipe() - cmd = spawn.get_command_line(tracker_fd=tracker_fd, - pipe_handle=child_r) - self._fds.extend([child_r, child_w]) - self.pid = util.spawnv_passfds(spawn.get_executable(), - cmd, self._fds) - self.sentinel = parent_r - with open(parent_w, 'wb', closefd=False) as f: - f.write(fp.getbuffer()) - finally: - if parent_r is not None: - util.Finalize(self, os.close, (parent_r,)) - for fd in (child_r, child_w, parent_w): - if fd is not None: - os.close(fd) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/popen_spawn_win32.py --- a/Lib/multiprocessing/popen_spawn_win32.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,99 +0,0 @@ -import os -import msvcrt -import signal -import sys -import _winapi - -from . import context -from . import spawn -from . import reduction -from . import util - -__all__ = ['Popen'] - -# -# -# - -TERMINATE = 0x10000 -WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) -WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") - -# -# We define a Popen class similar to the one from subprocess, but -# whose constructor takes a process object as its argument. -# - -class Popen(object): - ''' - Start a subprocess to run the code of a process object - ''' - method = 'spawn' - - def __init__(self, process_obj): - prep_data = spawn.get_preparation_data(process_obj._name) - - # read end of pipe will be "stolen" by the child process - # -- see spawn_main() in spawn.py. - rhandle, whandle = _winapi.CreatePipe(None, 0) - wfd = msvcrt.open_osfhandle(whandle, 0) - cmd = spawn.get_command_line(parent_pid=os.getpid(), - pipe_handle=rhandle) - cmd = ' '.join('"%s"' % x for x in cmd) - - with open(wfd, 'wb', closefd=True) as to_child: - # start process - try: - hp, ht, pid, tid = _winapi.CreateProcess( - spawn.get_executable(), cmd, - None, None, False, 0, None, None, None) - _winapi.CloseHandle(ht) - except: - _winapi.CloseHandle(rhandle) - raise - - # set attributes of self - self.pid = pid - self.returncode = None - self._handle = hp - self.sentinel = int(hp) - util.Finalize(self, _winapi.CloseHandle, (self.sentinel,)) - - # send information to child - context.set_spawning_popen(self) - try: - reduction.dump(prep_data, to_child) - reduction.dump(process_obj, to_child) - finally: - context.set_spawning_popen(None) - - def duplicate_for_child(self, handle): - assert self is context.get_spawning_popen() - return reduction.duplicate(handle, self.sentinel) - - def wait(self, timeout=None): - if self.returncode is None: - if timeout is None: - msecs = _winapi.INFINITE - else: - msecs = max(0, int(timeout * 1000 + 0.5)) - - res = _winapi.WaitForSingleObject(int(self._handle), msecs) - if res == _winapi.WAIT_OBJECT_0: - code = _winapi.GetExitCodeProcess(self._handle) - if code == TERMINATE: - code = -signal.SIGTERM - self.returncode = code - - return self.returncode - - def poll(self): - return self.wait(timeout=0) - - def terminate(self): - if self.returncode is None: - try: - _winapi.TerminateProcess(int(self._handle), TERMINATE) - except OSError: - if self.wait(timeout=1.0) is None: - raise diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/process.py --- a/Lib/multiprocessing/process.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/process.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,10 +4,35 @@ # multiprocessing/process.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # -__all__ = ['BaseProcess', 'current_process', 'active_children'] +__all__ = ['Process', 'current_process', 'active_children'] # # Imports @@ -43,7 +68,7 @@ Return list of process objects corresponding to live child processes ''' _cleanup() - return list(_children) + return list(_current_process._children) # # @@ -51,29 +76,33 @@ def _cleanup(): # check for processes which have finished - for p in list(_children): + for p in list(_current_process._children): if p._popen.poll() is not None: - _children.discard(p) + _current_process._children.discard(p) # # The `Process` class # -class BaseProcess(object): +class Process(object): ''' Process objects represent activity that is run in a separate process - The class is analogous to `threading.Thread` + The class is analagous to `threading.Thread` ''' - def _Popen(self): - raise NotImplementedError + _Popen = None def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, *, daemon=None): assert group is None, 'group argument must be None for now' - count = next(_process_counter) + count = next(_current_process._counter) self._identity = _current_process._identity + (count,) - self._config = _current_process._config.copy() + self._authkey = _current_process._authkey + if daemon is not None: + self._daemonic = daemon + else: + self._daemonic = _current_process._daemonic + self._tempdir = _current_process._tempdir self._parent_pid = os.getpid() self._popen = None self._target = target @@ -81,8 +110,6 @@ self._kwargs = dict(kwargs) self._name = name or type(self).__name__ + '-' + \ ':'.join(str(i) for i in self._identity) - if daemon is not None: - self.daemon = daemon _dangling.add(self) def run(self): @@ -99,12 +126,16 @@ assert self._popen is None, 'cannot start a process twice' assert self._parent_pid == os.getpid(), \ 'can only start a process object created by current process' - assert not _current_process._config.get('daemon'), \ + assert not _current_process._daemonic, \ 'daemonic processes are not allowed to have children' _cleanup() - self._popen = self._Popen(self) + if self._Popen is not None: + Popen = self._Popen + else: + from .forking import Popen + self._popen = Popen(self) self._sentinel = self._popen.sentinel - _children.add(self) + _current_process._children.add(self) def terminate(self): ''' @@ -120,7 +151,7 @@ assert self._popen is not None, 'can only join a started process' res = self._popen.wait(timeout) if res is not None: - _children.discard(self) + _current_process._children.discard(self) def is_alive(self): ''' @@ -148,7 +179,7 @@ ''' Return whether process is a daemon ''' - return self._config.get('daemon', False) + return self._daemonic @daemon.setter def daemon(self, daemonic): @@ -156,18 +187,18 @@ Set whether process is a daemon ''' assert self._popen is None, 'process has already started' - self._config['daemon'] = daemonic + self._daemonic = daemonic @property def authkey(self): - return self._config['authkey'] + return self._authkey @authkey.setter def authkey(self, authkey): ''' Set authorization key of process ''' - self._config['authkey'] = AuthenticationString(authkey) + self._authkey = AuthenticationString(authkey) @property def exitcode(self): @@ -221,19 +252,17 @@ status = 'stopped[%s]' % _exitcode_to_name.get(status, status) return '<%s(%s, %s%s)>' % (type(self).__name__, self._name, - status, self.daemon and ' daemon' or '') + status, self._daemonic and ' daemon' or '') ## def _bootstrap(self): - from . import util, context - global _current_process, _process_counter, _children + from . import util + global _current_process try: - if self._start_method is not None: - context._force_start_method(self._start_method) - _process_counter = itertools.count(1) - _children = set() + self._children = set() + self._counter = itertools.count(1) if sys.stdin is not None: try: sys.stdin.close() @@ -258,10 +287,10 @@ except SystemExit as e: if not e.args: exitcode = 1 - elif isinstance(e.args[0], int): + elif type(e.args[0]) is int: exitcode = e.args[0] else: - sys.stderr.write(str(e.args[0]) + '\n') + sys.stderr.write(e.args[0] + '\n') exitcode = 1 except: exitcode = 1 @@ -281,8 +310,8 @@ class AuthenticationString(bytes): def __reduce__(self): - from .context import get_spawning_popen - if get_spawning_popen() is None: + from .forking import Popen + if not Popen.thread_is_spawning(): raise TypeError( 'Pickling an AuthenticationString object is ' 'disallowed for security reasons' @@ -293,29 +322,20 @@ # Create object representing the main process # -class _MainProcess(BaseProcess): +class _MainProcess(Process): def __init__(self): self._identity = () + self._daemonic = False self._name = 'MainProcess' self._parent_pid = None self._popen = None - self._config = {'authkey': AuthenticationString(os.urandom(32)), - 'semprefix': '/mp'} - # Note that some versions of FreeBSD only allow named - # semaphores to have names of up to 14 characters. Therefore - # we choose a short prefix. - # - # On MacOSX in a sandbox it may be necessary to use a - # different prefix -- see #19478. - # - # Everything in self._config will be inherited by descendant - # processes. - + self._counter = itertools.count(1) + self._children = set() + self._authkey = AuthenticationString(os.urandom(32)) + self._tempdir = None _current_process = _MainProcess() -_process_counter = itertools.count(1) -_children = set() del _MainProcess # diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/queues.py --- a/Lib/multiprocessing/queues.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/queues.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,32 @@ # multiprocessing/queues.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = ['Queue', 'SimpleQueue', 'JoinableQueue'] @@ -18,14 +43,11 @@ import errno from queue import Empty, Full - import _multiprocessing - -from . import connection -from . import context - -from .util import debug, info, Finalize, register_after_fork, is_exiting -from .reduction import ForkingPickler +from multiprocessing.connection import Pipe +from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition +from multiprocessing.util import debug, info, Finalize, register_after_fork +from multiprocessing.forking import assert_spawning # # Queue type using a pipe, buffer and thread @@ -33,19 +55,18 @@ class Queue(object): - def __init__(self, maxsize=0, *, ctx): + def __init__(self, maxsize=0): if maxsize <= 0: - # Can raise ImportError (see issues #3770 and #23400) - from .synchronize import SEM_VALUE_MAX as maxsize + maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX self._maxsize = maxsize - self._reader, self._writer = connection.Pipe(duplex=False) - self._rlock = ctx.Lock() + self._reader, self._writer = Pipe(duplex=False) + self._rlock = Lock() self._opid = os.getpid() if sys.platform == 'win32': self._wlock = None else: - self._wlock = ctx.Lock() - self._sem = ctx.BoundedSemaphore(maxsize) + self._wlock = Lock() + self._sem = BoundedSemaphore(maxsize) # For use by concurrent.futures self._ignore_epipe = False @@ -55,7 +76,7 @@ register_after_fork(self, Queue._after_fork) def __getstate__(self): - context.assert_spawning(self) + assert_spawning(self) return (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock, self._wlock, self._sem, self._opid) @@ -73,8 +94,8 @@ self._joincancelled = False self._closed = False self._close = None - self._send_bytes = self._writer.send_bytes - self._recv_bytes = self._reader.recv_bytes + self._send = self._writer.send + self._recv = self._reader.recv self._poll = self._reader.poll def put(self, obj, block=True, timeout=None): @@ -82,17 +103,25 @@ if not self._sem.acquire(block, timeout): raise Full - with self._notempty: + self._notempty.acquire() + try: if self._thread is None: self._start_thread() self._buffer.append(obj) self._notempty.notify() + finally: + self._notempty.release() def get(self, block=True, timeout=None): if block and timeout is None: - with self._rlock: - res = self._recv_bytes() - self._sem.release() + self._rlock.acquire() + try: + res = self._recv() + self._sem.release() + return res + finally: + self._rlock.release() + else: if block: deadline = time.time() + timeout @@ -105,12 +134,11 @@ raise Empty elif not self._poll(): raise Empty - res = self._recv_bytes() + res = self._recv() self._sem.release() + return res finally: self._rlock.release() - # unserialize the data after having released the lock - return ForkingPickler.loads(res) def qsize(self): # Raises NotImplementedError on Mac OSX because of broken sem_getvalue() @@ -130,13 +158,9 @@ def close(self): self._closed = True - try: - self._reader.close() - finally: - close = self._close - if close: - self._close = None - close() + self._reader.close() + if self._close: + self._close() def join_thread(self): debug('Queue.join_thread()') @@ -159,7 +183,7 @@ self._buffer.clear() self._thread = threading.Thread( target=Queue._feed, - args=(self._buffer, self._notempty, self._send_bytes, + args=(self._buffer, self._notempty, self._send, self._wlock, self._writer.close, self._ignore_epipe), name='QueueFeederThread' ) @@ -203,13 +227,18 @@ @staticmethod def _finalize_close(buffer, notempty): debug('telling queue thread to quit') - with notempty: + notempty.acquire() + try: buffer.append(_sentinel) notempty.notify() + finally: + notempty.release() @staticmethod - def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): + def _feed(buffer, notempty, send, writelock, close, ignore_epipe): debug('starting thread to feed data to pipe') + from .util import is_exiting + nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait @@ -237,14 +266,12 @@ close() return - # serialize the data before acquiring the lock - obj = ForkingPickler.dumps(obj) if wacquire is None: - send_bytes(obj) + send(obj) else: wacquire() try: - send_bytes(obj) + send(obj) finally: wrelease() except IndexError: @@ -277,10 +304,10 @@ class JoinableQueue(Queue): - def __init__(self, maxsize=0, *, ctx): - Queue.__init__(self, maxsize, ctx=ctx) - self._unfinished_tasks = ctx.Semaphore(0) - self._cond = ctx.Condition() + def __init__(self, maxsize=0): + Queue.__init__(self, maxsize) + self._unfinished_tasks = Semaphore(0) + self._cond = Condition() def __getstate__(self): return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks) @@ -294,24 +321,35 @@ if not self._sem.acquire(block, timeout): raise Full - with self._notempty, self._cond: + self._notempty.acquire() + self._cond.acquire() + try: if self._thread is None: self._start_thread() self._buffer.append(obj) self._unfinished_tasks.release() self._notempty.notify() + finally: + self._cond.release() + self._notempty.release() def task_done(self): - with self._cond: + self._cond.acquire() + try: if not self._unfinished_tasks.acquire(False): raise ValueError('task_done() called too many times') if self._unfinished_tasks._semlock._is_zero(): self._cond.notify_all() + finally: + self._cond.release() def join(self): - with self._cond: + self._cond.acquire() + try: if not self._unfinished_tasks._semlock._is_zero(): self._cond.wait() + finally: + self._cond.release() # # Simplified Queue type -- really just a locked pipe @@ -319,37 +357,48 @@ class SimpleQueue(object): - def __init__(self, *, ctx): - self._reader, self._writer = connection.Pipe(duplex=False) - self._rlock = ctx.Lock() + def __init__(self): + self._reader, self._writer = Pipe(duplex=False) + self._rlock = Lock() self._poll = self._reader.poll if sys.platform == 'win32': self._wlock = None else: - self._wlock = ctx.Lock() + self._wlock = Lock() + self._make_methods() def empty(self): return not self._poll() def __getstate__(self): - context.assert_spawning(self) + assert_spawning(self) return (self._reader, self._writer, self._rlock, self._wlock) def __setstate__(self, state): (self._reader, self._writer, self._rlock, self._wlock) = state + self._make_methods() - def get(self): - with self._rlock: - res = self._reader.recv_bytes() - # unserialize the data after having released the lock - return ForkingPickler.loads(res) + def _make_methods(self): + recv = self._reader.recv + racquire, rrelease = self._rlock.acquire, self._rlock.release + def get(): + racquire() + try: + return recv() + finally: + rrelease() + self.get = get - def put(self, obj): - # serialize the data before acquiring the lock - obj = ForkingPickler.dumps(obj) if self._wlock is None: # writes to a message oriented win32 pipe are atomic - self._writer.send_bytes(obj) + self.put = self._writer.send else: - with self._wlock: - self._writer.send_bytes(obj) + send = self._writer.send + wacquire, wrelease = self._wlock.acquire, self._wlock.release + def put(obj): + wacquire() + try: + return send(obj) + finally: + wrelease() + self.put = put diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/reduction.py --- a/Lib/multiprocessing/reduction.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/reduction.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,240 +1,229 @@ # -# Module which deals with pickling of objects. +# Module to allow connection and socket objects to be transferred +# between processes # # multiprocessing/reduction.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # -import copyreg -import functools -import io +__all__ = [] + import os -import pickle +import sys import socket -import sys +import threading +import struct -from . import context +from multiprocessing import current_process +from multiprocessing.forking import Popen, duplicate, close, ForkingPickler +from multiprocessing.util import register_after_fork, debug, sub_debug +from multiprocessing.connection import Client, Listener, Connection -__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump'] - - -HAVE_SEND_HANDLE = (sys.platform == 'win32' or - (hasattr(socket, 'CMSG_LEN') and - hasattr(socket, 'SCM_RIGHTS') and - hasattr(socket.socket, 'sendmsg'))) # -# Pickler subclass +# # -class ForkingPickler(pickle.Pickler): - '''Pickler subclass used by multiprocessing.''' - _extra_reducers = {} - _copyreg_dispatch_table = copyreg.dispatch_table - - def __init__(self, *args): - super().__init__(*args) - self.dispatch_table = self._copyreg_dispatch_table.copy() - self.dispatch_table.update(self._extra_reducers) - - @classmethod - def register(cls, type, reduce): - '''Register a reduce function for a type.''' - cls._extra_reducers[type] = reduce - - @classmethod - def dumps(cls, obj, protocol=None): - buf = io.BytesIO() - cls(buf, protocol).dump(obj) - return buf.getbuffer() - - loads = pickle.loads - -register = ForkingPickler.register - -def dump(obj, file, protocol=None): - '''Replacement for pickle.dump() using ForkingPickler.''' - ForkingPickler(file, protocol).dump(obj) +if not(sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and + hasattr(socket, 'SCM_RIGHTS'))): + raise ImportError('pickling of connections not supported') # # Platform specific definitions # if sys.platform == 'win32': - # Windows - __all__ += ['DupHandle', 'duplicate', 'steal_handle'] - import _winapi - - def duplicate(handle, target_process=None, inheritable=False): - '''Duplicate a handle. (target_process is a handle not a pid!)''' - if target_process is None: - target_process = _winapi.GetCurrentProcess() - return _winapi.DuplicateHandle( - _winapi.GetCurrentProcess(), handle, target_process, - 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS) - - def steal_handle(source_pid, handle): - '''Steal a handle from process identified by source_pid.''' - source_process_handle = _winapi.OpenProcess( - _winapi.PROCESS_DUP_HANDLE, False, source_pid) - try: - return _winapi.DuplicateHandle( - source_process_handle, handle, - _winapi.GetCurrentProcess(), 0, False, - _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE) - finally: - _winapi.CloseHandle(source_process_handle) + from _multiprocessing import win32 def send_handle(conn, handle, destination_pid): - '''Send a handle over a local connection.''' - dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid) - conn.send(dh) + process_handle = win32.OpenProcess( + win32.PROCESS_ALL_ACCESS, False, destination_pid + ) + try: + new_handle = duplicate(handle, process_handle) + conn.send(new_handle) + finally: + close(process_handle) def recv_handle(conn): - '''Receive a handle over a local connection.''' - return conn.recv().detach() - - class DupHandle(object): - '''Picklable wrapper for a handle.''' - def __init__(self, handle, access, pid=None): - if pid is None: - # We just duplicate the handle in the current process and - # let the receiving process steal the handle. - pid = os.getpid() - proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid) - try: - self._handle = _winapi.DuplicateHandle( - _winapi.GetCurrentProcess(), - handle, proc, access, False, 0) - finally: - _winapi.CloseHandle(proc) - self._access = access - self._pid = pid - - def detach(self): - '''Get the handle. This should only be called once.''' - # retrieve handle from process which currently owns it - if self._pid == os.getpid(): - # The handle has already been duplicated for this process. - return self._handle - # We must steal the handle from the process whose pid is self._pid. - proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, - self._pid) - try: - return _winapi.DuplicateHandle( - proc, self._handle, _winapi.GetCurrentProcess(), - self._access, False, _winapi.DUPLICATE_CLOSE_SOURCE) - finally: - _winapi.CloseHandle(proc) + return conn.recv() else: - # Unix - __all__ += ['DupFd', 'sendfds', 'recvfds'] - import array - - # On MacOSX we should acknowledge receipt of fds -- see Issue14669 - ACKNOWLEDGE = sys.platform == 'darwin' - - def sendfds(sock, fds): - '''Send an array of fds over an AF_UNIX socket.''' - fds = array.array('i', fds) - msg = bytes([len(fds) % 256]) - sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) - if ACKNOWLEDGE and sock.recv(1) != b'A': - raise RuntimeError('did not receive acknowledgement of fd') - - def recvfds(sock, size): - '''Receive an array of fds over an AF_UNIX socket.''' - a = array.array('i') - bytes_size = a.itemsize * size - msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size)) - if not msg and not ancdata: - raise EOFError - try: - if ACKNOWLEDGE: - sock.send(b'A') - if len(ancdata) != 1: - raise RuntimeError('received %d items of ancdata' % - len(ancdata)) - cmsg_level, cmsg_type, cmsg_data = ancdata[0] - if (cmsg_level == socket.SOL_SOCKET and - cmsg_type == socket.SCM_RIGHTS): - if len(cmsg_data) % a.itemsize != 0: - raise ValueError - a.frombytes(cmsg_data) - assert len(a) % 256 == msg[0] - return list(a) - except (ValueError, IndexError): - pass - raise RuntimeError('Invalid data received') - def send_handle(conn, handle, destination_pid): - '''Send a handle over a local connection.''' with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: - sendfds(s, [handle]) + s.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, + struct.pack("@i", handle))]) def recv_handle(conn): - '''Receive a handle over a local connection.''' + size = struct.calcsize("@i") with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s: - return recvfds(s, 1)[0] + msg, ancdata, flags, addr = s.recvmsg(1, socket.CMSG_LEN(size)) + try: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + return struct.unpack("@i", cmsg_data[:size])[0] + except (ValueError, IndexError, struct.error): + pass + raise RuntimeError('Invalid data received') - def DupFd(fd): - '''Return a wrapper for an fd.''' - popen_obj = context.get_spawning_popen() - if popen_obj is not None: - return popen_obj.DupFd(popen_obj.duplicate_for_child(fd)) - elif HAVE_SEND_HANDLE: - from . import resource_sharer - return resource_sharer.DupFd(fd) - else: - raise ValueError('SCM_RIGHTS appears not to be available') # -# Try making some callable types picklable +# Support for a per-process server thread which caches pickled handles # -def _reduce_method(m): - if m.__self__ is None: - return getattr, (m.__class__, m.__func__.__name__) - else: - return getattr, (m.__self__, m.__func__.__name__) -class _C: - def f(self): - pass -register(type(_C().f), _reduce_method) +_cache = set() +def _reset(obj): + global _lock, _listener, _cache + for h in _cache: + close(h) + _cache.clear() + _lock = threading.Lock() + _listener = None -def _reduce_method_descriptor(m): - return getattr, (m.__objclass__, m.__name__) -register(type(list.append), _reduce_method_descriptor) -register(type(int.__add__), _reduce_method_descriptor) +_reset(None) +register_after_fork(_reset, _reset) +def _get_listener(): + global _listener -def _reduce_partial(p): - return _rebuild_partial, (p.func, p.args, p.keywords or {}) -def _rebuild_partial(func, args, keywords): - return functools.partial(func, *args, **keywords) -register(functools.partial, _reduce_partial) + if _listener is None: + _lock.acquire() + try: + if _listener is None: + debug('starting listener and thread for sending handles') + _listener = Listener(authkey=current_process().authkey) + t = threading.Thread(target=_serve) + t.daemon = True + t.start() + finally: + _lock.release() + + return _listener + +def _serve(): + from .util import is_exiting, sub_warning + + while 1: + try: + conn = _listener.accept() + handle_wanted, destination_pid = conn.recv() + _cache.remove(handle_wanted) + send_handle(conn, handle_wanted, destination_pid) + close(handle_wanted) + conn.close() + except: + if not is_exiting(): + import traceback + sub_warning( + 'thread for sharing handles raised exception :\n' + + '-'*79 + '\n' + traceback.format_exc() + '-'*79 + ) # -# Make sockets picklable +# Functions to be used for pickling/unpickling objects with handles +# + +def reduce_handle(handle): + if Popen.thread_is_spawning(): + return (None, Popen.duplicate_for_child(handle), True) + dup_handle = duplicate(handle) + _cache.add(dup_handle) + sub_debug('reducing handle %d', handle) + return (_get_listener().address, dup_handle, False) + +def rebuild_handle(pickled_data): + address, handle, inherited = pickled_data + if inherited: + return handle + sub_debug('rebuilding handle %d', handle) + conn = Client(address, authkey=current_process().authkey) + conn.send((handle, os.getpid())) + new_handle = recv_handle(conn) + conn.close() + return new_handle + +# +# Register `Connection` with `ForkingPickler` +# + +def reduce_connection(conn): + rh = reduce_handle(conn.fileno()) + return rebuild_connection, (rh, conn.readable, conn.writable) + +def rebuild_connection(reduced_handle, readable, writable): + handle = rebuild_handle(reduced_handle) + return Connection( + handle, readable=readable, writable=writable + ) + +ForkingPickler.register(Connection, reduce_connection) + +# +# Register `socket.socket` with `ForkingPickler` +# + +def fromfd(fd, family, type_, proto=0): + s = socket.fromfd(fd, family, type_, proto) + if s.__class__ is not socket.socket: + s = socket.socket(_sock=s) + return s + +def reduce_socket(s): + reduced_handle = reduce_handle(s.fileno()) + return rebuild_socket, (reduced_handle, s.family, s.type, s.proto) + +def rebuild_socket(reduced_handle, family, type_, proto): + fd = rebuild_handle(reduced_handle) + _sock = fromfd(fd, family, type_, proto) + close(fd) + return _sock + +ForkingPickler.register(socket.socket, reduce_socket) + +# +# Register `_multiprocessing.PipeConnection` with `ForkingPickler` # if sys.platform == 'win32': - def _reduce_socket(s): - from .resource_sharer import DupSocket - return _rebuild_socket, (DupSocket(s),) - def _rebuild_socket(ds): - return ds.detach() - register(socket.socket, _reduce_socket) + from multiprocessing.connection import PipeConnection -else: - def _reduce_socket(s): - df = DupFd(s.fileno()) - return _rebuild_socket, (df, s.family, s.type, s.proto) - def _rebuild_socket(df, family, type, proto): - fd = df.detach() - return socket.socket(family, type, proto, fileno=fd) - register(socket.socket, _reduce_socket) + def reduce_pipe_connection(conn): + rh = reduce_handle(conn.fileno()) + return rebuild_pipe_connection, (rh, conn.readable, conn.writable) + + def rebuild_pipe_connection(reduced_handle, readable, writable): + handle = rebuild_handle(reduced_handle) + return PipeConnection( + handle, readable=readable, writable=writable + ) + + ForkingPickler.register(PipeConnection, reduce_pipe_connection) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/resource_sharer.py --- a/Lib/multiprocessing/resource_sharer.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,158 +0,0 @@ -# -# We use a background thread for sharing fds on Unix, and for sharing sockets on -# Windows. -# -# A client which wants to pickle a resource registers it with the resource -# sharer and gets an identifier in return. The unpickling process will connect -# to the resource sharer, sends the identifier and its pid, and then receives -# the resource. -# - -import os -import signal -import socket -import sys -import threading - -from . import process -from . import reduction -from . import util - -__all__ = ['stop'] - - -if sys.platform == 'win32': - __all__ += ['DupSocket'] - - class DupSocket(object): - '''Picklable wrapper for a socket.''' - def __init__(self, sock): - new_sock = sock.dup() - def send(conn, pid): - share = new_sock.share(pid) - conn.send_bytes(share) - self._id = _resource_sharer.register(send, new_sock.close) - - def detach(self): - '''Get the socket. This should only be called once.''' - with _resource_sharer.get_connection(self._id) as conn: - share = conn.recv_bytes() - return socket.fromshare(share) - -else: - __all__ += ['DupFd'] - - class DupFd(object): - '''Wrapper for fd which can be used at any time.''' - def __init__(self, fd): - new_fd = os.dup(fd) - def send(conn, pid): - reduction.send_handle(conn, new_fd, pid) - def close(): - os.close(new_fd) - self._id = _resource_sharer.register(send, close) - - def detach(self): - '''Get the fd. This should only be called once.''' - with _resource_sharer.get_connection(self._id) as conn: - return reduction.recv_handle(conn) - - -class _ResourceSharer(object): - '''Manager for resouces using background thread.''' - def __init__(self): - self._key = 0 - self._cache = {} - self._old_locks = [] - self._lock = threading.Lock() - self._listener = None - self._address = None - self._thread = None - util.register_after_fork(self, _ResourceSharer._afterfork) - - def register(self, send, close): - '''Register resource, returning an identifier.''' - with self._lock: - if self._address is None: - self._start() - self._key += 1 - self._cache[self._key] = (send, close) - return (self._address, self._key) - - @staticmethod - def get_connection(ident): - '''Return connection from which to receive identified resource.''' - from .connection import Client - address, key = ident - c = Client(address, authkey=process.current_process().authkey) - c.send((key, os.getpid())) - return c - - def stop(self, timeout=None): - '''Stop the background thread and clear registered resources.''' - from .connection import Client - with self._lock: - if self._address is not None: - c = Client(self._address, - authkey=process.current_process().authkey) - c.send(None) - c.close() - self._thread.join(timeout) - if self._thread.is_alive(): - util.sub_warning('_ResourceSharer thread did ' - 'not stop when asked') - self._listener.close() - self._thread = None - self._address = None - self._listener = None - for key, (send, close) in self._cache.items(): - close() - self._cache.clear() - - def _afterfork(self): - for key, (send, close) in self._cache.items(): - close() - self._cache.clear() - # If self._lock was locked at the time of the fork, it may be broken - # -- see issue 6721. Replace it without letting it be gc'ed. - self._old_locks.append(self._lock) - self._lock = threading.Lock() - if self._listener is not None: - self._listener.close() - self._listener = None - self._address = None - self._thread = None - - def _start(self): - from .connection import Listener - assert self._listener is None - util.debug('starting listener and thread for sending handles') - self._listener = Listener(authkey=process.current_process().authkey) - self._address = self._listener.address - t = threading.Thread(target=self._serve) - t.daemon = True - t.start() - self._thread = t - - def _serve(self): - if hasattr(signal, 'pthread_sigmask'): - signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG)) - while 1: - try: - with self._listener.accept() as conn: - msg = conn.recv() - if msg is None: - break - key, destination_pid = msg - send, close = self._cache.pop(key) - try: - send(conn, destination_pid) - finally: - close() - except: - if not util.is_exiting(): - sys.excepthook(*sys.exc_info()) - - -_resource_sharer = _ResourceSharer() -stop = _resource_sharer.stop diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/semaphore_tracker.py --- a/Lib/multiprocessing/semaphore_tracker.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,143 +0,0 @@ -# -# On Unix we run a server process which keeps track of unlinked -# semaphores. The server ignores SIGINT and SIGTERM and reads from a -# pipe. Every other process of the program has a copy of the writable -# end of the pipe, so we get EOF when all other processes have exited. -# Then the server process unlinks any remaining semaphore names. -# -# This is important because the system only supports a limited number -# of named semaphores, and they will not be automatically removed till -# the next reboot. Without this semaphore tracker process, "killall -# python" would probably leave unlinked semaphores. -# - -import os -import signal -import sys -import threading -import warnings -import _multiprocessing - -from . import spawn -from . import util - -__all__ = ['ensure_running', 'register', 'unregister'] - - -class SemaphoreTracker(object): - - def __init__(self): - self._lock = threading.Lock() - self._fd = None - - def getfd(self): - self.ensure_running() - return self._fd - - def ensure_running(self): - '''Make sure that semaphore tracker process is running. - - This can be run from any process. Usually a child process will use - the semaphore created by its parent.''' - with self._lock: - if self._fd is not None: - return - fds_to_pass = [] - try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)' - r, w = os.pipe() - try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - util.spawnv_passfds(exe, args, fds_to_pass) - except: - os.close(w) - raise - else: - self._fd = w - finally: - os.close(r) - - def register(self, name): - '''Register name of semaphore with semaphore tracker.''' - self._send('REGISTER', name) - - def unregister(self, name): - '''Unregister name of semaphore with semaphore tracker.''' - self._send('UNREGISTER', name) - - def _send(self, cmd, name): - self.ensure_running() - msg = '{0}:{1}\n'.format(cmd, name).encode('ascii') - if len(name) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('name too long') - nbytes = os.write(self._fd, msg) - assert nbytes == len(msg) - - -_semaphore_tracker = SemaphoreTracker() -ensure_running = _semaphore_tracker.ensure_running -register = _semaphore_tracker.register -unregister = _semaphore_tracker.unregister -getfd = _semaphore_tracker.getfd - - -def main(fd): - '''Run semaphore tracker.''' - # protect the process from ^C and "killall python" etc - signal.signal(signal.SIGINT, signal.SIG_IGN) - signal.signal(signal.SIGTERM, signal.SIG_IGN) - - for f in (sys.stdin, sys.stdout): - try: - f.close() - except Exception: - pass - - cache = set() - try: - # keep track of registered/unregistered semaphores - with open(fd, 'rb') as f: - for line in f: - try: - cmd, name = line.strip().split(b':') - if cmd == b'REGISTER': - cache.add(name) - elif cmd == b'UNREGISTER': - cache.remove(name) - else: - raise RuntimeError('unrecognized command %r' % cmd) - except Exception: - try: - sys.excepthook(*sys.exc_info()) - except: - pass - finally: - # all processes have terminated; cleanup any remaining semaphores - if cache: - try: - warnings.warn('semaphore_tracker: There appear to be %d ' - 'leaked semaphores to clean up at shutdown' % - len(cache)) - except Exception: - pass - for name in cache: - # For some reason the process which created and registered this - # semaphore has failed to unregister it. Presumably it has died. - # We therefore unlink it. - try: - name = name.decode('ascii') - try: - _multiprocessing.sem_unlink(name) - except Exception as e: - warnings.warn('semaphore_tracker: %r: %s' % (name, e)) - finally: - pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/sharedctypes.py --- a/Lib/multiprocessing/sharedctypes.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/sharedctypes.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,17 +4,39 @@ # multiprocessing/sharedctypes.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # import ctypes import weakref -from . import heap -from . import get_context - -from .context import assert_spawning -from .reduction import ForkingPickler +from multiprocessing import heap, RLock +from multiprocessing.forking import assert_spawning, ForkingPickler __all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized'] @@ -66,7 +88,7 @@ result.__init__(*size_or_initializer) return result -def Value(typecode_or_type, *args, lock=True, ctx=None): +def Value(typecode_or_type, *args, lock=None): ''' Return a synchronization wrapper for a Value ''' @@ -74,41 +96,41 @@ if lock is False: return obj if lock in (True, None): - ctx = ctx or get_context() - lock = ctx.RLock() + lock = RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) - return synchronized(obj, lock, ctx=ctx) + return synchronized(obj, lock) -def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): +def Array(typecode_or_type, size_or_initializer, **kwds): ''' Return a synchronization wrapper for a RawArray ''' + lock = kwds.pop('lock', None) + if kwds: + raise ValueError('unrecognized keyword argument(s): %s' % list(kwds.keys())) obj = RawArray(typecode_or_type, size_or_initializer) if lock is False: return obj if lock in (True, None): - ctx = ctx or get_context() - lock = ctx.RLock() + lock = RLock() if not hasattr(lock, 'acquire'): raise AttributeError("'%r' has no method 'acquire'" % lock) - return synchronized(obj, lock, ctx=ctx) + return synchronized(obj, lock) def copy(obj): new_obj = _new_value(type(obj)) ctypes.pointer(new_obj)[0] = obj return new_obj -def synchronized(obj, lock=None, ctx=None): +def synchronized(obj, lock=None): assert not isinstance(obj, SynchronizedBase), 'object already synchronized' - ctx = ctx or get_context() if isinstance(obj, ctypes._SimpleCData): - return Synchronized(obj, lock, ctx) + return Synchronized(obj, lock) elif isinstance(obj, ctypes.Array): if obj._type_ is ctypes.c_char: - return SynchronizedString(obj, lock, ctx) - return SynchronizedArray(obj, lock, ctx) + return SynchronizedString(obj, lock) + return SynchronizedArray(obj, lock) else: cls = type(obj) try: @@ -118,7 +140,7 @@ d = dict((name, make_property(name)) for name in names) classname = 'Synchronized' + cls.__name__ scls = class_cache[cls] = type(classname, (SynchronizedBase,), d) - return scls(obj, lock, ctx) + return scls(obj, lock) # # Functions for pickling/unpickling @@ -135,8 +157,7 @@ if length is not None: type_ = type_ * length ForkingPickler.register(type_, reduce_ctype) - buf = wrapper.create_memoryview() - obj = type_.from_buffer(buf) + obj = type_.from_address(wrapper.get_address()) obj._wrapper = wrapper return obj @@ -178,22 +199,12 @@ class SynchronizedBase(object): - def __init__(self, obj, lock=None, ctx=None): + def __init__(self, obj, lock=None): self._obj = obj - if lock: - self._lock = lock - else: - ctx = ctx or get_context(force=True) - self._lock = ctx.RLock() + self._lock = lock or RLock() self.acquire = self._lock.acquire self.release = self._lock.release - def __enter__(self): - return self._lock.__enter__() - - def __exit__(self, *args): - return self._lock.__exit__(*args) - def __reduce__(self): assert_spawning(self) return synchronized, (self._obj, self._lock) @@ -218,20 +229,32 @@ return len(self._obj) def __getitem__(self, i): - with self: + self.acquire() + try: return self._obj[i] + finally: + self.release() def __setitem__(self, i, value): - with self: + self.acquire() + try: self._obj[i] = value + finally: + self.release() def __getslice__(self, start, stop): - with self: + self.acquire() + try: return self._obj[start:stop] + finally: + self.release() def __setslice__(self, start, stop, values): - with self: + self.acquire() + try: self._obj[start:stop] = values + finally: + self.release() class SynchronizedString(SynchronizedArray): diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/spawn.py --- a/Lib/multiprocessing/spawn.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,287 +0,0 @@ -# -# Code used to start processes when using the spawn or forkserver -# start methods. -# -# multiprocessing/spawn.py -# -# Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. -# - -import os -import pickle -import sys -import runpy -import types - -from . import get_start_method, set_start_method -from . import process -from . import util - -__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', - 'get_preparation_data', 'get_command_line', 'import_main_path'] - -# -# _python_exe is the assumed path to the python executable. -# People embedding Python want to modify it. -# - -if sys.platform != 'win32': - WINEXE = False - WINSERVICE = False -else: - WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False)) - WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") - -if WINSERVICE: - _python_exe = os.path.join(sys.exec_prefix, 'python.exe') -else: - _python_exe = sys.executable - -def set_executable(exe): - global _python_exe - _python_exe = exe - -def get_executable(): - return _python_exe - -# -# -# - -def is_forking(argv): - ''' - Return whether commandline indicates we are forking - ''' - if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': - return True - else: - return False - - -def freeze_support(): - ''' - Run code for process object if this in not the main process - ''' - if is_forking(sys.argv): - kwds = {} - for arg in sys.argv[2:]: - name, value = arg.split('=') - if value == 'None': - kwds[name] = None - else: - kwds[name] = int(value) - spawn_main(**kwds) - sys.exit() - - -def get_command_line(**kwds): - ''' - Returns prefix of command line used for spawning a child process - ''' - if getattr(sys, 'frozen', False): - return ([sys.executable, '--multiprocessing-fork'] + - ['%s=%r' % item for item in kwds.items()]) - else: - prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' - prog %= ', '.join('%s=%r' % item for item in kwds.items()) - opts = util._args_from_interpreter_flags() - return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] - - -def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): - ''' - Run code specified by data received over pipe - ''' - assert is_forking(sys.argv) - if sys.platform == 'win32': - import msvcrt - from .reduction import steal_handle - new_handle = steal_handle(parent_pid, pipe_handle) - fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) - else: - from . import semaphore_tracker - semaphore_tracker._semaphore_tracker._fd = tracker_fd - fd = pipe_handle - exitcode = _main(fd) - sys.exit(exitcode) - - -def _main(fd): - with os.fdopen(fd, 'rb', closefd=True) as from_parent: - process.current_process()._inheriting = True - try: - preparation_data = pickle.load(from_parent) - prepare(preparation_data) - self = pickle.load(from_parent) - finally: - del process.current_process()._inheriting - return self._bootstrap() - - -def _check_not_importing_main(): - if getattr(process.current_process(), '_inheriting', False): - raise RuntimeError(''' - An attempt has been made to start a new process before the - current process has finished its bootstrapping phase. - - This probably means that you are not using fork to start your - child processes and you have forgotten to use the proper idiom - in the main module: - - if __name__ == '__main__': - freeze_support() - ... - - The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') - - -def get_preparation_data(name): - ''' - Return info about parent needed by child to unpickle process object - ''' - _check_not_importing_main() - d = dict( - log_to_stderr=util._log_to_stderr, - authkey=process.current_process().authkey, - ) - - if util._logger is not None: - d['log_level'] = util._logger.getEffectiveLevel() - - sys_path=sys.path.copy() - try: - i = sys_path.index('') - except ValueError: - pass - else: - sys_path[i] = process.ORIGINAL_DIR - - d.update( - name=name, - sys_path=sys_path, - sys_argv=sys.argv, - orig_dir=process.ORIGINAL_DIR, - dir=os.getcwd(), - start_method=get_start_method(), - ) - - # Figure out whether to initialise main in the subprocess as a module - # or through direct execution (or to leave it alone entirely) - main_module = sys.modules['__main__'] - main_mod_name = getattr(main_module.__spec__, "name", None) - if main_mod_name is not None: - d['init_main_from_name'] = main_mod_name - elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): - main_path = getattr(main_module, '__file__', None) - if main_path is not None: - if (not os.path.isabs(main_path) and - process.ORIGINAL_DIR is not None): - main_path = os.path.join(process.ORIGINAL_DIR, main_path) - d['init_main_from_path'] = os.path.normpath(main_path) - - return d - -# -# Prepare current process -# - -old_main_modules = [] - -def prepare(data): - ''' - Try to get current process ready to unpickle process object - ''' - if 'name' in data: - process.current_process().name = data['name'] - - if 'authkey' in data: - process.current_process().authkey = data['authkey'] - - if 'log_to_stderr' in data and data['log_to_stderr']: - util.log_to_stderr() - - if 'log_level' in data: - util.get_logger().setLevel(data['log_level']) - - if 'sys_path' in data: - sys.path = data['sys_path'] - - if 'sys_argv' in data: - sys.argv = data['sys_argv'] - - if 'dir' in data: - os.chdir(data['dir']) - - if 'orig_dir' in data: - process.ORIGINAL_DIR = data['orig_dir'] - - if 'start_method' in data: - set_start_method(data['start_method']) - - if 'init_main_from_name' in data: - _fixup_main_from_name(data['init_main_from_name']) - elif 'init_main_from_path' in data: - _fixup_main_from_path(data['init_main_from_path']) - -# Multiprocessing module helpers to fix up the main module in -# spawned subprocesses -def _fixup_main_from_name(mod_name): - # __main__.py files for packages, directories, zip archives, etc, run - # their "main only" code unconditionally, so we don't even try to - # populate anything in __main__, nor do we make any changes to - # __main__ attributes - current_main = sys.modules['__main__'] - if mod_name == "__main__" or mod_name.endswith(".__main__"): - return - - # If this process was forked, __main__ may already be populated - if getattr(current_main.__spec__, "name", None) == mod_name: - return - - # Otherwise, __main__ may contain some non-main code where we need to - # support unpickling it properly. We rerun it as __mp_main__ and make - # the normal __main__ an alias to that - old_main_modules.append(current_main) - main_module = types.ModuleType("__mp_main__") - main_content = runpy.run_module(mod_name, - run_name="__mp_main__", - alter_sys=True) - main_module.__dict__.update(main_content) - sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module - - -def _fixup_main_from_path(main_path): - # If this process was forked, __main__ may already be populated - current_main = sys.modules['__main__'] - - # Unfortunately, the main ipython launch script historically had no - # "if __name__ == '__main__'" guard, so we work around that - # by treating it like a __main__.py file - # See https://github.com/ipython/ipython/issues/4698 - main_name = os.path.splitext(os.path.basename(main_path))[0] - if main_name == 'ipython': - return - - # Otherwise, if __file__ already has the setting we expect, - # there's nothing more to do - if getattr(current_main, '__file__', None) == main_path: - return - - # If the parent process has sent a path through rather than a module - # name we assume it is an executable script that may contain - # non-main code that needs to be executed - old_main_modules.append(current_main) - main_module = types.ModuleType("__mp_main__") - main_content = runpy.run_path(main_path, - run_name="__mp_main__") - main_module.__dict__.update(main_content) - sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module - - -def import_main_path(main_path): - ''' - Set sys.modules['__main__'] to module at main_path - ''' - _fixup_main_from_path(main_path) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/synchronize.py --- a/Lib/multiprocessing/synchronize.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/synchronize.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,7 +4,32 @@ # multiprocessing/synchronize.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # __all__ = [ @@ -13,20 +38,17 @@ import threading import sys -import tempfile + import _multiprocessing - -from time import time as _time - -from . import context -from . import process -from . import util +from multiprocessing.process import current_process +from multiprocessing.util import register_after_fork, debug +from multiprocessing.forking import assert_spawning, Popen # Try to import the mp.synchronize module cleanly, if it fails # raise ImportError for platforms lacking a working sem_open implementation. # See issue 3770 try: - from _multiprocessing import SemLock, sem_unlink + from _multiprocessing import SemLock except (ImportError): raise ImportError("This platform lacks a functioning sem_open" + " implementation, therefore, the required" + @@ -46,47 +68,15 @@ class SemLock(object): - _rand = tempfile._RandomNameSequence() - - def __init__(self, kind, value, maxvalue, *, ctx): - if ctx is None: - ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' - for i in range(100): - try: - sl = self._semlock = _multiprocessing.SemLock( - kind, value, maxvalue, self._make_name(), - unlink_now) - except FileExistsError: - pass - else: - break - else: - raise FileExistsError('cannot find name for semaphore') - - util.debug('created semlock with handle %s' % sl.handle) + def __init__(self, kind, value, maxvalue): + sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue) + debug('created semlock with handle %s' % sl.handle) self._make_methods() if sys.platform != 'win32': def _after_fork(obj): obj._semlock._after_fork() - util.register_after_fork(self, _after_fork) - - if self._semlock.name is not None: - # We only get here if we are on Unix with forking - # disabled. When the object is garbage collected or the - # process shuts down we unlink the semaphore name - from .semaphore_tracker import register - register(self._semlock.name) - util.Finalize(self, SemLock._cleanup, (self._semlock.name,), - exitpriority=0) - - @staticmethod - def _cleanup(name): - from .semaphore_tracker import unregister - sem_unlink(name) - unregister(name) + register_after_fork(self, _after_fork) def _make_methods(self): self.acquire = self._semlock.acquire @@ -99,32 +89,23 @@ return self._semlock.__exit__(*args) def __getstate__(self): - context.assert_spawning(self) + assert_spawning(self) sl = self._semlock - if sys.platform == 'win32': - h = context.get_spawning_popen().duplicate_for_child(sl.handle) - else: - h = sl.handle - return (h, sl.kind, sl.maxvalue, sl.name) + return (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue) def __setstate__(self, state): self._semlock = _multiprocessing.SemLock._rebuild(*state) - util.debug('recreated blocker with handle %r' % state[0]) + debug('recreated blocker with handle %r' % state[0]) self._make_methods() - @staticmethod - def _make_name(): - return '%s-%s' % (process.current_process()._config['semprefix'], - next(SemLock._rand)) - # # Semaphore # class Semaphore(SemLock): - def __init__(self, value=1, *, ctx): - SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) + def __init__(self, value=1): + SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX) def get_value(self): return self._semlock._get_value() @@ -134,7 +115,7 @@ value = self._semlock._get_value() except Exception: value = 'unknown' - return '<%s(value=%s)>' % (self.__class__.__name__, value) + return '' % value # # Bounded semaphore @@ -142,16 +123,16 @@ class BoundedSemaphore(Semaphore): - def __init__(self, value=1, *, ctx): - SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx) + def __init__(self, value=1): + SemLock.__init__(self, SEMAPHORE, value, value) def __repr__(self): try: value = self._semlock._get_value() except Exception: value = 'unknown' - return '<%s(value=%s, maxvalue=%s)>' % \ - (self.__class__.__name__, value, self._semlock.maxvalue) + return '' % \ + (value, self._semlock.maxvalue) # # Non-recursive lock @@ -159,13 +140,13 @@ class Lock(SemLock): - def __init__(self, *, ctx): - SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx) + def __init__(self): + SemLock.__init__(self, SEMAPHORE, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): - name = process.current_process().name + name = current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name elif self._semlock._get_value() == 1: @@ -176,7 +157,7 @@ name = 'SomeOtherProcess' except Exception: name = 'unknown' - return '<%s(owner=%s)>' % (self.__class__.__name__, name) + return '' % name # # Recursive lock @@ -184,13 +165,13 @@ class RLock(SemLock): - def __init__(self, *, ctx): - SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx) + def __init__(self): + SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1) def __repr__(self): try: if self._semlock._is_mine(): - name = process.current_process().name + name = current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() @@ -202,7 +183,7 @@ name, count = 'SomeOtherProcess', 'nonzero' except Exception: name, count = 'unknown', 'unknown' - return '<%s(%s, %s)>' % (self.__class__.__name__, name, count) + return '' % (name, count) # # Condition variable @@ -210,15 +191,15 @@ class Condition(object): - def __init__(self, lock=None, *, ctx): - self._lock = lock or ctx.RLock() - self._sleeping_count = ctx.Semaphore(0) - self._woken_count = ctx.Semaphore(0) - self._wait_semaphore = ctx.Semaphore(0) + def __init__(self, lock=None): + self._lock = lock or RLock() + self._sleeping_count = Semaphore(0) + self._woken_count = Semaphore(0) + self._wait_semaphore = Semaphore(0) self._make_methods() def __getstate__(self): - context.assert_spawning(self) + assert_spawning(self) return (self._lock, self._sleeping_count, self._woken_count, self._wait_semaphore) @@ -242,8 +223,8 @@ num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: - num_waiters = 'unknown' - return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters) + num_waiters = 'unkown' + return '' % (self._lock, num_waiters) def wait(self, timeout=None): assert self._lock._semlock._is_mine(), \ @@ -259,7 +240,7 @@ try: # wait for notification or timeout - return self._wait_semaphore.acquire(True, timeout) + ret = self._wait_semaphore.acquire(True, timeout) finally: # indicate that this thread has woken self._woken_count.release() @@ -267,6 +248,7 @@ # reacquire lock for i in range(count): self._lock.acquire() + return ret def notify(self): assert self._lock._semlock._is_mine(), 'lock is not owned' @@ -308,53 +290,45 @@ while self._wait_semaphore.acquire(False): pass - def wait_for(self, predicate, timeout=None): - result = predicate() - if result: - return result - if timeout is not None: - endtime = _time() + timeout - else: - endtime = None - waittime = None - while not result: - if endtime is not None: - waittime = endtime - _time() - if waittime <= 0: - break - self.wait(waittime) - result = predicate() - return result - # # Event # class Event(object): - def __init__(self, *, ctx): - self._cond = ctx.Condition(ctx.Lock()) - self._flag = ctx.Semaphore(0) + def __init__(self): + self._cond = Condition(Lock()) + self._flag = Semaphore(0) def is_set(self): - with self._cond: + self._cond.acquire() + try: if self._flag.acquire(False): self._flag.release() return True return False + finally: + self._cond.release() def set(self): - with self._cond: + self._cond.acquire() + try: self._flag.acquire(False) self._flag.release() self._cond.notify_all() + finally: + self._cond.release() def clear(self): - with self._cond: + self._cond.acquire() + try: self._flag.acquire(False) + finally: + self._cond.release() def wait(self, timeout=None): - with self._cond: + self._cond.acquire() + try: if self._flag.acquire(False): self._flag.release() else: @@ -364,43 +338,5 @@ self._flag.release() return True return False - -# -# Barrier -# - -class Barrier(threading.Barrier): - - def __init__(self, parties, action=None, timeout=None, *, ctx): - import struct - from .heap import BufferWrapper - wrapper = BufferWrapper(struct.calcsize('i') * 2) - cond = ctx.Condition() - self.__setstate__((parties, action, timeout, cond, wrapper)) - self._state = 0 - self._count = 0 - - def __setstate__(self, state): - (self._parties, self._action, self._timeout, - self._cond, self._wrapper) = state - self._array = self._wrapper.create_memoryview().cast('i') - - def __getstate__(self): - return (self._parties, self._action, self._timeout, - self._cond, self._wrapper) - - @property - def _state(self): - return self._array[0] - - @_state.setter - def _state(self, value): - self._array[0] = value - - @property - def _count(self): - return self._array[1] - - @_count.setter - def _count(self, value): - self._array[1] = value + finally: + self._cond.release() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/multiprocessing/util.py --- a/Lib/multiprocessing/util.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/multiprocessing/util.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,24 +4,48 @@ # multiprocessing/util.py # # Copyright (c) 2006-2008, R Oudkerk -# Licensed to PSF under a Contributor Agreement. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of author nor the names of any contributors may be +# used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. # -import os +import functools import itertools import weakref import atexit import threading # we want threading to install it's # cleanup function before multiprocessing does -from subprocess import _args_from_interpreter_flags -from . import process +from multiprocessing.process import current_process, active_children __all__ = [ 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', - 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', + 'SUBDEBUG', 'SUBWARNING', ] # @@ -69,6 +93,8 @@ _logger = logging.getLogger(LOGGER_NAME) _logger.propagate = 0 + logging.addLevelName(SUBDEBUG, 'SUBDEBUG') + logging.addLevelName(SUBWARNING, 'SUBWARNING') # XXX multiprocessing should cleanup before logging if hasattr(atexit, 'unregister'): @@ -107,14 +133,13 @@ def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up - tempdir = process.current_process()._config.get('tempdir') - if tempdir is None: + if current_process()._tempdir is None: import shutil, tempfile tempdir = tempfile.mkdtemp(prefix='pymp-') info('created temp directory %s', tempdir) Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) - process.current_process()._config['tempdir'] = tempdir - return tempdir + current_process()._tempdir = tempdir + return current_process()._tempdir # # Support for reinitialization of objects when bootstrapping a child process @@ -159,7 +184,6 @@ self._args = args self._kwargs = kwargs or {} self._key = (exitpriority, next(_finalizer_counter)) - self._pid = os.getpid() _finalizer_registry[self._key] = self @@ -167,7 +191,7 @@ # Need to bind these locally because the globals can have # been cleared at shutdown _finalizer_registry=_finalizer_registry, - sub_debug=sub_debug, getpid=os.getpid): + sub_debug=sub_debug): ''' Run the callback unless it has already been called or cancelled ''' @@ -176,13 +200,9 @@ except KeyError: sub_debug('finalizer no longer registered') else: - if self._pid != getpid(): - sub_debug('finalizer ignored because different process') - res = None - else: - sub_debug('finalizer calling %s with args %s and kwargs %s', - self._callback, self._args, self._kwargs) - res = self._callback(*self._args, **self._kwargs) + sub_debug('finalizer calling %s with args %s and kwargs %s', + self._callback, self._args, self._kwargs) + res = self._callback(*self._args, **self._kwargs) self._weakref = self._callback = self._args = \ self._kwargs = self._key = None return res @@ -212,11 +232,10 @@ obj = None if obj is None: - return '<%s object, dead>' % self.__class__.__name__ + return '' - x = '<%s object, callback=%s' % ( - self.__class__.__name__, - getattr(self._callback, '__name__', self._callback)) + x = '= 0') + _run_finalizers(0) - info('process shutting down') - debug('running all "atexit" finalizers with priority >= 0') - _run_finalizers(0) + for p in active_children(): + if p._daemonic: + info('calling terminate() for daemon %s', p.name) + p._popen.terminate() - if current_process() is not None: - # We check if the current process is None here because if - # it's None, any call to ``active_children()`` will raise - # an AttributeError (active_children winds up trying to - # get attributes from util._current_process). One - # situation where this can happen is if someone has - # manipulated sys.modules, causing this module to be - # garbage collected. The destructor for the module type - # then replaces all values in the module dict with None. - # For instance, after setuptools runs a test it replaces - # sys.modules with a copy created earlier. See issues - # #9775 and #15881. Also related: #4106, #9205, and - # #9207. + for p in active_children(): + info('calling join() for process %s', p.name) + p.join() - for p in active_children(): - if p.daemon: - info('calling terminate() for daemon %s', p.name) - p._popen.terminate() - - for p in active_children(): - info('calling join() for process %s', p.name) - p.join() - - debug('running the remaining "atexit" finalizers') - _run_finalizers() + debug('running the remaining "atexit" finalizers') + _run_finalizers() atexit.register(_exit_function) @@ -320,20 +310,10 @@ class ForkAwareThreadLock(object): def __init__(self): - self._reset() - register_after_fork(self, ForkAwareThreadLock._reset) - - def _reset(self): self._lock = threading.Lock() self.acquire = self._lock.acquire self.release = self._lock.release - - def __enter__(self): - return self._lock.__enter__() - - def __exit__(self, *args): - return self._lock.__exit__(*args) - + register_after_fork(self, ForkAwareThreadLock.__init__) class ForkAwareLocal(threading.local): def __init__(self): @@ -341,35 +321,17 @@ def __reduce__(self): return type(self), () + # -# Close fds except those specified +# Automatic retry after EINTR # -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except Exception: - MAXFD = 256 - -def close_all_fds_except(fds): - fds = list(fds) + [-1, MAXFD] - fds.sort() - assert fds[-1] == MAXFD, 'fd too large' - for i in range(len(fds) - 1): - os.closerange(fds[i]+1, fds[i+1]) - -# -# Start a program with only specified fds kept open -# - -def spawnv_passfds(path, args, passfds): - import _posixsubprocess - passfds = sorted(passfds) - errpipe_read, errpipe_write = os.pipe() - try: - return _posixsubprocess.fork_exec( - args, [os.fsencode(path)], True, passfds, None, None, - -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, - False, False, None) - finally: - os.close(errpipe_read) - os.close(errpipe_write) +def _eintr_retry(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + while True: + try: + return func(*args, **kwargs) + except InterruptedError: + continue + return wrapped diff -r 6db40a9955dc -r 0d413f60cc23 Lib/netrc.py --- a/Lib/netrc.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/netrc.py Mon Jan 25 17:05:13 2016 +0100 @@ -2,7 +2,7 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, shlex, stat +import io, os, shlex __all__ = ["netrc", "NetrcParseError"] @@ -21,18 +21,17 @@ class netrc: def __init__(self, file=None): - default_netrc = file is None if file is None: try: file = os.path.join(os.environ['HOME'], ".netrc") except KeyError: - raise OSError("Could not find .netrc: $HOME is not set") + raise IOError("Could not find .netrc: $HOME is not set") self.hosts = {} self.macros = {} with open(file) as fp: - self._parse(file, fp, default_netrc) + self._parse(file, fp) - def _parse(self, file, fp, default_netrc): + def _parse(self, file, fp): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" lexer.commenters = lexer.commenters.replace('#', '') @@ -87,27 +86,6 @@ elif tt == 'account': account = lexer.get_token() elif tt == 'password': - if os.name == 'posix' and default_netrc: - prop = os.fstat(fp.fileno()) - if prop.st_uid != os.getuid(): - import pwd - try: - fowner = pwd.getpwuid(prop.st_uid)[0] - except KeyError: - fowner = 'uid %s' % prop.st_uid - try: - user = pwd.getpwuid(os.getuid())[0] - except KeyError: - user = 'uid %s' % os.getuid() - raise NetrcParseError( - ("~/.netrc file owner (%s) does not match" - " current user (%s)") % (fowner, user), - file, lexer.lineno) - if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): - raise NetrcParseError( - "~/.netrc access too permissive: access" - " permissions must restrict access to only" - " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, diff -r 6db40a9955dc -r 0d413f60cc23 Lib/nntplib.py --- a/Lib/nntplib.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/nntplib.py Mon Jan 25 17:05:13 2016 +0100 @@ -80,18 +80,11 @@ from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["NNTP", - "NNTPError", "NNTPReplyError", "NNTPTemporaryError", - "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", + "NNTPReplyError", "NNTPTemporaryError", "NNTPPermanentError", + "NNTPProtocolError", "NNTPDataError", "decode_header", ] -# maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3977 limits NNTP line length to -# 512 characters, including CRLF. We have selected 2048 just to be on -# the safe side. -_MAXLINE = 2048 - - # Exceptions raised when an error or invalid response is received class NNTPError(Exception): """Base class for all nntplib exceptions""" @@ -173,7 +166,7 @@ parts.append(v.decode(enc or 'ascii')) else: parts.append(v) - return ''.join(parts) + return ' '.join(parts) def _parse_overview_fmt(lines): """Parse a list of string representing the response to LIST OVERVIEW.FMT @@ -201,7 +194,7 @@ return fmt def _parse_overview(lines, fmt, data_process_func=None): - """Parse the response to an OVER or XOVER command according to the + """Parse the response to a OVER or XOVER command according to the overview format `fmt`.""" n_defaults = len(_DEFAULT_OVERVIEW_FMT) overview = [] @@ -279,7 +272,7 @@ if _have_ssl: - def _encrypt_on(sock, context, hostname): + def _encrypt_on(sock, context): """Wrap a socket in SSL/TLS. Arguments: - sock: Socket to wrap - context: SSL context to use for the encrypted connection @@ -288,8 +281,10 @@ """ # Generate a default SSL context if none was passed. if context is None: - context = ssl._create_stdlib_context() - return context.wrap_socket(sock, server_hostname=hostname) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + # SSLv2 considered harmful. + context.options |= ssl.OP_NO_SSLv2 + return context.wrap_socket(sock) # The classes themselves @@ -364,7 +359,7 @@ if is_connected(): try: self.quit() - except (OSError, EOFError): + except (socket.error, EOFError): pass finally: if is_connected(): @@ -429,9 +424,7 @@ """Internal: return one line from the server, stripping _CRLF. Raise EOFError if the connection is closed. Returns a bytes object.""" - line = self.file.readline(_MAXLINE +1) - if len(line) > _MAXLINE: - raise NNTPDataError('line too long') + line = self.file.readline() if self.debugging > 1: print('*get*', repr(line)) if not line: raise EOFError @@ -954,7 +947,7 @@ if auth: user = auth[0] password = auth[2] - except OSError: + except IOError: pass # Perform NNTP authentication if needed. if not user: @@ -1005,7 +998,7 @@ resp = self._shortcmd('STARTTLS') if resp.startswith('382'): self.file.close() - self.sock = _encrypt_on(self.sock, context, self.host) + self.sock = _encrypt_on(self.sock, context) self.file = self.sock.makefile("rwb") self.tls_on = True # Capabilities may change after TLS starts up, so ask for them @@ -1041,18 +1034,11 @@ self.host = host self.port = port self.sock = socket.create_connection((host, port), timeout) - file = None - try: - file = self.sock.makefile("rwb") - _NNTPBase.__init__(self, file, host, - readermode, timeout) - if user or usenetrc: - self.login(user, password, usenetrc) - except: - if file: - file.close() - self.sock.close() - raise + file = self.sock.makefile("rwb") + _NNTPBase.__init__(self, file, host, + readermode, timeout) + if user or usenetrc: + self.login(user, password, usenetrc) def _close(self): try: @@ -1072,19 +1058,12 @@ in default port and the `ssl_context` argument for SSL connections. """ self.sock = socket.create_connection((host, port), timeout) - file = None - try: - self.sock = _encrypt_on(self.sock, ssl_context, host) - file = self.sock.makefile("rwb") - _NNTPBase.__init__(self, file, host, - readermode=readermode, timeout=timeout) - if user or usenetrc: - self.login(user, password, usenetrc) - except: - if file: - file.close() - self.sock.close() - raise + self.sock = _encrypt_on(self.sock, ssl_context) + file = self.sock.makefile("rwb") + _NNTPBase.__init__(self, file, host, + readermode=readermode, timeout=timeout) + if user or usenetrc: + self.login(user, password, usenetrc) def _close(self): try: @@ -1098,6 +1077,7 @@ # Test retrieval when run as a script. if __name__ == '__main__': import argparse + from email.utils import parsedate parser = argparse.ArgumentParser(description="""\ nntplib built-in demo - display the latest articles in a newsgroup""") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/ntpath.py --- a/Lib/ntpath.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/ntpath.py Mon Jan 25 17:05:13 2016 +0100 @@ -17,7 +17,7 @@ "ismount", "expanduser","expandvars","normpath","abspath", "splitunc","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath"] + "samefile", "sameopenfile",] # strings representing various path-related bits and pieces # These are primarily for export; internally, they are hardcoded. @@ -30,14 +30,53 @@ defpath = '.;C:\\bin' if 'ce' in sys.builtin_module_names: defpath = '\\Windows' +elif 'os2' in sys.builtin_module_names: + # OS/2 w/ VACPP + altsep = '/' devnull = 'nul' +def _get_empty(path): + if isinstance(path, bytes): + return b'' + else: + return '' + +def _get_sep(path): + if isinstance(path, bytes): + return b'\\' + else: + return '\\' + +def _get_altsep(path): + if isinstance(path, bytes): + return b'/' + else: + return '/' + def _get_bothseps(path): if isinstance(path, bytes): return b'\\/' else: return '\\/' +def _get_dot(path): + if isinstance(path, bytes): + return b'.' + else: + return '.' + +def _get_colon(path): + if isinstance(path, bytes): + return b':' + else: + return ':' + +def _get_special(path): + if isinstance(path, bytes): + return (b'\\\\.\\', b'\\\\?\\') + else: + return ('\\\\.\\', '\\\\?\\') + # Normalize the case of a pathname and map slashes to backslashes. # Other normalizations (such as optimizing '../' away) are not done # (this is done by normpath). @@ -46,16 +85,10 @@ """Normalize case of pathname. Makes all characters lowercase and all slashes into backslashes.""" - try: - if isinstance(s, bytes): - return s.replace(b'/', b'\\').lower() - else: - return s.replace('/', '\\').lower() - except (TypeError, AttributeError): - if not isinstance(s, (bytes, str)): - raise TypeError("normcase() argument must be str or bytes, " - "not %r" % s.__class__.__name__) from None - raise + if not isinstance(s, (bytes, str)): + raise TypeError("normcase() argument must be str or bytes, " + "not '{}'".format(s.__class__.__name__)) + return s.replace(_get_altsep(s), _get_sep(s)).lower() # Return whether a path is absolute. @@ -67,51 +100,86 @@ def isabs(s): """Test whether a path is absolute""" s = splitdrive(s)[1] - return len(s) > 0 and s[0] in _get_bothseps(s) + return len(s) > 0 and s[:1] in _get_bothseps(s) # Join two (or more) paths. -def join(path, *paths): - if isinstance(path, bytes): - sep = b'\\' - seps = b'\\/' - colon = b':' - else: - sep = '\\' - seps = '\\/' - colon = ':' - try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - result_drive, result_path = splitdrive(path) - for p in paths: - p_drive, p_path = splitdrive(p) - if p_path and p_path[0] in seps: - # Second path is absolute - if p_drive or not result_drive: - result_drive = p_drive - result_path = p_path - continue - elif p_drive and p_drive != result_drive: - if p_drive.lower() != result_drive.lower(): - # Different drives => ignore the first path entirely - result_drive = p_drive - result_path = p_path - continue - # Same drive in different case - result_drive = p_drive - # Second path is relative to the first - if result_path and result_path[-1] not in seps: - result_path = result_path + sep - result_path = result_path + p_path - ## add separator between UNC and non-absolute path - if (result_path and result_path[0] not in seps and - result_drive and result_drive[-1:] != colon): - return result_drive + sep + result_path - return result_drive + result_path - except (TypeError, AttributeError, BytesWarning): - genericpath._check_arg_types('join', path, *paths) - raise + +def join(a, *p): + """Join two or more pathname components, inserting "\\" as needed. + If any component is an absolute path, all previous path components + will be discarded.""" + sep = _get_sep(a) + seps = _get_bothseps(a) + colon = _get_colon(a) + path = a + for b in p: + b_wins = 0 # set to 1 iff b makes path irrelevant + if not path: + b_wins = 1 + + elif isabs(b): + # This probably wipes out path so far. However, it's more + # complicated if path begins with a drive letter. You get a+b + # (minus redundant slashes) in these four cases: + # 1. join('c:', '/a') == 'c:/a' + # 2. join('//computer/share', '/a') == '//computer/share/a' + # 3. join('c:/', '/a') == 'c:/a' + # 4. join('//computer/share/', '/a') == '//computer/share/a' + # But b wins in all of these cases: + # 5. join('c:/a', '/b') == '/b' + # 6. join('//computer/share/a', '/b') == '/b' + # 7. join('c:', 'd:/') == 'd:/' + # 8. join('c:', '//computer/share/') == '//computer/share/' + # 9. join('//computer/share', 'd:/') == 'd:/' + # 10. join('//computer/share', '//computer/share/') == '//computer/share/' + # 11. join('c:/', 'd:/') == 'd:/' + # 12. join('c:/', '//computer/share/') == '//computer/share/' + # 13. join('//computer/share/', 'd:/') == 'd:/' + # 14. join('//computer/share/', '//computer/share/') == '//computer/share/' + b_prefix, b_rest = splitdrive(b) + + # if b has a prefix, it always wins. + if b_prefix: + b_wins = 1 + else: + # b doesn't have a prefix. + # but isabs(b) returned true. + # and therefore b_rest[0] must be a slash. + # (but let's check that.) + assert(b_rest and b_rest[0] in seps) + + # so, b still wins if path has a rest that's more than a sep. + # you get a+b if path_rest is empty or only has a sep. + # (see cases 1-4 for times when b loses.) + path_rest = splitdrive(path)[1] + b_wins = path_rest and path_rest not in seps + + if b_wins: + path = b + else: + # Join, and ensure there's a separator. + assert len(path) > 0 + if path[-1:] in seps: + if b and b[:1] in seps: + path += b[1:] + else: + path += b + elif path[-1:] == colon: + path += b + elif b: + if b[:1] in seps: + path += b + else: + path += sep + b + else: + # path is not empty and does not end with a backslash, + # but b is empty; since, e.g., split('a/') produces + # ('a', ''), it's best if join() adds a backslash in + # this case. + path += sep + + return path # Split a path in a drive specification (a drive letter followed by a @@ -136,16 +204,10 @@ Paths cannot contain both a drive letter and a UNC path. """ - if len(p) >= 2: - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - else: - sep = '\\' - altsep = '/' - colon = ':' - normp = p.replace(altsep, sep) + empty = _get_empty(p) + if len(p) > 1: + sep = _get_sep(p) + normp = normcase(p) if (normp[0:2] == sep*2) and (normp[2:3] != sep): # is a UNC path: # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path @@ -153,18 +215,18 @@ # directory ^^^^^^^^^^^^^^^ index = normp.find(sep, 2) if index == -1: - return p[:0], p + return empty, p index2 = normp.find(sep, index + 1) # a UNC path can't have two slashes in a row # (after the initial two) if index2 == index + 1: - return p[:0], p + return empty, p if index2 == -1: index2 = len(p) return p[:index2], p[index2:] - if normp[1:2] == colon: + if normp[1:2] == _get_colon(p): return p[:2], p[2:] - return p[:0], p + return empty, p # Parse UNC paths @@ -181,12 +243,26 @@ """ import warnings warnings.warn("ntpath.splitunc is deprecated, use ntpath.splitdrive instead", - DeprecationWarning, 2) - drive, path = splitdrive(p) - if len(drive) == 2: - # Drive letter present - return p[:0], p - return drive, path + DeprecationWarning) + sep = _get_sep(p) + if not p[1:2]: + return p[:0], p # Drive letter present + firstTwo = p[0:2] + if normcase(firstTwo) == sep + sep: + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv equivalent to drive letter + # \\machine\mountpoint\directories... + # directory ^^^^^^^^^^^^^^^ + normp = normcase(p) + index = normp.find(sep, 2) + if index == -1: + ##raise RuntimeError, 'illegal UNC path: "' + p + '"' + return (p[:0], p) + index = normp.find(sep, index + 1) + if index == -1: + index = len(p) + return p[:index], p[index:] + return p[:0], p # Split a path in head (everything up to the last '/') and tail (the @@ -208,7 +284,10 @@ i -= 1 head, tail = p[:i], p[i:] # now tail has no slashes # remove trailing slashes from head, unless it's all slashes - head = head.rstrip(seps) or head + head2 = head + while head2 and head2[-1:] in seps: + head2 = head2[:-1] + head = head2 or head return d + head, tail @@ -218,10 +297,8 @@ # It is always true that root + ext == p. def splitext(p): - if isinstance(p, bytes): - return genericpath._splitext(p, b'\\', b'/', b'.') - else: - return genericpath._splitext(p, '\\', '/', '.') + return genericpath._splitext(p, _get_sep(p), _get_altsep(p), + _get_dot(p)) splitext.__doc__ = genericpath._splitext.__doc__ @@ -243,11 +320,12 @@ def islink(path): """Test whether a path is a symbolic link. - This will always return false for Windows prior to 6.0. + This will always return false for Windows prior to 6.0 + and for OS/2. """ try: st = os.lstat(path) - except (OSError, AttributeError): + except (os.error, AttributeError): return False return stat.S_ISLNK(st.st_mode) @@ -257,39 +335,20 @@ """Test whether a path exists. Returns True for broken symbolic links""" try: st = os.lstat(path) - except OSError: + except (os.error, WindowsError): return False return True -# Is a path a mount point? -# Any drive letter root (eg c:\) -# Any share UNC (eg \\server\share) -# Any volume mounted on a filesystem folder -# -# No one method detects all three situations. Historically we've lexically -# detected drive letter roots and share UNCs. The canonical approach to -# detecting mounted volumes (querying the reparse tag) fails for the most -# common case: drive letter roots. The alternative which uses GetVolumePathName -# fails if the drive letter is the result of a SUBST. -try: - from nt import _getvolumepathname -except ImportError: - _getvolumepathname = None +# Is a path a mount point? Either a root (with or without drive letter) +# or an UNC path with at most a / or \ after the mount point. + def ismount(path): - """Test whether a path is a mount point (a drive root, the root of a - share, or a mounted volume)""" + """Test whether a path is a mount point (defined as root of drive)""" seps = _get_bothseps(path) - path = abspath(path) root, rest = splitdrive(path) if root and root[0] in seps: return (not rest) or (rest in seps) - if rest in seps: - return True - - if _getvolumepathname: - return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) - else: - return False + return rest in seps # Expand paths beginning with '~' or '~user'. @@ -329,7 +388,7 @@ userhome = join(drive, os.environ['HOMEPATH']) if isinstance(path, bytes): - userhome = os.fsencode(userhome) + userhome = userhome.encode(sys.getfilesystemencoding()) if i != 1: #~user userhome = join(dirname(userhome), path[1:i]) @@ -355,16 +414,14 @@ Unknown variables are left unchanged.""" if isinstance(path, bytes): - if b'$' not in path and b'%' not in path: + if ord('$') not in path and ord('%') not in path: return path import string varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') quote = b'\'' percent = b'%' brace = b'{' - rbrace = b'}' dollar = b'$' - environ = getattr(os, 'environb', None) else: if '$' not in path and '%' not in path: return path @@ -373,9 +430,7 @@ quote = '\'' percent = '%' brace = '{' - rbrace = '}' dollar = '$' - environ = os.environ res = path[:0] index = 0 pathlen = len(path) @@ -388,7 +443,7 @@ index = path.index(c) res += c + path[:index + 1] except ValueError: - res += c + path + res += path index = pathlen - 1 elif c == percent: # variable or '%' if path[index + 1:index + 2] == percent: @@ -404,13 +459,14 @@ index = pathlen - 1 else: var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent + if isinstance(path, bytes): + var = var.decode('ascii') + if var in os.environ: + value = os.environ[var] + else: + value = '%' + var + '%' + if isinstance(path, bytes): + value = value.encode('ascii') res += value elif c == dollar: # variable or '$$' if path[index + 1:index + 2] == dollar: @@ -420,35 +476,43 @@ path = path[index+2:] pathlen = len(path) try: - index = path.index(rbrace) + if isinstance(path, bytes): + index = path.index(b'}') + else: + index = path.index('}') + var = path[:index] + if isinstance(path, bytes): + var = var.decode('ascii') + if var in os.environ: + value = os.environ[var] + else: + value = '${' + var + '}' + if isinstance(path, bytes): + value = value.encode('ascii') + res += value except ValueError: - res += dollar + brace + path + if isinstance(path, bytes): + res += b'${' + path + else: + res += '${' + path index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value else: - var = path[:0] + var = '' index += 1 c = path[index:index + 1] while c and c in varchars: - var += c + if isinstance(path, bytes): + var += c.decode('ascii') + else: + var += c index += 1 c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var + if var in os.environ: + value = os.environ[var] + else: + value = '$' + var + if isinstance(path, bytes): + value = value.encode('ascii') res += value if c: index -= 1 @@ -464,25 +528,16 @@ def normpath(path): """Normalize path, eliminating double slashes, etc.""" - if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' - curdir = b'.' - pardir = b'..' - special_prefixes = (b'\\\\.\\', b'\\\\?\\') - else: - sep = '\\' - altsep = '/' - curdir = '.' - pardir = '..' - special_prefixes = ('\\\\.\\', '\\\\?\\') + sep = _get_sep(path) + dotdot = _get_dot(path) * 2 + special_prefixes = _get_special(path) if path.startswith(special_prefixes): # in the case of paths with these prefixes: # \\.\ -> device names # \\?\ -> literal paths # do not do any normalization, but return the path unchanged return path - path = path.replace(altsep, sep) + path = path.replace(_get_altsep(path), sep) prefix, path = splitdrive(path) # collapse initial backslashes @@ -493,13 +548,13 @@ comps = path.split(sep) i = 0 while i < len(comps): - if not comps[i] or comps[i] == curdir: + if not comps[i] or comps[i] == _get_dot(path): del comps[i] - elif comps[i] == pardir: - if i > 0 and comps[i-1] != pardir: + elif comps[i] == dotdot: + if i > 0 and comps[i-1] != dotdot: del comps[i-1:i+1] i -= 1 - elif i == 0 and prefix.endswith(sep): + elif i == 0 and prefix.endswith(_get_sep(path)): del comps[i] else: i += 1 @@ -507,7 +562,7 @@ i += 1 # If the path is now empty, substitute '.' if not prefix and not comps: - comps.append(curdir) + comps.append(_get_dot(path)) return prefix + sep.join(comps) @@ -533,7 +588,7 @@ if path: # Empty path must return current working directory. try: path = _getfullpathname(path) - except OSError: + except WindowsError: pass # Bad path - return unchanged. elif isinstance(path, bytes): path = os.getcwdb() @@ -547,109 +602,42 @@ supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and sys.getwindowsversion()[3] >= 2) -def relpath(path, start=None): +def relpath(path, start=curdir): """Return a relative version of a path""" - if isinstance(path, bytes): - sep = b'\\' - curdir = b'.' - pardir = b'..' - else: - sep = '\\' - curdir = '.' - pardir = '..' + sep = _get_sep(path) - if start is None: - start = curdir + if start is curdir: + start = _get_dot(path) if not path: raise ValueError("no path specified") - try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) - start_drive, start_rest = splitdrive(start_abs) - path_drive, path_rest = splitdrive(path_abs) - if normcase(start_drive) != normcase(path_drive): - raise ValueError("path is on mount %r, start on mount %r" % ( - path_drive, start_drive)) + start_abs = abspath(normpath(start)) + path_abs = abspath(normpath(path)) + start_drive, start_rest = splitdrive(start_abs) + path_drive, path_rest = splitdrive(path_abs) + if normcase(start_drive) != normcase(path_drive): + error = "path is on mount '{0}', start on mount '{1}'".format( + path_drive, start_drive) + raise ValueError(error) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] - # Work out how much of the filepath is shared by start and path. - i = 0 - for e1, e2 in zip(start_list, path_list): - if normcase(e1) != normcase(e2): - break - i += 1 + start_list = [x for x in start_rest.split(sep) if x] + path_list = [x for x in path_rest.split(sep) if x] + # Work out how much of the filepath is shared by start and path. + i = 0 + for e1, e2 in zip(start_list, path_list): + if normcase(e1) != normcase(e2): + break + i += 1 - rel_list = [pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return curdir - return join(*rel_list) - except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): - genericpath._check_arg_types('relpath', path, start) - raise - - -# Return the longest common sub-path of the sequence of paths given as input. -# The function is case-insensitive and 'separator-insensitive', i.e. if the -# only difference between two paths is the use of '\' versus '/' as separator, -# they are deemed to be equal. -# -# However, the returned path will have the standard '\' separator (even if the -# given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is -# stripped from the returned path. - -def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - - if not paths: - raise ValueError('commonpath() arg is an empty sequence') - - if isinstance(paths[0], bytes): - sep = b'\\' - altsep = b'/' - curdir = b'.' + if isinstance(path, bytes): + pardir = b'..' else: - sep = '\\' - altsep = '/' - curdir = '.' - - try: - drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] - split_paths = [p.split(sep) for d, p in drivesplits] - - try: - isabs, = set(p[:1] == sep for d, p in drivesplits) - except ValueError: - raise ValueError("Can't mix absolute and relative paths") from None - - # Check that all drive letters or UNC paths match. The check is made only - # now otherwise type errors for mixing strings and bytes would not be - # caught. - if len(set(d for d, p in drivesplits)) != 1: - raise ValueError("Paths don't have the same drive") - - drive, path = splitdrive(paths[0].replace(altsep, sep)) - common = path.split(sep) - common = [c for c in common if c and c != curdir] - - split_paths = [[c for c in s if c and c != curdir] for s in split_paths] - s1 = min(split_paths) - s2 = max(split_paths) - for i, c in enumerate(s1): - if c != s2[i]: - common = common[:i] - break - else: - common = common[:len(s1)] - - prefix = drive + sep if isabs else drive - return prefix + sep.join(common) - except (TypeError, AttributeError): - genericpath._check_arg_types('commonpath', *paths) - raise + pardir = '..' + rel_list = [pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return _get_dot(path) + return join(*rel_list) # determine if two files are in fact the same file @@ -668,6 +656,23 @@ def _getfinalpathname(f): return normcase(abspath(f)) +def samefile(f1, f2): + "Test whether two pathnames reference the same actual file" + return _getfinalpathname(f1) == _getfinalpathname(f2) + + +try: + from nt import _getfileinformation +except ImportError: + # On other operating systems, just return the fd and see that + # it compares equal in sameopenfile. + def _getfileinformation(fd): + return fd + +def sameopenfile(f1, f2): + """Test whether two file objects reference the same file""" + return _getfileinformation(f1) == _getfileinformation(f2) + try: # The genericpath.isdir implementation uses os.stat and checks the mode diff -r 6db40a9955dc -r 0d413f60cc23 Lib/nturl2path.py --- a/Lib/nturl2path.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/nturl2path.py Mon Jan 25 17:05:13 2016 +0100 @@ -4,11 +4,9 @@ """OS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.""" # e.g. - # ///C|/foo/bar/spam.foo - # and - # ///C:/foo/bar/spam.foo - # become - # C:\foo\bar\spam.foo + # ///C|/foo/bar/spam.foo + # becomes + # C:\foo\bar\spam.foo import string, urllib.parse # Windows itself uses ":" even in URLs. url = url.replace(':', '|') @@ -25,7 +23,7 @@ comp = url.split('|') if len(comp) != 2 or comp[0][-1] not in string.ascii_letters: error = 'Bad URL: ' + url - raise OSError(error) + raise IOError(error) drive = comp[0][-1].upper() components = comp[1].split('/') path = drive + ':' @@ -41,9 +39,9 @@ """OS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.""" # e.g. - # C:\foo\bar\spam.foo + # C:\foo\bar\spam.foo # becomes - # ///C:/foo/bar/spam.foo + # ///C|/foo/bar/spam.foo import urllib.parse if not ':' in p: # No drive specifier, just convert slashes and quote the name @@ -57,7 +55,7 @@ comp = p.split(':') if len(comp) != 2 or len(comp[0]) > 1: error = 'Bad path: ' + p - raise OSError(error) + raise IOError(error) drive = urllib.parse.quote(comp[0].upper()) components = comp[1].split('\\') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/numbers.py --- a/Lib/numbers.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/numbers.py Mon Jan 25 17:05:13 2016 +0100 @@ -141,6 +141,11 @@ """self == other""" raise NotImplementedError + def __ne__(self, other): + """self != other""" + # The default __ne__ doesn't negate __eq__ until 3.0. + return not (self == other) + Complex.register(complex) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/opcode.py --- a/Lib/opcode.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/opcode.py Mon Jan 25 17:05:13 2016 +0100 @@ -6,20 +6,7 @@ __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs", "haslocal", "hascompare", "hasfree", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"] - -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 - -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass + "HAVE_ARGUMENT", "EXTENDED_ARG"] cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') @@ -31,7 +18,6 @@ haslocal = [] hascompare = [] hasfree = [] -hasnargs = [] opmap = {} opname = [''] * 256 @@ -70,9 +56,6 @@ def_op('UNARY_INVERT', 15) -def_op('BINARY_MATRIX_MULTIPLY', 16) -def_op('INPLACE_MATRIX_MULTIPLY', 17) - def_op('BINARY_POWER', 19) def_op('BINARY_MULTIPLY', 20) @@ -85,10 +68,7 @@ def_op('INPLACE_FLOOR_DIVIDE', 28) def_op('INPLACE_TRUE_DIVIDE', 29) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) - +def_op('STORE_MAP', 54) def_op('INPLACE_ADD', 55) def_op('INPLACE_SUBTRACT', 56) def_op('INPLACE_MULTIPLY', 57) @@ -103,12 +83,11 @@ def_op('BINARY_OR', 66) def_op('INPLACE_POWER', 67) def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) +def_op('STORE_LOCALS', 69) def_op('PRINT_EXPR', 70) def_op('LOAD_BUILD_CLASS', 71) def_op('YIELD_FROM', 72) -def_op('GET_AWAITABLE', 73) def_op('INPLACE_LSHIFT', 75) def_op('INPLACE_RSHIFT', 76) @@ -116,8 +95,7 @@ def_op('INPLACE_XOR', 78) def_op('INPLACE_OR', 79) def_op('BREAK_LOOP', 80) -def_op('WITH_CLEANUP_START', 81) -def_op('WITH_CLEANUP_FINISH', 82) +def_op('WITH_CLEANUP', 81) def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) @@ -174,7 +152,6 @@ def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) -hasnargs.append(131) def_op('MAKE_FUNCTION', 132) # Number of args with default values def_op('BUILD_SLICE', 133) # Number of items def_op('MAKE_CLOSURE', 134) @@ -188,11 +165,8 @@ hasfree.append(138) def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) -hasnargs.append(140) def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) -hasnargs.append(141) def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) -hasnargs.append(142) jrel_op('SETUP_WITH', 143) @@ -200,20 +174,7 @@ def_op('SET_ADD', 146) def_op('MAP_ADD', 147) -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) - -jrel_op('SETUP_ASYNC_WITH', 154) - def_op('EXTENDED_ARG', 144) EXTENDED_ARG = 144 -def_op('BUILD_LIST_UNPACK', 149) -def_op('BUILD_MAP_UNPACK', 150) -def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) -def_op('BUILD_TUPLE_UNPACK', 152) -def_op('BUILD_SET_UNPACK', 153) - -def_op('FORMAT_VALUE', 155) - del def_op, name_op, jrel_op, jabs_op diff -r 6db40a9955dc -r 0d413f60cc23 Lib/operator.py --- a/Lib/operator.py Sun Jan 24 22:15:20 2016 -0800 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,464 +0,0 @@ -""" -Operator Interface - -This module exports a set of functions corresponding to the intrinsic -operators of Python. For example, operator.add(x, y) is equivalent -to the expression x+y. The function names are those used for special -methods; variants without leading and trailing '__' are also provided -for convenience. - -This is the pure Python implementation of the module. -""" - -__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', - 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', - 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', - 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', - 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] - -from builtins import abs as _abs - - -# Comparison Operations *******************************************************# - -def lt(a, b): - "Same as a < b." - return a < b - -def le(a, b): - "Same as a <= b." - return a <= b - -def eq(a, b): - "Same as a == b." - return a == b - -def ne(a, b): - "Same as a != b." - return a != b - -def ge(a, b): - "Same as a >= b." - return a >= b - -def gt(a, b): - "Same as a > b." - return a > b - -# Logical Operations **********************************************************# - -def not_(a): - "Same as not a." - return not a - -def truth(a): - "Return True if a is true, False otherwise." - return True if a else False - -def is_(a, b): - "Same as a is b." - return a is b - -def is_not(a, b): - "Same as a is not b." - return a is not b - -# Mathematical/Bitwise Operations *********************************************# - -def abs(a): - "Same as abs(a)." - return _abs(a) - -def add(a, b): - "Same as a + b." - return a + b - -def and_(a, b): - "Same as a & b." - return a & b - -def floordiv(a, b): - "Same as a // b." - return a // b - -def index(a): - "Same as a.__index__()." - return a.__index__() - -def inv(a): - "Same as ~a." - return ~a -invert = inv - -def lshift(a, b): - "Same as a << b." - return a << b - -def mod(a, b): - "Same as a % b." - return a % b - -def mul(a, b): - "Same as a * b." - return a * b - -def matmul(a, b): - "Same as a @ b." - return a @ b - -def neg(a): - "Same as -a." - return -a - -def or_(a, b): - "Same as a | b." - return a | b - -def pos(a): - "Same as +a." - return +a - -def pow(a, b): - "Same as a ** b." - return a ** b - -def rshift(a, b): - "Same as a >> b." - return a >> b - -def sub(a, b): - "Same as a - b." - return a - b - -def truediv(a, b): - "Same as a / b." - return a / b - -def xor(a, b): - "Same as a ^ b." - return a ^ b - -# Sequence Operations *********************************************************# - -def concat(a, b): - "Same as a + b, for a and b sequences." - if not hasattr(a, '__getitem__'): - msg = "'%s' object can't be concatenated" % type(a).__name__ - raise TypeError(msg) - return a + b - -def contains(a, b): - "Same as b in a (note reversed operands)." - return b in a - -def countOf(a, b): - "Return the number of times b occurs in a." - count = 0 - for i in a: - if i == b: - count += 1 - return count - -def delitem(a, b): - "Same as del a[b]." - del a[b] - -def getitem(a, b): - "Same as a[b]." - return a[b] - -def indexOf(a, b): - "Return the first index of b in a." - for i, j in enumerate(a): - if j == b: - return i - else: - raise ValueError('sequence.index(x): x not in sequence') - -def setitem(a, b, c): - "Same as a[b] = c." - a[b] = c - -def length_hint(obj, default=0): - """ - Return an estimate of the number of items in obj. - This is useful for presizing containers when building from an iterable. - - If the object supports len(), the result will be exact. Otherwise, it may - over- or under-estimate by an arbitrary amount. The result will be an - integer >= 0. - """ - if not isinstance(default, int): - msg = ("'%s' object cannot be interpreted as an integer" % - type(default).__name__) - raise TypeError(msg) - - try: - return len(obj) - except TypeError: - pass - - try: - hint = type(obj).__length_hint__ - except AttributeError: - return default - - try: - val = hint(obj) - except TypeError: - return default - if val is NotImplemented: - return default - if not isinstance(val, int): - msg = ('__length_hint__ must be integer, not %s' % - type(val).__name__) - raise TypeError(msg) - if val < 0: - msg = '__length_hint__() should return >= 0' - raise ValueError(msg) - return val - -# Generalized Lookup Objects **************************************************# - -class attrgetter: - """ - Return a callable object that fetches the given attribute(s) from its operand. - After f = attrgetter('name'), the call f(r) returns r.name. - After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). - After h = attrgetter('name.first', 'name.last'), the call h(r) returns - (r.name.first, r.name.last). - """ - __slots__ = ('_attrs', '_call') - - def __init__(self, attr, *attrs): - if not attrs: - if not isinstance(attr, str): - raise TypeError('attribute name must be a string') - self._attrs = (attr,) - names = attr.split('.') - def func(obj): - for name in names: - obj = getattr(obj, name) - return obj - self._call = func - else: - self._attrs = (attr,) + attrs - getters = tuple(map(attrgetter, self._attrs)) - def func(obj): - return tuple(getter(obj) for getter in getters) - self._call = func - - def __call__(self, obj): - return self._call(obj) - - def __repr__(self): - return '%s.%s(%s)' % (self.__class__.__module__, - self.__class__.__qualname__, - ', '.join(map(repr, self._attrs))) - - def __reduce__(self): - return self.__class__, self._attrs - -class itemgetter: - """ - Return a callable object that fetches the given item(s) from its operand. - After f = itemgetter(2), the call f(r) returns r[2]. - After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) - """ - __slots__ = ('_items', '_call') - - def __init__(self, item, *items): - if not items: - self._items = (item,) - def func(obj): - return obj[item] - self._call = func - else: - self._items = items = (item,) + items - def func(obj): - return tuple(obj[i] for i in items) - self._call = func - - def __call__(self, obj): - return self._call(obj) - - def __repr__(self): - return '%s.%s(%s)' % (self.__class__.__module__, - self.__class__.__name__, - ', '.join(map(repr, self._items))) - - def __reduce__(self): - return self.__class__, self._items - -class methodcaller: - """ - Return a callable object that calls the given method on its operand. - After f = methodcaller('name'), the call f(r) returns r.name(). - After g = methodcaller('name', 'date', foo=1), the call g(r) returns - r.name('date', foo=1). - """ - __slots__ = ('_name', '_args', '_kwargs') - - def __init__(*args, **kwargs): - if len(args) < 2: - msg = "methodcaller needs at least one argument, the method name" - raise TypeError(msg) - self = args[0] - self._name = args[1] - if not isinstance(self._name, str): - raise TypeError('method name must be a string') - self._args = args[2:] - self._kwargs = kwargs - - def __call__(self, obj): - return getattr(obj, self._name)(*self._args, **self._kwargs) - - def __repr__(self): - args = [repr(self._name)] - args.extend(map(repr, self._args)) - args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) - return '%s.%s(%s)' % (self.__class__.__module__, - self.__class__.__name__, - ', '.join(args)) - - def __reduce__(self): - if not self._kwargs: - return self.__class__, (self._name,) + self._args - else: - from functools import partial - return partial(self.__class__, self._name, **self._kwargs), self._args - - -# In-place Operations *********************************************************# - -def iadd(a, b): - "Same as a += b." - a += b - return a - -def iand(a, b): - "Same as a &= b." - a &= b - return a - -def iconcat(a, b): - "Same as a += b, for a and b sequences." - if not hasattr(a, '__getitem__'): - msg = "'%s' object can't be concatenated" % type(a).__name__ - raise TypeError(msg) - a += b - return a - -def ifloordiv(a, b): - "Same as a //= b." - a //= b - return a - -def ilshift(a, b): - "Same as a <<= b." - a <<= b - return a - -def imod(a, b): - "Same as a %= b." - a %= b - return a - -def imul(a, b): - "Same as a *= b." - a *= b - return a - -def imatmul(a, b): - "Same as a @= b." - a @= b - return a - -def ior(a, b): - "Same as a |= b." - a |= b - return a - -def ipow(a, b): - "Same as a **= b." - a **=b - return a - -def irshift(a, b): - "Same as a >>= b." - a >>= b - return a - -def isub(a, b): - "Same as a -= b." - a -= b - return a - -def itruediv(a, b): - "Same as a /= b." - a /= b - return a - -def ixor(a, b): - "Same as a ^= b." - a ^= b - return a - - -try: - from _operator import * -except ImportError: - pass -else: - from _operator import __doc__ - -# All of these "__func__ = func" assignments have to happen after importing -# from _operator to make sure they're set to the right function -__lt__ = lt -__le__ = le -__eq__ = eq -__ne__ = ne -__ge__ = ge -__gt__ = gt -__not__ = not_ -__abs__ = abs -__add__ = add -__and__ = and_ -__floordiv__ = floordiv -__index__ = index -__inv__ = inv -__invert__ = invert -__lshift__ = lshift -__mod__ = mod -__mul__ = mul -__matmul__ = matmul -__neg__ = neg -__or__ = or_ -__pos__ = pos -__pow__ = pow -__rshift__ = rshift -__sub__ = sub -__truediv__ = truediv -__xor__ = xor -__concat__ = concat -__contains__ = contains -__delitem__ = delitem -__getitem__ = getitem -__setitem__ = setitem -__iadd__ = iadd -__iand__ = iand -__iconcat__ = iconcat -__ifloordiv__ = ifloordiv -__ilshift__ = ilshift -__imod__ = imod -__imul__ = imul -__imatmul__ = imatmul -__ior__ = ior -__ipow__ = ipow -__irshift__ = irshift -__isub__ = isub -__itruediv__ = itruediv -__ixor__ = ixor diff -r 6db40a9955dc -r 0d413f60cc23 Lib/optparse.py --- a/Lib/optparse.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/optparse.py Mon Jan 25 17:05:13 2016 +0100 @@ -38,8 +38,7 @@ 'OptionError', 'OptionConflictError', 'OptionValueError', - 'BadOptionError', - 'check_choice'] + 'BadOptionError'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. @@ -210,6 +209,7 @@ short_first): self.parser = None self.indent_increment = indent_increment + self.help_position = self.max_help_position = max_help_position if width is None: try: width = int(os.environ['COLUMNS']) @@ -217,8 +217,6 @@ width = 80 width -= 2 self.width = width - self.help_position = self.max_help_position = \ - min(max_help_position, max(width - 20, indent_increment * 2)) self.current_indent = 0 self.level = 0 self.help_width = None # computed later @@ -263,7 +261,7 @@ Format a paragraph of free-form text for inclusion in the help output at the current indentation level. """ - text_width = max(self.width - self.current_indent, 11) + text_width = self.width - self.current_indent indent = " "*self.current_indent return textwrap.fill(text, text_width, @@ -344,7 +342,7 @@ self.dedent() self.dedent() self.help_position = min(max_len + 2, self.max_help_position) - self.help_width = max(self.width - self.help_position, 11) + self.help_width = self.width - self.help_position def format_option_strings(self, option): """Return a comma-separated list of option strings & metavariables.""" @@ -646,8 +644,14 @@ self.type = "string" else: # Allow type objects or builtin type conversion functions - # (int, str, etc.) as an alternative to their names. - if isinstance(self.type, type): + # (int, str, etc.) as an alternative to their names. (The + # complicated check of builtins is only necessary for + # Python 2.1 and earlier, and is short-circuited by the + # first check on modern Pythons.) + import builtins + if ( isinstance(self.type, type) or + (hasattr(self.type, "__name__") and + getattr(builtins, self.type.__name__, None) is self.type) ): self.type = self.type.__name__ if self.type == "str": @@ -1459,7 +1463,7 @@ """_match_long_opt(opt : string) -> string Determine which long option string 'opt' matches, ie. which one - it is an unambiguous abbreviation for. Raises BadOptionError if + it is an unambiguous abbrevation for. Raises BadOptionError if 'opt' doesn't unambiguously match any long option string. """ return _match_abbrev(opt, self._long_opt) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/os.py --- a/Lib/os.py Sun Jan 24 22:15:20 2016 -0800 +++ b/Lib/os.py Mon Jan 25 17:05:13 2016 +0100 @@ -1,9 +1,9 @@ -r"""OS routines for NT or Posix depending on what system we're on. +r"""OS routines for Mac, NT, or Posix depending on what system we're on. This exports: - - all functions from posix, nt or ce, e.g. unlink, stat, etc. + - all functions from posix, nt, os2, or ce, e.g. unlink, stat, etc. - os.path is either posixpath or ntpath - - os.name is either 'posix', 'nt' or 'ce'. + - os.name is either 'posix', 'nt', 'os2' or 'ce'. - os.curdir is a string representing the current directory ('.' or ':') - os.pardir is a string representing the parent directory ('..' or '::') - os.sep is the (or a most common) pathname separator ('/' or ':' or '\\') @@ -30,9 +30,8 @@ # Note: more names are added to __all__ later. __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep", - "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR", - "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen", - "popen", "extsep"] + "defpath", "name", "path", "devnull", + "SEEK_SET", "SEEK_CUR", "SEEK_END"] def _exists(name): return name in globals() @@ -43,24 +42,16 @@ except AttributeError: return [n for n in dir(module) if n[0] != '_'] -# Any new dependencies of the os module and/or changes in path separator -# requires updating importlib as well. if 'posix' in _names: name = 'posix' linesep = '\n' from posix import * try: from posix import _exit - __all__.append('_exit') except ImportError: pass import posixpath as path - try: - from posix import _have_functions - except ImportError: - pass - import posix __all__.extend(_get_exports_list(posix)) del posix @@ -71,7 +62,6 @@ from nt import * try: from nt import _exit - __all__.append('_exit') except ImportError: pass import ntpath as path @@ -80,10 +70,23 @@ __all__.extend(_get_exports_list(nt)) del nt +elif 'os2' in _names: + name = 'os2' + linesep = '\r\n' + from os2 import * try: - from nt import _have_functions + from os2 import _exit except ImportError: pass + if sys.version.find('EMX GCC') == -1: + import ntpath as path + else: + import os2emxpath as path + from _emx_link import link + + import os2 + __all__.extend(_get_exports_list(os2)) + del os2 elif 'ce' in _names: name = 'ce' @@ -91,7 +94,6 @@ from ce import * try: from ce import _exit - __all__.append('_exit') except ImportError: pass # We can use the standard Windows path. @@ -101,11 +103,6 @@ __all__.extend(_get_exports_list(ce)) del ce - try: - from ce import _have_functions - except ImportError: - pass - else: raise ImportError('no os specific module found') @@ -115,111 +112,36 @@ del _names - -if _exists("_have_functions"): - _globals = globals() - def _add(str, fn): - if (fn in _globals) and (str in _have_functions): - _set.add(_globals[fn]) - - _set = set() - _add("HAVE_FACCESSAT", "access") - _add("HAVE_FCHMODAT", "chmod") - _add("HAVE_FCHOWNAT", "chown") - _add("HAVE_FSTATAT", "stat") - _add("HAVE_FUTIMESAT", "utime") - _add("HAVE_LINKAT", "link") - _add("HAVE_MKDIRAT", "mkdir") - _add("HAVE_MKFIFOAT", "mkfifo") - _add("HAVE_MKNODAT", "mknod") - _add("HAVE_OPENAT", "open") - _add("HAVE_READLINKAT", "readlink") - _add("HAVE_RENAMEAT", "rename") - _add("HAVE_SYMLINKAT", "symlink") - _add("HAVE_UNLINKAT", "unlink") - _add("HAVE_UNLINKAT", "rmdir") - _add("HAVE_UTIMENSAT", "utime") - supports_dir_fd = _set - - _set = set() - _add("HAVE_FACCESSAT", "access") - supports_effective_ids = _set - - _set = set() - _add("HAVE_FCHDIR", "chdir") - _add("HAVE_FCHMOD", "chmod") - _add("HAVE_FCHOWN", "chown") - _add("HAVE_FDOPENDIR", "listdir") - _add("HAVE_FEXECVE", "execve") - _set.add(stat) # fstat always works - _add("HAVE_FTRUNCATE", "truncate") - _add("HAVE_FUTIMENS", "utime") - _add("HAVE_FUTIMES", "utime") - _add("HAVE_FPATHCONF", "pathconf") - if _exists("statvfs") and _exists("fstatvfs"): # mac os x10.3 - _add("HAVE_FSTATVFS", "statvfs") - supports_fd = _set - - _set = set() - _add("HAVE_FACCESSAT", "access") - # Some platforms don't support lchmod(). Often the function exists - # anyway, as a stub that always returns ENOSUP or perhaps EOPNOTSUPP. - # (No, I don't know why that's a good design.) ./configure will detect - # this and reject it--so HAVE_LCHMOD still won't be defined on such - # platforms. This is Very Helpful. - # - # However, sometimes platforms without a working lchmod() *do* have - # fchmodat(). (Examples: Linux kernel 3.2 with glibc 2.15, - # OpenIndiana 3.x.) And fchmodat() has a flag that theoretically makes - # it behave like lchmod(). So in theory it would be a suitable - # replacement for lchmod(). But when lchmod() doesn't work, fchmodat()'s - # flag doesn't work *either*. Sadly ./configure isn't sophisticated - # enough to detect this condition--it only determines whether or not - # fchmodat() minimally works. - # - # Therefore we simply ignore fchmodat() when deciding whether or not - # os.chmod supports follow_symlinks. Just checking lchmod() is - # sufficient. After all--if you have a working fchmodat(), your - # lchmod() almost certainly works too. - # - # _add("HAVE_FCHMODAT", "chmod") - _add("HAVE_FCHOWNAT", "chown") - _add("HAVE_FSTATAT", "stat") - _add("HAVE_LCHFLAGS", "chflags") - _add("HAVE_LCHMOD", "chmod") - if _exists("lchown"): # mac os x10.3 - _add("HAVE_LCHOWN", "chown") - _add("HAVE_LINKAT", "link") - _add("HAVE_LUTIMES", "utime") - _add("HAVE_LSTAT", "stat") - _add("HAVE_FSTATAT", "stat") - _add("HAVE_UTIMENSAT", "utime") - _add("MS_WINDOWS", "stat") - supports_follow_symlinks = _set - - del _set - del _have_functions - del _globals - del _add - - # Python uses fixed values for the SEEK_ constants; they are mapped # to native constants if necessary in posixmodule.c -# Other possible SEEK values are directly imported from posixmodule.c SEEK_SET = 0 SEEK_CUR = 1 SEEK_END = 2 + +def _get_masked_mode(mode): + mask = umask(0) + umask(mask) + return mode & ~mask + +def _are_same_file(stat1, stat2): + """Helper function that checks whether two stat results refer to the same + file. + """ + return (stat1.st_ino == stat2.st_ino and stat1.st_dev == stat2.st_dev) +# + # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) def makedirs(name, mode=0o777, exist_ok=False): - """makedirs(name [, mode=0o777][, exist_ok=False]) + """makedirs(path [, mode=0o777][, exist_ok=False]) - Super-mkdir; create a leaf directory and all intermediate ones. Works like - mkdir, except that any intermediate path segment (not just the rightmost) - will be created if it does not exist. If the target directory already - exists, raise an OSError if exist_ok is False. Otherwise no exception is + Super-mkdir; create a leaf directory and all intermediate ones. + Works like mkdir, except that any intermediate path segment (not + just the rightmost) will be created if it does not exist. If the + target directory with the same mode as we specified already exists, + raises an OSError if exist_ok is False, otherwise no exception is raised. This is recursive. """ @@ -229,24 +151,21 @@ if head and tail and not path.exists(head): try: makedirs(head, mode, exist_ok) - except FileExistsError: - # Defeats race condition when another thread created the path - pass - cdir = curdir - if isinstance(tail, bytes): - cdir = bytes(curdir, 'ASCII') - if tail == cdir: # xxx/newdir/. exists if xxx/newdir exists + except OSError as e: + # be happy if someone already created the path + if e.errno != errno.EEXIST: + raise + if tail == curdir: # xxx/newdir/. exists if xxx/newdir exists return try: mkdir(name, mode) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not path.isdir(name): + except OSError as e: + if not (e.errno == errno.EEXIST and exist_ok and path.isdir(name) and + st.S_IMODE(lstat(name).st_mode) == _get_masked_mode(mode)): raise def removedirs(name): - """removedirs(name) + """removedirs(path) Super-rmdir; remove a leaf directory and all empty intermediate ones. Works like rmdir except that, if the leaf directory is @@ -263,7 +182,7 @@ while head and tail: try: rmdir(head) - except OSError: + except error: break head, tail = path.split(head) @@ -274,7 +193,7 @@ empty. Works like rename, except creation of any intermediate directories needed to make the new pathname good is attempted first. After the rename, directories corresponding to rightmost - path segments of the old name will be pruned until either the + path segments of the old name will be pruned way until either the whole path is consumed or a nonempty directory is found. Note: this function can fail with the new directory structure made @@ -290,7 +209,7 @@ if head and tail: try: removedirs(head) - except OSError: + except error: pass __all__.extend(["makedirs", "removedirs", "renames"]) @@ -318,16 +237,15 @@ When topdown is true, the caller can modify the dirnames list in-place (e.g., via del or slice assignment), and walk will only recurse into the - subdirectories whose names remain in dirnames; this can be used to prune the - search, or to impose a specific order of visiting. Modifying dirnames when - topdown is false is ineffective, since the directories in dirnames have - already been generated by the time dirnames itself is generated. No matter - the value of topdown, the list of subdirectories is retrieved before the - tuples for the directory and its subdirectories are generated. + subdirectories whose names remain in dirnames; this can be used to prune + the search, or to impose a specific order of visiting. Modifying + dirnames when topdown is false is ineffective, since the directories in + dirnames have already been generated by the time dirnames itself is + generated. - By default errors from the os.scandir() call are ignored. If + By default errors from the os.listdir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it - will be called with one argument, an OSError instance. It can + will be called with one argument, an os.error instance. It can report the error to continue with the walk, or raise the exception to abort the walk. Note that the filename is available as the filename attribute of the exception object. @@ -351,11 +269,9 @@ print("bytes in", len(files), "non-directory files") if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories - """ - dirs = [] - nondirs = [] + islink, join, isdir = path.islink, path.join, path.isdir # We may not have read permission for top, in which case we can't # get a list of the files the directory contains. os.walk @@ -363,78 +279,36 @@ # minor reason when (say) a thousand readable directories are still # left to visit. That logic is copied here. try: - # Note that scandir is global in this module due + # Note that listdir and error are globals in this module due # to earlier import-*. - scandir_it = scandir(top) - except OSError as error: + names = listdir(top) + except error as err: if onerror is not None: - onerror(error) + onerror(err) return - while True: - try: - try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - return + dirs, nondirs = [], [] + for name in names: + if isdir(join(top, name)): + dirs.append(name) + else: + nondirs.append(name) - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) - - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True - else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink - - if walk_into: - yield from walk(entry.path, topdown, onerror, followlinks) - - # Yield before recursion if going top down if topdown: yield top, dirs, nondirs - - # Recurse into sub-directories - islink, join = path.islink, path.join - for name in dirs: - new_path = join(top, name) - # Issue #23605: os.path.islink() is used instead of caching - # entry.is_symlink() result during the loop on os.scandir() because - # the caller can replace the directory entry during the "yield" - # above. - if followlinks or not islink(new_path): - yield from walk(new_path, topdown, onerror, followlinks) - else: - # Yield after recursion if going bottom up + for name in dirs: + new_path = join(top, name) + if followlinks or not islink(new_path): + for x in walk(new_path, topdown, onerror, followlinks): + yield x + if not topdown: yield top, dirs, nondirs __all__.append("walk") -if {open, stat} <= supports_dir_fd and {listdir, stat} <= supports_fd: +if _exists("openat"): - def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): + def fwalk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. This behaves exactly like walk(), except that it yields a 4-tuple @@ -444,12 +318,8 @@ `dirpath`, `dirnames` and `filenames` are identical to walk() output, and `dirfd` is a file descriptor referring to the directory `dirpath`. - The advantage of fwalk() over walk() is that it's safe against symlink - races (when follow_symlinks is False). - - If dir_fd is not None, it should be a file descriptor open to a directory, - and top should be relative; top will then be relative to that directory. - (dir_fd is always supported for fwalk.) + The advantage of walkfd() over walk() is that it's safe against symlink + races (when followlinks is False). Caution: Since fwalk() yields file descriptors, those are only valid until the @@ -461,7 +331,7 @@ import os for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): print(root, "consumes", end="") - print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), + print(sum([os.fstatat(rootfd, name).st_size for name in files]), end="") print("bytes in", len(files), "non-directory files") if 'CVS' in dirs: @@ -469,56 +339,51 @@ """ # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY, dir_fd=dir_fd) + orig_st = lstat(top) + topfd = open(top, O_RDONLY) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, topdown, onerror, follow_symlinks) + if (followlinks or (st.S_ISDIR(orig_st.st_mode) and + _are_same_file(orig_st, fstat(topfd)))): + for x in _fwalk(topfd, top, topdown, onerror, followlinks): + yield x finally: close(topfd) - def _fwalk(topfd, toppath, topdown, onerror, follow_symlinks): + def _fwalk(topfd, toppath, topdown, onerror, followlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. - names = listdir(topfd) + # whether to follow symlinks + flag = 0 if followlinks else AT_SYMLINK_NOFOLLOW + + names = flistdir(topfd) dirs, nondirs = [], [] for name in names: - try: - # Here, we don't use AT_SYMLINK_NOFOLLOW to be consistent with - # walk() which reports symlinks to directories as directories. - # We do however check for symlinks before recursing into - # a subdirectory. - if st.S_ISDIR(stat(name, dir_fd=topfd).st_mode): - dirs.append(name) - else: - nondirs.append(name) - except FileNotFoundError: - try: - # Add dangling symlinks, ignore disappeared files - if st.S_ISLNK(stat(name, dir_fd=topfd, follow_symlinks=False) - .st_mode): - nondirs.append(name) - except FileNotFoundError: - continue + # Here, we don't use AT_SYMLINK_NOFOLLOW to be consistent with + # walk() which reports symlinks to directories as directories. We do + # however check for symlinks before recursing into a subdirectory. + if st.S_ISDIR(fstatat(topfd, name).st_mode): + dirs.append(name) + else: + nondirs.append(name) if topdown: yield toppath, dirs, nondirs, topfd for name in dirs: try: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=follow_symlinks) - dirfd = open(name, O_RDONLY, dir_fd=topfd) - except OSError as err: + orig_st = fstatat(topfd, name, flag) + dirfd = openat(topfd, name, O_RDONLY) + except error as err: if onerror is not None: onerror(err) - continue + return try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): + if followlinks or _are_same_file(orig_st, fstat(dirfd)): dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, topdown, onerror, follow_symlinks) + for x in _fwalk(dirfd, dirpath, topdown, onerror, followlinks): + yield x finally: close(dirfd) @@ -606,7 +471,7 @@ fullname = path.join(dir, file) try: exec_func(fullname, *argrest) - except OSError as e: + except error as e: last_exc = e tb = sys.exc_info()[2] if (e.errno != errno.ENOENT and e.errno != errno.ENOTDIR @@ -663,7 +528,7 @@ # Change environ to automatically call putenv(), unsetenv if they exist. -from _collections_abc import MutableMapping +from collections.abc import MutableMapping class _Environ(MutableMapping): def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): @@ -676,11 +541,7 @@ self._data = data def __getitem__(self, key): - try: - value = self._data[self.encodekey(key)] - except KeyError: - # raise KeyError with the original key value - raise KeyError(key) from None + value = self._data[self.encodekey(key)] return self.decodevalue(value) def __setitem__(self, key, value): @@ -690,13 +551,9 @@ self._data[key] = value def __delitem__(self, key): - encodedkey = self.encodekey(key) - self.unsetenv(encodedkey) - try: - del self._data[encodedkey] - except KeyError: - # raise KeyError with the original key value - raise KeyError(key) from None + key = self.encodekey(key) + self.unsetenv(key) + del self._data[key] def __iter__(self): for key in self._data: @@ -723,19 +580,17 @@ except NameError: _putenv = lambda key, value: None else: - if "putenv" not in __all__: - __all__.append("putenv") + __all__.append("putenv") try: _unsetenv = unsetenv except NameError: _unsetenv = lambda key: _putenv(key, "") else: - if "unsetenv" not in __all__: - __all__.append("unsetenv") + __all__.append("unsetenv") def _createenviron(): - if name == 'nt': + if name in ('os2', 'nt'): # Where Env Var Names Must Be UPPERCASE def check_str(value): if not isinstance(value, str): @@ -775,7 +630,7 @@ key, default and the result are str.""" return environ.get(key, default) -supports_bytes_environ = (name != 'nt') +supports_bytes_environ = name not in ('os2', 'nt') __all__.extend(("getenv", "supports_bytes_environ")) if supports_bytes_environ: @@ -843,8 +698,6 @@ P_WAIT = 0 P_NOWAIT = P_NOWAITO = 1 - __all__.extend(["P_WAIT", "P_NOWAIT", "P_NOWAITO"]) - # XXX Should we support P_DETACH? I suppose it could fork()**2 # and close the std I/O streams. Also, P_OVERLAY is the same # as execv*()? @@ -874,7 +727,7 @@ elif WIFEXITED(sts): return WEXITSTATUS(sts) else: - raise OSError("Not stopped, signaled or exited???") + raise error("Not stopped, signaled or exited???") def spawnv(mode, file, args): """spawnv(mode, file, args) -> integer @@ -917,10 +770,6 @@ otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, env, execvpe) - - __all__.extend(["spawnv", "spawnve", "spawnvp", "spawnvpe"]) - - if _exists("spawnv"): # These aren't supplied by the basic Windows code # but can be easily implemented in Python @@ -946,7 +795,7 @@ return spawnve(mode, file, args[:-1], env) - __all__.extend(["spawnl", "spawnle"]) + __all__.extend(["spawnv", "spawnve", "spawnl", "spawnle",]) if _exists("spawnvp"): @@ -974,8 +823,34 @@ return spawnvpe(mode, file, args[:-1], env) - __all__.extend(["spawnlp", "spawnlpe"]) + __all__.extend(["spawnvp", "spawnvpe", "spawnlp", "spawnlpe",]) +import copyreg as _copyreg + +def _make_stat_result(tup, dict): + return stat_result(tup, dict) + +def _pickle_stat_result(sr): + (type, args) = sr.__reduce__() + return (_make_stat_result, args) + +try: + _copyreg.pickle(stat_result, _pickle_stat_result, _make_stat_result) +except NameError: # stat_result may not exist + pass + +def _make_statvfs_result(tup, dict): + return statvfs_result(tup, dict) + +def _pickle_statvfs_result(sr): + (type, args) = sr.__reduce__() + return (_make_statvfs_result, args) + +try: + _copyreg.pickle(statvfs_result, _pickle_statvfs_result, + _make_statvfs_result) +except NameError: # statvfs_result may not exist + pass # Supply os.popen() def popen(cmd, mode="r", buffering=-1): @@ -983,7 +858,7 @@ raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) if mode not in ("r", "w"): raise ValueError("invalid mode %r" % mode) - if buffering == 0 or buffering is None: + if buffering == 0 or buffering == None: raise ValueError("popen() does not support unbuffered streams") import subprocess, io if mode == "r": diff -r 6db40a9955dc -r 0d413f60cc23 Lib/os2emxpath.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/os2emxpath.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,158 @@ +# Module 'os2emxpath' -- common operations on OS/2 pathnames +"""Common pathname manipulations, OS/2 EMX version. + +Instead of importing this module directly, import os and refer to this +module as os.path. +""" + +import os +import stat +from genericpath import * +from ntpath import (expanduser, expandvars, isabs, islink, splitdrive, + splitext, split) + +__all__ = ["normcase","isabs","join","splitdrive","split","splitext", + "basename","dirname","commonprefix","getsize","getmtime", + "getatime","getctime", "islink","exists","lexists","isdir","isfile", + "ismount","expanduser","expandvars","normpath","abspath", + "splitunc","curdir","pardir","sep","pathsep","defpath","altsep", + "extsep","devnull","realpath","supports_unicode_filenames"] + +# strings representing various path-related bits and pieces +curdir = '.' +pardir = '..' +extsep = '.' +sep = '/' +altsep = '\\' +pathsep = ';' +defpath = '.;C:\\bin' +devnull = 'nul' + +# Normalize the case of a pathname and map slashes to backslashes. +# Other normalizations (such as optimizing '../' away) are not done +# (this is done by normpath). + +def normcase(s): + """Normalize case of pathname. + + Makes all characters lowercase and all altseps into seps.""" + if not isinstance(s, (bytes, str)): + raise TypeError("normcase() argument must be str or bytes, " + "not '{}'".format(s.__class__.__name__)) + return s.replace('\\', '/').lower() + + +# Join two (or more) paths. + +def join(a, *p): + """Join two or more pathname components, inserting sep as needed""" + path = a + for b in p: + if isabs(b): + path = b + elif path == '' or path[-1:] in '/\\:': + path = path + b + else: + path = path + '/' + b + return path + + +# Parse UNC paths +def splitunc(p): + """Split a pathname into UNC mount point and relative path specifiers. + + Return a 2-tuple (unc, rest); either part may be empty. + If unc is not empty, it has the form '//host/mount' (or similar + using backslashes). unc+rest is always the input path. + Paths containing drive letters never have an UNC part. + """ + if p[1:2] == ':': + return '', p # Drive letter present + firstTwo = p[0:2] + if firstTwo == '/' * 2 or firstTwo == '\\' * 2: + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv equivalent to drive letter + # \\machine\mountpoint\directories... + # directory ^^^^^^^^^^^^^^^ + normp = normcase(p) + index = normp.find('/', 2) + if index == -1: + ##raise RuntimeError, 'illegal UNC path: "' + p + '"' + return ("", p) + index = normp.find('/', index + 1) + if index == -1: + index = len(p) + return p[:index], p[index:] + return '', p + + +# Return the tail (basename) part of a path. + +def basename(p): + """Returns the final component of a pathname""" + return split(p)[1] + + +# Return the head (dirname) part of a path. + +def dirname(p): + """Returns the directory component of a pathname""" + return split(p)[0] + + +# alias exists to lexists +lexists = exists + + +# Is a path a directory? + +# Is a path a mount point? Either a root (with or without drive letter) +# or an UNC path with at most a / or \ after the mount point. + +def ismount(path): + """Test whether a path is a mount point (defined as root of drive)""" + unc, rest = splitunc(path) + if unc: + return rest in ("", "/", "\\") + p = splitdrive(path)[1] + return len(p) == 1 and p[0] in '/\\' + + +# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. + +def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = path.replace('\\', '/') + prefix, path = splitdrive(path) + while path[:1] == '/': + prefix = prefix + '/' + path = path[1:] + comps = path.split('/') + i = 0 + while i < len(comps): + if comps[i] == '.': + del comps[i] + elif comps[i] == '..' and i > 0 and comps[i-1] not in ('', '..'): + del comps[i-1:i+1] + i = i - 1 + elif comps[i] == '' and i > 0 and comps[i-1] != '': + del comps[i] + else: + i = i + 1 + # If the path is now empty, substitute '.' + if not prefix and not comps: + comps.append('.') + return prefix + '/'.join(comps) + + +# Return an absolute path. +def abspath(path): + """Return the absolute version of a path""" + if not isabs(path): + path = join(os.getcwd(), path) + return normpath(path) + +# realpath is a no-op on systems without islink support +realpath = abspath + +supports_unicode_filenames = False diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,17 @@ +"""Support for packaging, distribution and installation of Python projects. + +Third-party tools can use parts of packaging as building blocks +without causing the other modules to be imported: + + import packaging.version + import packaging.metadata + import packaging.pypi.simple + import packaging.tests.pypi_server +""" + +from logging import getLogger + +__all__ = ['__version__', 'logger'] + +__version__ = "1.0a3" +logger = getLogger('packaging') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/_trove.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/_trove.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,571 @@ +"""Temporary helper for create.""" + +# XXX get the list from PyPI and cache it instead of hardcoding + +# XXX see if it would be more useful to store it as another structure +# than a list of strings + +all_classifiers = [ +'Development Status :: 1 - Planning', +'Development Status :: 2 - Pre-Alpha', +'Development Status :: 3 - Alpha', +'Development Status :: 4 - Beta', +'Development Status :: 5 - Production/Stable', +'Development Status :: 6 - Mature', +'Development Status :: 7 - Inactive', +'Environment :: Console', +'Environment :: Console :: Curses', +'Environment :: Console :: Framebuffer', +'Environment :: Console :: Newt', +'Environment :: Console :: svgalib', +"Environment :: Handhelds/PDA's", +'Environment :: MacOS X', +'Environment :: MacOS X :: Aqua', +'Environment :: MacOS X :: Carbon', +'Environment :: MacOS X :: Cocoa', +'Environment :: No Input/Output (Daemon)', +'Environment :: Other Environment', +'Environment :: Plugins', +'Environment :: Web Environment', +'Environment :: Web Environment :: Buffet', +'Environment :: Web Environment :: Mozilla', +'Environment :: Web Environment :: ToscaWidgets', +'Environment :: Win32 (MS Windows)', +'Environment :: X11 Applications', +'Environment :: X11 Applications :: Gnome', +'Environment :: X11 Applications :: GTK', +'Environment :: X11 Applications :: KDE', +'Environment :: X11 Applications :: Qt', +'Framework :: BFG', +'Framework :: Buildout', +'Framework :: Buildout :: Extension', +'Framework :: Buildout :: Recipe', +'Framework :: Chandler', +'Framework :: CherryPy', +'Framework :: CubicWeb', +'Framework :: Django', +'Framework :: IDLE', +'Framework :: Paste', +'Framework :: Plone', +'Framework :: Plone :: 3.2', +'Framework :: Plone :: 3.3', +'Framework :: Plone :: 4.0', +'Framework :: Plone :: 4.1', +'Framework :: Plone :: 4.2', +'Framework :: Plone :: 4.3', +'Framework :: Pylons', +'Framework :: Setuptools Plugin', +'Framework :: Trac', +'Framework :: Tryton', +'Framework :: TurboGears', +'Framework :: TurboGears :: Applications', +'Framework :: TurboGears :: Widgets', +'Framework :: Twisted', +'Framework :: ZODB', +'Framework :: Zope2', +'Framework :: Zope3', +'Intended Audience :: Customer Service', +'Intended Audience :: Developers', +'Intended Audience :: Education', +'Intended Audience :: End Users/Desktop', +'Intended Audience :: Financial and Insurance Industry', +'Intended Audience :: Healthcare Industry', +'Intended Audience :: Information Technology', +'Intended Audience :: Legal Industry', +'Intended Audience :: Manufacturing', +'Intended Audience :: Other Audience', +'Intended Audience :: Religion', +'Intended Audience :: Science/Research', +'Intended Audience :: System Administrators', +'Intended Audience :: Telecommunications Industry', +'License :: Aladdin Free Public License (AFPL)', +'License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication', +'License :: DFSG approved', +'License :: Eiffel Forum License (EFL)', +'License :: Free For Educational Use', +'License :: Free For Home Use', +'License :: Free for non-commercial use', +'License :: Freely Distributable', +'License :: Free To Use But Restricted', +'License :: Freeware', +'License :: Netscape Public License (NPL)', +'License :: Nokia Open Source License (NOKOS)', +'License :: OSI Approved', +'License :: OSI Approved :: Academic Free License (AFL)', +'License :: OSI Approved :: Apache Software License', +'License :: OSI Approved :: Apple Public Source License', +'License :: OSI Approved :: Artistic License', +'License :: OSI Approved :: Attribution Assurance License', +'License :: OSI Approved :: BSD License', +'License :: OSI Approved :: Common Public License', +'License :: OSI Approved :: Eiffel Forum License', +'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)', +'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)', +'License :: OSI Approved :: GNU Affero General Public License v3', +'License :: OSI Approved :: GNU Free Documentation License (FDL)', +'License :: OSI Approved :: GNU General Public License (GPL)', +'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', +'License :: OSI Approved :: IBM Public License', +'License :: OSI Approved :: Intel Open Source License', +'License :: OSI Approved :: ISC License (ISCL)', +'License :: OSI Approved :: Jabber Open Source License', +'License :: OSI Approved :: MIT License', +'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)', +'License :: OSI Approved :: Motosoto License', +'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)', +'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)', +'License :: OSI Approved :: Nethack General Public License', +'License :: OSI Approved :: Nokia Open Source License', +'License :: OSI Approved :: Open Group Test Suite License', +'License :: OSI Approved :: Python License (CNRI Python License)', +'License :: OSI Approved :: Python Software Foundation License', +'License :: OSI Approved :: Qt Public License (QPL)', +'License :: OSI Approved :: Ricoh Source Code Public License', +'License :: OSI Approved :: Sleepycat License', +'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)', +'License :: OSI Approved :: Sun Public License', +'License :: OSI Approved :: University of Illinois/NCSA Open Source License', +'License :: OSI Approved :: Vovida Software License 1.0', +'License :: OSI Approved :: W3C License', +'License :: OSI Approved :: X.Net License', +'License :: OSI Approved :: zlib/libpng License', +'License :: OSI Approved :: Zope Public License', +'License :: Other/Proprietary License', +'License :: Public Domain', +'License :: Repoze Public License', +'Natural Language :: Afrikaans', +'Natural Language :: Arabic', +'Natural Language :: Bengali', +'Natural Language :: Bosnian', +'Natural Language :: Bulgarian', +'Natural Language :: Catalan', +'Natural Language :: Chinese (Simplified)', +'Natural Language :: Chinese (Traditional)', +'Natural Language :: Croatian', +'Natural Language :: Czech', +'Natural Language :: Danish', +'Natural Language :: Dutch', +'Natural Language :: English', +'Natural Language :: Esperanto', +'Natural Language :: Finnish', +'Natural Language :: French', +'Natural Language :: German', +'Natural Language :: Greek', +'Natural Language :: Hebrew', +'Natural Language :: Hindi', +'Natural Language :: Hungarian', +'Natural Language :: Icelandic', +'Natural Language :: Indonesian', +'Natural Language :: Italian', +'Natural Language :: Japanese', +'Natural Language :: Javanese', +'Natural Language :: Korean', +'Natural Language :: Latin', +'Natural Language :: Latvian', +'Natural Language :: Macedonian', +'Natural Language :: Malay', +'Natural Language :: Marathi', +'Natural Language :: Norwegian', +'Natural Language :: Panjabi', +'Natural Language :: Persian', +'Natural Language :: Polish', +'Natural Language :: Portuguese', +'Natural Language :: Portuguese (Brazilian)', +'Natural Language :: Romanian', +'Natural Language :: Russian', +'Natural Language :: Serbian', +'Natural Language :: Slovak', +'Natural Language :: Slovenian', +'Natural Language :: Spanish', +'Natural Language :: Swedish', +'Natural Language :: Tamil', +'Natural Language :: Telugu', +'Natural Language :: Thai', +'Natural Language :: Turkish', +'Natural Language :: Ukranian', +'Natural Language :: Urdu', +'Natural Language :: Vietnamese', +'Operating System :: BeOS', +'Operating System :: MacOS', +'Operating System :: MacOS :: MacOS 9', +'Operating System :: MacOS :: MacOS X', +'Operating System :: Microsoft', +'Operating System :: Microsoft :: MS-DOS', +'Operating System :: Microsoft :: Windows', +'Operating System :: Microsoft :: Windows :: Windows 3.1 or Earlier', +'Operating System :: Microsoft :: Windows :: Windows 95/98/2000', +'Operating System :: Microsoft :: Windows :: Windows CE', +'Operating System :: Microsoft :: Windows :: Windows NT/2000', +'Operating System :: OS/2', +'Operating System :: OS Independent', +'Operating System :: Other OS', +'Operating System :: PalmOS', +'Operating System :: PDA Systems', +'Operating System :: POSIX', +'Operating System :: POSIX :: AIX', +'Operating System :: POSIX :: BSD', +'Operating System :: POSIX :: BSD :: BSD/OS', +'Operating System :: POSIX :: BSD :: FreeBSD', +'Operating System :: POSIX :: BSD :: NetBSD', +'Operating System :: POSIX :: BSD :: OpenBSD', +'Operating System :: POSIX :: GNU Hurd', +'Operating System :: POSIX :: HP-UX', +'Operating System :: POSIX :: IRIX', +'Operating System :: POSIX :: Linux', +'Operating System :: POSIX :: Other', +'Operating System :: POSIX :: SCO', +'Operating System :: POSIX :: SunOS/Solaris', +'Operating System :: Unix', +'Programming Language :: Ada', +'Programming Language :: APL', +'Programming Language :: ASP', +'Programming Language :: Assembly', +'Programming Language :: Awk', +'Programming Language :: Basic', +'Programming Language :: C', +'Programming Language :: C#', +'Programming Language :: C++', +'Programming Language :: Cold Fusion', +'Programming Language :: Cython', +'Programming Language :: Delphi/Kylix', +'Programming Language :: Dylan', +'Programming Language :: Eiffel', +'Programming Language :: Emacs-Lisp', +'Programming Language :: Erlang', +'Programming Language :: Euler', +'Programming Language :: Euphoria', +'Programming Language :: Forth', +'Programming Language :: Fortran', +'Programming Language :: Haskell', +'Programming Language :: Java', +'Programming Language :: JavaScript', +'Programming Language :: Lisp', +'Programming Language :: Logo', +'Programming Language :: ML', +'Programming Language :: Modula', +'Programming Language :: Objective C', +'Programming Language :: Object Pascal', +'Programming Language :: OCaml', +'Programming Language :: Other', +'Programming Language :: Other Scripting Engines', +'Programming Language :: Pascal', +'Programming Language :: Perl', +'Programming Language :: PHP', +'Programming Language :: Pike', +'Programming Language :: Pliant', +'Programming Language :: PL/SQL', +'Programming Language :: PROGRESS', +'Programming Language :: Prolog', +'Programming Language :: Python', +'Programming Language :: Python :: 2', +'Programming Language :: Python :: 2.3', +'Programming Language :: Python :: 2.4', +'Programming Language :: Python :: 2.5', +'Programming Language :: Python :: 2.6', +'Programming Language :: Python :: 2.7', +'Programming Language :: Python :: 3', +'Programming Language :: Python :: 3.0', +'Programming Language :: Python :: 3.1', +'Programming Language :: Python :: 3.2', +'Programming Language :: Python :: Implementation', +'Programming Language :: Python :: Implementation :: CPython', +'Programming Language :: Python :: Implementation :: IronPython', +'Programming Language :: Python :: Implementation :: Jython', +'Programming Language :: Python :: Implementation :: PyPy', +'Programming Language :: Python :: Implementation :: Stackless', +'Programming Language :: REBOL', +'Programming Language :: Rexx', +'Programming Language :: Ruby', +'Programming Language :: Scheme', +'Programming Language :: Simula', +'Programming Language :: Smalltalk', +'Programming Language :: SQL', +'Programming Language :: Tcl', +'Programming Language :: Unix Shell', +'Programming Language :: Visual Basic', +'Programming Language :: XBasic', +'Programming Language :: YACC', +'Programming Language :: Zope', +'Topic :: Adaptive Technologies', +'Topic :: Artistic Software', +'Topic :: Communications', +'Topic :: Communications :: BBS', +'Topic :: Communications :: Chat', +'Topic :: Communications :: Chat :: AOL Instant Messenger', +'Topic :: Communications :: Chat :: ICQ', +'Topic :: Communications :: Chat :: Internet Relay Chat', +'Topic :: Communications :: Chat :: Unix Talk', +'Topic :: Communications :: Conferencing', +'Topic :: Communications :: Email', +'Topic :: Communications :: Email :: Address Book', +'Topic :: Communications :: Email :: Email Clients (MUA)', +'Topic :: Communications :: Email :: Filters', +'Topic :: Communications :: Email :: Mailing List Servers', +'Topic :: Communications :: Email :: Mail Transport Agents', +'Topic :: Communications :: Email :: Post-Office', +'Topic :: Communications :: Email :: Post-Office :: IMAP', +'Topic :: Communications :: Email :: Post-Office :: POP3', +'Topic :: Communications :: Fax', +'Topic :: Communications :: FIDO', +'Topic :: Communications :: File Sharing', +'Topic :: Communications :: File Sharing :: Gnutella', +'Topic :: Communications :: File Sharing :: Napster', +'Topic :: Communications :: Ham Radio', +'Topic :: Communications :: Internet Phone', +'Topic :: Communications :: Telephony', +'Topic :: Communications :: Usenet News', +'Topic :: Database', +'Topic :: Database :: Database Engines/Servers', +'Topic :: Database :: Front-Ends', +'Topic :: Desktop Environment', +'Topic :: Desktop Environment :: File Managers', +'Topic :: Desktop Environment :: Gnome', +'Topic :: Desktop Environment :: GNUstep', +'Topic :: Desktop Environment :: K Desktop Environment (KDE)', +'Topic :: Desktop Environment :: K Desktop Environment (KDE) :: Themes', +'Topic :: Desktop Environment :: PicoGUI', +'Topic :: Desktop Environment :: PicoGUI :: Applications', +'Topic :: Desktop Environment :: PicoGUI :: Themes', +'Topic :: Desktop Environment :: Screen Savers', +'Topic :: Desktop Environment :: Window Managers', +'Topic :: Desktop Environment :: Window Managers :: Afterstep', +'Topic :: Desktop Environment :: Window Managers :: Afterstep :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Applets', +'Topic :: Desktop Environment :: Window Managers :: Blackbox', +'Topic :: Desktop Environment :: Window Managers :: Blackbox :: Themes', +'Topic :: Desktop Environment :: Window Managers :: CTWM', +'Topic :: Desktop Environment :: Window Managers :: CTWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Epplets', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR15', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR16', +'Topic :: Desktop Environment :: Window Managers :: Enlightenment :: Themes DR17', +'Topic :: Desktop Environment :: Window Managers :: Fluxbox', +'Topic :: Desktop Environment :: Window Managers :: Fluxbox :: Themes', +'Topic :: Desktop Environment :: Window Managers :: FVWM', +'Topic :: Desktop Environment :: Window Managers :: FVWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: IceWM', +'Topic :: Desktop Environment :: Window Managers :: IceWM :: Themes', +'Topic :: Desktop Environment :: Window Managers :: MetaCity', +'Topic :: Desktop Environment :: Window Managers :: MetaCity :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Oroborus', +'Topic :: Desktop Environment :: Window Managers :: Oroborus :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Sawfish', +'Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes 0.30', +'Topic :: Desktop Environment :: Window Managers :: Sawfish :: Themes pre-0.30', +'Topic :: Desktop Environment :: Window Managers :: Waimea', +'Topic :: Desktop Environment :: Window Managers :: Waimea :: Themes', +'Topic :: Desktop Environment :: Window Managers :: Window Maker', +'Topic :: Desktop Environment :: Window Managers :: Window Maker :: Applets', +'Topic :: Desktop Environment :: Window Managers :: Window Maker :: Themes', +'Topic :: Desktop Environment :: Window Managers :: XFCE', +'Topic :: Desktop Environment :: Window Managers :: XFCE :: Themes', +'Topic :: Documentation', +'Topic :: Education', +'Topic :: Education :: Computer Aided Instruction (CAI)', +'Topic :: Education :: Testing', +'Topic :: Games/Entertainment', +'Topic :: Games/Entertainment :: Arcade', +'Topic :: Games/Entertainment :: Board Games', +'Topic :: Games/Entertainment :: First Person Shooters', +'Topic :: Games/Entertainment :: Fortune Cookies', +'Topic :: Games/Entertainment :: Multi-User Dungeons (MUD)', +'Topic :: Games/Entertainment :: Puzzle Games', +'Topic :: Games/Entertainment :: Real Time Strategy', +'Topic :: Games/Entertainment :: Role-Playing', +'Topic :: Games/Entertainment :: Side-Scrolling/Arcade Games', +'Topic :: Games/Entertainment :: Simulation', +'Topic :: Games/Entertainment :: Turn Based Strategy', +'Topic :: Home Automation', +'Topic :: Internet', +'Topic :: Internet :: File Transfer Protocol (FTP)', +'Topic :: Internet :: Finger', +'Topic :: Internet :: Log Analysis', +'Topic :: Internet :: Name Service (DNS)', +'Topic :: Internet :: Proxy Servers', +'Topic :: Internet :: WAP', +'Topic :: Internet :: WWW/HTTP', +'Topic :: Internet :: WWW/HTTP :: Browsers', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Message Boards', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary', +'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: Page Counters', +'Topic :: Internet :: WWW/HTTP :: HTTP Servers', +'Topic :: Internet :: WWW/HTTP :: Indexing/Search', +'Topic :: Internet :: WWW/HTTP :: Session', +'Topic :: Internet :: WWW/HTTP :: Site Management', +'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking', +'Topic :: Internet :: WWW/HTTP :: WSGI', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Application', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware', +'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', +'Topic :: Internet :: Z39.50', +'Topic :: Multimedia', +'Topic :: Multimedia :: Graphics', +'Topic :: Multimedia :: Graphics :: 3D Modeling', +'Topic :: Multimedia :: Graphics :: 3D Rendering', +'Topic :: Multimedia :: Graphics :: Capture', +'Topic :: Multimedia :: Graphics :: Capture :: Digital Camera', +'Topic :: Multimedia :: Graphics :: Capture :: Scanners', +'Topic :: Multimedia :: Graphics :: Capture :: Screen Capture', +'Topic :: Multimedia :: Graphics :: Editors', +'Topic :: Multimedia :: Graphics :: Editors :: Raster-Based', +'Topic :: Multimedia :: Graphics :: Editors :: Vector-Based', +'Topic :: Multimedia :: Graphics :: Graphics Conversion', +'Topic :: Multimedia :: Graphics :: Presentation', +'Topic :: Multimedia :: Graphics :: Viewers', +'Topic :: Multimedia :: Sound/Audio', +'Topic :: Multimedia :: Sound/Audio :: Analysis', +'Topic :: Multimedia :: Sound/Audio :: Capture/Recording', +'Topic :: Multimedia :: Sound/Audio :: CD Audio', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Playing', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Ripping', +'Topic :: Multimedia :: Sound/Audio :: CD Audio :: CD Writing', +'Topic :: Multimedia :: Sound/Audio :: Conversion', +'Topic :: Multimedia :: Sound/Audio :: Editors', +'Topic :: Multimedia :: Sound/Audio :: MIDI', +'Topic :: Multimedia :: Sound/Audio :: Mixers', +'Topic :: Multimedia :: Sound/Audio :: Players', +'Topic :: Multimedia :: Sound/Audio :: Players :: MP3', +'Topic :: Multimedia :: Sound/Audio :: Sound Synthesis', +'Topic :: Multimedia :: Sound/Audio :: Speech', +'Topic :: Multimedia :: Video', +'Topic :: Multimedia :: Video :: Capture', +'Topic :: Multimedia :: Video :: Conversion', +'Topic :: Multimedia :: Video :: Display', +'Topic :: Multimedia :: Video :: Non-Linear Editor', +'Topic :: Office/Business', +'Topic :: Office/Business :: Financial', +'Topic :: Office/Business :: Financial :: Accounting', +'Topic :: Office/Business :: Financial :: Investment', +'Topic :: Office/Business :: Financial :: Point-Of-Sale', +'Topic :: Office/Business :: Financial :: Spreadsheet', +'Topic :: Office/Business :: Groupware', +'Topic :: Office/Business :: News/Diary', +'Topic :: Office/Business :: Office Suites', +'Topic :: Office/Business :: Scheduling', +'Topic :: Other/Nonlisted Topic', +'Topic :: Printing', +'Topic :: Religion', +'Topic :: Scientific/Engineering', +'Topic :: Scientific/Engineering :: Artificial Life', +'Topic :: Scientific/Engineering :: Artificial Intelligence', +'Topic :: Scientific/Engineering :: Astronomy', +'Topic :: Scientific/Engineering :: Atmospheric Science', +'Topic :: Scientific/Engineering :: Bio-Informatics', +'Topic :: Scientific/Engineering :: Chemistry', +'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', +'Topic :: Scientific/Engineering :: GIS', +'Topic :: Scientific/Engineering :: Human Machine Interfaces', +'Topic :: Scientific/Engineering :: Image Recognition', +'Topic :: Scientific/Engineering :: Information Analysis', +'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator', +'Topic :: Scientific/Engineering :: Mathematics', +'Topic :: Scientific/Engineering :: Medical Science Apps.', +'Topic :: Scientific/Engineering :: Physics', +'Topic :: Scientific/Engineering :: Visualization', +'Topic :: Security', +'Topic :: Security :: Cryptography', +'Topic :: Sociology', +'Topic :: Sociology :: Genealogy', +'Topic :: Sociology :: History', +'Topic :: Software Development', +'Topic :: Software Development :: Assemblers', +'Topic :: Software Development :: Bug Tracking', +'Topic :: Software Development :: Build Tools', +'Topic :: Software Development :: Code Generators', +'Topic :: Software Development :: Compilers', +'Topic :: Software Development :: Debuggers', +'Topic :: Software Development :: Disassemblers', +'Topic :: Software Development :: Documentation', +'Topic :: Software Development :: Embedded Systems', +'Topic :: Software Development :: Internationalization', +'Topic :: Software Development :: Interpreters', +'Topic :: Software Development :: Libraries', +'Topic :: Software Development :: Libraries :: Application Frameworks', +'Topic :: Software Development :: Libraries :: Java Libraries', +'Topic :: Software Development :: Libraries :: Perl Modules', +'Topic :: Software Development :: Libraries :: PHP Classes', +'Topic :: Software Development :: Libraries :: Pike Modules', +'Topic :: Software Development :: Libraries :: pygame', +'Topic :: Software Development :: Libraries :: Python Modules', +'Topic :: Software Development :: Libraries :: Ruby Modules', +'Topic :: Software Development :: Libraries :: Tcl Extensions', +'Topic :: Software Development :: Localization', +'Topic :: Software Development :: Object Brokering', +'Topic :: Software Development :: Object Brokering :: CORBA', +'Topic :: Software Development :: Pre-processors', +'Topic :: Software Development :: Quality Assurance', +'Topic :: Software Development :: Testing', +'Topic :: Software Development :: Testing :: Traffic Generation', +'Topic :: Software Development :: User Interfaces', +'Topic :: Software Development :: Version Control', +'Topic :: Software Development :: Version Control :: CVS', +'Topic :: Software Development :: Version Control :: RCS', +'Topic :: Software Development :: Version Control :: SCCS', +'Topic :: Software Development :: Widget Sets', +'Topic :: System', +'Topic :: System :: Archiving', +'Topic :: System :: Archiving :: Backup', +'Topic :: System :: Archiving :: Compression', +'Topic :: System :: Archiving :: Mirroring', +'Topic :: System :: Archiving :: Packaging', +'Topic :: System :: Benchmark', +'Topic :: System :: Boot', +'Topic :: System :: Boot :: Init', +'Topic :: System :: Clustering', +'Topic :: System :: Console Fonts', +'Topic :: System :: Distributed Computing', +'Topic :: System :: Emulators', +'Topic :: System :: Filesystems', +'Topic :: System :: Hardware', +'Topic :: System :: Hardware :: Hardware Drivers', +'Topic :: System :: Hardware :: Mainframes', +'Topic :: System :: Hardware :: Symmetric Multi-processing', +'Topic :: System :: Installation/Setup', +'Topic :: System :: Logging', +'Topic :: System :: Monitoring', +'Topic :: System :: Networking', +'Topic :: System :: Networking :: Firewalls', +'Topic :: System :: Networking :: Monitoring', +'Topic :: System :: Networking :: Monitoring :: Hardware Watchdog', +'Topic :: System :: Networking :: Time Synchronization', +'Topic :: System :: Operating System', +'Topic :: System :: Operating System Kernels', +'Topic :: System :: Operating System Kernels :: BSD', +'Topic :: System :: Operating System Kernels :: GNU Hurd', +'Topic :: System :: Operating System Kernels :: Linux', +'Topic :: System :: Power (UPS)', +'Topic :: System :: Recovery Tools', +'Topic :: System :: Shells', +'Topic :: System :: Software Distribution', +'Topic :: System :: Systems Administration', +'Topic :: System :: Systems Administration :: Authentication/Directory', +'Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP', +'Topic :: System :: Systems Administration :: Authentication/Directory :: NIS', +'Topic :: System :: System Shells', +'Topic :: Terminals', +'Topic :: Terminals :: Serial', +'Topic :: Terminals :: Telnet', +'Topic :: Terminals :: Terminal Emulators/X Terminals', +'Topic :: Text Editors', +'Topic :: Text Editors :: Documentation', +'Topic :: Text Editors :: Emacs', +'Topic :: Text Editors :: Integrated Development Environments (IDE)', +'Topic :: Text Editors :: Text Processing', +'Topic :: Text Editors :: Word Processors', +'Topic :: Text Processing', +'Topic :: Text Processing :: Filters', +'Topic :: Text Processing :: Fonts', +'Topic :: Text Processing :: General', +'Topic :: Text Processing :: Indexing', +'Topic :: Text Processing :: Linguistic', +'Topic :: Text Processing :: Markup', +'Topic :: Text Processing :: Markup :: HTML', +'Topic :: Text Processing :: Markup :: LaTeX', +'Topic :: Text Processing :: Markup :: SGML', +'Topic :: Text Processing :: Markup :: VRML', +'Topic :: Text Processing :: Markup :: XML', +'Topic :: Utilities', +] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,53 @@ +"""Subpackage containing all standard commands.""" +import os +from packaging.errors import PackagingModuleError +from packaging.util import resolve_name + +__all__ = ['get_command_names', 'set_command', 'get_command_class', + 'STANDARD_COMMANDS'] + + +STANDARD_COMMANDS = [ + # packaging + 'check', 'test', + # building + 'build', 'build_py', 'build_ext', 'build_clib', 'build_scripts', 'clean', + # installing + 'install_dist', 'install_lib', 'install_headers', 'install_scripts', + 'install_data', 'install_distinfo', + # distributing + 'sdist', 'bdist', 'bdist_dumb', 'bdist_wininst', + 'register', 'upload', 'upload_docs', + ] + +if os.name == 'nt': + STANDARD_COMMANDS.insert(STANDARD_COMMANDS.index('bdist_wininst'), + 'bdist_msi') + +# XXX maybe we need more than one registry, so that --list-comands can display +# standard, custom and overriden standard commands differently +_COMMANDS = dict((name, 'packaging.command.%s.%s' % (name, name)) + for name in STANDARD_COMMANDS) + + +def get_command_names(): + """Return registered commands""" + return sorted(_COMMANDS) + + +def set_command(location): + cls = resolve_name(location) + # XXX we want to do the duck-type checking here + _COMMANDS[cls.get_command_name()] = cls + + +def get_command_class(name): + """Return the registered command""" + try: + cls = _COMMANDS[name] + except KeyError: + raise PackagingModuleError("Invalid command %s" % name) + if isinstance(cls, str): + cls = resolve_name(cls) + _COMMANDS[name] = cls + return cls diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/bdist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/bdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,141 @@ +"""Create a built (binary) distribution. + +If a --formats option was given on the command line, this command will +call the corresponding bdist_* commands; if the option was absent, a +bdist_* command depending on the current platform will be called. +""" + +import os + +from packaging import util +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError, PackagingOptionError + + +def show_formats(): + """Print list of available formats (arguments to "--format" option). + """ + from packaging.fancy_getopt import FancyGetopt + formats = [] + for format in bdist.format_commands: + formats.append(("formats=" + format, None, + bdist.format_command[format][1])) + pretty_printer = FancyGetopt(formats) + pretty_printer.print_help("List of available distribution formats:") + + +class bdist(Command): + + description = "create a built (binary) distribution" + + user_options = [('bdist-base=', 'b', + "temporary directory for creating built distributions"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % util.get_platform()), + ('formats=', None, + "formats for distribution (comma-separated list)"), + ('dist-dir=', 'd', + "directory to put final built distributions in " + "[default: dist]"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('owner=', 'u', + "Owner name used when creating a tar file" + " [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file" + " [default: current group]"), + ] + + boolean_options = ['skip-build'] + + help_options = [ + ('help-formats', None, + "lists available distribution formats", show_formats), + ] + + # This is of course very simplistic. The various UNIX family operating + # systems have their specific formats, but they are out of scope for us; + # bdist_dumb is, well, dumb; it's more a building block for other + # packaging tools than a real end-user binary format. + default_format = {'posix': 'gztar', + 'nt': 'zip', + 'os2': 'zip'} + + # Establish the preferred order (for the --help-formats option). + format_commands = ['gztar', 'bztar', 'tar', + 'wininst', 'zip', 'msi'] + + # And the real information. + format_command = {'gztar': ('bdist_dumb', "gzip'ed tar file"), + 'bztar': ('bdist_dumb', "bzip2'ed tar file"), + 'tar': ('bdist_dumb', "tar file"), + 'wininst': ('bdist_wininst', + "Windows executable installer"), + 'zip': ('bdist_dumb', "ZIP file"), + 'msi': ('bdist_msi', "Microsoft Installer"), + } + + def initialize_options(self): + self.bdist_base = None + self.plat_name = None + self.formats = None + self.dist_dir = None + self.skip_build = False + self.group = None + self.owner = None + + def finalize_options(self): + # have to finalize 'plat_name' before 'bdist_base' + if self.plat_name is None: + if self.skip_build: + self.plat_name = util.get_platform() + else: + self.plat_name = self.get_finalized_command('build').plat_name + + # 'bdist_base' -- parent of per-built-distribution-format + # temporary directories (eg. we'll probably have + # "build/bdist./dumb", etc.) + if self.bdist_base is None: + build_base = self.get_finalized_command('build').build_base + self.bdist_base = os.path.join(build_base, + 'bdist.' + self.plat_name) + + self.ensure_string_list('formats') + if self.formats is None: + try: + self.formats = [self.default_format[os.name]] + except KeyError: + raise PackagingPlatformError( + "don't know how to create built distributions " + "on platform %s" % os.name) + + if self.dist_dir is None: + self.dist_dir = "dist" + + def run(self): + # Figure out which sub-commands we need to run. + commands = [] + for format in self.formats: + try: + commands.append(self.format_command[format][0]) + except KeyError: + raise PackagingOptionError("invalid format '%s'" % format) + + # Reinitialize and run each command. + for i in range(len(self.formats)): + cmd_name = commands[i] + sub_cmd = self.reinitialize_command(cmd_name) + sub_cmd.format = self.formats[i] + + # passing the owner and group names for tar archiving + if cmd_name == 'bdist_dumb': + sub_cmd.owner = self.owner + sub_cmd.group = self.group + + # If we're going to need to run this command again, tell it to + # keep its temporary files around so subsequent runs go faster. + if cmd_name in commands[i+1:]: + sub_cmd.keep_temp = True + self.run_command(cmd_name) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/bdist_dumb.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/bdist_dumb.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,139 @@ +"""Create a "dumb" built distribution. + +A dumb distribution is just an archive meant to be unpacked under +sys.prefix or sys.exec_prefix. +""" + +import os +from shutil import rmtree +from sysconfig import get_python_version + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError +from packaging import logger + + +class bdist_dumb(Command): + + description = 'create a "dumb" built distribution' + + user_options = [('bdist-dir=', 'd', + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('format=', 'f', + "archive format to create (tar, gztar, bztar, zip)"), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('relative', None, + "build the archive using relative paths" + "(default: false)"), + ('owner=', 'u', + "Owner name used when creating a tar file" + " [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file" + " [default: current group]"), + ] + + boolean_options = ['keep-temp', 'skip-build', 'relative'] + + default_format = {'posix': 'gztar', + 'nt': 'zip', + 'os2': 'zip'} + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.format = None + self.keep_temp = False + self.dist_dir = None + self.skip_build = None + self.relative = False + self.owner = None + self.group = None + + def finalize_options(self): + if self.bdist_dir is None: + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'dumb') + + if self.format is None: + try: + self.format = self.default_format[os.name] + except KeyError: + raise PackagingPlatformError( + "don't know how to create dumb built distributions " + "on platform %s" % os.name) + + self.set_undefined_options('bdist', + 'dist_dir', 'plat_name', 'skip_build') + + def run(self): + if not self.skip_build: + self.run_command('build') + + install = self.reinitialize_command('install_dist', + reinit_subcommands=True) + install.root = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + + logger.info("installing to %s", self.bdist_dir) + self.run_command('install_dist') + + # And make an archive relative to the root of the + # pseudo-installation tree. + archive_basename = "%s.%s" % (self.distribution.get_fullname(), + self.plat_name) + + # OS/2 objects to any ":" characters in a filename (such as when + # a timestamp is used in a version) so change them to hyphens. + if os.name == "os2": + archive_basename = archive_basename.replace(":", "-") + + pseudoinstall_root = os.path.join(self.dist_dir, archive_basename) + if not self.relative: + archive_root = self.bdist_dir + else: + if (self.distribution.has_ext_modules() and + (install.install_base != install.install_platbase)): + raise PackagingPlatformError( + "can't make a dumb built distribution where base and " + "platbase are different (%r, %r)" % + (install.install_base, install.install_platbase)) + else: + archive_root = os.path.join( + self.bdist_dir, + self._ensure_relative(install.install_base)) + + # Make the archive + filename = self.make_archive(pseudoinstall_root, + self.format, root_dir=archive_root, + owner=self.owner, group=self.group) + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + self.distribution.dist_files.append(('bdist_dumb', pyversion, + filename)) + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', self.bdist_dir) + else: + rmtree(self.bdist_dir) + + def _ensure_relative(self, path): + # copied from dir_util, deleted + drive, path = os.path.splitdrive(path) + if path[0:1] == os.sep: + path = drive + path[1:] + return path diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/bdist_msi.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/bdist_msi.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,743 @@ +"""Create a Microsoft Installer (.msi) binary distribution.""" + +# Copyright (C) 2005, 2006 Martin von Löwis +# Licensed to PSF under a Contributor Agreement. + +import sys +import os +import msilib + +from shutil import rmtree +from sysconfig import get_python_version +from packaging.command.cmd import Command +from packaging.version import NormalizedVersion +from packaging.errors import PackagingOptionError +from packaging import logger as log +from packaging.util import get_platform +from msilib import schema, sequence, text +from msilib import Directory, Feature, Dialog, add_data + +class MSIVersion(NormalizedVersion): + """ + MSI ProductVersion must be strictly numeric. + MSIVersion disallows prerelease and postrelease versions. + """ + def __init__(self, *args, **kwargs): + super(MSIVersion, self).__init__(*args, **kwargs) + if not self.is_final: + raise ValueError("ProductVersion must be strictly numeric") + +class PyDialog(Dialog): + """Dialog class with a fixed layout: controls at the top, then a ruler, + then a list of buttons: back, next, cancel. Optionally a bitmap at the + left.""" + def __init__(self, *args, **kw): + """Dialog(database, name, x, y, w, h, attributes, title, first, + default, cancel, bitmap=true)""" + super(PyDialog, self).__init__(*args) + ruler = self.h - 36 + #if kw.get("bitmap", True): + # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") + self.line("BottomLine", 0, ruler, self.w, 0) + + def title(self, title): + "Set the title text of the dialog at the top." + # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, + # text, in VerdanaBold10 + self.text("Title", 15, 10, 320, 60, 0x30003, + r"{\VerdanaBold10}%s" % title) + + def back(self, title, next, name = "Back", active = 1): + """Add a back button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) + + def cancel(self, title, next, name = "Cancel", active = 1): + """Add a cancel button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) + + def next(self, title, next, name = "Next", active = 1): + """Add a Next button with a given title, the tab-next button, + its name in the Control table, possibly initially disabled. + + Return the button, so that events can be associated""" + if active: + flags = 3 # Visible|Enabled + else: + flags = 1 # Visible + return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) + + def xbutton(self, name, title, next, xpos): + """Add a button with a given title, the tab-next button, + its name in the Control table, giving its x position; the + y-position is aligned with the other buttons. + + Return the button, so that events can be associated""" + return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) + +class bdist_msi(Command): + + description = "create a Microsoft Installer (.msi) binary distribution" + + user_options = [('bdist-dir=', None, + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('target-version=', None, + "require a specific python version" + + " on the target system"), + ('no-target-compile', 'c', + "do not compile .py to .pyc on the target system"), + ('no-target-optimize', 'o', + "do not compile .py to .pyo (optimized)" + "on the target system"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('install-script=', None, + "basename of installation script to be run after" + "installation or before deinstallation"), + ('pre-install-script=', None, + "Fully qualified filename of a script to be run before " + "any files are installed. This script need not be in the " + "distribution"), + ] + + boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', + 'skip-build'] + + all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', + '2.5', '2.6', '2.7', '2.8', '2.9', + '3.0', '3.1', '3.2', '3.3', '3.4', + '3.5', '3.6', '3.7', '3.8', '3.9'] + other_version = 'X' + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.keep_temp = False + self.no_target_compile = False + self.no_target_optimize = False + self.target_version = None + self.dist_dir = None + self.skip_build = None + self.install_script = None + self.pre_install_script = None + self.versions = None + + def finalize_options(self): + self.set_undefined_options('bdist', 'skip_build') + + if self.bdist_dir is None: + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'msi') + + short_version = get_python_version() + if (not self.target_version) and self.distribution.has_ext_modules(): + self.target_version = short_version + + if self.target_version: + self.versions = [self.target_version] + if not self.skip_build and self.distribution.has_ext_modules()\ + and self.target_version != short_version: + raise PackagingOptionError("target version can only be %s, or the '--skip-build'" \ + " option must be specified" % (short_version,)) + else: + self.versions = list(self.all_versions) + + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') + + if self.pre_install_script: + raise PackagingOptionError("the pre-install-script feature is not yet implemented") + + if self.install_script: + for script in self.distribution.scripts: + if self.install_script == os.path.basename(script): + break + else: + raise PackagingOptionError("install_script '%s' not found in scripts" % \ + self.install_script) + self.install_script_key = None + + + def run(self): + if not self.skip_build: + self.run_command('build') + + install = self.reinitialize_command('install_dist', + reinit_subcommands=True) + install.prefix = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + + install_lib = self.reinitialize_command('install_lib') + # we do not want to include pyc or pyo files + install_lib.compile = False + install_lib.optimize = 0 + + if self.distribution.has_ext_modules(): + # If we are building an installer for a Python version other + # than the one we are currently running, then we need to ensure + # our build_lib reflects the other Python version rather than ours. + # Note that for target_version!=sys.version, we must have skipped the + # build step, so there is no issue with enforcing the build of this + # version. + target_version = self.target_version + if not target_version: + assert self.skip_build, "Should have already checked this" + target_version = '%s.%s' % sys.version_info[:2] + plat_specifier = ".%s-%s" % (self.plat_name, target_version) + build = self.get_finalized_command('build') + build.build_lib = os.path.join(build.build_base, + 'lib' + plat_specifier) + + log.info("installing to %s", self.bdist_dir) + install.ensure_finalized() + + # avoid warning of 'install_lib' about installing + # into a directory not in sys.path + sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) + + install.run() + + del sys.path[0] + + self.mkpath(self.dist_dir) + fullname = self.distribution.get_fullname() + installer_name = self.get_installer_filename(fullname) + installer_name = os.path.abspath(installer_name) + if os.path.exists(installer_name): os.unlink(installer_name) + + metadata = self.distribution.metadata + author = metadata.author + if not author: + author = metadata.maintainer + if not author: + author = "UNKNOWN" + version = MSIVersion(metadata.get_version()) + # Prefix ProductName with Python x.y, so that + # it sorts together with the other Python packages + # in Add-Remove-Programs (APR) + fullname = self.distribution.get_fullname() + if self.target_version: + product_name = "Python %s %s" % (self.target_version, fullname) + else: + product_name = "Python %s" % (fullname) + self.db = msilib.init_database(installer_name, schema, + product_name, msilib.gen_uuid(), + str(version), author) + msilib.add_tables(self.db, sequence) + props = [('DistVersion', version)] + email = metadata.author_email or metadata.maintainer_email + if email: + props.append(("ARPCONTACT", email)) + if metadata.url: + props.append(("ARPURLINFOABOUT", metadata.url)) + if props: + add_data(self.db, 'Property', props) + + self.add_find_python() + self.add_files() + self.add_scripts() + self.add_ui() + self.db.Commit() + + if hasattr(self.distribution, 'dist_files'): + tup = 'bdist_msi', self.target_version or 'any', fullname + self.distribution.dist_files.append(tup) + + if not self.keep_temp: + log.info("removing temporary build directory %s", self.bdist_dir) + if not self.dry_run: + rmtree(self.bdist_dir) + + def add_files(self): + db = self.db + cab = msilib.CAB("distfiles") + rootdir = os.path.abspath(self.bdist_dir) + + root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") + f = Feature(db, "Python", "Python", "Everything", + 0, 1, directory="TARGETDIR") + + items = [(f, root, '')] + for version in self.versions + [self.other_version]: + target = "TARGETDIR" + version + name = default = "Python" + version + desc = "Everything" + if version is self.other_version: + title = "Python from another location" + level = 2 + else: + title = "Python %s from registry" % version + level = 1 + f = Feature(db, name, title, desc, 1, level, directory=target) + dir = Directory(db, cab, root, rootdir, target, default) + items.append((f, dir, version)) + db.Commit() + + seen = {} + for feature, dir, version in items: + todo = [dir] + while todo: + dir = todo.pop() + for file in os.listdir(dir.absolute): + afile = os.path.join(dir.absolute, file) + if os.path.isdir(afile): + short = "%s|%s" % (dir.make_short(file), file) + default = file + version + newdir = Directory(db, cab, dir, file, default, short) + todo.append(newdir) + else: + if not dir.component: + dir.start_component(dir.logical, feature, 0) + if afile not in seen: + key = seen[afile] = dir.add_file(file) + if file==self.install_script: + if self.install_script_key: + raise PackagingOptionError( + "Multiple files with name %s" % file) + self.install_script_key = '[#%s]' % key + else: + key = seen[afile] + add_data(self.db, "DuplicateFile", + [(key + version, dir.component, key, None, dir.logical)]) + db.Commit() + cab.commit(db) + + def add_find_python(self): + """Adds code to the installer to compute the location of Python. + + Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the + registry for each version of Python. + + Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, + else from PYTHON.MACHINE.X.Y. + + Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" + + start = 402 + for ver in self.versions: + install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver + machine_reg = "python.machine." + ver + user_reg = "python.user." + ver + machine_prop = "PYTHON.MACHINE." + ver + user_prop = "PYTHON.USER." + ver + machine_action = "PythonFromMachine" + ver + user_action = "PythonFromUser" + ver + exe_action = "PythonExe" + ver + target_dir_prop = "TARGETDIR" + ver + exe_prop = "PYTHON" + ver + if msilib.Win64: + # type: msidbLocatorTypeRawValue + msidbLocatorType64bit + Type = 2+16 + else: + Type = 2 + add_data(self.db, "RegLocator", + [(machine_reg, 2, install_path, None, Type), + (user_reg, 1, install_path, None, Type)]) + add_data(self.db, "AppSearch", + [(machine_prop, machine_reg), + (user_prop, user_reg)]) + add_data(self.db, "CustomAction", + [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), + (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), + (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), + ]) + add_data(self.db, "InstallExecuteSequence", + [(machine_action, machine_prop, start), + (user_action, user_prop, start + 1), + (exe_action, None, start + 2), + ]) + add_data(self.db, "InstallUISequence", + [(machine_action, machine_prop, start), + (user_action, user_prop, start + 1), + (exe_action, None, start + 2), + ]) + add_data(self.db, "Condition", + [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) + start += 4 + assert start < 500 + + def add_scripts(self): + if self.install_script: + start = 6800 + for ver in self.versions + [self.other_version]: + install_action = "install_script." + ver + exe_prop = "PYTHON" + ver + add_data(self.db, "CustomAction", + [(install_action, 50, exe_prop, self.install_script_key)]) + add_data(self.db, "InstallExecuteSequence", + [(install_action, "&Python%s=3" % ver, start)]) + start += 1 + # XXX pre-install scripts are currently refused in finalize_options() + # but if this feature is completed, it will also need to add + # entries for each version as the above code does + if self.pre_install_script: + scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") + with open(scriptfn, "w") as f: + # The batch file will be executed with [PYTHON], so that %1 + # is the path to the Python interpreter; %0 will be the path + # of the batch file. + # rem =""" + # %1 %0 + # exit + # """ + # + f.write('rem ="""\n%1 %0\nexit\n"""\n') + with open(self.pre_install_script) as fp: + f.write(fp.read()) + add_data(self.db, "Binary", + [("PreInstall", msilib.Binary(scriptfn)), + ]) + add_data(self.db, "CustomAction", + [("PreInstall", 2, "PreInstall", None), + ]) + add_data(self.db, "InstallExecuteSequence", + [("PreInstall", "NOT Installed", 450), + ]) + + def add_ui(self): + db = self.db + x = y = 50 + w = 370 + h = 300 + title = "[ProductName] Setup" + + # see "Dialog Style Bits" + modal = 3 # visible | modal + modeless = 1 # visible + + # UI customization properties + add_data(db, "Property", + # See "DefaultUIFont Property" + [("DefaultUIFont", "DlgFont8"), + # See "ErrorDialog Style Bit" + ("ErrorDialog", "ErrorDlg"), + ("Progress1", "Install"), # modified in maintenance type dlg + ("Progress2", "installs"), + ("MaintenanceForm_Action", "Repair"), + # possible values: ALL, JUSTME + ("WhichUsers", "ALL") + ]) + + # Fonts, see "TextStyle Table" + add_data(db, "TextStyle", + [("DlgFont8", "Tahoma", 9, None, 0), + ("DlgFontBold8", "Tahoma", 8, None, 1), #bold + ("VerdanaBold10", "Verdana", 10, None, 1), + ("VerdanaRed9", "Verdana", 9, 255, 0), + ]) + + # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" + # Numbers indicate sequence; see sequence.py for how these action integrate + add_data(db, "InstallUISequence", + [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), + ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), + # In the user interface, assume all-users installation if privileged. + ("SelectFeaturesDlg", "Not Installed", 1230), + # XXX no support for resume installations yet + #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), + ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), + ("ProgressDlg", None, 1280)]) + + add_data(db, 'ActionText', text.ActionText) + add_data(db, 'UIText', text.UIText) + ##################################################################### + # Standard dialogs: FatalError, UserExit, ExitDialog + fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + fatal.title("[ProductName] Installer ended prematurely") + fatal.back("< Back", "Finish", active = 0) + fatal.cancel("Cancel", "Back", active = 0) + fatal.text("Description1", 15, 70, 320, 80, 0x30003, + "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") + fatal.text("Description2", 15, 155, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c=fatal.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Exit") + + user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + user_exit.title("[ProductName] Installer was interrupted") + user_exit.back("< Back", "Finish", active = 0) + user_exit.cancel("Cancel", "Back", active = 0) + user_exit.text("Description1", 15, 70, 320, 80, 0x30003, + "[ProductName] setup was interrupted. Your system has not been modified. " + "To install this program at a later time, please run the installation again.") + user_exit.text("Description2", 15, 155, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c = user_exit.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Exit") + + exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, + "Finish", "Finish", "Finish") + exit_dialog.title("Completing the [ProductName] Installer") + exit_dialog.back("< Back", "Finish", active = 0) + exit_dialog.cancel("Cancel", "Back", active = 0) + exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, + "Click the Finish button to exit the Installer.") + c = exit_dialog.next("Finish", "Cancel", name="Finish") + c.event("EndDialog", "Return") + + ##################################################################### + # Required dialog: FilesInUse, ErrorDlg + inuse = PyDialog(db, "FilesInUse", + x, y, w, h, + 19, # KeepModeless|Modal|Visible + title, + "Retry", "Retry", "Retry", bitmap=False) + inuse.text("Title", 15, 6, 200, 15, 0x30003, + r"{\DlgFontBold8}Files in Use") + inuse.text("Description", 20, 23, 280, 20, 0x30003, + "Some files that need to be updated are currently in use.") + inuse.text("Text", 20, 55, 330, 50, 3, + "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") + inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", + None, None, None) + c=inuse.back("Exit", "Ignore", name="Exit") + c.event("EndDialog", "Exit") + c=inuse.next("Ignore", "Retry", name="Ignore") + c.event("EndDialog", "Ignore") + c=inuse.cancel("Retry", "Exit", name="Retry") + c.event("EndDialog","Retry") + + # See "Error Dialog". See "ICE20" for the required names of the controls. + error = Dialog(db, "ErrorDlg", + 50, 10, 330, 101, + 65543, # Error|Minimize|Modal|Visible + title, + "ErrorText", None, None) + error.text("ErrorText", 50,9,280,48,3, "") + #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) + error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") + error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") + error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") + error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") + error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") + error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") + error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") + + ##################################################################### + # Global "Query Cancel" dialog + cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, + "No", "No", "No") + cancel.text("Text", 48, 15, 194, 30, 3, + "Are you sure you want to cancel [ProductName] installation?") + #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, + # "py.ico", None, None) + c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") + c.event("EndDialog", "Exit") + + c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") + c.event("EndDialog", "Return") + + ##################################################################### + # Global "Wait for costing" dialog + costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, + "Return", "Return", "Return") + costing.text("Text", 48, 15, 194, 30, 3, + "Please wait while the installer finishes determining your disk space requirements.") + c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) + c.event("EndDialog", "Exit") + + ##################################################################### + # Preparation dialog: no user input except cancellation + prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, + "Cancel", "Cancel", "Cancel") + prep.text("Description", 15, 70, 320, 40, 0x30003, + "Please wait while the Installer prepares to guide you through the installation.") + prep.title("Welcome to the [ProductName] Installer") + c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") + c.mapping("ActionText", "Text") + c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) + c.mapping("ActionData", "Text") + prep.back("Back", None, active=0) + prep.next("Next", None, active=0) + c=prep.cancel("Cancel", None) + c.event("SpawnDialog", "CancelDlg") + + ##################################################################### + # Feature (Python directory) selection + seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, + "Next", "Next", "Cancel") + seldlg.title("Select Python Installations") + + seldlg.text("Hint", 15, 30, 300, 20, 3, + "Select the Python locations where %s should be installed." + % self.distribution.get_fullname()) + + seldlg.back("< Back", None, active=0) + c = seldlg.next("Next >", "Cancel") + order = 1 + c.event("[TARGETDIR]", "[SourceDir]", ordering=order) + for version in self.versions + [self.other_version]: + order += 1 + c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, + "FEATURE_SELECTED AND &Python%s=3" % version, + ordering=order) + c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) + c.event("EndDialog", "Return", ordering=order + 2) + c = seldlg.cancel("Cancel", "Features") + c.event("SpawnDialog", "CancelDlg") + + c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, + "FEATURE", None, "PathEdit", None) + c.event("[FEATURE_SELECTED]", "1") + ver = self.other_version + install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver + dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver + + c = seldlg.text("Other", 15, 200, 300, 15, 3, + "Provide an alternate Python location") + c.condition("Enable", install_other_cond) + c.condition("Show", install_other_cond) + c.condition("Disable", dont_install_other_cond) + c.condition("Hide", dont_install_other_cond) + + c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, + "TARGETDIR" + ver, None, "Next", None) + c.condition("Enable", install_other_cond) + c.condition("Show", install_other_cond) + c.condition("Disable", dont_install_other_cond) + c.condition("Hide", dont_install_other_cond) + + ##################################################################### + # Disk cost + cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, + "OK", "OK", "OK", bitmap=False) + cost.text("Title", 15, 6, 200, 15, 0x30003, + "{\DlgFontBold8}Disk Space Requirements") + cost.text("Description", 20, 20, 280, 20, 0x30003, + "The disk space required for the installation of the selected features.") + cost.text("Text", 20, 53, 330, 60, 3, + "The highlighted volumes (if any) do not have enough disk space " + "available for the currently selected features. You can either " + "remove some files from the highlighted volumes, or choose to " + "install less features onto local drive(s), or select different " + "destination drive(s).") + cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, + None, "{120}{70}{70}{70}{70}", None, None) + cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") + + ##################################################################### + # WhichUsers Dialog. Only available on NT, and for privileged users. + # This must be run before FindRelatedProducts, because that will + # take into account whether the previous installation was per-user + # or per-machine. We currently don't support going back to this + # dialog after "Next" was selected; to support this, we would need to + # find how to reset the ALLUSERS property, and how to re-run + # FindRelatedProducts. + # On Windows9x, the ALLUSERS property is ignored on the command line + # and in the Property table, but installer fails according to the documentation + # if a dialog attempts to set ALLUSERS. + whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, + "AdminInstall", "Next", "Cancel") + whichusers.title("Select whether to install [ProductName] for all users of this computer.") + # A radio group with two options: allusers, justme + g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, + "WhichUsers", "", "Next") + g.add("ALL", 0, 5, 150, 20, "Install for all users") + g.add("JUSTME", 0, 25, 150, 20, "Install just for me") + + whichusers.back("Back", None, active=0) + + c = whichusers.next("Next >", "Cancel") + c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) + c.event("EndDialog", "Return", ordering = 2) + + c = whichusers.cancel("Cancel", "AdminInstall") + c.event("SpawnDialog", "CancelDlg") + + ##################################################################### + # Installation Progress dialog (modeless) + progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, + "Cancel", "Cancel", "Cancel", bitmap=False) + progress.text("Title", 20, 15, 200, 15, 0x30003, + "{\DlgFontBold8}[Progress1] [ProductName]") + progress.text("Text", 35, 65, 300, 30, 3, + "Please wait while the Installer [Progress2] [ProductName]. " + "This may take several minutes.") + progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") + + c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") + c.mapping("ActionText", "Text") + + #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) + #c.mapping("ActionData", "Text") + + c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, + None, "Progress done", None, None) + c.mapping("SetProgress", "Progress") + + progress.back("< Back", "Next", active=False) + progress.next("Next >", "Cancel", active=False) + progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") + + ################################################################### + # Maintenance type: repair/uninstall + maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, + "Next", "Next", "Cancel") + maint.title("Welcome to the [ProductName] Setup Wizard") + maint.text("BodyText", 15, 63, 330, 42, 3, + "Select whether you want to repair or remove [ProductName].") + g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, + "MaintenanceForm_Action", "", "Next") + #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") + g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") + g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") + + maint.back("< Back", None, active=False) + c=maint.next("Finish", "Cancel") + # Change installation: Change progress dialog to "Change", then ask + # for feature selection + #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) + #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) + + # Reinstall: Change progress dialog to "Repair", then invoke reinstall + # Also set list of reinstalled features to "ALL" + c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) + c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) + c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) + c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) + + # Uninstall: Change progress to "Remove", then invoke uninstall + # Also set list of removed features to "ALL" + c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) + c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) + c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) + c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) + + # Close dialog when maintenance action scheduled + c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) + #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) + + maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") + + def get_installer_filename(self, fullname): + # Factored out to allow overriding in subclasses + if self.target_version: + base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, + self.target_version) + else: + base_name = "%s.%s.msi" % (fullname, self.plat_name) + installer_name = os.path.join(self.dist_dir, base_name) + return installer_name diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/bdist_wininst.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/bdist_wininst.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,345 @@ +"""Create an executable installer for Windows.""" + +import sys +import os + +from shutil import rmtree +from sysconfig import get_python_version +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError, PackagingPlatformError +from packaging import logger +from packaging.util import get_platform + + +class bdist_wininst(Command): + + description = "create an executable installer for Windows" + + user_options = [('bdist-dir=', None, + "temporary directory for creating the distribution"), + ('plat-name=', 'p', + "platform name to embed in generated filenames " + "(default: %s)" % get_platform()), + ('keep-temp', 'k', + "keep the pseudo-installation tree around after " + + "creating the distribution archive"), + ('target-version=', None, + "require a specific python version" + + " on the target system"), + ('no-target-compile', 'c', + "do not compile .py to .pyc on the target system"), + ('no-target-optimize', 'o', + "do not compile .py to .pyo (optimized)" + "on the target system"), + ('dist-dir=', 'd', + "directory to put final built distributions in"), + ('bitmap=', 'b', + "bitmap to use for the installer instead of python-powered logo"), + ('title=', 't', + "title to display on the installer background instead of default"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + ('install-script=', None, + "basename of installation script to be run after" + "installation or before deinstallation"), + ('pre-install-script=', None, + "Fully qualified filename of a script to be run before " + "any files are installed. This script need not be in the " + "distribution"), + ('user-access-control=', None, + "specify Vista's UAC handling - 'none'/default=no " + "handling, 'auto'=use UAC if target Python installed for " + "all users, 'force'=always use UAC"), + ] + + boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', + 'skip-build'] + + def initialize_options(self): + self.bdist_dir = None + self.plat_name = None + self.keep_temp = False + self.no_target_compile = False + self.no_target_optimize = False + self.target_version = None + self.dist_dir = None + self.bitmap = None + self.title = None + self.skip_build = None + self.install_script = None + self.pre_install_script = None + self.user_access_control = None + + + def finalize_options(self): + self.set_undefined_options('bdist', 'skip_build') + + if self.bdist_dir is None: + if self.skip_build and self.plat_name: + # If build is skipped and plat_name is overridden, bdist will + # not see the correct 'plat_name' - so set that up manually. + bdist = self.distribution.get_command_obj('bdist') + bdist.plat_name = self.plat_name + # next the command will be initialized using that name + bdist_base = self.get_finalized_command('bdist').bdist_base + self.bdist_dir = os.path.join(bdist_base, 'wininst') + + if not self.target_version: + self.target_version = "" + + if not self.skip_build and self.distribution.has_ext_modules(): + short_version = get_python_version() + if self.target_version and self.target_version != short_version: + raise PackagingOptionError("target version can only be %s, or the '--skip-build'" \ + " option must be specified" % (short_version,)) + self.target_version = short_version + + self.set_undefined_options('bdist', 'dist_dir', 'plat_name') + + if self.install_script: + for script in self.distribution.scripts: + if self.install_script == os.path.basename(script): + break + else: + raise PackagingOptionError("install_script '%s' not found in scripts" % \ + self.install_script) + + def run(self): + if (sys.platform != "win32" and + (self.distribution.has_ext_modules() or + self.distribution.has_c_libraries())): + raise PackagingPlatformError \ + ("distribution contains extensions and/or C libraries; " + "must be compiled on a Windows 32 platform") + + if not self.skip_build: + self.run_command('build') + + install = self.reinitialize_command('install', reinit_subcommands=True) + install.root = self.bdist_dir + install.skip_build = self.skip_build + install.warn_dir = False + install.plat_name = self.plat_name + + install_lib = self.reinitialize_command('install_lib') + # we do not want to include pyc or pyo files + install_lib.compile = False + install_lib.optimize = 0 + + if self.distribution.has_ext_modules(): + # If we are building an installer for a Python version other + # than the one we are currently running, then we need to ensure + # our build_lib reflects the other Python version rather than ours. + # Note that for target_version!=sys.version, we must have skipped the + # build step, so there is no issue with enforcing the build of this + # version. + target_version = self.target_version + if not target_version: + assert self.skip_build, "Should have already checked this" + target_version = '%s.%s' % sys.version_info[:2] + plat_specifier = ".%s-%s" % (self.plat_name, target_version) + build = self.get_finalized_command('build') + build.build_lib = os.path.join(build.build_base, + 'lib' + plat_specifier) + + # Use a custom scheme for the zip-file, because we have to decide + # at installation time which scheme to use. + for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): + value = key.upper() + if key == 'headers': + value = value + '/Include/$dist_name' + setattr(install, + 'install_' + key, + value) + + logger.info("installing to %s", self.bdist_dir) + install.ensure_finalized() + + # avoid warning of 'install_lib' about installing + # into a directory not in sys.path + sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) + + install.run() + + del sys.path[0] + + # And make an archive relative to the root of the + # pseudo-installation tree. + from tempfile import NamedTemporaryFile + archive_basename = NamedTemporaryFile().name + fullname = self.distribution.get_fullname() + arcname = self.make_archive(archive_basename, "zip", + root_dir=self.bdist_dir) + # create an exe containing the zip-file + self.create_exe(arcname, fullname, self.bitmap) + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + self.distribution.dist_files.append(('bdist_wininst', pyversion, + self.get_installer_filename(fullname))) + # remove the zip-file again + logger.debug("removing temporary file '%s'", arcname) + os.remove(arcname) + + if not self.keep_temp: + logger.info('removing %s', self.bdist_dir) + if not self.dry_run: + rmtree(self.bdist_dir) + + def get_inidata(self): + # Return data describing the installation. + + lines = [] + metadata = self.distribution.metadata + + # Write the [metadata] section. + lines.append("[metadata]") + + # 'info' will be displayed in the installer's dialog box, + # describing the items to be installed. + info = (metadata.long_description or '') + '\n' + + # Escape newline characters + def escape(s): + return s.replace("\n", "\\n") + + for name in ["author", "author_email", "description", "maintainer", + "maintainer_email", "name", "url", "version"]: + data = getattr(metadata, name, "") + if data: + info = info + ("\n %s: %s" % \ + (name.capitalize(), escape(data))) + lines.append("%s=%s" % (name, escape(data))) + + # The [setup] section contains entries controlling + # the installer runtime. + lines.append("\n[Setup]") + if self.install_script: + lines.append("install_script=%s" % self.install_script) + lines.append("info=%s" % escape(info)) + lines.append("target_compile=%d" % (not self.no_target_compile)) + lines.append("target_optimize=%d" % (not self.no_target_optimize)) + if self.target_version: + lines.append("target_version=%s" % self.target_version) + if self.user_access_control: + lines.append("user_access_control=%s" % self.user_access_control) + + title = self.title or self.distribution.get_fullname() + lines.append("title=%s" % escape(title)) + import time + import packaging + build_info = "Built %s with packaging-%s" % \ + (time.ctime(time.time()), packaging.__version__) + lines.append("build_info=%s" % build_info) + return "\n".join(lines) + + def create_exe(self, arcname, fullname, bitmap=None): + import struct + + self.mkpath(self.dist_dir) + + cfgdata = self.get_inidata() + + installer_name = self.get_installer_filename(fullname) + logger.info("creating %s", installer_name) + + if bitmap: + with open(bitmap, "rb") as fp: + bitmapdata = fp.read() + bitmaplen = len(bitmapdata) + else: + bitmaplen = 0 + + with open(installer_name, "wb") as file: + file.write(self.get_exe_bytes()) + if bitmap: + file.write(bitmapdata) + + # Convert cfgdata from unicode to ascii, mbcs encoded + if isinstance(cfgdata, str): + cfgdata = cfgdata.encode("mbcs") + + # Append the pre-install script + cfgdata = cfgdata + b"\0" + if self.pre_install_script: + # We need to normalize newlines, so we open in text mode and + # convert back to bytes. "latin-1" simply avoids any possible + # failures. + with open(self.pre_install_script, encoding="latin-1") as fp: + script_data = fp.read().encode("latin-1") + cfgdata = cfgdata + script_data + b"\n\0" + else: + # empty pre-install script + cfgdata = cfgdata + b"\0" + file.write(cfgdata) + + # The 'magic number' 0x1234567B is used to make sure that the + # binary layout of 'cfgdata' is what the wininst.exe binary + # expects. If the layout changes, increment that number, make + # the corresponding changes to the wininst.exe sources, and + # recompile them. + header = struct.pack(" cur_version: + bv = get_build_version() + else: + if self.target_version < "2.4": + bv = 6.0 + else: + bv = 7.1 + else: + # for current version - use authoritative check. + bv = get_build_version() + + # wininst-x.y.exe is in the same directory as this file + directory = os.path.dirname(__file__) + # we must use a wininst-x.y.exe built with the same C compiler + # used for python. XXX What about mingw, borland, and so on? + + # if plat_name starts with "win" but is not "win32" + # we want to strip "win" and leave the rest (e.g. -amd64) + # for all other cases, we don't want any suffix + if self.plat_name != 'win32' and self.plat_name[:3] == 'win': + sfix = self.plat_name[3:] + else: + sfix = '' + + filename = os.path.join(directory, "wininst-%.1f%s.exe" % (bv, sfix)) + with open(filename, "rb") as fp: + return fp.read() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/build.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/build.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,151 @@ +"""Main build command, which calls the other build_* commands.""" + +import sys +import os + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError +from packaging.compiler import show_compilers + + +class build(Command): + + description = "build everything needed to install" + + user_options = [ + ('build-base=', 'b', + "base directory for build library"), + ('build-purelib=', None, + "build directory for platform-neutral distributions"), + ('build-platlib=', None, + "build directory for platform-specific distributions"), + ('build-lib=', None, + "build directory for all distribution (defaults to either " + + "build-purelib or build-platlib"), + ('build-scripts=', None, + "build directory for scripts"), + ('build-temp=', 't', + "temporary build directory"), + ('plat-name=', 'p', + "platform name to build for, if supported " + "(default: %s)" % get_platform()), + ('compiler=', 'c', + "specify the compiler type"), + ('debug', 'g', + "compile extensions and libraries with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('executable=', 'e', + "specify final destination interpreter path (build.py)"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in separate text files"), + ('use-2to3-fixers', None, + "list additional fixers opted for during 2to3 conversion"), + ] + + boolean_options = ['debug', 'force'] + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.build_base = 'build' + # these are decided only after 'build_base' has its final value + # (unless overridden by the user or client) + self.build_purelib = None + self.build_platlib = None + self.build_lib = None + self.build_temp = None + self.build_scripts = None + self.compiler = None + self.plat_name = None + self.debug = None + self.force = False + self.executable = None + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + if self.plat_name is None: + self.plat_name = get_platform() + else: + # plat-name only supported for windows (other platforms are + # supported via ./configure flags, if at all). Avoid misleading + # other platforms. + if os.name != 'nt': + raise PackagingOptionError( + "--plat-name only supported on Windows (try " + "using './configure --help' on your platform)") + pyversion = '%s.%s' % sys.version_info[:2] + plat_specifier = ".%s-%s" % (self.plat_name, pyversion) + + # Make it so Python 2.x and Python 2.x with --with-pydebug don't + # share the same build directories. Doing so confuses the build + # process for C modules + if hasattr(sys, 'gettotalrefcount'): + plat_specifier += '-pydebug' + + # 'build_purelib' and 'build_platlib' just default to 'lib' and + # 'lib.' under the base build directory. We only use one of + # them for a given distribution, though -- + if self.build_purelib is None: + self.build_purelib = os.path.join(self.build_base, 'lib') + if self.build_platlib is None: + self.build_platlib = os.path.join(self.build_base, + 'lib' + plat_specifier) + + # 'build_lib' is the actual directory that we will use for this + # particular module distribution -- if user didn't supply it, pick + # one of 'build_purelib' or 'build_platlib'. + if self.build_lib is None: + if self.distribution.ext_modules: + self.build_lib = self.build_platlib + else: + self.build_lib = self.build_purelib + + # 'build_temp' -- temporary directory for compiler turds, + # "build/temp." + if self.build_temp is None: + self.build_temp = os.path.join(self.build_base, + 'temp' + plat_specifier) + if self.build_scripts is None: + self.build_scripts = os.path.join(self.build_base, + 'scripts-' + pyversion) + + if self.executable is None: + self.executable = os.path.normpath(sys.executable) + + def run(self): + # Run all relevant sub-commands. This will be some subset of: + # - build_py - pure Python modules + # - build_clib - standalone C libraries + # - build_ext - Python extension modules + # - build_scripts - Python scripts + for cmd_name in self.get_sub_commands(): + self.run_command(cmd_name) + + # -- Predicates for the sub-command list --------------------------- + + def has_pure_modules(self): + return self.distribution.has_pure_modules() + + def has_c_libraries(self): + return self.distribution.has_c_libraries() + + def has_ext_modules(self): + return self.distribution.has_ext_modules() + + def has_scripts(self): + return self.distribution.has_scripts() + + sub_commands = [('build_py', has_pure_modules), + ('build_clib', has_c_libraries), + ('build_ext', has_ext_modules), + ('build_scripts', has_scripts), + ] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/build_clib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/build_clib.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,197 @@ +"""Build C/C++ libraries. + +This command is useful to build libraries that are included in the +distribution and needed by extension modules. +""" + +# XXX this module has *lots* of code ripped-off quite transparently from +# build_ext.py -- not surprisingly really, as the work required to build +# a static library from a collection of C source files is not really all +# that different from what's required to build a shared object file from +# a collection of C source files. Nevertheless, I haven't done the +# necessary refactoring to account for the overlap in code between the +# two modules, mainly because a number of subtle details changed in the +# cut 'n paste. Sigh. + +import os +from packaging.command.cmd import Command +from packaging.errors import PackagingSetupError +from packaging.compiler import customize_compiler, new_compiler +from packaging import logger + + +def show_compilers(): + from packaging.compiler import show_compilers + show_compilers() + + +class build_clib(Command): + + description = "build C/C++ libraries used by extension modules" + + user_options = [ + ('build-clib=', 'b', + "directory to build C/C++ libraries to"), + ('build-temp=', 't', + "directory to put temporary build by-products"), + ('debug', 'g', + "compile with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('compiler=', 'c', + "specify the compiler type"), + ] + + boolean_options = ['debug', 'force'] + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.build_clib = None + self.build_temp = None + + # List of libraries to build + self.libraries = None + + # Compilation options for all libraries + self.include_dirs = None + self.define = None + self.undef = None + self.debug = None + self.force = False + self.compiler = None + + + def finalize_options(self): + # This might be confusing: both build-clib and build-temp default + # to build-temp as defined by the "build" command. This is because + # I think that C libraries are really just temporary build + # by-products, at least from the point of view of building Python + # extensions -- but I want to keep my options open. + self.set_undefined_options('build', + ('build_temp', 'build_clib'), + ('build_temp', 'build_temp'), + 'compiler', 'debug', 'force') + + self.libraries = self.distribution.libraries + if self.libraries: + self.check_library_list(self.libraries) + + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + if isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + # XXX same as for build_ext -- what about 'self.define' and + # 'self.undef' ? + + def run(self): + if not self.libraries: + return + + # Yech -- this is cut 'n pasted from build_ext.py! + self.compiler = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, + force=self.force) + customize_compiler(self.compiler) + + if self.include_dirs is not None: + self.compiler.set_include_dirs(self.include_dirs) + if self.define is not None: + # 'define' option is a list of (name,value) tuples + for name, value in self.define: + self.compiler.define_macro(name, value) + if self.undef is not None: + for macro in self.undef: + self.compiler.undefine_macro(macro) + + self.build_libraries(self.libraries) + + + def check_library_list(self, libraries): + """Ensure that the list of libraries is valid. + + `library` is presumably provided as a command option 'libraries'. + This method checks that it is a list of 2-tuples, where the tuples + are (library_name, build_info_dict). + + Raise PackagingSetupError if the structure is invalid anywhere; + just returns otherwise. + """ + if not isinstance(libraries, list): + raise PackagingSetupError("'libraries' option must be a list of tuples") + + for lib in libraries: + if not isinstance(lib, tuple) and len(lib) != 2: + raise PackagingSetupError("each element of 'libraries' must a 2-tuple") + + name, build_info = lib + + if not isinstance(name, str): + raise PackagingSetupError("first element of each tuple in 'libraries' " + \ + "must be a string (the library name)") + if '/' in name or (os.sep != '/' and os.sep in name): + raise PackagingSetupError(("bad library name '%s': " + + "may not contain directory separators") % \ + lib[0]) + + if not isinstance(build_info, dict): + raise PackagingSetupError("second element of each tuple in 'libraries' " + \ + "must be a dictionary (build info)") + + def get_library_names(self): + # Assume the library list is valid -- 'check_library_list()' is + # called from 'finalize_options()', so it should be! + if not self.libraries: + return None + + lib_names = [] + for lib_name, build_info in self.libraries: + lib_names.append(lib_name) + return lib_names + + + def get_source_files(self): + self.check_library_list(self.libraries) + filenames = [] + for lib_name, build_info in self.libraries: + sources = build_info.get('sources') + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'libraries' option (library '%s'), " + "'sources' must be present and must be " + "a list of source filenames") % lib_name) + + filenames.extend(sources) + return filenames + + def build_libraries(self, libraries): + for lib_name, build_info in libraries: + sources = build_info.get('sources') + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'libraries' option (library '%s'), " + + "'sources' must be present and must be " + + "a list of source filenames") % lib_name) + sources = list(sources) + + logger.info("building '%s' library", lib_name) + + # First, compile the source code to object files in the library + # directory. (This should probably change to putting object + # files in a temporary build directory.) + macros = build_info.get('macros') + include_dirs = build_info.get('include_dirs') + objects = self.compiler.compile(sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=include_dirs, + debug=self.debug) + + # Now "link" the object files together into a static library. + # (On Unix at least, this isn't really linking -- it just + # builds an archive. Whatever.) + self.compiler.create_static_lib(objects, lib_name, + output_dir=self.build_clib, + debug=self.debug) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/build_ext.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/build_ext.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,641 @@ +"""Build extension modules.""" + +import os +import re +import sys +import site +import sysconfig + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import (CCompilerError, CompileError, PackagingError, + PackagingPlatformError, PackagingSetupError) +from packaging.compiler import customize_compiler, show_compilers +from packaging.util import newer_group +from packaging.compiler.extension import Extension +from packaging import logger + +if os.name == 'nt': + from packaging.compiler.msvccompiler import get_build_version + MSVC_VERSION = int(get_build_version()) + +# An extension name is just a dot-separated list of Python NAMEs (ie. +# the same as a fully-qualified module name). +extension_name_re = re.compile \ + (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') + + +class build_ext(Command): + + description = "build C/C++ extension modules (compile/link to build directory)" + + # XXX thoughts on how to deal with complex command-line options like + # these, i.e. how to make it so fancy_getopt can suck them off the + # command line and turn them into the appropriate + # lists of tuples of what-have-you. + # - each command needs a callback to process its command-line options + # - Command.__init__() needs access to its share of the whole + # command line (must ultimately come from + # Distribution.parse_command_line()) + # - it then calls the current command class' option-parsing + # callback to deal with weird options like -D, which have to + # parse the option text and churn out some custom data + # structure + # - that data structure (in this case, a list of 2-tuples) + # will then be present in the command object by the time + # we get to finalize_options() (i.e. the constructor + # takes care of both command-line and client options + # in between initialize_options() and finalize_options()) + + sep_by = " (separated by '%s')" % os.pathsep + user_options = [ + ('build-lib=', 'b', + "directory for compiled extension modules"), + ('build-temp=', 't', + "directory for temporary files (build by-products)"), + ('plat-name=', 'p', + "platform name to cross-compile for, if supported " + "(default: %s)" % get_platform()), + ('inplace', 'i', + "ignore build-lib and put compiled extensions into the source " + + "directory alongside your pure Python modules"), + ('user', None, + "add user include, library and rpath"), + ('include-dirs=', 'I', + "list of directories to search for header files" + sep_by), + ('define=', 'D', + "C preprocessor macros to define"), + ('undef=', 'U', + "C preprocessor macros to undefine"), + ('libraries=', 'l', + "external C libraries to link with"), + ('library-dirs=', 'L', + "directories to search for external C libraries" + sep_by), + ('rpath=', 'R', + "directories to search for shared C libraries at runtime"), + ('link-objects=', 'O', + "extra explicit link objects to include in the link"), + ('debug', 'g', + "compile/link with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('compiler=', 'c', + "specify the compiler type"), + ('swig-opts=', None, + "list of SWIG command-line options"), + ('swig=', None, + "path to the SWIG executable"), + ] + + boolean_options = ['inplace', 'debug', 'force', 'user'] + + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.extensions = None + self.build_lib = None + self.plat_name = None + self.build_temp = None + self.inplace = False + self.package = None + + self.include_dirs = None + self.define = None + self.undef = None + self.libraries = None + self.library_dirs = None + self.rpath = None + self.link_objects = None + self.debug = None + self.force = None + self.compiler = None + self.swig = None + self.swig_opts = None + self.user = None + + def finalize_options(self): + self.set_undefined_options('build', + 'build_lib', 'build_temp', 'compiler', + 'debug', 'force', 'plat_name') + + if self.package is None: + self.package = self.distribution.ext_package + + # Ensure that the list of extensions is valid, i.e. it is a list of + # Extension objects. + self.extensions = self.distribution.ext_modules + if self.extensions: + if not isinstance(self.extensions, (list, tuple)): + type_name = (self.extensions is None and 'None' + or type(self.extensions).__name__) + raise PackagingSetupError( + "'ext_modules' must be a sequence of Extension instances," + " not %s" % (type_name,)) + for i, ext in enumerate(self.extensions): + if isinstance(ext, Extension): + continue # OK! (assume type-checking done + # by Extension constructor) + type_name = (ext is None and 'None' or type(ext).__name__) + raise PackagingSetupError( + "'ext_modules' item %d must be an Extension instance," + " not %s" % (i, type_name)) + + # Make sure Python's include directories (for Python.h, pyconfig.h, + # etc.) are in the include search path. + py_include = sysconfig.get_path('include') + plat_py_include = sysconfig.get_path('platinclude') + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + if isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + # Put the Python "system" include dir at the end, so that + # any local include dirs take precedence. + self.include_dirs.append(py_include) + if plat_py_include != py_include: + self.include_dirs.append(plat_py_include) + + self.ensure_string_list('libraries') + + # Life is easier if we're not forever checking for None, so + # simplify these options to empty lists if unset + if self.libraries is None: + self.libraries = [] + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, str): + self.library_dirs = self.library_dirs.split(os.pathsep) + + if self.rpath is None: + self.rpath = [] + elif isinstance(self.rpath, str): + self.rpath = self.rpath.split(os.pathsep) + + # for extensions under windows use different directories + # for Release and Debug builds. + # also Python's library directory must be appended to library_dirs + if os.name == 'nt': + # the 'libs' directory is for binary installs - we assume that + # must be the *native* platform. But we don't really support + # cross-compiling via a binary install anyway, so we let it go. + self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + if self.debug: + self.build_temp = os.path.join(self.build_temp, "Debug") + else: + self.build_temp = os.path.join(self.build_temp, "Release") + + # Append the source distribution include and library directories, + # this allows distutils on windows to work in the source tree + self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + if MSVC_VERSION == 9: + # Use the .lib files for the correct architecture + if self.plat_name == 'win32': + suffix = '' + else: + # win-amd64 or win-ia64 + suffix = self.plat_name[4:] + new_lib = os.path.join(sys.exec_prefix, 'PCbuild') + if suffix: + new_lib = os.path.join(new_lib, suffix) + self.library_dirs.append(new_lib) + + elif MSVC_VERSION == 8: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS8.0')) + elif MSVC_VERSION == 7: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS7.1')) + else: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VC6')) + + # OS/2 (EMX) doesn't support Debug vs Release builds, but has the + # import libraries in its "Config" subdirectory + if os.name == 'os2': + self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) + + # for extensions under Cygwin and AtheOS Python's library directory must be + # appended to library_dirs + if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(os.path.join(sys.prefix, "lib", + "python" + sysconfig.get_python_version(), + "config")) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # for extensions under Linux or Solaris with a shared Python library, + # Python's library directory must be appended to library_dirs + sysconfig.get_config_var('Py_ENABLE_SHARED') + if (sys.platform.startswith(('linux', 'gnu', 'sunos')) + and sysconfig.get_config_var('Py_ENABLE_SHARED')): + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # The argument parsing will result in self.define being a string, but + # it has to be a list of 2-tuples. All the preprocessor symbols + # specified by the 'define' option will be set to '1'. Multiple + # symbols can be separated with commas. + + if self.define: + defines = self.define.split(',') + self.define = [(symbol, '1') for symbol in defines] + + # The option for macros to undefine is also a string from the + # option parsing, but has to be a list. Multiple symbols can also + # be separated with commas here. + if self.undef: + self.undef = self.undef.split(',') + + if self.swig_opts is None: + self.swig_opts = [] + else: + self.swig_opts = self.swig_opts.split(' ') + + # Finally add the user include and library directories if requested + if self.user: + user_include = os.path.join(site.USER_BASE, "include") + user_lib = os.path.join(site.USER_BASE, "lib") + if os.path.isdir(user_include): + self.include_dirs.append(user_include) + if os.path.isdir(user_lib): + self.library_dirs.append(user_lib) + self.rpath.append(user_lib) + + def run(self): + from packaging.compiler import new_compiler + + if not self.extensions: + return + + # If we were asked to build any C/C++ libraries, make sure that the + # directory where we put them is in the library search path for + # linking extensions. + if self.distribution.has_c_libraries(): + build_clib = self.get_finalized_command('build_clib') + self.libraries.extend(build_clib.get_library_names() or []) + self.library_dirs.append(build_clib.build_clib) + + # Setup the CCompiler object that we'll use to do all the + # compiling and linking + self.compiler_obj = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, + force=self.force) + + customize_compiler(self.compiler_obj) + # If we are cross-compiling, init the compiler now (if we are not + # cross-compiling, init would not hurt, but people may rely on + # late initialization of compiler even if they shouldn't...) + if os.name == 'nt' and self.plat_name != get_platform(): + self.compiler_obj.initialize(self.plat_name) + + # And make sure that any compile/link-related options (which might + # come from the command line or from the setup script) are set in + # that CCompiler object -- that way, they automatically apply to + # all compiling and linking done here. + if self.include_dirs is not None: + self.compiler_obj.set_include_dirs(self.include_dirs) + if self.define is not None: + # 'define' option is a list of (name,value) tuples + for name, value in self.define: + self.compiler_obj.define_macro(name, value) + if self.undef is not None: + for macro in self.undef: + self.compiler_obj.undefine_macro(macro) + if self.libraries is not None: + self.compiler_obj.set_libraries(self.libraries) + if self.library_dirs is not None: + self.compiler_obj.set_library_dirs(self.library_dirs) + if self.rpath is not None: + self.compiler_obj.set_runtime_library_dirs(self.rpath) + if self.link_objects is not None: + self.compiler_obj.set_link_objects(self.link_objects) + + # Now actually compile and link everything. + self.build_extensions() + + def get_source_files(self): + filenames = [] + + # Wouldn't it be neat if we knew the names of header files too... + for ext in self.extensions: + filenames.extend(ext.sources) + + return filenames + + def get_outputs(self): + # And build the list of output (built) filenames. Note that this + # ignores the 'inplace' flag, and assumes everything goes in the + # "build" tree. + outputs = [] + for ext in self.extensions: + outputs.append(self.get_ext_fullpath(ext.name)) + return outputs + + def build_extensions(self): + for ext in self.extensions: + try: + self.build_extension(ext) + except (CCompilerError, PackagingError, CompileError) as e: + if not ext.optional: + raise + logger.warning('%s: building extension %r failed: %s', + self.get_command_name(), ext.name, e) + + def build_extension(self, ext): + sources = ext.sources + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " + + "'sources' must be present and must be " + + "a list of source filenames") % ext.name) + sources = list(sources) + + ext_path = self.get_ext_fullpath(ext.name) + depends = sources + ext.depends + if not (self.force or newer_group(depends, ext_path, 'newer')): + logger.debug("skipping '%s' extension (up-to-date)", ext.name) + return + else: + logger.info("building '%s' extension", ext.name) + + # First, scan the sources for SWIG definition files (.i), run + # SWIG on 'em to create .c files, and modify the sources list + # accordingly. + sources = self.swig_sources(sources, ext) + + # Next, compile the source code to object files. + + # XXX not honouring 'define_macros' or 'undef_macros' -- the + # CCompiler API needs to change to accommodate this, and I + # want to do one thing at a time! + + # Two possible sources for extra compiler arguments: + # - 'extra_compile_args' in Extension object + # - CFLAGS environment variable (not particularly + # elegant, but people seem to expect it and I + # guess it's useful) + # The environment variable should take precedence, and + # any sensible compiler will give precedence to later + # command-line args. Hence we combine them in order: + extra_args = ext.extra_compile_args or [] + + macros = ext.define_macros[:] + for undef in ext.undef_macros: + macros.append((undef,)) + + objects = self.compiler_obj.compile(sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=ext.include_dirs, + debug=self.debug, + extra_postargs=extra_args, + depends=ext.depends) + + # XXX -- this is a Vile HACK! + # + # The setup.py script for Python on Unix needs to be able to + # get this list so it can perform all the clean up needed to + # avoid keeping object files around when cleaning out a failed + # build of an extension module. Since Packaging does not + # track dependencies, we have to get rid of intermediates to + # ensure all the intermediates will be properly re-built. + # + self._built_objects = objects[:] + + # Now link the object files together into a "shared object" -- + # of course, first we have to figure out all the other things + # that go into the mix. + if ext.extra_objects: + objects.extend(ext.extra_objects) + extra_args = ext.extra_link_args or [] + + # Detect target language, if not provided + language = ext.language or self.compiler_obj.detect_language(sources) + + self.compiler_obj.link_shared_object( + objects, ext_path, + libraries=self.get_libraries(ext), + library_dirs=ext.library_dirs, + runtime_library_dirs=ext.runtime_library_dirs, + extra_postargs=extra_args, + export_symbols=self.get_export_symbols(ext), + debug=self.debug, + build_temp=self.build_temp, + target_lang=language) + + + def swig_sources(self, sources, extension): + """Walk the list of source files in 'sources', looking for SWIG + interface (.i) files. Run SWIG on all that are found, and + return a modified 'sources' list with SWIG source files replaced + by the generated C (or C++) files. + """ + new_sources = [] + swig_sources = [] + swig_targets = {} + + # XXX this drops generated C/C++ files into the source tree, which + # is fine for developers who want to distribute the generated + # source -- but there should be an option to put SWIG output in + # the temp dir. + + if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts): + target_ext = '.cpp' + else: + target_ext = '.c' + + for source in sources: + base, ext = os.path.splitext(source) + if ext == ".i": # SWIG interface file + new_sources.append(base + '_wrap' + target_ext) + swig_sources.append(source) + swig_targets[source] = new_sources[-1] + else: + new_sources.append(source) + + if not swig_sources: + return new_sources + + swig = self.swig or self.find_swig() + swig_cmd = [swig, "-python"] + swig_cmd.extend(self.swig_opts) + + # Do not override commandline arguments + if not self.swig_opts: + for o in extension.swig_opts: + swig_cmd.append(o) + + for source in swig_sources: + target = swig_targets[source] + logger.info("swigging %s to %s", source, target) + self.spawn(swig_cmd + ["-o", target, source]) + + return new_sources + + def find_swig(self): + """Return the name of the SWIG executable. On Unix, this is + just "swig" -- it should be in the PATH. Tries a bit harder on + Windows. + """ + + if os.name == "posix": + return "swig" + elif os.name == "nt": + + # Look for SWIG in its standard installation directory on + # Windows (or so I presume!). If we find it there, great; + # if not, act like Unix and assume it's in the PATH. + for vers in ("1.3", "1.2", "1.1"): + fn = os.path.join("c:\\swig%s" % vers, "swig.exe") + if os.path.isfile(fn): + return fn + else: + return "swig.exe" + + elif os.name == "os2": + # assume swig available in the PATH. + return "swig.exe" + + else: + raise PackagingPlatformError(("I don't know how to find (much less run) SWIG " + "on platform '%s'") % os.name) + + # -- Name generators ----------------------------------------------- + # (extension names, filenames, whatever) + def get_ext_fullpath(self, ext_name): + """Returns the path of the filename for a given extension. + + The file is located in `build_lib` or directly in the package + (inplace option). + """ + fullname = self.get_ext_fullname(ext_name) + modpath = fullname.split('.') + filename = self.get_ext_filename(modpath[-1]) + + if not self.inplace: + # no further work needed + # returning : + # build_dir/package/path/filename + filename = os.path.join(*modpath[:-1]+[filename]) + return os.path.join(self.build_lib, filename) + + # the inplace option requires to find the package directory + # using the build_py command for that + package = '.'.join(modpath[0:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = os.path.abspath(build_py.get_package_dir(package)) + + # returning + # package_dir/filename + return os.path.join(package_dir, filename) + + def get_ext_fullname(self, ext_name): + """Returns the fullname of a given extension name. + + Adds the `package.` prefix""" + if self.package is None: + return ext_name + else: + return self.package + '.' + ext_name + + def get_ext_filename(self, ext_name): + r"""Convert the name of an extension (eg. "foo.bar") into the name + of the file from which it will be loaded (eg. "foo/bar.so", or + "foo\bar.pyd"). + """ + ext_path = ext_name.split('.') + # OS/2 has an 8 character module (extension) limit :-( + if os.name == "os2": + ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] + # extensions in debug_mode are named 'module_d.pyd' under windows + so_ext = sysconfig.get_config_var('SO') + if os.name == 'nt' and self.debug: + return os.path.join(*ext_path) + '_d' + so_ext + return os.path.join(*ext_path) + so_ext + + def get_export_symbols(self, ext): + """Return the list of symbols that a shared extension has to + export. This either uses 'ext.export_symbols' or, if it's not + provided, "init" + module_name. Only relevant on Windows, where + the .pyd file (DLL) must export the module "init" function. + """ + initfunc_name = "PyInit_" + ext.name.split('.')[-1] + if initfunc_name not in ext.export_symbols: + ext.export_symbols.append(initfunc_name) + return ext.export_symbols + + def get_libraries(self, ext): + """Return the list of libraries to link against when building a + shared extension. On most platforms, this is just 'ext.libraries'; + on Windows and OS/2, we add the Python library (eg. python20.dll). + """ + # The python library is always needed on Windows. For MSVC, this + # is redundant, since the library is mentioned in a pragma in + # pyconfig.h that MSVC groks. The other Windows compilers all seem + # to need it mentioned explicitly, though, so that's what we do. + # Append '_d' to the python import library on debug builds. + if sys.platform == "win32": + from packaging.compiler.msvccompiler import MSVCCompiler + if not isinstance(self.compiler_obj, MSVCCompiler): + template = "python%d%d" + if self.debug: + template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + else: + return ext.libraries + elif sys.platform == "os2emx": + # EMX/GCC requires the python library explicitly, and I + # believe VACPP does as well (though not confirmed) - AIM Apr01 + template = "python%d%d" + # debug versions of the main DLL aren't supported, at least + # not at this time - AIM Apr01 + #if self.debug: + # template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "cygwin": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "atheos": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # Get SHLIBS from Makefile + extra = [] + for lib in sysconfig.get_config_var('SHLIBS').split(): + if lib.startswith('-l'): + extra.append(lib[2:]) + else: + extra.append(lib) + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib, "m"] + extra + + elif sys.platform == 'darwin': + # Don't use the default code below + return ext.libraries + + else: + if sysconfig.get_config_var('Py_ENABLE_SHARED'): + template = 'python%d.%d' + sys.abiflags + pythonlib = template % sys.version_info[:2] + return ext.libraries + [pythonlib] + else: + return ext.libraries diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/build_py.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/build_py.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,392 @@ +"""Build pure Python modules (just copy to build directory).""" + +import os +import imp +from glob import glob + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError, PackagingFileError +from packaging.util import convert_path +from packaging.compat import Mixin2to3 + +# marking public APIs +__all__ = ['build_py'] + + +class build_py(Command, Mixin2to3): + + description = "build pure Python modules (copy to build directory)" + + # The options for controlling byte compilation are two independent sets; + # more info in install_lib or the reST docs + + user_options = [ + ('build-lib=', 'd', "directory to build (copy) to"), + ('compile', 'c', "compile .py to .pyc"), + ('no-compile', None, "don't compile .py files [default]"), + ('optimize=', 'O', + "also compile with optimization: -O1 for \"python -O\", " + "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + ('force', 'f', "forcibly build everything (ignore file timestamps)"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in separate text files"), + ('use-2to3-fixers', None, + "list additional fixers opted for during 2to3 conversion"), + ] + + boolean_options = ['compile', 'force'] + + negative_opt = {'no-compile': 'compile'} + + def initialize_options(self): + self.build_lib = None + self.py_modules = None + self.package = None + self.package_data = None + self.package_dir = None + self.compile = False + self.optimize = 0 + self.force = None + self._updated_files = [] + self._doctests_2to3 = [] + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + self.set_undefined_options('build', + 'use_2to3', 'use_2to3_fixers', + 'convert_2to3_doctests', 'build_lib', + 'force') + + # Get the distribution options that are aliases for build_py + # options -- list of packages and list of modules. + self.packages = self.distribution.packages + self.py_modules = self.distribution.py_modules + self.package_data = self.distribution.package_data + self.package_dir = None + if self.distribution.package_dir is not None: + self.package_dir = convert_path(self.distribution.package_dir) + self.data_files = self.get_data_files() + + # Ick, copied straight from install_lib.py (fancy_getopt needs a + # type system! Hell, *everything* needs a type system!!!) + if not isinstance(self.optimize, int): + try: + self.optimize = int(self.optimize) + assert 0 <= self.optimize <= 2 + except (ValueError, AssertionError): + raise PackagingOptionError("optimize must be 0, 1, or 2") + + def run(self): + # XXX copy_file by default preserves atime and mtime. IMHO this is + # the right thing to do, but perhaps it should be an option -- in + # particular, a site administrator might want installed files to + # reflect the time of installation rather than the last + # modification time before the installed release. + + # XXX copy_file by default preserves mode, which appears to be the + # wrong thing to do: if a file is read-only in the working + # directory, we want it to be installed read/write so that the next + # installation of the same module distribution can overwrite it + # without problems. (This might be a Unix-specific issue.) Thus + # we turn off 'preserve_mode' when copying to the build directory, + # since the build directory is supposed to be exactly what the + # installation will look like (ie. we preserve mode when + # installing). + + # Two options control which modules will be installed: 'packages' + # and 'py_modules'. The former lets us work with whole packages, not + # specifying individual modules at all; the latter is for + # specifying modules one-at-a-time. + + if self.py_modules: + self.build_modules() + if self.packages: + self.build_packages() + self.build_package_data() + + if self.use_2to3 and self._updated_files: + self.run_2to3(self._updated_files, self._doctests_2to3, + self.use_2to3_fixers) + + self.byte_compile(self.get_outputs(include_bytecode=False), + prefix=self.build_lib) + + # -- Top-level worker functions ------------------------------------ + + def get_data_files(self): + """Generate list of '(package,src_dir,build_dir,filenames)' tuples. + + Helper function for finalize_options. + """ + data = [] + if not self.packages: + return data + for package in self.packages: + # Locate package source directory + src_dir = self.get_package_dir(package) + + # Compute package build directory + build_dir = os.path.join(*([self.build_lib] + package.split('.'))) + + # Length of path to strip from found files + plen = 0 + if src_dir: + plen = len(src_dir) + 1 + + # Strip directory from globbed filenames + filenames = [ + file[plen:] for file in self.find_data_files(package, src_dir) + ] + data.append((package, src_dir, build_dir, filenames)) + return data + + def find_data_files(self, package, src_dir): + """Return filenames for package's data files in 'src_dir'. + + Helper function for get_data_files. + """ + globs = (self.package_data.get('', []) + + self.package_data.get(package, [])) + files = [] + for pattern in globs: + # Each pattern has to be converted to a platform-specific path + filelist = glob(os.path.join(src_dir, convert_path(pattern))) + # Files that match more than one pattern are only added once + files.extend(fn for fn in filelist if fn not in files) + return files + + def build_package_data(self): + """Copy data files into build directory. + + Helper function for run. + """ + # FIXME add tests for this method + for package, src_dir, build_dir, filenames in self.data_files: + for filename in filenames: + target = os.path.join(build_dir, filename) + srcfile = os.path.join(src_dir, filename) + self.mkpath(os.path.dirname(target)) + outf, copied = self.copy_file(srcfile, + target, preserve_mode=False) + doctests = self.distribution.convert_2to3_doctests + if copied and srcfile in doctests: + self._doctests_2to3.append(outf) + + # XXX - this should be moved to the Distribution class as it is not + # only needed for build_py. It also has no dependencies on this class. + def get_package_dir(self, package): + """Return the directory, relative to the top of the source + distribution, where package 'package' should be found + (at least according to the 'package_dir' option, if any). + """ + path = package.split('.') + if self.package_dir is not None: + path.insert(0, self.package_dir) + + if len(path) > 0: + return os.path.join(*path) + + return '' + + def check_package(self, package, package_dir): + """Helper function for find_package_modules and find_modules.""" + # Empty dir name means current directory, which we can probably + # assume exists. Also, os.path.exists and isdir don't know about + # my "empty string means current dir" convention, so we have to + # circumvent them. + if package_dir != "": + if not os.path.exists(package_dir): + raise PackagingFileError( + "package directory '%s' does not exist" % package_dir) + if not os.path.isdir(package_dir): + raise PackagingFileError( + "supposed package directory '%s' exists, " + "but is not a directory" % package_dir) + + # Require __init__.py for all but the "root package" + if package: + init_py = os.path.join(package_dir, "__init__.py") + if os.path.isfile(init_py): + return init_py + else: + logger.warning("package init file %r not found " + "(or not a regular file)", init_py) + + # Either not in a package at all (__init__.py not expected), or + # __init__.py doesn't exist -- so don't return the filename. + return None + + def check_module(self, module, module_file): + if not os.path.isfile(module_file): + logger.warning("file %r (for module %r) not found", + module_file, module) + return False + else: + return True + + def find_package_modules(self, package, package_dir): + self.check_package(package, package_dir) + module_files = glob(os.path.join(package_dir, "*.py")) + modules = [] + if self.distribution.script_name is not None: + setup_script = os.path.abspath(self.distribution.script_name) + else: + setup_script = None + + for f in module_files: + abs_f = os.path.abspath(f) + if abs_f != setup_script: + module = os.path.splitext(os.path.basename(f))[0] + modules.append((package, module, f)) + else: + logger.debug("excluding %r", setup_script) + return modules + + def find_modules(self): + """Finds individually-specified Python modules, ie. those listed by + module name in 'self.py_modules'. Returns a list of tuples (package, + module_base, filename): 'package' is a tuple of the path through + package-space to the module; 'module_base' is the bare (no + packages, no dots) module name, and 'filename' is the path to the + ".py" file (relative to the distribution root) that implements the + module. + """ + # Map package names to tuples of useful info about the package: + # (package_dir, checked) + # package_dir - the directory where we'll find source files for + # this package + # checked - true if we have checked that the package directory + # is valid (exists, contains __init__.py, ... ?) + packages = {} + + # List of (package, module, filename) tuples to return + modules = [] + + # We treat modules-in-packages almost the same as toplevel modules, + # just the "package" for a toplevel is empty (either an empty + # string or empty list, depending on context). Differences: + # - don't check for __init__.py in directory for empty package + for module in self.py_modules: + path = module.split('.') + package = '.'.join(path[0:-1]) + module_base = path[-1] + + try: + package_dir, checked = packages[package] + except KeyError: + package_dir = self.get_package_dir(package) + checked = False + + if not checked: + init_py = self.check_package(package, package_dir) + packages[package] = (package_dir, 1) + if init_py: + modules.append((package, "__init__", init_py)) + + # XXX perhaps we should also check for just .pyc files + # (so greedy closed-source bastards can distribute Python + # modules too) + module_file = os.path.join(package_dir, module_base + ".py") + if not self.check_module(module, module_file): + continue + + modules.append((package, module_base, module_file)) + + return modules + + def find_all_modules(self): + """Compute the list of all modules that will be built, whether + they are specified one-module-at-a-time ('self.py_modules') or + by whole packages ('self.packages'). Return a list of tuples + (package, module, module_file), just like 'find_modules()' and + 'find_package_modules()' do.""" + modules = [] + if self.py_modules: + modules.extend(self.find_modules()) + if self.packages: + for package in self.packages: + package_dir = self.get_package_dir(package) + m = self.find_package_modules(package, package_dir) + modules.extend(m) + return modules + + def get_source_files(self): + sources = [module[-1] for module in self.find_all_modules()] + sources += [ + os.path.join(src_dir, filename) + for package, src_dir, build_dir, filenames in self.data_files + for filename in filenames] + return sources + + def get_module_outfile(self, build_dir, package, module): + outfile_path = [build_dir] + list(package) + [module + ".py"] + return os.path.join(*outfile_path) + + def get_outputs(self, include_bytecode=True): + modules = self.find_all_modules() + outputs = [] + for package, module, module_file in modules: + package = package.split('.') + filename = self.get_module_outfile(self.build_lib, package, module) + outputs.append(filename) + if include_bytecode: + if self.compile: + outputs.append(imp.cache_from_source(filename, True)) + if self.optimize: + outputs.append(imp.cache_from_source(filename, False)) + + outputs += [ + os.path.join(build_dir, filename) + for package, src_dir, build_dir, filenames in self.data_files + for filename in filenames] + + return outputs + + def build_module(self, module, module_file, package): + if isinstance(package, str): + package = package.split('.') + elif not isinstance(package, (list, tuple)): + raise TypeError( + "'package' must be a string (dot-separated), list, or tuple") + + # Now put the module source file into the "build" area -- this is + # easy, we just copy it somewhere under self.build_lib (the build + # directory for Python source). + outfile = self.get_module_outfile(self.build_lib, package, module) + dir = os.path.dirname(outfile) + self.mkpath(dir) + return self.copy_file(module_file, outfile, preserve_mode=False) + + def build_modules(self): + modules = self.find_modules() + for package, module, module_file in modules: + # Now "build" the module -- ie. copy the source file to + # self.build_lib (the build directory for Python source). + # (Actually, it gets copied to the directory for this package + # under self.build_lib.) + self.build_module(module, module_file, package) + + def build_packages(self): + for package in self.packages: + # Get list of (package, module, module_file) tuples based on + # scanning the package directory. 'package' is only included + # in the tuple so that 'find_modules()' and + # 'find_package_tuples()' have a consistent interface; it's + # ignored here (apart from a sanity check). Also, 'module' is + # the *unqualified* module name (ie. no dots, no package -- we + # already know its package!), and 'module_file' is the path to + # the .py file, relative to the current directory + # (ie. including 'package_dir'). + package_dir = self.get_package_dir(package) + modules = self.find_package_modules(package, package_dir) + + # Now loop over the modules we found, "building" each one (just + # copy it to self.build_lib). + for package_, module, module_file in modules: + assert package == package_ + self.build_module(module, module_file, package) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/build_scripts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/build_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,154 @@ +"""Build scripts (copy to build dir and fix up shebang line).""" + +import os +import re +import sysconfig +from tokenize import detect_encoding + +from packaging.command.cmd import Command +from packaging.util import convert_path, newer +from packaging import logger +from packaging.compat import Mixin2to3 + + +# check if Python is called on the first line with this expression +first_line_re = re.compile(b'^#!.*python[0-9.]*([ \t].*)?$') + +class build_scripts(Command, Mixin2to3): + + description = "build scripts (copy and fix up shebang line)" + + user_options = [ + ('build-dir=', 'd', "directory to build (copy) to"), + ('force', 'f', "forcibly build everything (ignore file timestamps"), + ('executable=', 'e', "specify final destination interpreter path"), + ] + + boolean_options = ['force'] + + + def initialize_options(self): + self.build_dir = None + self.scripts = None + self.force = None + self.executable = None + self.outfiles = None + self.use_2to3 = False + self.convert_2to3_doctests = None + self.use_2to3_fixers = None + + def finalize_options(self): + self.set_undefined_options('build', + ('build_scripts', 'build_dir'), + 'use_2to3', 'use_2to3_fixers', + 'convert_2to3_doctests', 'force', + 'executable') + self.scripts = self.distribution.scripts + + def get_source_files(self): + return self.scripts + + def run(self): + if not self.scripts: + return + copied_files = self.copy_scripts() + if self.use_2to3 and copied_files: + self._run_2to3(copied_files, fixers=self.use_2to3_fixers) + + def copy_scripts(self): + """Copy each script listed in 'self.scripts'; if it's marked as a + Python script in the Unix way (first line matches 'first_line_re', + ie. starts with "\#!" and contains "python"), then adjust the first + line to refer to the current Python interpreter as we copy. + """ + self.mkpath(self.build_dir) + outfiles = [] + for script in self.scripts: + adjust = False + script = convert_path(script) + outfile = os.path.join(self.build_dir, os.path.basename(script)) + outfiles.append(outfile) + + if not self.force and not newer(script, outfile): + logger.debug("not copying %s (up-to-date)", script) + continue + + # Always open the file, but ignore failures in dry-run mode -- + # that way, we'll get accurate feedback if we can read the + # script. + try: + f = open(script, "rb") + except IOError: + if not self.dry_run: + raise + f = None + else: + encoding, lines = detect_encoding(f.readline) + f.seek(0) + first_line = f.readline() + if not first_line: + logger.warning('%s: %s is an empty file (skipping)', + self.get_command_name(), script) + continue + + match = first_line_re.match(first_line) + if match: + adjust = True + post_interp = match.group(1) or b'' + + if adjust: + logger.info("copying and adjusting %s -> %s", script, + self.build_dir) + if not self.dry_run: + if not sysconfig.is_python_build(): + executable = self.executable + else: + executable = os.path.join( + sysconfig.get_config_var("BINDIR"), + "python%s%s" % (sysconfig.get_config_var("VERSION"), + sysconfig.get_config_var("EXE"))) + executable = os.fsencode(executable) + shebang = b"#!" + executable + post_interp + b"\n" + # Python parser starts to read a script using UTF-8 until + # it gets a #coding:xxx cookie. The shebang has to be the + # first line of a file, the #coding:xxx cookie cannot be + # written before. So the shebang has to be decodable from + # UTF-8. + try: + shebang.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + "The shebang ({!r}) is not decodable " + "from utf-8".format(shebang)) + # If the script is encoded to a custom encoding (use a + # #coding:xxx cookie), the shebang has to be decodable from + # the script encoding too. + try: + shebang.decode(encoding) + except UnicodeDecodeError: + raise ValueError( + "The shebang ({!r}) is not decodable " + "from the script encoding ({})" + .format(shebang, encoding)) + with open(outfile, "wb") as outf: + outf.write(shebang) + outf.writelines(f.readlines()) + if f: + f.close() + else: + if f: + f.close() + self.copy_file(script, outfile) + + if os.name == 'posix': + for file in outfiles: + if self.dry_run: + logger.info("changing mode of %s", file) + else: + oldmode = os.stat(file).st_mode & 0o7777 + newmode = (oldmode | 0o555) & 0o7777 + if newmode != oldmode: + logger.info("changing mode of %s from %o to %o", + file, oldmode, newmode) + os.chmod(file, newmode) + return outfiles diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/check.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/check.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,88 @@ +"""Check PEP compliance of metadata.""" + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingSetupError +from packaging.util import resolve_name + +class check(Command): + + description = "check PEP compliance of metadata" + + user_options = [('metadata', 'm', 'Verify metadata'), + ('all', 'a', + ('runs extended set of checks')), + ('strict', 's', + 'Will exit with an error if a check fails')] + + boolean_options = ['metadata', 'all', 'strict'] + + def initialize_options(self): + """Sets default values for options.""" + self.all = False + self.metadata = True + self.strict = False + self._warnings = [] + + def finalize_options(self): + pass + + def warn(self, msg, *args): + """Wrapper around logging that also remembers messages.""" + # XXX we could use a special handler for this, but would need to test + # if it works even if the logger has a too high level + self._warnings.append((msg, args)) + return logger.warning('%s: %s' % (self.get_command_name(), msg), *args) + + def run(self): + """Runs the command.""" + # perform the various tests + if self.metadata: + self.check_metadata() + if self.all: + self.check_restructuredtext() + self.check_hooks_resolvable() + + # let's raise an error in strict mode, if we have at least + # one warning + if self.strict and len(self._warnings) > 0: + msg = '\n'.join(msg % args for msg, args in self._warnings) + raise PackagingSetupError(msg) + + def check_metadata(self): + """Ensures that all required elements of metadata are supplied. + + name, version, URL, author + + Warns if any are missing. + """ + missing, warnings = self.distribution.metadata.check(strict=True) + if missing != []: + self.warn('missing required metadata: %s', ', '.join(missing)) + for warning in warnings: + self.warn(warning) + + def check_restructuredtext(self): + """Checks if the long string fields are reST-compliant.""" + missing, warnings = self.distribution.metadata.check(restructuredtext=True) + if self.distribution.metadata.docutils_support: + for warning in warnings: + line = warning[-1].get('line') + if line is None: + warning = warning[1] + else: + warning = '%s (line %s)' % (warning[1], line) + self.warn(warning) + elif self.strict: + raise PackagingSetupError('The docutils package is needed.') + + def check_hooks_resolvable(self): + for options in self.distribution.command_options.values(): + for hook_kind in ("pre_hook", "post_hook"): + if hook_kind not in options: + break + for hook_name in options[hook_kind][1].values(): + try: + resolve_name(hook_name) + except ImportError: + self.warn('name %r cannot be resolved', hook_name) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/clean.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/clean.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,76 @@ +"""Clean up temporary files created by the build command.""" + +# Contributed by Bastian Kleineidam + +import os +from shutil import rmtree +from packaging.command.cmd import Command +from packaging import logger + +class clean(Command): + + description = "clean up temporary files from 'build' command" + user_options = [ + ('build-base=', 'b', + "base build directory (default: 'build.build-base')"), + ('build-lib=', None, + "build directory for all modules (default: 'build.build-lib')"), + ('build-temp=', 't', + "temporary build directory (default: 'build.build-temp')"), + ('build-scripts=', None, + "build directory for scripts (default: 'build.build-scripts')"), + ('bdist-base=', None, + "temporary directory for built distributions"), + ('all', 'a', + "remove all build output, not just temporary by-products") + ] + + boolean_options = ['all'] + + def initialize_options(self): + self.build_base = None + self.build_lib = None + self.build_temp = None + self.build_scripts = None + self.bdist_base = None + self.all = None + + def finalize_options(self): + self.set_undefined_options('build', 'build_base', 'build_lib', + 'build_scripts', 'build_temp') + self.set_undefined_options('bdist', 'bdist_base') + + def run(self): + # remove the build/temp. directory (unless it's already + # gone) + if os.path.exists(self.build_temp): + if self.dry_run: + logger.info('removing %s', self.build_temp) + else: + rmtree(self.build_temp) + else: + logger.debug("'%s' does not exist -- can't clean it", + self.build_temp) + + if self.all: + # remove build directories + for directory in (self.build_lib, + self.bdist_base, + self.build_scripts): + if os.path.exists(directory): + if self.dry_run: + logger.info('removing %s', directory) + else: + rmtree(directory) + else: + logger.warning("'%s' does not exist -- can't clean it", + directory) + + # just for the heck of it, try to remove the base build directory: + # we might have emptied it right now, but if not we don't care + if not self.dry_run: + try: + os.rmdir(self.build_base) + logger.info("removing '%s'", self.build_base) + except OSError: + pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/cmd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/cmd.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,461 @@ +"""Base class for commands.""" + +import os +import re +from shutil import copyfile, move, make_archive +from packaging import util +from packaging import logger +from packaging.errors import PackagingOptionError + + +class Command: + """Abstract base class for defining command classes, the "worker bees" + of Packaging. A useful analogy for command classes is to think of + them as subroutines with local variables called "options". The options + are "declared" in 'initialize_options()' and "defined" (given their + final values, aka "finalized") in 'finalize_options()', both of which + must be defined by every command class. The distinction between the + two is necessary because option values might come from the outside + world (command line, config file, ...), and any options dependent on + other options must be computed *after* these outside influences have + been processed -- hence 'finalize_options()'. The "body" of the + subroutine, where it does all its work based on the values of its + options, is the 'run()' method, which must also be implemented by every + command class. + """ + + # 'sub_commands' formalizes the notion of a "family" of commands, + # eg. "install_dist" as the parent with sub-commands "install_lib", + # "install_headers", etc. The parent of a family of commands + # defines 'sub_commands' as a class attribute; it's a list of + # (command_name : string, predicate : unbound_method | string | None) + # tuples, where 'predicate' is a method of the parent command that + # determines whether the corresponding command is applicable in the + # current situation. (Eg. we "install_headers" is only applicable if + # we have any C header files to install.) If 'predicate' is None, + # that command is always applicable. + # + # 'sub_commands' is usually defined at the *end* of a class, because + # predicates can be unbound methods, so they must already have been + # defined. The canonical example is the "install_dist" command. + sub_commands = [] + + # Pre and post command hooks are run just before or just after the command + # itself. They are simple functions that receive the command instance. They + # are specified as callable objects or dotted strings (for lazy loading). + pre_hook = None + post_hook = None + + # -- Creation/initialization methods ------------------------------- + + def __init__(self, dist): + """Create and initialize a new Command object. Most importantly, + invokes the 'initialize_options()' method, which is the real + initializer and depends on the actual command being instantiated. + """ + # late import because of mutual dependence between these classes + from packaging.dist import Distribution + + if not isinstance(dist, Distribution): + raise TypeError("dist must be an instance of Distribution, not %r" + % type(dist)) + if self.__class__ is Command: + raise RuntimeError("Command is an abstract class") + + self.distribution = dist + self.initialize_options() + + # Per-command versions of the global flags, so that the user can + # customize Packaging' behaviour command-by-command and let some + # commands fall back on the Distribution's behaviour. None means + # "not defined, check self.distribution's copy", while 0 or 1 mean + # false and true (duh). Note that this means figuring out the real + # value of each flag is a touch complicated -- hence "self._dry_run" + # will be handled by a property, below. + # XXX This needs to be fixed. [I changed it to a property--does that + # "fix" it?] + self._dry_run = None + + # Some commands define a 'self.force' option to ignore file + # timestamps, but methods defined *here* assume that + # 'self.force' exists for all commands. So define it here + # just to be safe. + self.force = None + + # The 'help' flag is just used for command line parsing, so + # none of that complicated bureaucracy is needed. + self.help = False + + # 'finalized' records whether or not 'finalize_options()' has been + # called. 'finalize_options()' itself should not pay attention to + # this flag: it is the business of 'ensure_finalized()', which + # always calls 'finalize_options()', to respect/update it. + self.finalized = False + + # XXX A more explicit way to customize dry_run would be better. + @property + def dry_run(self): + if self._dry_run is None: + return getattr(self.distribution, 'dry_run') + else: + return self._dry_run + + def ensure_finalized(self): + if not self.finalized: + self.finalize_options() + self.finalized = True + + # Subclasses must define: + # initialize_options() + # provide default values for all options; may be customized by + # setup script, by options from config file(s), or by command-line + # options + # finalize_options() + # decide on the final values for all options; this is called + # after all possible intervention from the outside world + # (command line, option file, etc.) has been processed + # run() + # run the command: do whatever it is we're here to do, + # controlled by the command's various option values + + def initialize_options(self): + """Set default values for all the options that this command + supports. Note that these defaults may be overridden by other + commands, by the setup script, by config files, or by the + command line. Thus, this is not the place to code dependencies + between options; generally, 'initialize_options()' implementations + are just a bunch of "self.foo = None" assignments. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + def finalize_options(self): + """Set final values for all the options that this command supports. + This is always called as late as possible, ie. after any option + assignments from the command line or from other commands have been + done. Thus, this is the place to code option dependencies: if + 'foo' depends on 'bar', then it is safe to set 'foo' from 'bar' as + long as 'foo' still has the same value it was assigned in + 'initialize_options()'. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + def dump_options(self, header=None, indent=""): + if header is None: + header = "command options for '%s':" % self.get_command_name() + logger.info(indent + header) + indent = indent + " " + negative_opt = getattr(self, 'negative_opt', ()) + for option, _, _ in self.user_options: + if option in negative_opt: + continue + option = option.replace('-', '_') + if option[-1] == "=": + option = option[:-1] + value = getattr(self, option) + logger.info(indent + "%s = %s", option, value) + + def run(self): + """A command's raison d'etre: carry out the action it exists to + perform, controlled by the options initialized in + 'initialize_options()', customized by other commands, the setup + script, the command line and config files, and finalized in + 'finalize_options()'. All terminal output and filesystem + interaction should be done by 'run()'. + + This method must be implemented by all command classes. + """ + raise RuntimeError( + "abstract method -- subclass %s must override" % self.__class__) + + # -- External interface -------------------------------------------- + # (called by outsiders) + + def get_source_files(self): + """Return the list of files that are used as inputs to this command, + i.e. the files used to generate the output files. The result is used + by the `sdist` command in determining the set of default files. + + Command classes should implement this method if they operate on files + from the source tree. + """ + return [] + + def get_outputs(self): + """Return the list of files that would be produced if this command + were actually run. Not affected by the "dry-run" flag or whether + any other commands have been run. + + Command classes should implement this method if they produce any + output files that get consumed by another command. e.g., `build_ext` + returns the list of built extension modules, but not any temporary + files used in the compilation process. + """ + return [] + + # -- Option validation methods ------------------------------------- + # (these are very handy in writing the 'finalize_options()' method) + # + # NB. the general philosophy here is to ensure that a particular option + # value meets certain type and value constraints. If not, we try to + # force it into conformance (eg. if we expect a list but have a string, + # split the string on comma and/or whitespace). If we can't force the + # option into conformance, raise PackagingOptionError. Thus, command + # classes need do nothing more than (eg.) + # self.ensure_string_list('foo') + # and they can be guaranteed that thereafter, self.foo will be + # a list of strings. + + def _ensure_stringlike(self, option, what, default=None): + val = getattr(self, option) + if val is None: + setattr(self, option, default) + return default + elif not isinstance(val, str): + raise PackagingOptionError("'%s' must be a %s (got `%s`)" % + (option, what, val)) + return val + + def ensure_string(self, option, default=None): + """Ensure that 'option' is a string; if not defined, set it to + 'default'. + """ + self._ensure_stringlike(option, "string", default) + + def ensure_string_list(self, option): + r"""Ensure that 'option' is a list of strings. If 'option' is + currently a string, we split it either on /,\s*/ or /\s+/, so + "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become + ["foo", "bar", "baz"]. + """ + val = getattr(self, option) + if val is None: + return + elif isinstance(val, str): + setattr(self, option, re.split(r',\s*|\s+', val)) + else: + if isinstance(val, list): + # checks if all elements are str + ok = True + for element in val: + if not isinstance(element, str): + ok = False + break + else: + ok = False + + if not ok: + raise PackagingOptionError( + "'%s' must be a list of strings (got %r)" % (option, val)) + + def _ensure_tested_string(self, option, tester, + what, error_fmt, default=None): + val = self._ensure_stringlike(option, what, default) + if val is not None and not tester(val): + raise PackagingOptionError( + ("error in '%s' option: " + error_fmt) % (option, val)) + + def ensure_filename(self, option): + """Ensure that 'option' is the name of an existing file.""" + self._ensure_tested_string(option, os.path.isfile, + "filename", + "'%s' does not exist or is not a file") + + def ensure_dirname(self, option): + self._ensure_tested_string(option, os.path.isdir, + "directory name", + "'%s' does not exist or is not a directory") + + # -- Convenience methods for commands ------------------------------ + + @classmethod + def get_command_name(cls): + if hasattr(cls, 'command_name'): + return cls.command_name + else: + return cls.__name__ + + def set_undefined_options(self, src_cmd, *options): + """Set values of undefined options from another command. + + Undefined options are options set to None, which is the convention + used to indicate that an option has not been changed between + 'initialize_options()' and 'finalize_options()'. This method is + usually called from 'finalize_options()' for options that depend on + some other command rather than another option of the same command, + typically subcommands. + + The 'src_cmd' argument is the other command from which option values + will be taken (a command object will be created for it if necessary); + the remaining positional arguments are strings that give the name of + the option to set. If the name is different on the source and target + command, you can pass a tuple with '(name_on_source, name_on_dest)' so + that 'self.name_on_dest' will be set from 'src_cmd.name_on_source'. + """ + src_cmd_obj = self.distribution.get_command_obj(src_cmd) + src_cmd_obj.ensure_finalized() + for obj in options: + if isinstance(obj, tuple): + src_option, dst_option = obj + else: + src_option, dst_option = obj, obj + if getattr(self, dst_option) is None: + setattr(self, dst_option, + getattr(src_cmd_obj, src_option)) + + def get_finalized_command(self, command, create=True): + """Wrapper around Distribution's 'get_command_obj()' method: find + (create if necessary and 'create' is true) the command object for + 'command', call its 'ensure_finalized()' method, and return the + finalized command object. + """ + cmd_obj = self.distribution.get_command_obj(command, create) + cmd_obj.ensure_finalized() + return cmd_obj + + def reinitialize_command(self, command, reinit_subcommands=False): + return self.distribution.reinitialize_command( + command, reinit_subcommands) + + def run_command(self, command): + """Run some other command: uses the 'run_command()' method of + Distribution, which creates and finalizes the command object if + necessary and then invokes its 'run()' method. + """ + self.distribution.run_command(command) + + def get_sub_commands(self): + """Determine the sub-commands that are relevant in the current + distribution (ie., that need to be run). This is based on the + 'sub_commands' class attribute: each tuple in that list may include + a method that we call to determine if the subcommand needs to be + run for the current distribution. Return a list of command names. + """ + commands = [] + for sub_command in self.sub_commands: + if len(sub_command) == 2: + cmd_name, method = sub_command + if method is None or method(self): + commands.append(cmd_name) + else: + commands.append(sub_command) + return commands + + # -- External world manipulation ----------------------------------- + + def execute(self, func, args, msg=None, level=1): + util.execute(func, args, msg, dry_run=self.dry_run) + + def mkpath(self, name, mode=0o777, dry_run=None): + if dry_run is None: + dry_run = self.dry_run + name = os.path.normpath(name) + if os.path.isdir(name) or name == '': + return + if dry_run: + head = '' + for part in name.split(os.sep): + logger.info("created directory %s%s", head, part) + head += part + os.sep + return + os.makedirs(name, mode) + + def copy_file(self, infile, outfile, + preserve_mode=True, preserve_times=True, link=None, level=1): + """Copy a file respecting dry-run and force flags. + + (dry-run defaults to whatever is in the Distribution object, and + force to false for commands that don't define it.) + """ + if self.dry_run: + # XXX add a comment + return + if os.path.isdir(outfile): + outfile = os.path.join(outfile, os.path.split(infile)[-1]) + copyfile(infile, outfile) + return outfile, None # XXX + + def copy_tree(self, infile, outfile, preserve_mode=True, + preserve_times=True, preserve_symlinks=False, level=1): + """Copy an entire directory tree respecting dry-run + and force flags. + """ + if self.dry_run: + # XXX should not return but let copy_tree log and decide to execute + # or not based on its dry_run argument + return + + return util.copy_tree(infile, outfile, preserve_mode, preserve_times, + preserve_symlinks, not self.force, dry_run=self.dry_run) + + def move_file(self, src, dst, level=1): + """Move a file respecting the dry-run flag.""" + if self.dry_run: + return # XXX same thing + return move(src, dst) + + def spawn(self, cmd, search_path=True, level=1): + """Spawn an external command respecting dry-run flag.""" + from packaging.util import spawn + spawn(cmd, search_path, dry_run=self.dry_run) + + def make_archive(self, base_name, format, root_dir=None, base_dir=None, + owner=None, group=None): + return make_archive(base_name, format, root_dir, + base_dir, dry_run=self.dry_run, + owner=owner, group=group) + + def make_file(self, infiles, outfile, func, args, + exec_msg=None, skip_msg=None, level=1): + """Special case of 'execute()' for operations that process one or + more input files and generate one output file. Works just like + 'execute()', except the operation is skipped and a different + message printed if 'outfile' already exists and is newer than all + files listed in 'infiles'. If the command defined 'self.force', + and it is true, then the command is unconditionally run -- does no + timestamp checks. + """ + if skip_msg is None: + skip_msg = "skipping %s (inputs unchanged)" % outfile + + # Allow 'infiles' to be a single string + if isinstance(infiles, str): + infiles = (infiles,) + elif not isinstance(infiles, (list, tuple)): + raise TypeError( + "'infiles' must be a string, or a list or tuple of strings") + + if exec_msg is None: + exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles)) + + # If 'outfile' must be regenerated (either because it doesn't + # exist, is out-of-date, or the 'force' flag is true) then + # perform the action that presumably regenerates it + if self.force or util.newer_group(infiles, outfile): + self.execute(func, args, exec_msg, level) + + # Otherwise, print the "skip" message + else: + logger.debug(skip_msg) + + def byte_compile(self, files, prefix=None): + """Byte-compile files to pyc and/or pyo files. + + This method requires that the calling class define compile and + optimize options, like build_py and install_lib. It also + automatically respects the force and dry-run options. + + prefix, if given, is a string that will be stripped off the + filenames encoded in bytecode files. + """ + if self.compile: + util.byte_compile(files, optimize=False, prefix=prefix, + force=self.force, dry_run=self.dry_run) + if self.optimize: + util.byte_compile(files, optimize=self.optimize, prefix=prefix, + force=self.force, dry_run=self.dry_run) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/command_template --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/command_template Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,35 @@ +"""Do X and Y.""" + +from packaging import logger +from packaging.command.cmd import Command + + +class x(Command): + + # Brief (40-50 characters) description of the command + description = "" + + # List of option tuples: long name, short name (None if no short + # name), and help string. + user_options = [ + ('', '', # long option, short option (one letter) or None + ""), # help text + ] + + def initialize_options(self): + self. = None + self. = None + self. = None + + def finalize_options(self): + if self.x is None: + self.x = ... + + def run(self): + ... + logger.info(...) + + if not self.dry_run: + ... + + self.execute(..., dry_run=self.dry_run) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/config.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,349 @@ +"""Prepare the build. + +This module provides config, a (mostly) empty command class +that exists mainly to be sub-classed by specific module distributions and +applications. The idea is that while every "config" command is different, +at least they're all named the same, and users always see "config" in the +list of standard commands. Also, this is a good place to put common +configure-like tasks: "try to compile this C code", or "figure out where +this header file lives". +""" + +import os +import re + +from packaging.command.cmd import Command +from packaging.errors import PackagingExecError +from packaging.compiler import customize_compiler +from packaging import logger + +LANG_EXT = {'c': '.c', 'c++': '.cxx'} + +class config(Command): + + description = "prepare the build" + + user_options = [ + ('compiler=', None, + "specify the compiler type"), + ('cc=', None, + "specify the compiler executable"), + ('include-dirs=', 'I', + "list of directories to search for header files"), + ('define=', 'D', + "C preprocessor macros to define"), + ('undef=', 'U', + "C preprocessor macros to undefine"), + ('libraries=', 'l', + "external C libraries to link with"), + ('library-dirs=', 'L', + "directories to search for external C libraries"), + + ('noisy', None, + "show every action (compile, link, run, ...) taken"), + ('dump-source', None, + "dump generated source files before attempting to compile them"), + ] + + + # The three standard command methods: since the "config" command + # does nothing by default, these are empty. + + def initialize_options(self): + self.compiler = None + self.cc = None + self.include_dirs = None + self.libraries = None + self.library_dirs = None + + # maximal output for now + self.noisy = True + self.dump_source = True + + # list of temporary files generated along-the-way that we have + # to clean at some point + self.temp_files = [] + + def finalize_options(self): + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + elif isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + if self.libraries is None: + self.libraries = [] + elif isinstance(self.libraries, str): + self.libraries = [self.libraries] + + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, str): + self.library_dirs = self.library_dirs.split(os.pathsep) + + def run(self): + pass + + + # Utility methods for actual "config" commands. The interfaces are + # loosely based on Autoconf macros of similar names. Sub-classes + # may use these freely. + + def _check_compiler(self): + """Check that 'self.compiler' really is a CCompiler object; + if not, make it one. + """ + # We do this late, and only on-demand, because this is an expensive + # import. + from packaging.compiler.ccompiler import CCompiler + from packaging.compiler import new_compiler + if not isinstance(self.compiler, CCompiler): + self.compiler = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, force=True) + customize_compiler(self.compiler) + if self.include_dirs: + self.compiler.set_include_dirs(self.include_dirs) + if self.libraries: + self.compiler.set_libraries(self.libraries) + if self.library_dirs: + self.compiler.set_library_dirs(self.library_dirs) + + + def _gen_temp_sourcefile(self, body, headers, lang): + filename = "_configtest" + LANG_EXT[lang] + with open(filename, "w") as file: + if headers: + for header in headers: + file.write("#include <%s>\n" % header) + file.write("\n") + file.write(body) + if body[-1] != "\n": + file.write("\n") + return filename + + def _preprocess(self, body, headers, include_dirs, lang): + src = self._gen_temp_sourcefile(body, headers, lang) + out = "_configtest.i" + self.temp_files.extend((src, out)) + self.compiler.preprocess(src, out, include_dirs=include_dirs) + return src, out + + def _compile(self, body, headers, include_dirs, lang): + src = self._gen_temp_sourcefile(body, headers, lang) + if self.dump_source: + dump_file(src, "compiling '%s':" % src) + obj = self.compiler.object_filenames([src])[0] + self.temp_files.extend((src, obj)) + self.compiler.compile([src], include_dirs=include_dirs) + return src, obj + + def _link(self, body, headers, include_dirs, libraries, library_dirs, + lang): + src, obj = self._compile(body, headers, include_dirs, lang) + prog = os.path.splitext(os.path.basename(src))[0] + self.compiler.link_executable([obj], prog, + libraries=libraries, + library_dirs=library_dirs, + target_lang=lang) + + if self.compiler.exe_extension is not None: + prog = prog + self.compiler.exe_extension + self.temp_files.append(prog) + + return src, obj, prog + + def _clean(self, *filenames): + if not filenames: + filenames = self.temp_files + self.temp_files = [] + logger.info("removing: %s", ' '.join(filenames)) + for filename in filenames: + try: + os.remove(filename) + except OSError: + pass + + + # XXX these ignore the dry-run flag: what to do, what to do? even if + # you want a dry-run build, you still need some sort of configuration + # info. My inclination is to make it up to the real config command to + # consult 'dry_run', and assume a default (minimal) configuration if + # true. The problem with trying to do it here is that you'd have to + # return either true or false from all the 'try' methods, neither of + # which is correct. + + # XXX need access to the header search path and maybe default macros. + + def try_cpp(self, body=None, headers=None, include_dirs=None, lang="c"): + """Construct a source file from 'body' (a string containing lines + of C/C++ code) and 'headers' (a list of header files to include) + and run it through the preprocessor. Return true if the + preprocessor succeeded, false if there were any errors. + ('body' probably isn't of much use, but what the heck.) + """ + from packaging.compiler.ccompiler import CompileError + self._check_compiler() + ok = True + try: + self._preprocess(body, headers, include_dirs, lang) + except CompileError: + ok = False + + self._clean() + return ok + + def search_cpp(self, pattern, body=None, headers=None, include_dirs=None, + lang="c"): + """Construct a source file (just like 'try_cpp()'), run it through + the preprocessor, and return true if any line of the output matches + 'pattern'. 'pattern' should either be a compiled regex object or a + string containing a regex. If both 'body' and 'headers' are None, + preprocesses an empty file -- which can be useful to determine the + symbols the preprocessor and compiler set by default. + """ + self._check_compiler() + src, out = self._preprocess(body, headers, include_dirs, lang) + + if isinstance(pattern, str): + pattern = re.compile(pattern) + + with open(out) as file: + match = False + while True: + line = file.readline() + if line == '': + break + if pattern.search(line): + match = True + break + + self._clean() + return match + + def try_compile(self, body, headers=None, include_dirs=None, lang="c"): + """Try to compile a source file built from 'body' and 'headers'. + Return true on success, false otherwise. + """ + from packaging.compiler.ccompiler import CompileError + self._check_compiler() + try: + self._compile(body, headers, include_dirs, lang) + ok = True + except CompileError: + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + def try_link(self, body, headers=None, include_dirs=None, libraries=None, + library_dirs=None, lang="c"): + """Try to compile and link a source file, built from 'body' and + 'headers', to executable form. Return true on success, false + otherwise. + """ + from packaging.compiler.ccompiler import CompileError, LinkError + self._check_compiler() + try: + self._link(body, headers, include_dirs, + libraries, library_dirs, lang) + ok = True + except (CompileError, LinkError): + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + def try_run(self, body, headers=None, include_dirs=None, libraries=None, + library_dirs=None, lang="c"): + """Try to compile, link to an executable, and run a program + built from 'body' and 'headers'. Return true on success, false + otherwise. + """ + from packaging.compiler.ccompiler import CompileError, LinkError + self._check_compiler() + try: + src, obj, exe = self._link(body, headers, include_dirs, + libraries, library_dirs, lang) + self.spawn([exe]) + ok = True + except (CompileError, LinkError, PackagingExecError): + ok = False + + logger.info(ok and "success!" or "failure.") + self._clean() + return ok + + + # -- High-level methods -------------------------------------------- + # (these are the ones that are actually likely to be useful + # when implementing a real-world config command!) + + def check_func(self, func, headers=None, include_dirs=None, + libraries=None, library_dirs=None, decl=False, call=False): + + """Determine if function 'func' is available by constructing a + source file that refers to 'func', and compiles and links it. + If everything succeeds, returns true; otherwise returns false. + + The constructed source file starts out by including the header + files listed in 'headers'. If 'decl' is true, it then declares + 'func' (as "int func()"); you probably shouldn't supply 'headers' + and set 'decl' true in the same call, or you might get errors about + a conflicting declarations for 'func'. Finally, the constructed + 'main()' function either references 'func' or (if 'call' is true) + calls it. 'libraries' and 'library_dirs' are used when + linking. + """ + + self._check_compiler() + body = [] + if decl: + body.append("int %s ();" % func) + body.append("int main () {") + if call: + body.append(" %s();" % func) + else: + body.append(" %s;" % func) + body.append("}") + body = "\n".join(body) + "\n" + + return self.try_link(body, headers, include_dirs, + libraries, library_dirs) + + def check_lib(self, library, library_dirs=None, headers=None, + include_dirs=None, other_libraries=[]): + """Determine if 'library' is available to be linked against, + without actually checking that any particular symbols are provided + by it. 'headers' will be used in constructing the source file to + be compiled, but the only effect of this is to check if all the + header files listed are available. Any libraries listed in + 'other_libraries' will be included in the link, in case 'library' + has symbols that depend on other libraries. + """ + self._check_compiler() + return self.try_link("int main (void) { }", + headers, include_dirs, + [library]+other_libraries, library_dirs) + + def check_header(self, header, include_dirs=None, library_dirs=None, + lang="c"): + """Determine if the system header file named by 'header_file' + exists and can be found by the preprocessor; return true if so, + false otherwise. + """ + return self.try_cpp(body="/* No body */", headers=[header], + include_dirs=include_dirs) + + +def dump_file(filename, head=None): + """Dumps a file content into log.info. + + If head is not None, will be dumped before the file content. + """ + if head is None: + logger.info(filename) + else: + logger.info(head) + with open(filename) as file: + logger.info(file.read()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_data.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,79 @@ +"""Install platform-independent data files.""" + +# Contributed by Bastian Kleineidam + +import os +from shutil import Error +from sysconfig import get_paths, format_value +from packaging import logger +from packaging.util import convert_path +from packaging.command.cmd import Command + + +class install_data(Command): + + description = "install platform-independent data files" + + user_options = [ + ('install-dir=', 'd', + "base directory for installing data files " + "(default: installation base dir)"), + ('root=', None, + "install everything relative to this alternate root directory"), + ('force', 'f', "force installation (overwrite existing files)"), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.outfiles = [] + self.data_files_out = [] + self.root = None + self.force = False + self.data_files = self.distribution.data_files + self.warn_dir = True + + def finalize_options(self): + self.set_undefined_options('install_dist', + ('install_data', 'install_dir'), + 'root', 'force') + + def run(self): + self.mkpath(self.install_dir) + for _file in self.data_files.items(): + destination = convert_path(self.expand_categories(_file[1])) + dir_dest = os.path.abspath(os.path.dirname(destination)) + + self.mkpath(dir_dest) + try: + out = self.copy_file(_file[0], dir_dest)[0] + except Error as e: + logger.warning('%s: %s', self.get_command_name(), e) + out = destination + + self.outfiles.append(out) + self.data_files_out.append((_file[0], destination)) + + def expand_categories(self, path_with_categories): + local_vars = get_paths() + local_vars['distribution.name'] = self.distribution.metadata['Name'] + expanded_path = format_value(path_with_categories, local_vars) + expanded_path = format_value(expanded_path, local_vars) + if '{' in expanded_path and '}' in expanded_path: + logger.warning( + '%s: unable to expand %s, some categories may be missing', + self.get_command_name(), path_with_categories) + return expanded_path + + def get_source_files(self): + return list(self.data_files) + + def get_inputs(self): + return list(self.data_files) + + def get_outputs(self): + return self.outfiles + + def get_resources_out(self): + return self.data_files_out diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_dist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,605 @@ +"""Main install command, which calls the other install_* commands.""" + +import sys +import os + +import sysconfig +from sysconfig import get_config_vars, get_paths, get_path, get_config_var + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingPlatformError +from packaging.util import write_file +from packaging.util import convert_path, change_root, get_platform +from packaging.errors import PackagingOptionError + + +class install_dist(Command): + + description = "install everything from build directory" + + user_options = [ + # Select installation scheme and set base director(y|ies) + ('prefix=', None, + "installation prefix"), + ('exec-prefix=', None, + "(Unix only) prefix for platform-specific files"), + ('user', None, + "install in user site-packages directory [%s]" % + get_path('purelib', '%s_user' % os.name)), + ('home=', None, + "(Unix only) home directory to install under"), + + # Or just set the base director(y|ies) + ('install-base=', None, + "base installation directory (instead of --prefix or --home)"), + ('install-platbase=', None, + "base installation directory for platform-specific files " + + "(instead of --exec-prefix or --home)"), + ('root=', None, + "install everything relative to this alternate root directory"), + + # Or explicitly set the installation scheme + ('install-purelib=', None, + "installation directory for pure Python module distributions"), + ('install-platlib=', None, + "installation directory for non-pure module distributions"), + ('install-lib=', None, + "installation directory for all module distributions " + + "(overrides --install-purelib and --install-platlib)"), + + ('install-headers=', None, + "installation directory for C/C++ headers"), + ('install-scripts=', None, + "installation directory for Python scripts"), + ('install-data=', None, + "installation directory for data files"), + + # Byte-compilation options -- see install_lib for details + ('compile', 'c', "compile .py to .pyc [default]"), + ('no-compile', None, "don't compile .py files"), + ('optimize=', 'O', + 'also compile with optimization: -O1 for "python -O", ' + '-O2 for "python -OO", and -O0 to disable [default: -O0]'), + + # Miscellaneous control options + ('force', 'f', + "force installation (overwrite any existing files)"), + ('skip-build', None, + "skip rebuilding everything (for testing/debugging)"), + + # Where to install documentation (eventually!) + #('doc-format=', None, "format of documentation to generate"), + #('install-man=', None, "directory for Unix man pages"), + #('install-html=', None, "directory for HTML documentation"), + #('install-info=', None, "directory for GNU info files"), + + # XXX use a name that makes clear this is the old format + ('record=', None, + "filename in which to record a list of installed files " + "(not PEP 376-compliant)"), + ('resources=', None, + "data files mapping"), + + # .dist-info related arguments, read by install_dist_info + ('no-distinfo', None, + "do not create a .dist-info directory"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file (i.e."), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), + ] + + boolean_options = ['compile', 'force', 'skip-build', 'no-distinfo', + 'requested', 'no-record', 'user'] + + negative_opt = {'no-compile': 'compile', 'no-requested': 'requested'} + + def initialize_options(self): + # High-level options: these select both an installation base + # and scheme. + self.prefix = None + self.exec_prefix = None + self.home = None + self.user = False + + # These select only the installation base; it's up to the user to + # specify the installation scheme (currently, that means supplying + # the --install-{platlib,purelib,scripts,data} options). + self.install_base = None + self.install_platbase = None + self.root = None + + # These options are the actual installation directories; if not + # supplied by the user, they are filled in using the installation + # scheme implied by prefix/exec-prefix/home and the contents of + # that installation scheme. + self.install_purelib = None # for pure module distributions + self.install_platlib = None # non-pure (dists w/ extensions) + self.install_headers = None # for C/C++ headers + self.install_lib = None # set to either purelib or platlib + self.install_scripts = None + self.install_data = None + self.install_userbase = get_config_var('userbase') + self.install_usersite = get_path('purelib', '%s_user' % os.name) + + self.compile = None + self.optimize = None + + # These two are for putting non-packagized distributions into their + # own directory and creating a .pth file if it makes sense. + # 'extra_path' comes from the setup file; 'install_path_file' can + # be turned off if it makes no sense to install a .pth file. (But + # better to install it uselessly than to guess wrong and not + # install it when it's necessary and would be used!) Currently, + # 'install_path_file' is always true unless some outsider meddles + # with it. + self.extra_path = None + self.install_path_file = True + + # 'force' forces installation, even if target files are not + # out-of-date. 'skip_build' skips running the "build" command, + # handy if you know it's not necessary. 'warn_dir' (which is *not* + # a user option, it's just there so the bdist_* commands can turn + # it off) determines whether we warn about installing to a + # directory not in sys.path. + self.force = False + self.skip_build = False + self.warn_dir = True + + # These are only here as a conduit from the 'build' command to the + # 'install_*' commands that do the real work. ('build_base' isn't + # actually used anywhere, but it might be useful in future.) They + # are not user options, because if the user told the install + # command where the build directory is, that wouldn't affect the + # build command. + self.build_base = None + self.build_lib = None + + # Not defined yet because we don't know anything about + # documentation yet. + #self.install_man = None + #self.install_html = None + #self.install_info = None + + self.record = None + self.resources = None + + # .dist-info related options + self.no_distinfo = None + self.installer = None + self.requested = None + self.no_record = None + + # -- Option finalizing methods ------------------------------------- + # (This is rather more involved than for most commands, + # because this is where the policy for installing third- + # party Python modules on various platforms given a wide + # array of user input is decided. Yes, it's quite complex!) + + def finalize_options(self): + # This method (and its pliant slaves, like 'finalize_unix()', + # 'finalize_other()', and 'select_scheme()') is where the default + # installation directories for modules, extension modules, and + # anything else we care to install from a Python module + # distribution. Thus, this code makes a pretty important policy + # statement about how third-party stuff is added to a Python + # installation! Note that the actual work of installation is done + # by the relatively simple 'install_*' commands; they just take + # their orders from the installation directory options determined + # here. + + # Check for errors/inconsistencies in the options; first, stuff + # that's wrong on any platform. + + if ((self.prefix or self.exec_prefix or self.home) and + (self.install_base or self.install_platbase)): + raise PackagingOptionError( + "must supply either prefix/exec-prefix/home or " + "install-base/install-platbase -- not both") + + if self.home and (self.prefix or self.exec_prefix): + raise PackagingOptionError( + "must supply either home or prefix/exec-prefix -- not both") + + if self.user and (self.prefix or self.exec_prefix or self.home or + self.install_base or self.install_platbase): + raise PackagingOptionError( + "can't combine user with prefix/exec_prefix/home or " + "install_base/install_platbase") + + # Next, stuff that's wrong (or dubious) only on certain platforms. + if os.name != "posix": + if self.exec_prefix: + logger.warning( + '%s: exec-prefix option ignored on this platform', + self.get_command_name()) + self.exec_prefix = None + + # Now the interesting logic -- so interesting that we farm it out + # to other methods. The goal of these methods is to set the final + # values for the install_{lib,scripts,data,...} options, using as + # input a heady brew of prefix, exec_prefix, home, install_base, + # install_platbase, user-supplied versions of + # install_{purelib,platlib,lib,scripts,data,...}, and the + # INSTALL_SCHEME dictionary above. Phew! + + self.dump_dirs("pre-finalize_{unix,other}") + + if os.name == 'posix': + self.finalize_unix() + else: + self.finalize_other() + + self.dump_dirs("post-finalize_{unix,other}()") + + # Expand configuration variables, tilde, etc. in self.install_base + # and self.install_platbase -- that way, we can use $base or + # $platbase in the other installation directories and not worry + # about needing recursive variable expansion (shudder). + + py_version = '%s.%s' % sys.version_info[:2] + prefix, exec_prefix, srcdir, projectbase = get_config_vars( + 'prefix', 'exec_prefix', 'srcdir', 'projectbase') + + metadata = self.distribution.metadata + self.config_vars = { + 'dist_name': metadata['Name'], + 'dist_version': metadata['Version'], + 'dist_fullname': metadata.get_fullname(), + 'py_version': py_version, + 'py_version_short': py_version[:3], + 'py_version_nodot': py_version[:3:2], + 'sys_prefix': prefix, + 'prefix': prefix, + 'sys_exec_prefix': exec_prefix, + 'exec_prefix': exec_prefix, + 'srcdir': srcdir, + 'projectbase': projectbase, + 'userbase': self.install_userbase, + 'usersite': self.install_usersite, + } + + self.expand_basedirs() + + self.dump_dirs("post-expand_basedirs()") + + # Now define config vars for the base directories so we can expand + # everything else. + self.config_vars['base'] = self.install_base + self.config_vars['platbase'] = self.install_platbase + + # Expand "~" and configuration variables in the installation + # directories. + self.expand_dirs() + + self.dump_dirs("post-expand_dirs()") + + # Create directories under USERBASE + if self.user: + self.create_user_dirs() + + # Pick the actual directory to install all modules to: either + # install_purelib or install_platlib, depending on whether this + # module distribution is pure or not. Of course, if the user + # already specified install_lib, use their selection. + if self.install_lib is None: + if self.distribution.ext_modules: # has extensions: non-pure + self.install_lib = self.install_platlib + else: + self.install_lib = self.install_purelib + + # Convert directories from Unix /-separated syntax to the local + # convention. + self.convert_paths('lib', 'purelib', 'platlib', 'scripts', + 'data', 'headers', 'userbase', 'usersite') + + # Well, we're not actually fully completely finalized yet: we still + # have to deal with 'extra_path', which is the hack for allowing + # non-packagized module distributions (hello, Numerical Python!) to + # get their own directories. + self.handle_extra_path() + self.install_libbase = self.install_lib # needed for .pth file + self.install_lib = os.path.join(self.install_lib, self.extra_dirs) + + # If a new root directory was supplied, make all the installation + # dirs relative to it. + if self.root is not None: + self.change_roots('libbase', 'lib', 'purelib', 'platlib', + 'scripts', 'data', 'headers') + + self.dump_dirs("after prepending root") + + # Find out the build directories, ie. where to install from. + self.set_undefined_options('build', 'build_base', 'build_lib') + + # Punt on doc directories for now -- after all, we're punting on + # documentation completely! + + if self.no_distinfo is None: + self.no_distinfo = False + + def finalize_unix(self): + """Finalize options for posix platforms.""" + if self.install_base is not None or self.install_platbase is not None: + if ((self.install_lib is None and + self.install_purelib is None and + self.install_platlib is None) or + self.install_headers is None or + self.install_scripts is None or + self.install_data is None): + raise PackagingOptionError( + "install-base or install-platbase supplied, but " + "installation scheme is incomplete") + return + + if self.user: + if self.install_userbase is None: + raise PackagingPlatformError( + "user base directory is not specified") + self.install_base = self.install_platbase = self.install_userbase + self.select_scheme("posix_user") + elif self.home is not None: + self.install_base = self.install_platbase = self.home + self.select_scheme("posix_home") + else: + if self.prefix is None: + if self.exec_prefix is not None: + raise PackagingOptionError( + "must not supply exec-prefix without prefix") + + self.prefix = os.path.normpath(sys.prefix) + self.exec_prefix = os.path.normpath(sys.exec_prefix) + + else: + if self.exec_prefix is None: + self.exec_prefix = self.prefix + + self.install_base = self.prefix + self.install_platbase = self.exec_prefix + self.select_scheme("posix_prefix") + + def finalize_other(self): + """Finalize options for non-posix platforms""" + if self.user: + if self.install_userbase is None: + raise PackagingPlatformError( + "user base directory is not specified") + self.install_base = self.install_platbase = self.install_userbase + self.select_scheme(os.name + "_user") + elif self.home is not None: + self.install_base = self.install_platbase = self.home + self.select_scheme("posix_home") + else: + if self.prefix is None: + self.prefix = os.path.normpath(sys.prefix) + + self.install_base = self.install_platbase = self.prefix + try: + self.select_scheme(os.name) + except KeyError: + raise PackagingPlatformError( + "no support for installation on '%s'" % os.name) + + def dump_dirs(self, msg): + """Dump the list of user options.""" + logger.debug(msg + ":") + for opt in self.user_options: + opt_name = opt[0] + if opt_name[-1] == "=": + opt_name = opt_name[0:-1] + if opt_name in self.negative_opt: + opt_name = self.negative_opt[opt_name] + opt_name = opt_name.replace('-', '_') + val = not getattr(self, opt_name) + else: + opt_name = opt_name.replace('-', '_') + val = getattr(self, opt_name) + logger.debug(" %s: %s", opt_name, val) + + def select_scheme(self, name): + """Set the install directories by applying the install schemes.""" + # it's the caller's problem if they supply a bad name! + scheme = get_paths(name, expand=False) + for key, value in scheme.items(): + if key == 'platinclude': + key = 'headers' + value = os.path.join(value, self.distribution.metadata['Name']) + attrname = 'install_' + key + if hasattr(self, attrname): + if getattr(self, attrname) is None: + setattr(self, attrname, value) + + def _expand_attrs(self, attrs): + for attr in attrs: + val = getattr(self, attr) + if val is not None: + if os.name == 'posix' or os.name == 'nt': + val = os.path.expanduser(val) + # see if we want to push this work in sysconfig XXX + val = sysconfig._subst_vars(val, self.config_vars) + setattr(self, attr, val) + + def expand_basedirs(self): + """Call `os.path.expanduser` on install_{base,platbase} and root.""" + self._expand_attrs(['install_base', 'install_platbase', 'root']) + + def expand_dirs(self): + """Call `os.path.expanduser` on install dirs.""" + self._expand_attrs(['install_purelib', 'install_platlib', + 'install_lib', 'install_headers', + 'install_scripts', 'install_data']) + + def convert_paths(self, *names): + """Call `convert_path` over `names`.""" + for name in names: + attr = "install_" + name + setattr(self, attr, convert_path(getattr(self, attr))) + + def handle_extra_path(self): + """Set `path_file` and `extra_dirs` using `extra_path`.""" + if self.extra_path is None: + self.extra_path = self.distribution.extra_path + + if self.extra_path is not None: + if isinstance(self.extra_path, str): + self.extra_path = self.extra_path.split(',') + + if len(self.extra_path) == 1: + path_file = extra_dirs = self.extra_path[0] + elif len(self.extra_path) == 2: + path_file, extra_dirs = self.extra_path + else: + raise PackagingOptionError( + "'extra_path' option must be a list, tuple, or " + "comma-separated string with 1 or 2 elements") + + # convert to local form in case Unix notation used (as it + # should be in setup scripts) + extra_dirs = convert_path(extra_dirs) + else: + path_file = None + extra_dirs = '' + + # XXX should we warn if path_file and not extra_dirs? (in which + # case the path file would be harmless but pointless) + self.path_file = path_file + self.extra_dirs = extra_dirs + + def change_roots(self, *names): + """Change the install direcories pointed by name using root.""" + for name in names: + attr = "install_" + name + setattr(self, attr, change_root(self.root, getattr(self, attr))) + + def create_user_dirs(self): + """Create directories under USERBASE as needed.""" + home = convert_path(os.path.expanduser("~")) + for name, path in self.config_vars.items(): + if path.startswith(home) and not os.path.isdir(path): + os.makedirs(path, 0o700) + + # -- Command execution methods ------------------------------------- + + def run(self): + """Runs the command.""" + # Obviously have to build before we can install + if not self.skip_build: + self.run_command('build') + # If we built for any other platform, we can't install. + build_plat = self.distribution.get_command_obj('build').plat_name + # check warn_dir - it is a clue that the 'install_dist' is happening + # internally, and not to sys.path, so we don't check the platform + # matches what we are running. + if self.warn_dir and build_plat != get_platform(): + raise PackagingPlatformError("Can't install when " + "cross-compiling") + + # Run all sub-commands (at least those that need to be run) + for cmd_name in self.get_sub_commands(): + self.run_command(cmd_name) + + if self.path_file: + self.create_path_file() + + # write list of installed files, if requested. + if self.record: + outputs = self.get_outputs() + if self.root: # strip any package prefix + root_len = len(self.root) + for counter in range(len(outputs)): + outputs[counter] = outputs[counter][root_len:] + self.execute(write_file, + (self.record, outputs), + "writing list of installed files to '%s'" % + self.record) + + normpath, normcase = os.path.normpath, os.path.normcase + sys_path = [normcase(normpath(p)) for p in sys.path] + install_lib = normcase(normpath(self.install_lib)) + if (self.warn_dir and + not (self.path_file and self.install_path_file) and + install_lib not in sys_path): + logger.debug(("modules installed to '%s', which is not in " + "Python's module search path (sys.path) -- " + "you'll have to change the search path yourself"), + self.install_lib) + + def create_path_file(self): + """Creates the .pth file""" + filename = os.path.join(self.install_libbase, + self.path_file + ".pth") + if self.install_path_file: + self.execute(write_file, + (filename, [self.extra_dirs]), + "creating %s" % filename) + else: + logger.warning('%s: path file %r not created', + self.get_command_name(), filename) + + # -- Reporting methods --------------------------------------------- + + def get_outputs(self): + """Assembles the outputs of all the sub-commands.""" + outputs = [] + for cmd_name in self.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + # Add the contents of cmd.get_outputs(), ensuring + # that outputs doesn't contain duplicate entries + for filename in cmd.get_outputs(): + if filename not in outputs: + outputs.append(filename) + + if self.path_file and self.install_path_file: + outputs.append(os.path.join(self.install_libbase, + self.path_file + ".pth")) + + return outputs + + def get_inputs(self): + """Returns the inputs of all the sub-commands""" + # XXX gee, this looks familiar ;-( + inputs = [] + for cmd_name in self.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + inputs.extend(cmd.get_inputs()) + + return inputs + + # -- Predicates for sub-command list ------------------------------- + + def has_lib(self): + """Returns true if the current distribution has any Python + modules to install.""" + return (self.distribution.has_pure_modules() or + self.distribution.has_ext_modules()) + + def has_headers(self): + """Returns true if the current distribution has any headers to + install.""" + return self.distribution.has_headers() + + def has_scripts(self): + """Returns true if the current distribution has any scripts to. + install.""" + return self.distribution.has_scripts() + + def has_data(self): + """Returns true if the current distribution has any data to. + install.""" + return self.distribution.has_data_files() + + # 'sub_commands': a list of commands this command might have to run to + # get its work done. See cmd.py for more info. + sub_commands = [('install_lib', has_lib), + ('install_headers', has_headers), + ('install_scripts', has_scripts), + ('install_data', has_data), + # keep install_distinfo last, as it needs the record + # with files to be completely generated + ('install_distinfo', lambda self: not self.no_distinfo), + ] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_distinfo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_distinfo.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,143 @@ +"""Create the PEP 376-compliant .dist-info directory.""" + +# Forked from the former install_egg_info command by Josip Djolonga + +import os +import csv +import hashlib +from shutil import rmtree + +from packaging import logger +from packaging.command.cmd import Command + + +class install_distinfo(Command): + + description = 'create a .dist-info directory for the distribution' + + user_options = [ + ('install-dir=', None, + "directory where the the .dist-info directory will be created"), + ('installer=', None, + "the name of the installer"), + ('requested', None, + "generate a REQUESTED file"), + ('no-requested', None, + "do not generate a REQUESTED file"), + ('no-record', None, + "do not generate a RECORD file"), + ('no-resources', None, + "do not generate a RESOURCES file"), + ] + + boolean_options = ['requested', 'no-record', 'no-resources'] + + negative_opt = {'no-requested': 'requested'} + + def initialize_options(self): + self.install_dir = None + self.installer = None + self.requested = None + self.no_record = None + self.no_resources = None + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install_dist', + 'installer', 'requested', 'no_record') + + self.set_undefined_options('install_lib', 'install_dir') + + if self.installer is None: + # FIXME distutils or packaging? + # + document default in the option help text above and in install + self.installer = 'distutils' + if self.requested is None: + self.requested = True + if self.no_record is None: + self.no_record = False + if self.no_resources is None: + self.no_resources = False + + metadata = self.distribution.metadata + + basename = metadata.get_fullname(filesafe=True) + ".dist-info" + + self.install_dir = os.path.join(self.install_dir, basename) + + def run(self): + target = self.install_dir + + if os.path.isdir(target) and not os.path.islink(target): + if not self.dry_run: + rmtree(target) + elif os.path.exists(target): + self.execute(os.unlink, (self.install_dir,), + "removing " + target) + + self.execute(os.makedirs, (target,), "creating " + target) + + metadata_path = os.path.join(self.install_dir, 'METADATA') + self.execute(self.distribution.metadata.write, (metadata_path,), + "creating " + metadata_path) + self.outfiles.append(metadata_path) + + installer_path = os.path.join(self.install_dir, 'INSTALLER') + logger.info('creating %s', installer_path) + if not self.dry_run: + with open(installer_path, 'w') as f: + f.write(self.installer) + self.outfiles.append(installer_path) + + if self.requested: + requested_path = os.path.join(self.install_dir, 'REQUESTED') + logger.info('creating %s', requested_path) + if not self.dry_run: + open(requested_path, 'wb').close() + self.outfiles.append(requested_path) + + if not self.no_resources: + install_data = self.get_finalized_command('install_data') + if install_data.get_resources_out() != []: + resources_path = os.path.join(self.install_dir, + 'RESOURCES') + logger.info('creating %s', resources_path) + if not self.dry_run: + with open(resources_path, 'w') as f: + writer = csv.writer(f, delimiter=',', + lineterminator='\n', + quotechar='"') + for row in install_data.get_resources_out(): + writer.writerow(row) + + self.outfiles.append(resources_path) + + if not self.no_record: + record_path = os.path.join(self.install_dir, 'RECORD') + logger.info('creating %s', record_path) + if not self.dry_run: + with open(record_path, 'w', encoding='utf-8') as f: + writer = csv.writer(f, delimiter=',', + lineterminator='\n', + quotechar='"') + + install = self.get_finalized_command('install_dist') + + for fpath in install.get_outputs(): + if fpath.endswith('.pyc') or fpath.endswith('.pyo'): + # do not put size and md5 hash, as in PEP-376 + writer.writerow((fpath, '', '')) + else: + size = os.path.getsize(fpath) + with open(fpath, 'rb') as fp: + hash = hashlib.md5() + hash.update(fp.read()) + md5sum = hash.hexdigest() + writer.writerow((fpath, md5sum, size)) + + # add the RECORD file itself + writer.writerow((record_path, '', '')) + self.outfiles.append(record_path) + + def get_outputs(self): + return self.outfiles diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_headers.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_headers.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,43 @@ +"""Install C/C++ header files to the Python include directory.""" + +from packaging.command.cmd import Command + + +# XXX force is never used +class install_headers(Command): + + description = "install C/C++ header files" + + user_options = [('install-dir=', 'd', + "directory to install header files to"), + ('force', 'f', + "force installation (overwrite existing files)"), + ] + + boolean_options = ['force'] + + def initialize_options(self): + self.install_dir = None + self.force = False + self.outfiles = [] + + def finalize_options(self): + self.set_undefined_options('install_dist', + ('install_headers', 'install_dir'), + 'force') + + def run(self): + headers = self.distribution.headers + if not headers: + return + + self.mkpath(self.install_dir) + for header in headers: + out = self.copy_file(header, self.install_dir)[0] + self.outfiles.append(out) + + def get_inputs(self): + return self.distribution.headers or [] + + def get_outputs(self): + return self.outfiles diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_lib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_lib.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,188 @@ +"""Install all modules (extensions and pure Python).""" + +import os +import imp + +from packaging import logger +from packaging.command.cmd import Command +from packaging.errors import PackagingOptionError + + +# Extension for Python source files. +# XXX dead code? most of the codebase checks for literal '.py' +if hasattr(os, 'extsep'): + PYTHON_SOURCE_EXTENSION = os.extsep + "py" +else: + PYTHON_SOURCE_EXTENSION = ".py" + + +class install_lib(Command): + + description = "install all modules (extensions and pure Python)" + + # The options for controlling byte compilation are two independent sets: + # 'compile' is strictly boolean, and only decides whether to + # generate .pyc files. 'optimize' is three-way (0, 1, or 2), and + # decides both whether to generate .pyo files and what level of + # optimization to use. + + user_options = [ + ('install-dir=', 'd', "directory to install to"), + ('build-dir=', 'b', "build directory (where to install from)"), + ('force', 'f', "force installation (overwrite existing files)"), + ('compile', 'c', "compile .py to .pyc [default]"), + ('no-compile', None, "don't compile .py files"), + ('optimize=', 'O', + "also compile with optimization: -O1 for \"python -O\", " + "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"), + ('skip-build', None, "skip the build steps"), + ] + + boolean_options = ['force', 'compile', 'skip-build'] + + negative_opt = {'no-compile': 'compile'} + + def initialize_options(self): + # let the 'install_dist' command dictate our installation directory + self.install_dir = None + self.build_dir = None + self.force = False + self.compile = None + self.optimize = None + self.skip_build = None + + def finalize_options(self): + # Get all the information we need to install pure Python modules + # from the umbrella 'install_dist' command -- build (source) directory, + # install (target) directory, and whether to compile .py files. + self.set_undefined_options('install_dist', + ('build_lib', 'build_dir'), + ('install_lib', 'install_dir'), + 'force', 'compile', 'optimize', + 'skip_build') + + if self.compile is None: + self.compile = True + if self.optimize is None: + self.optimize = 0 + + if not isinstance(self.optimize, int): + try: + self.optimize = int(self.optimize) + if self.optimize not in (0, 1, 2): + raise AssertionError + except (ValueError, AssertionError): + raise PackagingOptionError("optimize must be 0, 1, or 2") + + def run(self): + # Make sure we have built everything we need first + self.build() + + # Install everything: simply dump the entire contents of the build + # directory to the installation directory (that's the beauty of + # having a build directory!) + outfiles = self.install() + + # (Optionally) compile .py to .pyc and/or .pyo + if outfiles is not None and self.distribution.has_pure_modules(): + # XXX comment from distutils: "This [prefix stripping] is far from + # complete, but it should at least generate usable bytecode in RPM + # distributions." -> need to find exact requirements for + # byte-compiled files and fix it + install_root = self.get_finalized_command('install_dist').root + self.byte_compile(outfiles, prefix=install_root) + + # -- Top-level worker functions ------------------------------------ + # (called from 'run()') + + def build(self): + if not self.skip_build: + if self.distribution.has_pure_modules(): + self.run_command('build_py') + if self.distribution.has_ext_modules(): + self.run_command('build_ext') + + def install(self): + if os.path.isdir(self.build_dir): + outfiles = self.copy_tree(self.build_dir, self.install_dir) + else: + logger.warning( + '%s: %r does not exist -- no Python modules to install', + self.get_command_name(), self.build_dir) + return + return outfiles + + # -- Utility methods ----------------------------------------------- + + def _mutate_outputs(self, has_any, build_cmd, cmd_option, output_dir): + if not has_any: + return [] + + build_cmd = self.get_finalized_command(build_cmd) + build_files = build_cmd.get_outputs() + build_dir = getattr(build_cmd, cmd_option) + + prefix_len = len(build_dir) + len(os.sep) + outputs = [] + for file in build_files: + outputs.append(os.path.join(output_dir, file[prefix_len:])) + + return outputs + + def _bytecode_filenames(self, py_filenames): + bytecode_files = [] + for py_file in py_filenames: + # Since build_py handles package data installation, the + # list of outputs can contain more than just .py files. + # Make sure we only report bytecode for the .py files. + ext = os.path.splitext(os.path.normcase(py_file))[1] + if ext != PYTHON_SOURCE_EXTENSION: + continue + if self.compile: + bytecode_files.append(imp.cache_from_source(py_file, True)) + if self.optimize: + bytecode_files.append(imp.cache_from_source(py_file, False)) + + return bytecode_files + + # -- External interface -------------------------------------------- + # (called by outsiders) + + def get_outputs(self): + """Return the list of files that would be installed if this command + were actually run. Not affected by the "dry-run" flag or whether + modules have actually been built yet. + """ + pure_outputs = \ + self._mutate_outputs(self.distribution.has_pure_modules(), + 'build_py', 'build_lib', + self.install_dir) + if self.compile: + bytecode_outputs = self._bytecode_filenames(pure_outputs) + else: + bytecode_outputs = [] + + ext_outputs = \ + self._mutate_outputs(self.distribution.has_ext_modules(), + 'build_ext', 'build_lib', + self.install_dir) + + return pure_outputs + bytecode_outputs + ext_outputs + + def get_inputs(self): + """Get the list of files that are input to this command, ie. the + files that get installed as they are named in the build tree. + The files in this list correspond one-to-one to the output + filenames returned by 'get_outputs()'. + """ + inputs = [] + + if self.distribution.has_pure_modules(): + build_py = self.get_finalized_command('build_py') + inputs.extend(build_py.get_outputs()) + + if self.distribution.has_ext_modules(): + build_ext = self.get_finalized_command('build_ext') + inputs.extend(build_ext.get_outputs()) + + return inputs diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/install_scripts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/install_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,59 @@ +"""Install scripts.""" + +# Contributed by Bastian Kleineidam + +import os +from packaging.command.cmd import Command +from packaging import logger + +class install_scripts(Command): + + description = "install scripts (Python or otherwise)" + + user_options = [ + ('install-dir=', 'd', "directory to install scripts to"), + ('build-dir=','b', "build directory (where to install from)"), + ('force', 'f', "force installation (overwrite existing files)"), + ('skip-build', None, "skip the build steps"), + ] + + boolean_options = ['force', 'skip-build'] + + + def initialize_options(self): + self.install_dir = None + self.force = False + self.build_dir = None + self.skip_build = None + + def finalize_options(self): + self.set_undefined_options('build', ('build_scripts', 'build_dir')) + self.set_undefined_options('install_dist', + ('install_scripts', 'install_dir'), + 'force', 'skip_build') + + def run(self): + if not self.skip_build: + self.run_command('build_scripts') + + if not os.path.exists(self.build_dir): + self.outfiles = [] + return + + self.outfiles = self.copy_tree(self.build_dir, self.install_dir) + if os.name == 'posix': + # Set the executable bits (owner, group, and world) on + # all the scripts we just installed. + for file in self.get_outputs(): + if self.dry_run: + logger.info("changing mode of %s", file) + else: + mode = (os.stat(file).st_mode | 0o555) & 0o7777 + logger.info("changing mode of %s to %o", file, mode) + os.chmod(file, mode) + + def get_inputs(self): + return self.distribution.scripts or [] + + def get_outputs(self): + return self.outfiles or [] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/register.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/register.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,263 @@ +"""Register a release with a project index.""" + +# Contributed by Richard Jones + +import getpass +import urllib.error +import urllib.parse +import urllib.request + +from packaging import logger +from packaging.util import (read_pypirc, generate_pypirc, DEFAULT_REPOSITORY, + DEFAULT_REALM, get_pypirc_path, encode_multipart) +from packaging.command.cmd import Command + +class register(Command): + + description = "register a release with PyPI" + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('list-classifiers', None, + "list valid Trove classifiers"), + ('strict', None , + "stop the registration if the metadata is not fully compliant") + ] + + boolean_options = ['show-response', 'list-classifiers', 'strict'] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.list_classifiers = False + self.strict = False + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + + def run(self): + self._set_config() + + # Check the package metadata + check = self.distribution.get_command_obj('check') + if check.strict != self.strict and not check.all: + # If check was already run but with different options, + # re-run it + check.strict = self.strict + check.all = True + self.distribution.have_run.pop('check', None) + self.run_command('check') + + if self.dry_run: + self.verify_metadata() + elif self.list_classifiers: + self.classifiers() + else: + self.send_metadata() + + def _set_config(self): + ''' Reads the configuration file and set attributes. + ''' + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + self.has_config = True + else: + if self.repository not in ('pypi', DEFAULT_REPOSITORY): + raise ValueError('%s not found in .pypirc' % self.repository) + if self.repository == 'pypi': + self.repository = DEFAULT_REPOSITORY + self.has_config = False + + def classifiers(self): + ''' Fetch the list of classifiers from the server. + ''' + response = urllib.request.urlopen(self.repository+'?:action=list_classifiers') + logger.info(response.read()) + + def verify_metadata(self): + ''' Send the metadata to the package index server to be checked. + ''' + # send the info to the server and report the result + code, result = self.post_to_server(self.build_post_data('verify')) + logger.info('server response (%s): %s', code, result) + + + def send_metadata(self): + ''' Send the metadata to the package index server. + + Well, do the following: + 1. figure who the user is, and then + 2. send the data as a Basic auth'ed POST. + + First we try to read the username/password from $HOME/.pypirc, + which is a ConfigParser-formatted file with a section + [distutils] containing username and password entries (both + in clear text). Eg: + + [distutils] + index-servers = + pypi + + [pypi] + username: fred + password: sekrit + + Otherwise, to figure who the user is, we offer the user three + choices: + + 1. use existing login, + 2. register as a new user, or + 3. set the password to a random string and email the user. + + ''' + # TODO factor registration out into another method + # TODO use print to print, not logging + + # see if we can short-cut and get the username/password from the + # config + if self.has_config: + choice = '1' + username = self.username + password = self.password + else: + choice = 'x' + username = password = '' + + # get the user's login info + choices = '1 2 3 4'.split() + while choice not in choices: + logger.info('''\ +We need to know who you are, so please choose either: + 1. use your existing login, + 2. register as a new user, + 3. have the server generate a new password for you (and email it to you), or + 4. quit +Your selection [default 1]: ''') + + choice = input() + if not choice: + choice = '1' + elif choice not in choices: + print('Please choose one of the four options!') + + if choice == '1': + # get the username and password + while not username: + username = input('Username: ') + while not password: + password = getpass.getpass('Password: ') + + # set up the authentication + auth = urllib.request.HTTPPasswordMgr() + host = urllib.parse.urlparse(self.repository)[1] + auth.add_password(self.realm, host, username, password) + # send the info to the server and report the result + code, result = self.post_to_server(self.build_post_data('submit'), + auth) + logger.info('Server response (%s): %s', code, result) + + # possibly save the login + if code == 200: + if self.has_config: + # sharing the password in the distribution instance + # so the upload command can reuse it + self.distribution.password = password + else: + logger.info( + 'I can store your PyPI login so future submissions ' + 'will be faster.\n(the login will be stored in %s)', + get_pypirc_path()) + choice = 'X' + while choice.lower() not in ('y', 'n'): + choice = input('Save your login (y/N)?') + if not choice: + choice = 'n' + if choice.lower() == 'y': + generate_pypirc(username, password) + + elif choice == '2': + data = {':action': 'user'} + data['name'] = data['password'] = data['email'] = '' + data['confirm'] = None + while not data['name']: + data['name'] = input('Username: ') + while data['password'] != data['confirm']: + while not data['password']: + data['password'] = getpass.getpass('Password: ') + while not data['confirm']: + data['confirm'] = getpass.getpass(' Confirm: ') + if data['password'] != data['confirm']: + data['password'] = '' + data['confirm'] = None + print("Password and confirm don't match!") + while not data['email']: + data['email'] = input(' EMail: ') + code, result = self.post_to_server(data) + if code != 200: + logger.info('server response (%s): %s', code, result) + else: + logger.info('you will receive an email shortly; follow the ' + 'instructions in it to complete registration.') + elif choice == '3': + data = {':action': 'password_reset'} + data['email'] = '' + while not data['email']: + data['email'] = input('Your email address: ') + code, result = self.post_to_server(data) + logger.info('server response (%s): %s', code, result) + + def build_post_data(self, action): + # figure the data to send - the metadata plus some additional + # information used by the package server + data = self.distribution.metadata.todict() + data[':action'] = action + return data + + # XXX to be refactored with upload.upload_file + def post_to_server(self, data, auth=None): + ''' Post a query to the server, and return a string response. + ''' + if 'name' in data: + logger.info('Registering %s to %s', data['name'], self.repository) + # Build up the MIME payload for the urllib2 POST data + content_type, body = encode_multipart(data.items(), []) + + # build the Request + headers = { + 'Content-type': content_type, + 'Content-length': str(len(body)) + } + req = urllib.request.Request(self.repository, body, headers) + + # handle HTTP and include the Basic Auth handler + opener = urllib.request.build_opener( + urllib.request.HTTPBasicAuthHandler(password_mgr=auth) + ) + data = '' + try: + result = opener.open(req) + except urllib.error.HTTPError as e: + if self.show_response: + data = e.fp.read() + result = e.code, e.msg + except urllib.error.URLError as e: + result = 500, str(e) + else: + if self.show_response: + data = result.read() + result = 200, 'OK' + if self.show_response: + dashes = '-' * 75 + logger.info('%s%s%s', dashes, data, dashes) + + return result diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/sdist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/sdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,347 @@ +"""Create a source distribution.""" + +import os +import re +import sys +from io import StringIO +from shutil import get_archive_formats, rmtree + +from packaging import logger +from packaging.util import resolve_name +from packaging.errors import (PackagingPlatformError, PackagingOptionError, + PackagingModuleError, PackagingFileError) +from packaging.command import get_command_names +from packaging.command.cmd import Command +from packaging.manifest import Manifest + + +def show_formats(): + """Print all possible values for the 'formats' option (used by + the "--help-formats" command-line option). + """ + from packaging.fancy_getopt import FancyGetopt + formats = sorted(('formats=' + name, None, desc) + for name, desc in get_archive_formats()) + FancyGetopt(formats).print_help( + "List of available source distribution formats:") + +# a \ followed by some spaces + EOL +_COLLAPSE_PATTERN = re.compile('\\\w\n', re.M) +_COMMENTED_LINE = re.compile('^#.*\n$|^\w*\n$', re.M) + + +class sdist(Command): + + description = "create a source distribution (tarball, zip file, etc.)" + + user_options = [ + ('manifest=', 'm', + "name of manifest file [default: MANIFEST]"), + ('use-defaults', None, + "include the default file set in the manifest " + "[default; disable with --no-defaults]"), + ('no-defaults', None, + "don't include the default file set"), + ('prune', None, + "specifically exclude files/directories that should not be " + "distributed (build tree, RCS/CVS dirs, etc.) " + "[default; disable with --no-prune]"), + ('no-prune', None, + "don't automatically exclude anything"), + ('manifest-only', 'o', + "just regenerate the manifest and then stop "), + ('formats=', None, + "formats for source distribution (comma-separated list)"), + ('keep-temp', 'k', + "keep the distribution tree around after creating " + + "archive file(s)"), + ('dist-dir=', 'd', + "directory to put the source distribution archive(s) in " + "[default: dist]"), + ('check-metadata', None, + "Ensure that all required elements of metadata " + "are supplied. Warn if any missing. [default]"), + ('owner=', 'u', + "Owner name used when creating a tar file [default: current user]"), + ('group=', 'g', + "Group name used when creating a tar file [default: current group]"), + ('manifest-builders=', None, + "manifest builders (comma-separated list)"), + ] + + boolean_options = ['use-defaults', 'prune', + 'manifest-only', 'keep-temp', 'check-metadata'] + + help_options = [ + ('help-formats', None, + "list available distribution formats", show_formats), + ] + + negative_opt = {'no-defaults': 'use-defaults', + 'no-prune': 'prune'} + + default_format = {'posix': 'gztar', + 'nt': 'zip'} + + def initialize_options(self): + self.manifest = None + # 'use_defaults': if true, we will include the default file set + # in the manifest + self.use_defaults = True + self.prune = True + self.manifest_only = False + self.formats = None + self.keep_temp = False + self.dist_dir = None + + self.archive_files = None + self.metadata_check = True + self.owner = None + self.group = None + self.filelist = None + self.manifest_builders = None + + def _check_archive_formats(self, formats): + supported_formats = [name for name, desc in get_archive_formats()] + for format in formats: + if format not in supported_formats: + return format + return None + + def finalize_options(self): + if self.manifest is None: + self.manifest = "MANIFEST" + + self.ensure_string_list('formats') + if self.formats is None: + try: + self.formats = [self.default_format[os.name]] + except KeyError: + raise PackagingPlatformError("don't know how to create source " + "distributions on platform %s" % os.name) + + bad_format = self._check_archive_formats(self.formats) + if bad_format: + raise PackagingOptionError("unknown archive format '%s'" \ + % bad_format) + + if self.dist_dir is None: + self.dist_dir = "dist" + + if self.filelist is None: + self.filelist = Manifest() + + if self.manifest_builders is None: + self.manifest_builders = [] + else: + if isinstance(self.manifest_builders, str): + self.manifest_builders = self.manifest_builders.split(',') + builders = [] + for builder in self.manifest_builders: + builder = builder.strip() + if builder == '': + continue + try: + builder = resolve_name(builder) + except ImportError as e: + raise PackagingModuleError(e) + + builders.append(builder) + + self.manifest_builders = builders + + def run(self): + # 'filelist' contains the list of files that will make up the + # manifest + self.filelist.clear() + + # Check the package metadata + if self.metadata_check: + self.run_command('check') + + # Do whatever it takes to get the list of files to process + # (process the manifest template, read an existing manifest, + # whatever). File list is accumulated in 'self.filelist'. + self.get_file_list() + + # If user just wanted us to regenerate the manifest, stop now. + if self.manifest_only: + return + + # Otherwise, go ahead and create the source distribution tarball, + # or zipfile, or whatever. + self.make_distribution() + + def get_file_list(self): + """Figure out the list of files to include in the source + distribution, and put it in 'self.filelist'. This might involve + reading the manifest template (and writing the manifest), or just + reading the manifest, or just using the default file set -- it all + depends on the user's options. + """ + template_exists = len(self.distribution.extra_files) > 0 + if not template_exists: + logger.warning('%s: using default file list', + self.get_command_name()) + self.filelist.findall() + + if self.use_defaults: + self.add_defaults() + if template_exists: + template = '\n'.join(self.distribution.extra_files) + self.filelist.read_template(StringIO(template)) + + # call manifest builders, if any. + for builder in self.manifest_builders: + builder(self.distribution, self.filelist) + + if self.prune: + self.prune_file_list() + + self.filelist.write(self.manifest) + + def add_defaults(self): + """Add all default files to self.filelist. + + In addition to the setup.cfg file, this will include all files returned + by the get_source_files of every registered command. This will find + Python modules and packages, data files listed in package_data_, + data_files and extra_files, scripts, C sources of extension modules or + C libraries (headers are missing). + """ + if os.path.exists('setup.cfg'): + self.filelist.append('setup.cfg') + else: + logger.warning("%s: standard 'setup.cfg' file not found", + self.get_command_name()) + + for cmd_name in get_command_names(): + try: + cmd_obj = self.get_finalized_command(cmd_name) + except PackagingOptionError: + pass + else: + self.filelist.extend(cmd_obj.get_source_files()) + + def prune_file_list(self): + """Prune off branches that might slip into the file list as created + by 'read_template()', but really don't belong there: + * the build tree (typically "build") + * the release tree itself (only an issue if we ran "sdist" + previously with --keep-temp, or it aborted) + * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories + """ + build = self.get_finalized_command('build') + base_dir = self.distribution.get_fullname() + + self.filelist.exclude_pattern(None, prefix=build.build_base) + self.filelist.exclude_pattern(None, prefix=base_dir) + + # pruning out vcs directories + # both separators are used under win32 + if sys.platform == 'win32': + seps = r'/|\\' + else: + seps = '/' + + vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', + '_darcs'] + vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) + self.filelist.exclude_pattern(vcs_ptrn, is_regex=True) + + def make_release_tree(self, base_dir, files): + """Create the directory tree that will become the source + distribution archive. All directories implied by the filenames in + 'files' are created under 'base_dir', and then we hard link or copy + (if hard linking is unavailable) those files into place. + Essentially, this duplicates the developer's source tree, but in a + directory named after the distribution, containing only the files + to be distributed. + """ + # Create all the directories under 'base_dir' necessary to + # put 'files' there; the 'mkpath()' is just so we don't die + # if the manifest happens to be empty. + self.mkpath(base_dir) + self.create_tree(base_dir, files, dry_run=self.dry_run) + + # And walk over the list of files, either making a hard link (if + # os.link exists) to each one that doesn't already exist in its + # corresponding location under 'base_dir', or copying each file + # that's out-of-date in 'base_dir'. (Usually, all files will be + # out-of-date, because by default we blow away 'base_dir' when + # we're done making the distribution archives.) + + if hasattr(os, 'link'): # can make hard links on this system + link = 'hard' + msg = "making hard links in %s..." % base_dir + else: # nope, have to copy + link = None + msg = "copying files to %s..." % base_dir + + if not files: + logger.warning("no files to distribute -- empty manifest?") + else: + logger.info(msg) + + for file in self.distribution.metadata.requires_files: + if file not in files: + msg = "'%s' must be included explicitly in 'extra_files'" \ + % file + raise PackagingFileError(msg) + + for file in files: + if not os.path.isfile(file): + logger.warning("'%s' not a regular file -- skipping", file) + else: + dest = os.path.join(base_dir, file) + self.copy_file(file, dest, link=link) + + self.distribution.metadata.write(os.path.join(base_dir, 'PKG-INFO')) + + def make_distribution(self): + """Create the source distribution(s). First, we create the release + tree with 'make_release_tree()'; then, we create all required + archive files (according to 'self.formats') from the release tree. + Finally, we clean up by blowing away the release tree (unless + 'self.keep_temp' is true). The list of archive files created is + stored so it can be retrieved later by 'get_archive_files()'. + """ + # Don't warn about missing metadata here -- should be (and is!) + # done elsewhere. + base_dir = self.distribution.get_fullname() + base_name = os.path.join(self.dist_dir, base_dir) + + self.make_release_tree(base_dir, self.filelist.files) + archive_files = [] # remember names of files we create + # tar archive must be created last to avoid overwrite and remove + if 'tar' in self.formats: + self.formats.append(self.formats.pop(self.formats.index('tar'))) + + for fmt in self.formats: + file = self.make_archive(base_name, fmt, base_dir=base_dir, + owner=self.owner, group=self.group) + archive_files.append(file) + self.distribution.dist_files.append(('sdist', '', file)) + + self.archive_files = archive_files + + if not self.keep_temp: + if self.dry_run: + logger.info('removing %s', base_dir) + else: + rmtree(base_dir) + + def get_archive_files(self): + """Return the list of archive files created when the command + was run, or None if the command hasn't run yet. + """ + return self.archive_files + + def create_tree(self, base_dir, files, mode=0o777, dry_run=False): + need_dir = set() + for file in files: + need_dir.add(os.path.join(base_dir, os.path.dirname(file))) + + # Now create them + for dir in sorted(need_dir): + self.mkpath(dir, mode, dry_run=dry_run) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/test.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/test.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,80 @@ +"""Run the project's test suite.""" + +import os +import sys +import logging +import unittest + +from packaging import logger +from packaging.command.cmd import Command +from packaging.database import get_distribution +from packaging.errors import PackagingOptionError +from packaging.util import resolve_name + + +class test(Command): + + description = "run the project's test suite" + + user_options = [ + ('suite=', 's', + "test suite to run (for example: 'some_module.test_suite')"), + ('runner=', None, + "test runner to be called."), + ('tests-require=', None, + "list of distributions required to run the test suite."), + ] + + def initialize_options(self): + self.suite = None + self.runner = None + self.tests_require = [] + + def finalize_options(self): + self.build_lib = self.get_finalized_command("build").build_lib + for requirement in self.tests_require: + if get_distribution(requirement) is None: + logger.warning("test dependency %s is not installed, " + "tests may fail", requirement) + if (not self.suite and not self.runner and + self.get_ut_with_discovery() is None): + raise PackagingOptionError( + "no test discovery available, please give a 'suite' or " + "'runner' option or install unittest2") + + def get_ut_with_discovery(self): + if hasattr(unittest.TestLoader, "discover"): + return unittest + else: + try: + import unittest2 + return unittest2 + except ImportError: + return None + + def run(self): + prev_syspath = sys.path[:] + try: + # build release + build = self.reinitialize_command('build') + self.run_command('build') + sys.path.insert(0, build.build_lib) + + # XXX maybe we could pass the verbose argument of pysetup here + logger = logging.getLogger('packaging') + verbose = logger.getEffectiveLevel() >= logging.DEBUG + verbosity = verbose + 1 + + # run the tests + if self.runner: + resolve_name(self.runner)() + elif self.suite: + runner = unittest.TextTestRunner(verbosity=verbosity) + runner.run(resolve_name(self.suite)()) + elif self.get_ut_with_discovery(): + ut = self.get_ut_with_discovery() + test_suite = ut.TestLoader().discover(os.curdir) + runner = ut.TextTestRunner(verbosity=verbosity) + runner.run(test_suite) + finally: + sys.path[:] = prev_syspath diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/upload.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/upload.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,168 @@ +"""Upload a distribution to a project index.""" + +import os +import socket +import logging +import platform +import urllib.parse +from base64 import standard_b64encode +from hashlib import md5 +from urllib.error import HTTPError +from urllib.request import urlopen, Request + +from packaging import logger +from packaging.errors import PackagingOptionError +from packaging.util import (spawn, read_pypirc, DEFAULT_REPOSITORY, + DEFAULT_REALM, encode_multipart) +from packaging.command.cmd import Command + + +class upload(Command): + + description = "upload distribution to PyPI" + + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('sign', 's', + "sign files to upload using gpg"), + ('identity=', 'i', + "GPG identity used to sign files"), + ('upload-docs', None, + "upload documentation too"), + ] + + boolean_options = ['show-response', 'sign'] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.username = '' + self.password = '' + self.show_response = False + self.sign = False + self.identity = None + self.upload_docs = False + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + if self.identity and not self.sign: + raise PackagingOptionError( + "Must use --sign for --identity to have meaning") + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + + # getting the password from the distribution + # if previously set by the register command + if not self.password and self.distribution.password: + self.password = self.distribution.password + + def run(self): + if not self.distribution.dist_files: + raise PackagingOptionError( + "No dist file created in earlier command") + for command, pyversion, filename in self.distribution.dist_files: + self.upload_file(command, pyversion, filename) + if self.upload_docs: + upload_docs = self.get_finalized_command("upload_docs") + upload_docs.repository = self.repository + upload_docs.username = self.username + upload_docs.password = self.password + upload_docs.run() + + # XXX to be refactored with register.post_to_server + def upload_file(self, command, pyversion, filename): + # Makes sure the repository URL is compliant + scheme, netloc, url, params, query, fragments = \ + urllib.parse.urlparse(self.repository) + if params or query or fragments: + raise AssertionError("Incompatible url %s" % self.repository) + + if scheme not in ('http', 'https'): + raise AssertionError("unsupported scheme " + scheme) + + # Sign if requested + if self.sign: + gpg_args = ["gpg", "--detach-sign", "-a", filename] + if self.identity: + gpg_args[2:2] = ["--local-user", self.identity] + spawn(gpg_args, + dry_run=self.dry_run) + + # Fill in the data - send all the metadata in case we need to + # register a new release + with open(filename, 'rb') as f: + content = f.read() + + data = self.distribution.metadata.todict() + + # extra upload infos + data[':action'] = 'file_upload' + data['protcol_version'] = '1' + data['content'] = (os.path.basename(filename), content) + data['filetype'] = command + data['pyversion'] = pyversion + data['md5_digest'] = md5(content).hexdigest() + + if command == 'bdist_dumb': + data['comment'] = 'built for %s' % platform.platform(terse=True) + + if self.sign: + with open(filename + '.asc') as fp: + sig = fp.read() + data['gpg_signature'] = [ + (os.path.basename(filename) + ".asc", sig)] + + # set up the authentication + # The exact encoding of the authentication string is debated. + # Anyway PyPI only accepts ascii for both username or password. + user_pass = (self.username + ":" + self.password).encode('ascii') + auth = b"Basic " + standard_b64encode(user_pass) + + # Build up the MIME payload for the POST data + files = [] + for key in ('content', 'gpg_signature'): + if key in data: + filename_, value = data.pop(key) + files.append((key, filename_, value)) + + content_type, body = encode_multipart(data.items(), files) + + logger.info("Submitting %s to %s", filename, self.repository) + + # build the Request + headers = {'Content-type': content_type, + 'Content-length': str(len(body)), + 'Authorization': auth} + + request = Request(self.repository, body, headers) + # send the data + try: + result = urlopen(request) + status = result.code + reason = result.msg + except socket.error as e: + logger.error(e) + return + except HTTPError as e: + status = e.code + reason = e.msg + + if status == 200: + logger.info('Server response (%s): %s', status, reason) + else: + logger.error('Upload failed (%s): %s', status, reason) + + if self.show_response and logger.isEnabledFor(logging.INFO): + sep = '-' * 75 + logger.info('%s\n%s\n%s', sep, result.read().decode(), sep) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/upload_docs.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/command/upload_docs.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,131 @@ +"""Upload HTML documentation to a project index.""" + +import os +import base64 +import socket +import zipfile +import logging +import http.client +import urllib.parse +from io import BytesIO + +from packaging import logger +from packaging.util import (read_pypirc, DEFAULT_REPOSITORY, DEFAULT_REALM, + encode_multipart) +from packaging.errors import PackagingFileError +from packaging.command.cmd import Command + + +def zip_dir(directory): + """Compresses recursively contents of directory into a BytesIO object""" + destination = BytesIO() + with zipfile.ZipFile(destination, "w") as zip_file: + for root, dirs, files in os.walk(directory): + for name in files: + full = os.path.join(root, name) + relative = root[len(directory):].lstrip(os.path.sep) + dest = os.path.join(relative, name) + zip_file.write(full, dest) + return destination + + +class upload_docs(Command): + + description = "upload HTML documentation to PyPI" + + user_options = [ + ('repository=', 'r', + "repository URL [default: %s]" % DEFAULT_REPOSITORY), + ('show-response', None, + "display full response text from server"), + ('upload-dir=', None, + "directory to upload"), + ] + + def initialize_options(self): + self.repository = None + self.realm = None + self.show_response = False + self.upload_dir = None + self.username = '' + self.password = '' + + def finalize_options(self): + if self.repository is None: + self.repository = DEFAULT_REPOSITORY + if self.realm is None: + self.realm = DEFAULT_REALM + if self.upload_dir is None: + build = self.get_finalized_command('build') + self.upload_dir = os.path.join(build.build_base, "docs") + if not os.path.isdir(self.upload_dir): + self.upload_dir = os.path.join(build.build_base, "doc") + logger.info('Using upload directory %s', self.upload_dir) + self.verify_upload_dir(self.upload_dir) + config = read_pypirc(self.repository, self.realm) + if config != {}: + self.username = config['username'] + self.password = config['password'] + self.repository = config['repository'] + self.realm = config['realm'] + + def verify_upload_dir(self, upload_dir): + self.ensure_dirname('upload_dir') + index_location = os.path.join(upload_dir, "index.html") + if not os.path.exists(index_location): + mesg = "No 'index.html found in docs directory (%s)" + raise PackagingFileError(mesg % upload_dir) + + def run(self): + name = self.distribution.metadata['Name'] + version = self.distribution.metadata['Version'] + zip_file = zip_dir(self.upload_dir) + + fields = [(':action', 'doc_upload'), + ('name', name), ('version', version)] + files = [('content', name, zip_file.getvalue())] + content_type, body = encode_multipart(fields, files) + + credentials = self.username + ':' + self.password + # FIXME should use explicit encoding + auth = b"Basic " + base64.encodebytes(credentials.encode()).strip() + + logger.info("Submitting documentation to %s", self.repository) + + scheme, netloc, url, params, query, fragments = urllib.parse.urlparse( + self.repository) + if scheme == "http": + conn = http.client.HTTPConnection(netloc) + elif scheme == "https": + conn = http.client.HTTPSConnection(netloc) + else: + raise AssertionError("unsupported scheme %r" % scheme) + + try: + conn.connect() + conn.putrequest("POST", url) + conn.putheader('Content-type', content_type) + conn.putheader('Content-length', str(len(body))) + conn.putheader('Authorization', auth) + conn.endheaders() + conn.send(body) + + except socket.error as e: + logger.error(e) + return + + r = conn.getresponse() + + if r.status == 200: + logger.info('Server response (%s): %s', r.status, r.reason) + elif r.status == 301: + location = r.getheader('Location') + if location is None: + location = 'http://packages.python.org/%s/' % name + logger.info('Upload successful. Visit %s', location) + else: + logger.error('Upload failed (%s): %s', r.status, r.reason) + + if self.show_response and logger.isEnabledFor(logging.INFO): + sep = '-' * 75 + logger.info('%s\n%s\n%s', sep, r.read().decode('utf-8'), sep) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-10.0-amd64.exe Binary file Lib/packaging/command/wininst-10.0-amd64.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-10.0.exe Binary file Lib/packaging/command/wininst-10.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-6.0.exe Binary file Lib/packaging/command/wininst-6.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-7.1.exe Binary file Lib/packaging/command/wininst-7.1.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-8.0.exe Binary file Lib/packaging/command/wininst-8.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-9.0-amd64.exe Binary file Lib/packaging/command/wininst-9.0-amd64.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/command/wininst-9.0.exe Binary file Lib/packaging/command/wininst-9.0.exe has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compat.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compat.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,50 @@ +"""Support for build-time 2to3 conversion.""" + +from packaging import logger + + +# XXX Having two classes with the same name is not a good thing. +# XXX 2to3-related code should move from util to this module + +try: + from packaging.util import Mixin2to3 as _Mixin2to3 + _CONVERT = True + _KLASS = _Mixin2to3 +except ImportError: + _CONVERT = False + _KLASS = object + +__all__ = ['Mixin2to3'] + + +class Mixin2to3(_KLASS): + """ The base class which can be used for refactoring. When run under + Python 3.0, the run_2to3 method provided by Mixin2to3 is overridden. + When run on Python 2.x, it merely creates a class which overrides run_2to3, + yet does nothing in particular with it. + """ + if _CONVERT: + + def _run_2to3(self, files=[], doctests=[], fixers=[]): + """ Takes a list of files and doctests, and performs conversion + on those. + - First, the files which contain the code(`files`) are converted. + - Second, the doctests in `files` are converted. + - Thirdly, the doctests in `doctests` are converted. + """ + if fixers: + self.fixer_names = fixers + + if files: + logger.info('converting Python code and doctests') + _KLASS.run_2to3(self, files) + _KLASS.run_2to3(self, files, doctests_only=True) + + if doctests: + logger.info('converting doctests in text files') + _KLASS.run_2to3(self, doctests, doctests_only=True) + else: + # If run on Python 2.x, there is nothing to do. + + def _run_2to3(self, files=[], doctests=[], fixers=[]): + pass diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,274 @@ +"""Compiler abstraction model used by packaging. + +An abstract base class is defined in the ccompiler submodule, and +concrete implementations suitable for various platforms are defined in +the other submodules. The extension module is also placed in this +package. + +In general, code should not instantiate compiler classes directly but +use the new_compiler and customize_compiler functions provided in this +module. + +The compiler system has a registration API: get_default_compiler, +set_compiler, show_compilers. +""" + +import os +import sys +import re +import sysconfig + +from packaging.util import resolve_name +from packaging.errors import PackagingPlatformError +from packaging import logger + +def customize_compiler(compiler): + """Do any platform-specific customization of a CCompiler instance. + + Mainly needed on Unix, so we can plug in the information that + varies across Unices and is stored in Python's Makefile. + """ + if compiler.name == "unix": + cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags = ( + sysconfig.get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS', + 'CCSHARED', 'LDSHARED', 'SO', 'AR', + 'ARFLAGS')) + + if 'CC' in os.environ: + cc = os.environ['CC'] + if 'CXX' in os.environ: + cxx = os.environ['CXX'] + if 'LDSHARED' in os.environ: + ldshared = os.environ['LDSHARED'] + if 'CPP' in os.environ: + cpp = os.environ['CPP'] + else: + cpp = cc + " -E" # not always + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: + cflags = opt + ' ' + os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] + cflags = cflags + ' ' + os.environ['CPPFLAGS'] + ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] + if 'AR' in os.environ: + ar = os.environ['AR'] + if 'ARFLAGS' in os.environ: + archiver = ar + ' ' + os.environ['ARFLAGS'] + else: + if ar_flags is not None: + archiver = ar + ' ' + ar_flags + else: + # see if its the proper default value + # mmm I don't want to backport the makefile + archiver = ar + ' rc' + + cc_cmd = cc + ' ' + cflags + compiler.set_executables( + preprocessor=cpp, + compiler=cc_cmd, + compiler_so=cc_cmd + ' ' + ccshared, + compiler_cxx=cxx, + linker_so=ldshared, + linker_exe=cc, + archiver=archiver) + + compiler.shared_lib_extension = so_ext + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), +) + +def get_default_compiler(osname=None, platform=None): + """ Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Defaults to Unix compiler + return 'unix' + + +# compiler mapping +# XXX useful to expose them? (i.e. get_compiler_names) +_COMPILERS = { + 'unix': 'packaging.compiler.unixccompiler.UnixCCompiler', + 'msvc': 'packaging.compiler.msvccompiler.MSVCCompiler', + 'cygwin': 'packaging.compiler.cygwinccompiler.CygwinCCompiler', + 'mingw32': 'packaging.compiler.cygwinccompiler.Mingw32CCompiler', + 'bcpp': 'packaging.compiler.bcppcompiler.BCPPCompiler', +} + +def set_compiler(location): + """Add or change a compiler""" + cls = resolve_name(location) + # XXX we want to check the class here + _COMPILERS[cls.name] = cls + + +def show_compilers(): + """Print list of available compilers (used by the "--help-compiler" + options to "build", "build_ext", "build_clib"). + """ + from packaging.fancy_getopt import FancyGetopt + compilers = [] + + for name, cls in _COMPILERS.items(): + if isinstance(cls, str): + cls = resolve_name(cls) + _COMPILERS[name] = cls + + compilers.append(("compiler=" + name, None, cls.description)) + + compilers.sort() + pretty_printer = FancyGetopt(compilers) + pretty_printer.print_help("List of available compilers:") + + +def new_compiler(plat=None, compiler=None, dry_run=False, force=False): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + cls = _COMPILERS[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise PackagingPlatformError(msg) + + if isinstance(cls, str): + cls = resolve_name(cls) + _COMPILERS[compiler] = cls + + return cls(dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + + pp_opts = [] + for macro in macros: + + if not isinstance(macro, tuple) and 1 <= len(macro) <= 2: + raise TypeError( + "bad macro definition '%s': each element of 'macros'" + "list must be a 1- or 2-tuple" % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + + return pp_opts + + +def gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries): + """Generate linker options for searching library directories and + linking with specific libraries. + + 'libraries' and 'library_dirs' are, respectively, lists of library names + (not filenames!) and search directories. Returns a list of command-line + options suitable for use with some compiler (depending on the two format + strings passed in). + """ + lib_opts = [] + + for dir in library_dirs: + lib_opts.append(compiler.library_dir_option(dir)) + + for dir in runtime_library_dirs: + opt = compiler.runtime_library_dir_option(dir) + if isinstance(opt, list): + lib_opts.extend(opt) + else: + lib_opts.append(opt) + + # XXX it's important that we *not* remove redundant library mentions! + # sometimes you really do have to say "-lfoo -lbar -lfoo" in order to + # resolve all symbols. I just hope we never have to say "-lfoo obj.o + # -lbar" to get things to work -- that's certainly a possibility, but a + # pretty nasty way to arrange your C code. + + for lib in libraries: + lib_dir, lib_name = os.path.split(lib) + if lib_dir != '': + lib_file = compiler.find_library_file([lib_dir], lib_name) + if lib_file is not None: + lib_opts.append(lib_file) + else: + logger.warning("no library file corresponding to " + "'%s' found (skipping)" % lib) + else: + lib_opts.append(compiler.library_option(lib)) + + return lib_opts diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/bcppcompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/bcppcompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,355 @@ +"""CCompiler implementation for the Borland C++ compiler.""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + +import os + +from packaging.errors import (PackagingExecError, CompileError, LibError, + LinkError, UnknownFileError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_preprocess_options +from packaging.file_util import write_file +from packaging.dep_util import newer +from packaging import logger + + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + name = 'bcpp' + description = 'Borland C++ Compiler' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, dry_run=False, force=False): + super(BCPPCompiler, self).__init__(dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + compile_opts = extra_preargs or [] + compile_opts.append('-c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + # XXX why do the normpath here? + src = os.path.normpath(src) + obj = os.path.normpath(obj) + # XXX _setup_compile() did a mkpath() too but before the normpath. + # Is it possible to skip the normpath? + self.mkpath(os.path.dirname(obj)) + + if ext == '.res': + # This is already a binary file -- skip it. + continue # the 'for' loop + if ext == '.rc': + # This needs to be compiled to a .res file -- do it now. + try: + self.spawn(["brcc32", "-fo", obj, src]) + except PackagingExecError as msg: + raise CompileError(msg) + continue # the 'for' loop + + # The next two are both for the real compiler. + if ext in self._c_extensions: + input_opt = "" + elif ext in self._cpp_extensions: + input_opt = "-P" + else: + # Unknown file type -- no extra options. The compiler + # will probably fail, but let it just in case this is a + # file the compiler recognizes even if we don't. + input_opt = "" + + output_opt = "-o" + obj + + # Compiler command line syntax is: "bcc32 [options] file(s)". + # Note that the source file names must appear at the end of + # the command line. + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs + [src]) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = [output_filename, '/u'] + objects + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + + # XXX this ignores 'build_temp'! should follow the lead of + # msvccompiler.py + + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + if runtime_library_dirs: + logger.warning("don't know what to do with " + "'runtime_library_dirs': %r", runtime_library_dirs) + + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + + # Figure out linker args based on type of target. + if target_desc == CCompiler.EXECUTABLE: + startup_obj = 'c0w32' + if debug: + ld_args = self.ldflags_exe_debug[:] + else: + ld_args = self.ldflags_exe[:] + else: + startup_obj = 'c0d32' + if debug: + ld_args = self.ldflags_shared_debug[:] + else: + ld_args = self.ldflags_shared[:] + + + # Create a temporary exports file for use by the linker + if export_symbols is None: + def_file = '' + else: + head, tail = os.path.split(output_filename) + modname, ext = os.path.splitext(tail) + temp_dir = os.path.dirname(objects[0]) # preserve tree structure + def_file = os.path.join(temp_dir, '%s.def' % modname) + contents = ['EXPORTS'] + for sym in (export_symbols or []): + contents.append(' %s=_%s' % (sym, sym)) + self.execute(write_file, (def_file, contents), + "writing %s" % def_file) + + # Borland C++ has problems with '/' in paths + objects2 = [os.path.normpath(o) for o in objects] + # split objects in .obj and .res files + # Borland C++ needs them at different positions in the command line + objects = [startup_obj] + resources = [] + for file in objects2: + base, ext = os.path.splitext(os.path.normcase(file)) + if ext == '.res': + resources.append(file) + else: + objects.append(file) + + + for l in library_dirs: + ld_args.append("/L%s" % os.path.normpath(l)) + ld_args.append("/L.") # we sometimes use relative paths + + # list of object files + ld_args.extend(objects) + + # XXX the command line syntax for Borland C++ is a bit wonky; + # certain filenames are jammed together in one big string, but + # comma-delimited. This doesn't mesh too well with the + # Unix-centric attitude (with a DOS/Windows quoting hack) of + # 'spawn()', so constructing the argument list is a bit + # awkward. Note that doing the obvious thing and jamming all + # the filenames and commas into one argument would be wrong, + # because 'spawn()' would quote any filenames with spaces in + # them. Arghghh!. Apparently it works fine as coded... + + # name of dll/exe file + ld_args.extend((',',output_filename)) + # no map file and start libraries + ld_args.append(',,') + + for lib in libraries: + # see if we find it and if there is a bcpp specific lib + # (xxx_bcpp.lib) + libfile = self.find_library_file(library_dirs, lib, debug) + if libfile is None: + ld_args.append(lib) + # probably a BCPP internal library -- don't warn + else: + # full name which prefers bcpp_xxx.lib over xxx.lib + ld_args.append(libfile) + + # some default libraries + ld_args.append('import32') + ld_args.append('cw32mt') + + # def file for export symbols + ld_args.extend((',',def_file)) + # add resource files + ld_args.append(',') + ld_args.extend(resources) + + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + + + def find_library_file(self, dirs, lib, debug=False): + # List of effective library names to try, in order of preference: + # xxx_bcpp.lib is better than xxx.lib + # and xxx_d.lib is better than xxx.lib if debug is set + # + # The "_bcpp" suffix is to handle a Python installation for people + # with multiple compilers (primarily Packaging hackers, I suspect + # ;-). The idea is they'd have one static library for each + # compiler they care about, since (almost?) every Windows compiler + # seems to have a different format for static libraries. + if debug: + dlib = (lib + "_d") + try_names = (dlib + "_bcpp", lib + "_bcpp", dlib, lib) + else: + try_names = (lib + "_bcpp", lib) + + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # overwrite the one from CCompiler to support rc and res-files + def object_filenames(self, source_filenames, strip_dir=False, + output_dir=''): + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + # use normcase to make sure '.rc' is really '.rc' and not '.RC' + base, ext = os.path.splitext(os.path.normcase(src_name)) + if ext not in (self.src_extensions + ['.rc','.res']): + raise UnknownFileError("unknown file type '%s' (from '%s')" % \ + (ext, src_name)) + if strip_dir: + base = os.path.basename(base) + if ext == '.res': + # these can go unchanged + obj_names.append(os.path.join(output_dir, base + ext)) + elif ext == '.rc': + # these need to be compiled to .res-files + obj_names.append(os.path.join(output_dir, base + '.res')) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, + extra_postargs=None): + _, macros, include_dirs = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except PackagingExecError as msg: + raise CompileError(msg) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/ccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/ccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,863 @@ +"""Abstract base class for compilers. + +This modules contains CCompiler, an abstract base class that defines the +interface for the compiler abstraction model used by packaging. +""" + +import os +from shutil import move +from packaging import logger +from packaging.util import split_quoted, execute, newer_group, spawn +from packaging.errors import (CompileError, LinkError, UnknownFileError) +from packaging.compiler import gen_preprocess_options + + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'name' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. + name = None + description = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c": "c", + ".cc": "c++", + ".cpp": "c++", + ".cxx": "c++", + ".m": "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, dry_run=False, force=False): + self.dry_run = dry_run + self.force = force + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key, value in self.executables.items(): + self.set_executable(key, value) + + def set_executables(self, **args): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key, value in args.items(): + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % \ + (key, self.__class__.__name__)) + self.set_executable(key, value) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i = i + 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) == 1 or + (len(defn) == 2 and + (isinstance(defn[1], str) or defn[1] is None))) and + isinstance(defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro(name) + if i is not None: + del self.macros[i] + + defn = (name, value) + self.macros.append(defn) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro(name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + def add_library(self, libname): + """Add 'libname' to the list of libraries that will be included in + all links driven by this compiler object. Note that 'libname' + should *not* be the name of a file containing a library, but the + name of the library itself: the actual filename will be inferred by + the linker, the compiler, or the compiler class (depending on the + platform). + + The linker will be instructed to link against libraries in the + order they were supplied to 'add_library()' and/or + 'set_libraries()'. It is perfectly valid to duplicate library + names; the linker will be instructed to link against libraries as + many times as they are mentioned. + """ + self.libraries.append(libname) + + def set_libraries(self, libnames): + """Set the list of libraries to be included in all links driven by + this compiler object to 'libnames' (a list of strings). This does + not affect any standard system libraries that the linker may + include by default. + """ + self.libraries = libnames[:] + + + def add_library_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + libraries specified to 'add_library()' and 'set_libraries()'. The + linker will be instructed to search for libraries in the order they + are supplied to 'add_library_dir()' and/or 'set_library_dirs()'. + """ + self.library_dirs.append(dir) + + def set_library_dirs(self, dirs): + """Set the list of library search directories to 'dirs' (a list of + strings). This does not affect any standard library search path + that the linker may search by default. + """ + self.library_dirs = dirs[:] + + def add_runtime_library_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + shared libraries at runtime. + """ + self.runtime_library_dirs.append(dir) + + def set_runtime_library_dirs(self, dirs): + """Set the list of directories to search for shared libraries at + runtime to 'dirs' (a list of strings). This does not affect any + standard search path that the runtime linker may search by + default. + """ + self.runtime_library_dirs = dirs[:] + + def add_link_object(self, object): + """Add 'object' to the list of object files (or analogues, such as + explicitly named library files or the output of "resource + compilers") to be included in every link driven by this compiler + object. + """ + self.objects.append(object) + + def set_link_objects(self, objects): + """Set the list of object files (or analogues) to be included in + every link to 'objects'. This does not affect any standard object + files that the linker may include by default (such as system + libraries). + """ + self.objects = objects[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + def _setup_compile(self, outdir, macros, incdirs, sources, depends, + extra): + """Process arguments and decide which source files to compile.""" + if outdir is None: + outdir = self.output_dir + elif not isinstance(outdir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if incdirs is None: + incdirs = self.include_dirs + elif isinstance(incdirs, (list, tuple)): + incdirs = list(incdirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + if extra is None: + extra = [] + + # Get the list of expected output (object) files + objects = self.object_filenames(sources, + strip_dir=False, + output_dir=outdir) + assert len(objects) == len(sources) + + pp_opts = gen_preprocess_options(macros, incdirs) + + build = {} + for i in range(len(sources)): + src = sources[i] + obj = objects[i] + ext = os.path.splitext(src)[1] + self.mkpath(os.path.dirname(obj)) + build[obj] = (src, ext) + + return macros, objects, extra, pp_opts, build + + def _get_cc_args(self, pp_opts, debug, before): + # works for unixccompiler and cygwinccompiler + cc_args = pp_opts + ['-c'] + if debug: + cc_args[:0] = ['-g'] + if before: + cc_args[:0] = before + return cc_args + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + def _fix_object_args(self, objects, output_dir): + """Typecheck and fix up some arguments supplied to various methods. + Specifically: ensure that 'objects' is a list; if output_dir is + None, replace with self.output_dir. Return fixed versions of + 'objects' and 'output_dir'. + """ + if not isinstance(objects, (list, tuple)): + raise TypeError("'objects' must be a list or tuple of strings") + objects = list(objects) + + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + return objects, output_dir + + def _fix_lib_args(self, libraries, library_dirs, runtime_library_dirs): + """Typecheck and fix up some of the arguments supplied to the + 'link_*' methods. Specifically: ensure that all arguments are + lists, and augment them with their permanent versions + (eg. 'self.libraries' augments 'libraries'). Return a tuple with + fixed versions of all arguments. + """ + if libraries is None: + libraries = self.libraries + elif isinstance(libraries, (list, tuple)): + libraries = list(libraries) + (self.libraries or []) + else: + raise TypeError( + "'libraries' (if supplied) must be a list of strings") + + if library_dirs is None: + library_dirs = self.library_dirs + elif isinstance(library_dirs, (list, tuple)): + library_dirs = list(library_dirs) + (self.library_dirs or []) + else: + raise TypeError( + "'library_dirs' (if supplied) must be a list of strings") + + if runtime_library_dirs is None: + runtime_library_dirs = self.runtime_library_dirs + elif isinstance(runtime_library_dirs, (list, tuple)): + runtime_library_dirs = (list(runtime_library_dirs) + + (self.runtime_library_dirs or [])) + else: + raise TypeError("'runtime_library_dirs' (if supplied) " + "must be a list of strings") + + return libraries, library_dirs, runtime_library_dirs + + def _need_link(self, objects, output_file): + """Return true if we need to relink the files listed in 'objects' + to recreate 'output_file'. + """ + if self.force: + return True + else: + if self.dry_run: + newer = newer_group(objects, output_file, missing='newer') + else: + newer = newer_group(objects, output_file) + return newer + + def detect_language(self, sources): + """Detect the language of a given file, or list of files. Uses + language_map, and language_order to do the job. + """ + if not isinstance(sources, list): + sources = [sources] + lang = None + index = len(self.language_order) + for source in sources: + base, ext = os.path.splitext(source) + extlang = self.language_map.get(ext) + try: + extindex = self.language_order.index(extlang) + if extindex < index: + lang = extlang + index = extindex + except ValueError: + pass + return lang + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + def compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=False, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + 'sources' must be a list of filenames, most likely C/C++ + files, but in reality anything that can be handled by a + particular compiler and compiler class (eg. MSVCCompiler can + handle resource files in 'sources'). Return a list of object + filenames, one per source filename in 'sources'. Depending on + the implementation, not all source files will necessarily be + compiled, but all corresponding object filenames will be + returned. + + If 'output_dir' is given, object files will be put under it, while + retaining their original path component. That is, "foo/bar.c" + normally compiles to "foo/bar.o" (for a Unix implementation); if + 'output_dir' is "build", then it would compile to + "build/foo/bar.o". + + 'macros', if given, must be a list of macro definitions. A macro + definition is either a (name, value) 2-tuple or a (name,) 1-tuple. + The former defines a macro; if the value is None, the macro is + defined without an explicit value. The 1-tuple case undefines a + macro. Later definitions/redefinitions/ undefinitions take + precedence. + + 'include_dirs', if given, must be a list of strings, the + directories to add to the default include file search path for this + compilation only. + + 'debug' is a boolean; if true, the compiler will be instructed to + output debug symbols in (or alongside) the object file(s). + + 'extra_preargs' and 'extra_postargs' are implementation- dependent. + On platforms that have the notion of a command line (e.g. Unix, + DOS/Windows), they are most likely lists of strings: extra + command-line arguments to prepand/append to the compiler command + line. On other platforms, consult the implementation class + documentation. In any event, they are intended as an escape hatch + for those occasions when the abstract compiler framework doesn't + cut the mustard. + + 'depends', if given, is a list of filenames that all targets + depend on. If a source file is older than any file in + depends, then the source file will be recompiled. This + supports dependency tracking, but only at a coarse + granularity. + + Raises CompileError on failure. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # Return *all* object filenames, not just the ones we just built. + return objects + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + """Compile 'src' to product 'obj'.""" + + # A concrete compiler class that does not override compile() + # should implement _compile(). + pass + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + """Link a bunch of stuff together to create a static library file. + The "bunch of stuff" consists of the list of object files supplied + as 'objects', the extra object files supplied to + 'add_link_object()' and/or 'set_link_objects()', the libraries + supplied to 'add_library()' and/or 'set_libraries()', and the + libraries supplied as 'libraries' (if any). + + 'output_libname' should be a library name, not a filename; the + filename will be inferred from the library name. 'output_dir' is + the directory where the library file will be put. + + 'debug' is a boolean; if true, debugging information will be + included in the library (note that on most platforms, it is the + compile step where this matters: the 'debug' flag is included here + just for consistency). + + 'target_lang' is the target language for which the given objects + are being compiled. This allows specific linkage time treatment of + certain languages. + + Raises LibError on failure. + """ + pass + + # values for target_desc parameter in link() + SHARED_OBJECT = "shared_object" + SHARED_LIBRARY = "shared_library" + EXECUTABLE = "executable" + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + """Link a bunch of stuff together to create an executable or + shared library file. + + The "bunch of stuff" consists of the list of object files supplied + as 'objects'. 'output_filename' should be a filename. If + 'output_dir' is supplied, 'output_filename' is relative to it + (i.e. 'output_filename' can provide directory components if + needed). + + 'libraries' is a list of libraries to link against. These are + library names, not filenames, since they're translated into + filenames in a platform-specific way (eg. "foo" becomes "libfoo.a" + on Unix and "foo.lib" on DOS/Windows). However, they can include a + directory component, which means the linker will look in that + specific directory rather than searching all the normal locations. + + 'library_dirs', if supplied, should be a list of directories to + search for libraries that were specified as bare library names + (ie. no directory component). These are on top of the system + default and those supplied to 'add_library_dir()' and/or + 'set_library_dirs()'. 'runtime_library_dirs' is a list of + directories that will be embedded into the shared library and used + to search for other shared libraries that *it* depends on at + run-time. (This may only be relevant on Unix.) + + 'export_symbols' is a list of symbols that the shared library will + export. (This appears to be relevant only on Windows.) + + 'debug' is as for 'compile()' and 'create_static_lib()', with the + slight distinction that it actually matters on most platforms (as + opposed to 'create_static_lib()', which includes a 'debug' flag + mostly for form's sake). + + 'extra_preargs' and 'extra_postargs' are as for 'compile()' (except + of course that they supply command-line arguments for the + particular linker being used). + + 'target_lang' is the target language for which the given objects + are being compiled. This allows specific linkage time treatment of + certain languages. + + Raises LinkError on failure. + """ + raise NotImplementedError + + + # Old 'link_*()' methods, rewritten to use the new 'link()' method. + + def link_shared_lib(self, objects, output_libname, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, export_symbols=None, + debug=False, extra_preargs=None, extra_postargs=None, + build_temp=None, target_lang=None): + self.link(CCompiler.SHARED_LIBRARY, objects, + self.library_filename(output_libname, lib_type='shared'), + output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + + def link_shared_object(self, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, export_symbols=None, + debug=False, extra_preargs=None, extra_postargs=None, + build_temp=None, target_lang=None): + self.link(CCompiler.SHARED_OBJECT, objects, + output_filename, output_dir, + libraries, library_dirs, runtime_library_dirs, + export_symbols, debug, + extra_preargs, extra_postargs, build_temp, target_lang) + + def link_executable(self, objects, output_progname, output_dir=None, + libraries=None, library_dirs=None, + runtime_library_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, + target_lang=None): + self.link(CCompiler.EXECUTABLE, objects, + self.executable_filename(output_progname), output_dir, + libraries, library_dirs, runtime_library_dirs, None, + debug, extra_preargs, extra_postargs, None, target_lang) + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + + def library_dir_option(self, dir): + """Return the compiler option to add 'dir' to the list of + directories searched for libraries. + """ + raise NotImplementedError + + def runtime_library_dir_option(self, dir): + """Return the compiler option to add 'dir' to the list of + directories searched for runtime libraries. + """ + raise NotImplementedError + + def library_option(self, lib): + """Return the compiler option to add 'dir' to the list of libraries + linked into the shared library or executable. + """ + raise NotImplementedError + + def has_function(self, funcname, includes=None, include_dirs=None, + libraries=None, library_dirs=None): + """Return a boolean indicating whether funcname is supported on + the current platform. The optional arguments can be used to + augment the compilation environment. + """ + + # this can't be included at module scope because it tries to + # import math which might not be available at that point - maybe + # the necessary logic should just be inlined? + import tempfile + if includes is None: + includes = [] + if include_dirs is None: + include_dirs = [] + if libraries is None: + libraries = [] + if library_dirs is None: + library_dirs = [] + fd, fname = tempfile.mkstemp(".c", funcname, text=True) + with os.fdopen(fd, "w") as f: + for incl in includes: + f.write("""#include "%s"\n""" % incl) + f.write("""\ +main (int argc, char **argv) { + %s(); +} +""" % funcname) + try: + objects = self.compile([fname], include_dirs=include_dirs) + except CompileError: + return False + + try: + self.link_executable(objects, "a.out", + libraries=libraries, + library_dirs=library_dirs) + except (LinkError, TypeError): + return False + return True + + def find_library_file(self, dirs, lib, debug=False): + """Search the specified list of directories for a static or shared + library file 'lib' and return the full path to that file. If + 'debug' is true, look for a debugging version (if that makes sense on + the current platform). Return None if 'lib' wasn't found in any of + the specified directories. + """ + raise NotImplementedError + + # -- Filename generation methods ----------------------------------- + + # The default implementation of the filename generating methods are + # prejudiced towards the Unix/DOS/Windows view of the world: + # * object files are named by replacing the source file extension + # (eg. .c/.cpp -> .o/.obj) + # * library files (shared or static) are named by plugging the + # library name and extension into a format string, eg. + # "lib%s.%s" % (lib_name, ".a") for Unix static libraries + # * executables are named by appending an extension (possibly + # empty) to the program name: eg. progname + ".exe" for + # Windows + # + # To reduce redundant code, these methods expect to find + # several attributes in the current object (presumably defined + # as class attributes): + # * src_extensions - + # list of C/C++ source file extensions, eg. ['.c', '.cpp'] + # * obj_extension - + # object file extension, eg. '.o' or '.obj' + # * static_lib_extension - + # extension for static library files, eg. '.a' or '.lib' + # * shared_lib_extension - + # extension for shared library/object files, eg. '.so', '.dll' + # * static_lib_format - + # format string for generating static library filenames, + # eg. 'lib%s.%s' or '%s.%s' + # * shared_lib_format + # format string for generating shared library filenames + # (probably same as static_lib_format, since the extension + # is one of the intended parameters to the format string) + # * exe_extension - + # extension for executable files, eg. '' or '.exe' + + def object_filenames(self, source_filenames, strip_dir=False, output_dir=''): + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + raise UnknownFileError("unknown file type '%s' (from '%s')" % + (ext, src_name)) + if strip_dir: + base = os.path.basename(base) + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + def shared_object_filename(self, basename, strip_dir=False, output_dir=''): + assert output_dir is not None + if strip_dir: + basename = os.path.basename(basename) + return os.path.join(output_dir, basename + self.shared_lib_extension) + + def executable_filename(self, basename, strip_dir=False, output_dir=''): + assert output_dir is not None + if strip_dir: + basename = os.path.basename(basename) + return os.path.join(output_dir, basename + (self.exe_extension or '')) + + def library_filename(self, libname, lib_type='static', # or 'shared' + strip_dir=False, output_dir=''): + assert output_dir is not None + if lib_type not in ("static", "shared", "dylib"): + raise ValueError( + "'lib_type' must be 'static', 'shared' or 'dylib'") + fmt = getattr(self, lib_type + "_lib_format") + ext = getattr(self, lib_type + "_lib_extension") + + dir, base = os.path.split(libname) + filename = fmt % (base, ext) + if strip_dir: + dir = '' + + return os.path.join(output_dir, dir, filename) + + + # -- Utility methods ----------------------------------------------- + + def execute(self, func, args, msg=None, level=1): + execute(func, args, msg, self.dry_run) + + def spawn(self, cmd): + spawn(cmd, dry_run=self.dry_run) + + def move_file(self, src, dst): + logger.info("moving %r to %r", src, dst) + if self.dry_run: + return + return move(src, dst) + + def mkpath(self, name, mode=0o777): + name = os.path.normpath(name) + if os.path.isdir(name) or name == '': + return + if self.dry_run: + head = '' + for part in name.split(os.sep): + logger.info("created directory %s%s", head, part) + head += part + os.sep + return + os.makedirs(name, mode) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/cygwinccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/cygwinccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,355 @@ +"""CCompiler implementations for Cygwin and mingw32 versions of GCC. + +This module contains the CygwinCCompiler class, a subclass of +UnixCCompiler that handles the Cygwin port of the GNU C compiler to +Windows, and the Mingw32CCompiler class which handles the mingw32 port +of GCC (same as cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate a import library for its dll +# - create a def-file for python??.dll +# - create a import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + + +import os +import sys + +from packaging import logger +from packaging.compiler.unixccompiler import UnixCCompiler +from packaging.util import write_file +from packaging.errors import PackagingExecError, CompileError, UnknownFileError +from packaging.util import get_compiler_versions +import sysconfig + +# TODO use platform instead of sys.version +# (platform does unholy sys.version parsing too, but at least it gives other +# VMs a chance to override the returned values) + + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + name = 'cygwin' + description = 'Cygwin port of GNU C Compiler for Win32' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, dry_run=False, force=False): + super(CygwinCCompiler, self).__init__(dry_run, force) + + status, details = check_config_h() + logger.debug("Python's GCC status: %s (details: %s)", status, details) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_compiler_versions() + logger.debug(self.name + ": gcc %s, ld %s, dllwrap %s\n", + self.gcc_version, + self.ld_version, + self.dllwrap_version) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + """Compile the source by spawning GCC and windres if needed.""" + if ext == '.rc' or ext == '.res': + # gcc needs '.res' and '.rc' compiled to object files !!! + try: + self.spawn(["windres", "-i", src, "-o", obj]) + except PackagingExecError as msg: + raise CompileError(msg) + else: # for other files use the C-compiler + try: + self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + """Link the objects.""" + # use separate copies, so we can modify the lists + extra_preargs = list(extra_preargs or []) + libraries = list(libraries or []) + objects = list(objects or []) + + # Additional libraries + libraries.extend(self.dll_libraries) + + # handle export symbols by creating a def-file + # with executables this only works with gcc/ld as linker + if ((export_symbols is not None) and + (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): + # (The linker doesn't do anything if output is up-to-date. + # So it would probably better to check if we really need this, + # but for this we had to insert some unchanged parts of + # UnixCCompiler, and this is not what we want.) + + # we want to put some files in the same directory as the + # object files are, build_temp doesn't help much + # where are the object files + temp_dir = os.path.dirname(objects[0]) + # name of dll to give the helper files the same base name + dll_name, dll_extension = os.path.splitext( + os.path.basename(output_filename)) + + # generate the filenames for these files + def_file = os.path.join(temp_dir, dll_name + ".def") + lib_file = os.path.join(temp_dir, 'lib' + dll_name + ".a") + + # Generate .def file + contents = [ + "LIBRARY %s" % os.path.basename(output_filename), + "EXPORTS"] + for sym in export_symbols: + contents.append(sym) + self.execute(write_file, (def_file, contents), + "writing %s" % def_file) + + # next add options for def-file and to creating import libraries + + # dllwrap uses different options than gcc/ld + if self.linker_dll == "dllwrap": + extra_preargs.extend(("--output-lib", lib_file)) + # for dllwrap we have to use a special option + extra_preargs.extend(("--def", def_file)) + # we use gcc/ld here and can be sure ld is >= 2.9.10 + else: + # doesn't work: bfd_close build\...\libfoo.a: Invalid operation + #extra_preargs.extend(("-Wl,--out-implib,%s" % lib_file)) + # for gcc/ld the def-file is specified as any object files + objects.append(def_file) + + #end: if ((export_symbols is not None) and + # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): + + # who wants symbols and a many times larger output file + # should explicitly switch the debug mode on + # otherwise we let dllwrap/ld strip the output file + # (On my machine: 10KB < stripped_file < ??100KB + # unstripped_file = stripped_file + XXX KB + # ( XXX=254 for a typical python extension)) + if not debug: + extra_preargs.append("-s") + + super(CygwinCCompiler, self).link( + target_desc, objects, output_filename, output_dir, libraries, + library_dirs, runtime_library_dirs, + None, # export_symbols, we do this in our def-file + debug, extra_preargs, extra_postargs, build_temp, target_lang) + + # -- Miscellaneous methods ----------------------------------------- + + def object_filenames(self, source_filenames, strip_dir=False, + output_dir=''): + """Adds supports for rc and res files.""" + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + # use normcase to make sure '.rc' is really '.rc' and not '.RC' + base, ext = os.path.splitext(os.path.normcase(src_name)) + if ext not in (self.src_extensions + ['.rc','.res']): + raise UnknownFileError("unknown file type '%s' (from '%s')" % (ext, src_name)) + if strip_dir: + base = os.path.basename(base) + if ext in ('.res', '.rc'): + # these need to be compiled to object files + obj_names.append(os.path.join(output_dir, + base + ext + self.obj_extension)) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + name = 'mingw32' + description = 'MinGW32 compiler' + + def __init__(self, dry_run=False, force=False): + super(Mingw32CCompiler, self).__init__(dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + self.set_executables(compiler='gcc -mno-cygwin -O -Wall', + compiler_so='gcc -mno-cygwin -mdll -O -Wall', + compiler_cxx='g++ -mno-cygwin -O -Wall', + linker_exe='gcc -mno-cygwin', + linker_so='%s -mno-cygwin %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using a unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + with open(fn) as config_h: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + except IOError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/extension.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/extension.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,121 @@ +"""Class representing C/C++ extension modules.""" + +from packaging import logger + +# This class is really only used by the "build_ext" command, so it might +# make sense to put it in distutils.command.build_ext. However, that +# module is already big enough, and I want to make this class a bit more +# complex to simplify some common cases ("foo" module in "foo.c") and do +# better error-checking ("foo.c" actually exists). +# +# Also, putting this in build_ext.py means every setup script would have to +# import that large-ish module (indirectly, through distutils.core) in +# order to do anything. + + +class Extension: + """Just a collection of attributes that describes an extension + module and everything needed to build it (hopefully in a portable + way, but there are hooks that let you be as unportable as you need). + + Instance attributes: + name : string + the full name of the extension, including any packages -- ie. + *not* a filename or pathname, but Python dotted name + sources : [string] + list of source filenames, relative to the distribution root + (where the setup script lives), in Unix form (slash-separated) + for portability. Source files may be C, C++, SWIG (.i), + platform-specific resource files, or whatever else is recognized + by the "build_ext" command as source for a Python extension. + include_dirs : [string] + list of directories to search for C/C++ header files (in Unix + form for portability) + define_macros : [(name : string, value : string|None)] + list of macros to define; each macro is defined using a 2-tuple, + where 'value' is either the string to define it to or None to + define it without a particular value (equivalent of "#define + FOO" in source or -DFOO on Unix C compiler command line) + undef_macros : [string] + list of macros to undefine explicitly + library_dirs : [string] + list of directories to search for C/C++ libraries at link time + libraries : [string] + list of library names (not filenames or paths) to link against + runtime_library_dirs : [string] + list of directories to search for C/C++ libraries at run time + (for shared extensions, this is when the extension is loaded) + extra_objects : [string] + list of extra files to link with (eg. object files not implied + by 'sources', static library that must be explicitly specified, + binary resource files, etc.) + extra_compile_args : [string] + any extra platform- and compiler-specific information to use + when compiling the source files in 'sources'. For platforms and + compilers where "command line" makes sense, this is typically a + list of command-line arguments, but for other platforms it could + be anything. + extra_link_args : [string] + any extra platform- and compiler-specific information to use + when linking object files together to create the extension (or + to create a new static Python interpreter). Similar + interpretation as for 'extra_compile_args'. + export_symbols : [string] + list of symbols to be exported from a shared extension. Not + used on all platforms, and not generally necessary for Python + extensions, which typically export exactly one symbol: "init" + + extension_name. + swig_opts : [string] + any extra options to pass to SWIG if a source file has the .i + extension. + depends : [string] + list of files that the extension depends on + language : string + extension language (i.e. "c", "c++", "objc"). Will be detected + from the source extensions if not provided. + optional : boolean + specifies that a build failure in the extension should not abort the + build process, but simply not install the failing extension. + """ + + # **kwargs are allowed so that a warning is emitted instead of an + # exception + def __init__(self, name, sources, include_dirs=None, define_macros=None, + undef_macros=None, library_dirs=None, libraries=None, + runtime_library_dirs=None, extra_objects=None, + extra_compile_args=None, extra_link_args=None, + export_symbols=None, swig_opts=None, depends=None, + language=None, optional=None, **kw): + if not isinstance(name, str): + raise AssertionError("'name' must be a string") + + if not isinstance(sources, list): + raise AssertionError("'sources' must be a list of strings") + + for v in sources: + if not isinstance(v, str): + raise AssertionError("'sources' must be a list of strings") + + self.name = name + self.sources = sources + self.include_dirs = include_dirs or [] + self.define_macros = define_macros or [] + self.undef_macros = undef_macros or [] + self.library_dirs = library_dirs or [] + self.libraries = libraries or [] + self.runtime_library_dirs = runtime_library_dirs or [] + self.extra_objects = extra_objects or [] + self.extra_compile_args = extra_compile_args or [] + self.extra_link_args = extra_link_args or [] + self.export_symbols = export_symbols or [] + self.swig_opts = swig_opts or [] + self.depends = depends or [] + self.language = language + self.optional = optional + + # If there are unknown keyword options, warn about them + if len(kw) > 0: + options = [repr(option) for option in kw] + options = ', '.join(sorted(options)) + logger.warning( + 'unknown arguments given to Extension: %s', options) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/msvc9compiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/msvc9compiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,720 @@ +"""CCompiler implementation for the Microsoft Visual Studio 2008 compiler. + +The MSVCCompiler class is compatible with VS 2005 and VS 2008. Legacy +support for older versions of VS are in the msvccompiler module. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes +import os +import subprocess +import sys +import re + +from packaging.errors import (PackagingExecError, PackagingPlatformError, + CompileError, LibError, LinkError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_lib_options +from packaging import logger +from packaging.util import get_platform + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" +WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" +NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targetting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', + 'win-ia64' : 'ia64', +} + + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise PackagingPlatformError( +"""Python was built with Visual Studio 2008; extensions must be built with a +compiler than can generate compatible binaries. Visual Studio 2008 was not +found on this system. If you have Cygwin installed, you can try compiling +with MingW32, by passing "-c mingw32" to pysetup.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + logger.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + logger.debug("%s is not a valid directory", productdir) + return None + else: + logger.debug("env var %s is not set or invalid", toolskey) + if not productdir: + logger.debug("no productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + logger.debug("unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = set(("include", "lib", "libpath", "path")) + result = {} + + if vcvarsall is None: + raise PackagingPlatformError("Unable to find vcvarsall.bat") + logger.debug("calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise PackagingPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + if len(result) != len(interesting): + raise ValueError(str(list(result))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise PackagingPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + name = 'msvc' + description = 'Microsoft Visual C++' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, dry_run=False, force=False): + super(MSVCCompiler, self).__init__(dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + ok_plats = 'win32', 'win-amd64', 'win-ia64' + if plat_name not in ok_plats: + raise PackagingPlatformError("--plat-name must be one of %s" % + (ok_plats,)) + + if "DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and self.find_exe("cl.exe"): + # Assume that the SDK set up everything alright; don't try to be + # smarter + self.cc = "cl.exe" + self.linker = "link.exe" + self.lib = "lib.exe" + self.rc = "rc.exe" + self.mc = "mc.exe" + else: + # On x86, 'vcvars32.bat amd64' creates an env that doesn't work; + # to cross compile, you use 'x86_amd64'. + # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross + # compile use 'x86' (ie, it runs the x86 compiler directly) + # No idea how itanium handles this, if at all. + if plat_name == get_platform() or plat_name == 'win32': + # native build or cross-compile to win32 + plat_spec = PLAT_TO_VCVARS[plat_name] + else: + # cross compile from win32 -> some 64bit + plat_spec = PLAT_TO_VCVARS[get_platform()] + '_' + \ + PLAT_TO_VCVARS[plat_name] + + vc_env = query_vcvarsall(VERSION, plat_spec) + + # take care to only use strings in the environment. + self.__paths = vc_env['path'].split(os.pathsep) + os.environ['lib'] = vc_env['lib'] + os.environ['include'] = vc_env['include'] + + if len(self.__paths) == 0: + raise PackagingPlatformError("Python was built with %s, " + "and extensions need to be built with the same " + "version of the compiler, but it isn't installed." + % self.__product) + + self.cc = self.find_exe("cl.exe") + self.linker = self.find_exe("link.exe") + self.lib = self.find_exe("lib.exe") + self.rc = self.find_exe("rc.exe") # resource compiler + self.mc = self.find_exe("mc.exe") # message compiler + #self.set_path_env_var('lib') + #self.set_path_env_var('include') + + # extend the MSVC path with the current path + try: + for p in os.environ['path'].split(';'): + self.__paths.append(p) + except KeyError: + pass + self.__paths = normalize_and_reduce_paths(self.__paths) + os.environ['path'] = ";".join(self.__paths) + + self.preprocess_options = None + if self.__arch == "x86": + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', + '/Z7', '/D_DEBUG'] + else: + # Win64 + self.compile_options = [ '/nologo', '/Ox', '/MD', '/W3', '/GS-' , + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', + '/Z7', '/D_DEBUG'] + + self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] + if self.__version >= 7: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG', '/pdb:None' + ] + self.ldflags_static = [ '/nologo'] + + self.initialized = True + + # -- Worker methods ------------------------------------------------ + + def object_filenames(self, + source_filenames, + strip_dir=False, + output_dir=''): + # Copied from ccompiler.py, extended to return .res as 'object'-file + # for .rc input file + if output_dir is None: output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + # Better to raise an exception instead of silently continuing + # and later complain about sources and targets having + # different lengths + raise CompileError("Don't know how to compile %s" % src_name) + if strip_dir: + base = os.path.basename(base) + if ext in self._rc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + elif ext in self._mc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info + + compile_opts = extra_preargs or [] + compile_opts.append('/c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) + except PackagingExecError as msg: + raise CompileError(msg) + continue + elif ext in self._mc_extensions: + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the + # generated include file + # * '-r dir' specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) + + except PackagingExecError as msg: + raise CompileError(msg) + continue + else: + # how to handle this file? + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=False, + target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = objects + ['/OUT:' + output_filename] + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + libraries, library_dirs, runtime_library_dirs = fixed_args + + if runtime_library_dirs: + self.warn("don't know what to do with 'runtime_library_dirs': " + + str(runtime_library_dirs)) + + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + if target_desc == CCompiler.EXECUTABLE: + if debug: + ldflags = self.ldflags_shared_debug[1:] + else: + ldflags = self.ldflags_shared[1:] + else: + if debug: + ldflags = self.ldflags_shared_debug + else: + ldflags = self.ldflags_shared + + export_opts = [] + for sym in (export_symbols or []): + export_opts.append("/EXPORT:" + sym) + + ld_args = (ldflags + lib_opts + export_opts + + objects + ['/OUT:' + output_filename]) + + # The MSVC linker generates .lib and .exp files, which cannot be + # suppressed by any linker switches. The .lib files may even be + # needed! Make sure they are generated in the temporary build + # directory. Since they have different names for debug and release + # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) + if export_symbols is not None: + dll_name, dll_ext = os.path.splitext( + os.path.basename(output_filename)) + implib_file = os.path.join( + build_temp, + self.library_filename(dll_name)) + ld_args.append('/IMPLIB:' + implib_file) + + # Embedded manifests are recommended - see MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can embed it later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + # embed the manifest + # XXX - this is somewhat fragile - if mt.exe fails, distutils + # will still consider the DLL up-to-date, but it will not have a + # manifest. Maybe we should link to a temp file? OTOH, that + # implies a build environment error that shouldn't go undetected. + if target_desc == CCompiler.EXECUTABLE: + mfid = 1 + else: + mfid = 2 + self._remove_visual_c_ref(temp_manifest) + out_arg = '-outputresource:%s;%s' % (output_filename, mfid) + try: + self.spawn(['mt.exe', '-nologo', '-manifest', + temp_manifest, out_arg]) + except PackagingExecError as msg: + raise LinkError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + with open(manifest_file) as manifest_f: + manifest_buf = manifest_f.read() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = "\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + with open(manifest_file, 'w') as manifest_f: + manifest_f.write(manifest_buf) + except IOError: + pass + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "/LIBPATH:" + dir + + def runtime_library_dir_option(self, dir): + raise PackagingPlatformError( + "don't know how to set runtime library search path for MSVC++") + + def library_option(self, lib): + return self.library_filename(lib) + + + def find_library_file(self, dirs, lib, debug=False): + # Prefer a debugging library if found (and requested), but deal + # with it if we don't have one. + if debug: + try_names = [lib + "_d", lib] + else: + try_names = [lib] + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/msvccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/msvccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,635 @@ +"""CCompiler implementation for old Microsoft Visual Studio compilers. + +For a compiler compatible with VS 2005 and 2008, use msvc9compiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + + +import sys +import os + +from packaging.errors import (PackagingExecError, PackagingPlatformError, + CompileError, LibError, LinkError) +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_lib_options +from packaging import logger + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + + except ImportError: + logger.warning( + "can't read registry to find the necessary compiler setting;\n" + "make sure that Python modules _winreg, win32api or win32con " + "are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + + +def read_keys(base, key): + """Return list of registry keys.""" + + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i = i + 1 + return L + + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i = i + 1 + return d + + +def convert_mbcs(s): + enc = getattr(s, "encode", None) + if enc is not None: + try: + s = enc("mbcs") + except UnicodeError: + pass + return s + + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError: + raise PackagingPlatformError( +"""Python was built with Visual Studio 2003; extensions must be built with +a compiler than can generate compatible binaries. Visual Studio 2003 was +not found on this system. If you have Cygwin installed, you can try +compiling with MingW32, by passing "-c mingw32" to pysetup.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel", "Itanium", or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler): + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + name = 'msvc' + description = "Microsoft Visual C++" + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, dry_run=False, force=False): + super(MSVCCompiler, self).__init__(dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + def initialize(self): + self.__paths = [] + if ("DISTUTILS_USE_SDK" in os.environ and "MSSdk" in os.environ and + self.find_exe("cl.exe")): + # Assume that the SDK set up everything alright; don't try to be + # smarter + self.cc = "cl.exe" + self.linker = "link.exe" + self.lib = "lib.exe" + self.rc = "rc.exe" + self.mc = "mc.exe" + else: + self.__paths = self.get_msvc_paths("path") + + if len(self.__paths) == 0: + raise PackagingPlatformError("Python was built with %s " + "and extensions need to be built with the same " + "version of the compiler, but it isn't installed." % + self.__product) + + self.cc = self.find_exe("cl.exe") + self.linker = self.find_exe("link.exe") + self.lib = self.find_exe("lib.exe") + self.rc = self.find_exe("rc.exe") # resource compiler + self.mc = self.find_exe("mc.exe") # message compiler + self.set_path_env_var('lib') + self.set_path_env_var('include') + + # extend the MSVC path with the current path + try: + for p in os.environ['path'].split(';'): + self.__paths.append(p) + except KeyError: + pass + self.__paths = normalize_and_reduce_paths(self.__paths) + os.environ['path'] = ';'.join(self.__paths) + + self.preprocess_options = None + if self.__arch == "Intel": + self.compile_options = ['/nologo', '/Ox', '/MD', '/W3', '/GX', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GX', + '/Z7', '/D_DEBUG'] + else: + # Win64 + self.compile_options = ['/nologo', '/Ox', '/MD', '/W3', '/GS-', + '/DNDEBUG'] + self.compile_options_debug = ['/nologo', '/Od', '/MDd', '/W3', '/GS-', + '/Z7', '/D_DEBUG'] + + self.ldflags_shared = ['/DLL', '/nologo', '/INCREMENTAL:NO'] + if self.__version >= 7: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/DEBUG' + ] + else: + self.ldflags_shared_debug = [ + '/DLL', '/nologo', '/INCREMENTAL:no', '/pdb:None', '/DEBUG' + ] + self.ldflags_static = [ '/nologo'] + + self.initialized = True + + # -- Worker methods ------------------------------------------------ + + def object_filenames(self, source_filenames, strip_dir=False, output_dir=''): + # Copied from ccompiler.py, extended to return .res as 'object'-file + # for .rc input file + if output_dir is None: + output_dir = '' + obj_names = [] + for src_name in source_filenames: + base, ext = os.path.splitext(src_name) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / + if ext not in self.src_extensions: + # Better to raise an exception instead of silently continuing + # and later complain about sources and targets having + # different lengths + raise CompileError("Don't know how to compile %s" % src_name) + if strip_dir: + base = os.path.basename(base) + if ext in self._rc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + elif ext in self._mc_extensions: + obj_names.append(os.path.join(output_dir, + base + self.res_extension)) + else: + obj_names.append(os.path.join(output_dir, + base + self.obj_extension)) + return obj_names + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=False, + extra_preargs=None, extra_postargs=None, depends=None): + + if not self.initialized: + self.initialize() + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + + compile_opts = extra_preargs or [] + compile_opts.append('/c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) + except PackagingExecError as msg: + raise CompileError(msg) + continue + elif ext in self._mc_extensions: + + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the + # generated include file + # * '-r dir' specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) + + except PackagingExecError as msg: + raise CompileError(msg) + continue + else: + # how to handle this file? + raise CompileError( + "Don't know how to compile %s to %s" % + (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + return objects + + def create_static_lib(self, objects, output_libname, output_dir=None, + debug=False, target_lang=None): + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = objects + ['/OUT:' + output_filename] + if debug: + pass # XXX what goes here? + try: + self.spawn([self.lib] + lib_args) + except PackagingExecError as msg: + raise LibError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, output_filename, output_dir=None, + libraries=None, library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + if runtime_library_dirs: + self.warn("don't know what to do with 'runtime_library_dirs': %s" + % (runtime_library_dirs,)) + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + + if target_desc == CCompiler.EXECUTABLE: + if debug: + ldflags = self.ldflags_shared_debug[1:] + else: + ldflags = self.ldflags_shared[1:] + else: + if debug: + ldflags = self.ldflags_shared_debug + else: + ldflags = self.ldflags_shared + + export_opts = [] + for sym in (export_symbols or []): + export_opts.append("/EXPORT:" + sym) + + ld_args = (ldflags + lib_opts + export_opts + + objects + ['/OUT:' + output_filename]) + + # The MSVC linker generates .lib and .exp files, which cannot be + # suppressed by any linker switches. The .lib files may even be + # needed! Make sure they are generated in the temporary build + # directory. Since they have different names for debug and release + # builds, they can go into the same directory. + if export_symbols is not None: + dll_name, dll_ext = os.path.splitext( + os.path.basename(output_filename)) + implib_file = os.path.join( + os.path.dirname(objects[0]), + self.library_filename(dll_name)) + ld_args.append('/IMPLIB:' + implib_file) + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + self.mkpath(os.path.dirname(output_filename)) + try: + self.spawn([self.linker] + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "/LIBPATH:" + dir + + def runtime_library_dir_option(self, dir): + raise PackagingPlatformError("don't know how to set runtime library search path for MSVC++") + + def library_option(self, lib): + return self.library_filename(lib) + + def find_library_file(self, dirs, lib, debug=False): + # Prefer a debugging library if found (and requested), but deal + # with it if we don't have one. + if debug: + try_names = [lib + "_d", lib] + else: + try_names = [lib] + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.exists(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + logger.debug("importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from packaging.compiler.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from packaging.compiler.msvc9compiler import MacroExpander diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/compiler/unixccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/compiler/unixccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,339 @@ +"""CCompiler implementation for Unix compilers. + +This module contains the UnixCCompiler class, a subclass of CCompiler +that handles the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys + +from packaging.util import newer +from packaging.compiler.ccompiler import CCompiler +from packaging.compiler import gen_preprocess_options, gen_lib_options +from packaging.errors import (PackagingExecError, CompileError, + LibError, LinkError) +from packaging import logger +import sysconfig + + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the pysetup command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + +def _darwin_compiler_fixup(compiler_so, cc_args): + """ + This function will strip '-isysroot PATH' and '-arch ARCH' from the + compile flags if the user has specified one them in extra_compile_flags. + + This is needed because '-arch ARCH' adds another architecture to the + build, without a way to remove an architecture. Furthermore GCC will + barf if multiple '-isysroot' arguments are present. + """ + stripArch = stripSysroot = False + + compiler_so = list(compiler_so) + kernel_version = os.uname()[2] # 8.4.3 + major_version = int(kernel_version.split('.')[0]) + + if major_version < 8: + # OSX before 10.4.0, these don't support -arch and -isysroot at + # all. + stripArch = stripSysroot = True + else: + stripArch = '-arch' in cc_args + stripSysroot = '-isysroot' in cc_args + + if stripArch or 'ARCHFLAGS' in os.environ: + while True: + try: + index = compiler_so.index('-arch') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + break + + if 'ARCHFLAGS' in os.environ and not stripArch: + # User specified different -arch flags in the environ, + # see also the sysconfig + compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() + + if stripSysroot: + try: + index = compiler_so.index('-isysroot') + # Strip this argument and the next one: + del compiler_so[index:index+2] + except ValueError: + pass + + # Check if the SDK that is used during compilation actually exists, + # the universal build requires the usage of a universal SDK and not all + # users have that installed by default. + sysroot = None + if '-isysroot' in cc_args: + idx = cc_args.index('-isysroot') + sysroot = cc_args[idx+1] + elif '-isysroot' in compiler_so: + idx = compiler_so.index('-isysroot') + sysroot = compiler_so[idx+1] + + if sysroot and not os.path.isdir(sysroot): + logger.warning( + "compiling with an SDK that doesn't seem to exist: %r;\n" + "please check your Xcode installation", sysroot) + + return compiler_so + +class UnixCCompiler(CCompiler): + + name = 'unix' + description = 'Standard UNIX-style compiler' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. XXX whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, + output_file=None, macros=None, include_dirs=None, + extra_preargs=None, extra_postargs=None): + ignore, macros, include_dirs = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(('-o', output_file)) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except PackagingExecError as msg: + raise CompileError(msg) + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + compiler_so = self.compiler_so + if sys.platform == 'darwin': + compiler_so = _darwin_compiler_fixup(compiler_so, cc_args + extra_postargs) + try: + self.spawn(compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except PackagingExecError as msg: + raise CompileError(msg) + + def create_static_lib(self, objects, output_libname, + output_dir=None, debug=False, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + self.mkpath(os.path.dirname(output_filename)) + self.spawn(self.archiver + + [output_filename] + + objects + self.objects) + + # Not many Unices required ranlib anymore -- SunOS 4.x is, I + # think the only major Unix that does. Maybe we need some + # platform intelligence here to skip ranlib if it's not + # needed -- or maybe Python's configure script took care of + # it for us, hence the check for leading colon. + if self.ranlib: + try: + self.spawn(self.ranlib + [output_filename]) + except PackagingExecError as msg: + raise LibError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, + output_filename, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=False, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + libraries, library_dirs, runtime_library_dirs = \ + self._fix_lib_args(libraries, library_dirs, runtime_library_dirs) + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if type(output_dir) not in (str, type(None)): + raise TypeError("'output_dir' must be a string or None") + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + ld_args = (objects + self.objects + + lib_opts + ['-o', output_filename]) + if debug: + ld_args[:0] = ['-g'] + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + self.mkpath(os.path.dirname(output_filename)) + try: + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe[:] + else: + linker = self.linker_so[:] + if target_lang == "c++" and self.compiler_cxx: + # skip over environment variable settings if /usr/bin/env + # is used to set up the linker's environment. + # This is needed on OSX. Note: this assumes that the + # normal and C++ compiler have the same environment + # settings. + i = 0 + if os.path.basename(linker[0]) == "env": + i = 1 + while '=' in linker[i]: + i = i + 1 + + linker[i] = self.compiler_cxx[i] + + if sys.platform == 'darwin': + linker = _darwin_compiler_fixup(linker, ld_args) + + self.spawn(linker + ld_args) + except PackagingExecError as msg: + raise LinkError(msg) + else: + logger.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "-L" + dir + + def _is_gcc(self, compiler_name): + return "gcc" in compiler_name or "g++" in compiler_name + + def runtime_library_dir_option(self, dir): + # XXX Hackish, at the very least. See Python bug #445902: + # http://sourceforge.net/tracker/index.php + # ?func=detail&aid=445902&group_id=5470&atid=105470 + # Linkers on different platforms need different options to + # specify that directories need to be added to the list of + # directories searched for dependencies when a dynamic library + # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to + # be told to pass the -R option through to the linker, whereas + # other compilers and gcc on other systems just know this. + # Other compilers may need something slightly different. At + # this time, there's no way to determine this information from + # the configuration data stored in the Python installation, so + # we use this hack. + + compiler = os.path.basename(sysconfig.get_config_var("CC")) + if sys.platform[:6] == "darwin": + # MacOSX's linker doesn't understand the -R flag at all + return "-L" + dir + elif sys.platform[:5] == "hp-ux": + if self._is_gcc(compiler): + return ["-Wl,+s", "-L" + dir] + return ["+s", "-L" + dir] + elif sys.platform[:7] == "irix646" or sys.platform[:6] == "osf1V5": + return ["-rpath", dir] + elif self._is_gcc(compiler): + # gcc on non-GNU systems does not need -Wl, but can + # use it anyway. Since distutils has always passed in + # -Wl whenever gcc was used in the past it is probably + # safest to keep doing so. + if sysconfig.get_config_var("GNULD") == "yes": + # GNU ld needs an extra option to get a RUNPATH + # instead of just an RPATH. + return "-Wl,--enable-new-dtags,-R" + dir + else: + return "-Wl,-R" + dir + elif sys.platform[:3] == "aix": + return "-blibpath:" + dir + else: + # No idea how --enable-new-dtags would be passed on to + # ld if this system was using GNU ld. Don't know if a + # system like this even exists. + return "-R" + dir + + def library_option(self, lib): + return "-l" + lib + + def find_library_file(self, dirs, lib, debug=False): + shared_f = self.library_filename(lib, lib_type='shared') + dylib_f = self.library_filename(lib, lib_type='dylib') + static_f = self.library_filename(lib, lib_type='static') + + for dir in dirs: + shared = os.path.join(dir, shared_f) + dylib = os.path.join(dir, dylib_f) + static = os.path.join(dir, static_f) + # We're second-guessing the linker here, with not much hard + # data to go on: GCC seems to prefer the shared library, so I'm + # assuming that *all* Unix C compilers do. And of course I'm + # ignoring even GCC's "-static" option. So sue me. + if os.path.exists(dylib): + return dylib + elif os.path.exists(shared): + return shared + elif os.path.exists(static): + return static + + # Oops, didn't find it in *any* of 'dirs' + return None diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/config.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,391 @@ +"""Utilities to find and read config files used by packaging.""" + +import os +import sys +import logging + +from shlex import split +from configparser import RawConfigParser +from packaging import logger +from packaging.errors import PackagingOptionError +from packaging.compiler.extension import Extension +from packaging.util import (check_environ, iglob, resolve_name, strtobool, + split_multiline) +from packaging.compiler import set_compiler +from packaging.command import set_command +from packaging.markers import interpret + + +def _check_name(name, packages): + if '.' not in name: + return + parts = name.split('.') + parent = '.'.join(parts[:-1]) + if parent not in packages: + # we could log a warning instead of raising, but what's the use + # of letting people build modules they can't import? + raise PackagingOptionError( + 'parent package for extension %r not found' % name) + + +def _pop_values(values_dct, key): + """Remove values from the dictionary and convert them as a list""" + vals_str = values_dct.pop(key, '') + if not vals_str: + return + fields = [] + # the line separator is \n for setup.cfg files + for field in vals_str.split('\n'): + tmp_vals = field.split('--') + if len(tmp_vals) == 2 and not interpret(tmp_vals[1]): + continue + fields.append(tmp_vals[0]) + # Get bash options like `gcc -print-file-name=libgcc.a` XXX bash options? + vals = split(' '.join(fields)) + if vals: + return vals + + +def _rel_path(base, path): + # normalizes and returns a lstripped-/-separated path + base = base.replace(os.path.sep, '/') + path = path.replace(os.path.sep, '/') + assert path.startswith(base) + return path[len(base):].lstrip('/') + + +def get_resources_dests(resources_root, rules): + """Find destinations for resources files""" + destinations = {} + for base, suffix, dest in rules: + prefix = os.path.join(resources_root, base) + for abs_base in iglob(prefix): + abs_glob = os.path.join(abs_base, suffix) + for abs_path in iglob(abs_glob): + resource_file = _rel_path(resources_root, abs_path) + if dest is None: # remove the entry if it was here + destinations.pop(resource_file, None) + else: + rel_path = _rel_path(abs_base, abs_path) + rel_dest = dest.replace(os.path.sep, '/').rstrip('/') + destinations[resource_file] = rel_dest + '/' + rel_path + return destinations + + +class Config: + """Class used to work with configuration files""" + def __init__(self, dist): + self.dist = dist + self.setup_hooks = [] + + def run_hooks(self, config): + """Run setup hooks in the order defined in the spec.""" + for hook in self.setup_hooks: + hook(config) + + def find_config_files(self): + """Find as many configuration files as should be processed for this + platform, and return a list of filenames in the order in which they + should be parsed. The filenames returned are guaranteed to exist + (modulo nasty race conditions). + + There are three possible config files: packaging.cfg in the + Packaging installation directory (ie. where the top-level + Packaging __inst__.py file lives), a file in the user's home + directory named .pydistutils.cfg on Unix and pydistutils.cfg + on Windows/Mac; and setup.cfg in the current directory. + + The file in the user's home directory can be disabled with the + --no-user-cfg option. + """ + files = [] + check_environ() + + # Where to look for the system-wide Packaging config file + sys_dir = os.path.dirname(sys.modules['packaging'].__file__) + + # Look for the system config file + sys_file = os.path.join(sys_dir, "packaging.cfg") + if os.path.isfile(sys_file): + files.append(sys_file) + + # What to call the per-user config file + if os.name == 'posix': + user_filename = ".pydistutils.cfg" + else: + user_filename = "pydistutils.cfg" + + # And look for the user config file + if self.dist.want_user_cfg: + user_file = os.path.join(os.path.expanduser('~'), user_filename) + if os.path.isfile(user_file): + files.append(user_file) + + # All platforms support local setup.cfg + local_file = "setup.cfg" + if os.path.isfile(local_file): + files.append(local_file) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("using config files: %s", ', '.join(files)) + return files + + def _convert_metadata(self, name, value): + # converts a value found in setup.cfg into a valid metadata + # XXX + return value + + def _read_setup_cfg(self, parser, cfg_filename): + cfg_directory = os.path.dirname(os.path.abspath(cfg_filename)) + content = {} + for section in parser.sections(): + content[section] = dict(parser.items(section)) + + # global setup hooks are called first + if 'global' in content: + if 'setup_hooks' in content['global']: + setup_hooks = split_multiline(content['global']['setup_hooks']) + + # add project directory to sys.path, to allow hooks to be + # distributed with the project + sys.path.insert(0, cfg_directory) + try: + for line in setup_hooks: + try: + hook = resolve_name(line) + except ImportError as e: + logger.warning('cannot find setup hook: %s', + e.args[0]) + else: + self.setup_hooks.append(hook) + self.run_hooks(content) + finally: + sys.path.pop(0) + + metadata = self.dist.metadata + + # setting the metadata values + if 'metadata' in content: + for key, value in content['metadata'].items(): + key = key.replace('_', '-') + if metadata.is_multi_field(key): + value = split_multiline(value) + + if key == 'project-url': + value = [(label.strip(), url.strip()) + for label, url in + [v.split(',') for v in value]] + + if key == 'description-file': + if 'description' in content['metadata']: + msg = ("description and description-file' are " + "mutually exclusive") + raise PackagingOptionError(msg) + + filenames = value.split() + + # concatenate all files + value = [] + for filename in filenames: + # will raise if file not found + with open(filename) as description_file: + value.append(description_file.read().strip()) + # add filename as a required file + if filename not in metadata.requires_files: + metadata.requires_files.append(filename) + value = '\n'.join(value).strip() + key = 'description' + + if metadata.is_metadata_field(key): + metadata[key] = self._convert_metadata(key, value) + + if 'files' in content: + files = content['files'] + self.dist.package_dir = files.pop('packages_root', None) + + files = dict((key, split_multiline(value)) for key, value in + files.items()) + + self.dist.packages = [] + + packages = files.get('packages', []) + if isinstance(packages, str): + packages = [packages] + + for package in packages: + if ':' in package: + dir_, package = package.split(':') + self.dist.package_dir[package] = dir_ + self.dist.packages.append(package) + + self.dist.py_modules = files.get('modules', []) + if isinstance(self.dist.py_modules, str): + self.dist.py_modules = [self.dist.py_modules] + self.dist.scripts = files.get('scripts', []) + if isinstance(self.dist.scripts, str): + self.dist.scripts = [self.dist.scripts] + + self.dist.package_data = {} + # bookkeeping for the loop below + firstline = True + prev = None + + for line in files.get('package_data', []): + if '=' in line: + # package name -- file globs or specs + key, value = line.split('=') + prev = self.dist.package_data[key.strip()] = value.split() + elif firstline: + # invalid continuation on the first line + raise PackagingOptionError( + 'malformed package_data first line: %r (misses "=")' % + line) + else: + # continuation, add to last seen package name + prev.extend(line.split()) + + firstline = False + + self.dist.data_files = [] + for data in files.get('data_files', []): + data = data.split('=') + if len(data) != 2: + continue + key, value = data + values = [v.strip() for v in value.split(',')] + self.dist.data_files.append((key, values)) + + # manifest template + self.dist.extra_files = files.get('extra_files', []) + + resources = [] + for rule in files.get('resources', []): + glob, destination = rule.split('=', 1) + rich_glob = glob.strip().split(' ', 1) + if len(rich_glob) == 2: + prefix, suffix = rich_glob + else: + assert len(rich_glob) == 1 + prefix = '' + suffix = glob + if destination == '': + destination = None + resources.append( + (prefix.strip(), suffix.strip(), destination.strip())) + self.dist.data_files = get_resources_dests( + cfg_directory, resources) + + ext_modules = self.dist.ext_modules + for section_key in content: + # no str.partition in 2.4 :( + labels = section_key.split(':') + if len(labels) == 2 and labels[0] == 'extension': + values_dct = content[section_key] + if 'name' in values_dct: + raise PackagingOptionError( + 'extension name should be given as [extension: name], ' + 'not as key') + name = labels[1].strip() + _check_name(name, self.dist.packages) + ext_modules.append(Extension( + name, + _pop_values(values_dct, 'sources'), + _pop_values(values_dct, 'include_dirs'), + _pop_values(values_dct, 'define_macros'), + _pop_values(values_dct, 'undef_macros'), + _pop_values(values_dct, 'library_dirs'), + _pop_values(values_dct, 'libraries'), + _pop_values(values_dct, 'runtime_library_dirs'), + _pop_values(values_dct, 'extra_objects'), + _pop_values(values_dct, 'extra_compile_args'), + _pop_values(values_dct, 'extra_link_args'), + _pop_values(values_dct, 'export_symbols'), + _pop_values(values_dct, 'swig_opts'), + _pop_values(values_dct, 'depends'), + values_dct.pop('language', None), + values_dct.pop('optional', None), + **values_dct)) + + def parse_config_files(self, filenames=None): + if filenames is None: + filenames = self.find_config_files() + + logger.debug("Distribution.parse_config_files():") + + parser = RawConfigParser() + + for filename in filenames: + logger.debug(" reading %s", filename) + parser.read(filename, encoding='utf-8') + + if os.path.split(filename)[-1] == 'setup.cfg': + self._read_setup_cfg(parser, filename) + + for section in parser.sections(): + if section == 'global': + if parser.has_option('global', 'compilers'): + self._load_compilers(parser.get('global', 'compilers')) + + if parser.has_option('global', 'commands'): + self._load_commands(parser.get('global', 'commands')) + + options = parser.options(section) + opt_dict = self.dist.get_option_dict(section) + + for opt in options: + if opt == '__name__': + continue + val = parser.get(section, opt) + opt = opt.replace('-', '_') + + if opt == 'sub_commands': + val = split_multiline(val) + if isinstance(val, str): + val = [val] + + # Hooks use a suffix system to prevent being overriden + # by a config file processed later (i.e. a hook set in + # the user config file cannot be replaced by a hook + # set in a project config file, unless they have the + # same suffix). + if (opt.startswith("pre_hook.") or + opt.startswith("post_hook.")): + hook_type, alias = opt.split(".") + hook_dict = opt_dict.setdefault( + hook_type, (filename, {}))[1] + hook_dict[alias] = val + else: + opt_dict[opt] = filename, val + + # Make the RawConfigParser forget everything (so we retain + # the original filenames that options come from) + parser.__init__() + + # If there was a "global" section in the config file, use it + # to set Distribution options. + if 'global' in self.dist.command_options: + for opt, (src, val) in self.dist.command_options['global'].items(): + alias = self.dist.negative_opt.get(opt) + try: + if alias: + setattr(self.dist, alias, not strtobool(val)) + elif opt == 'dry_run': # FIXME ugh! + setattr(self.dist, opt, strtobool(val)) + else: + setattr(self.dist, opt, val) + except ValueError as msg: + raise PackagingOptionError(msg) + + def _load_compilers(self, compilers): + compilers = split_multiline(compilers) + if isinstance(compilers, str): + compilers = [compilers] + for compiler in compilers: + set_compiler(compiler.strip()) + + def _load_commands(self, commands): + commands = split_multiline(commands) + if isinstance(commands, str): + commands = [commands] + for command in commands: + set_command(command.strip()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/create.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/create.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,682 @@ +"""Interactive helper used to create a setup.cfg file. + +This script will generate a packaging configuration file by looking at +the current directory and asking the user questions. It is intended to +be called as *pysetup create*. +""" + +# Original code by Sean Reifschneider + +# Original TODO list: +# Look for a license file and automatically add the category. +# When a .c file is found during the walk, can we add it as an extension? +# Ask if there is a maintainer different that the author +# Ask for the platform (can we detect this via "import win32" or something?) +# Ask for the dependencies. +# Ask for the Requires-Dist +# Ask for the Provides-Dist +# Ask for a description +# Detect scripts (not sure how. #! outside of package?) + +import os +import re +import imp +import sys +import glob +import shutil +import sysconfig +from hashlib import md5 +from textwrap import dedent +from tokenize import detect_encoding +from configparser import RawConfigParser + +from packaging import logger +# importing this with an underscore as it should be replaced by the +# dict form or another structures for all purposes +from packaging._trove import all_classifiers as _CLASSIFIERS_LIST +from packaging.version import is_valid_version + +_FILENAME = 'setup.cfg' +_DEFAULT_CFG = '.pypkgcreate' # FIXME use a section in user .pydistutils.cfg + +_helptext = { + 'name': ''' +The name of the project to be packaged, usually a single word composed +of lower-case characters such as "zope.interface", "sqlalchemy" or +"CherryPy". +''', + 'version': ''' +Version number of the software, typically 2 or 3 numbers separated by +dots such as "1.0", "0.6b3", or "3.2.1". "0.1.0" is recommended for +initial development. +''', + 'summary': ''' +A one-line summary of what this project is or does, typically a sentence +80 characters or less in length. +''', + 'author': ''' +The full name of the author (typically you). +''', + 'author_email': ''' +Email address of the project author. +''', + 'do_classifier': ''' +Trove classifiers are optional identifiers that allow you to specify the +intended audience by saying things like "Beta software with a text UI +for Linux under the PSF license". However, this can be a somewhat +involved process. +''', + 'packages': ''' +Python packages included in the project. +''', + 'modules': ''' +Pure Python modules included in the project. +''', + 'extra_files': ''' +You can provide extra files/dirs contained in your project. +It has to follow the template syntax. XXX add help here. +''', + + 'home_page': ''' +The home page for the project, typically a public Web page. +''', + 'trove_license': ''' +Optionally you can specify a license. Type a string that identifies a +common license, and then you can select a list of license specifiers. +''', + 'trove_generic': ''' +Optionally, you can set other trove identifiers for things such as the +human language, programming language, user interface, etc. +''', + 'setup.py found': ''' +The setup.py script will be executed to retrieve the metadata. +An interactive helper will be run if you answer "n", +''', +} + +PROJECT_MATURITY = ['Development Status :: 1 - Planning', + 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', + 'Development Status :: 5 - Production/Stable', + 'Development Status :: 6 - Mature', + 'Development Status :: 7 - Inactive'] + +# XXX everything needs docstrings and tests (both low-level tests of various +# methods and functional tests of running the script) + + +def load_setup(): + """run the setup script (i.e the setup.py file) + + This function load the setup file in all cases (even if it have already + been loaded before, because we are monkey patching its setup function with + a particular one""" + with open("setup.py", "rb") as f: + encoding, lines = detect_encoding(f.readline) + with open("setup.py", encoding=encoding) as f: + imp.load_module("setup", f, "setup.py", (".py", "r", imp.PY_SOURCE)) + + +def ask_yn(question, default=None, helptext=None): + question += ' (y/n)' + while True: + answer = ask(question, default, helptext, required=True) + if answer and answer[0].lower() in ('y', 'n'): + return answer[0].lower() + + logger.error('You must select "Y" or "N".') + + +# XXX use util.ask +# FIXME: if prompt ends with '?', don't add ':' + + +def ask(question, default=None, helptext=None, required=True, + lengthy=False, multiline=False): + prompt = '%s: ' % (question,) + if default: + prompt = '%s [%s]: ' % (question, default) + if default and len(question) + len(default) > 70: + prompt = '%s\n [%s]: ' % (question, default) + if lengthy or multiline: + prompt += '\n > ' + + if not helptext: + helptext = 'No additional help available.' + + helptext = helptext.strip("\n") + + while True: + line = input(prompt).strip() + if line == '?': + print('=' * 70) + print(helptext) + print('=' * 70) + continue + if default and not line: + return default + if not line and required: + print('*' * 70) + print('This value cannot be empty.') + print('===========================') + if helptext: + print(helptext) + print('*' * 70) + continue + return line + + +def convert_yn_to_bool(yn, yes=True, no=False): + """Convert a y/yes or n/no to a boolean value.""" + if yn.lower().startswith('y'): + return yes + else: + return no + + +def _build_classifiers_dict(classifiers): + d = {} + for key in classifiers: + subdict = d + for subkey in key.split(' :: '): + if subkey not in subdict: + subdict[subkey] = {} + subdict = subdict[subkey] + return d + +CLASSIFIERS = _build_classifiers_dict(_CLASSIFIERS_LIST) + + +def _build_licences(classifiers): + res = [] + for index, item in enumerate(classifiers): + if not item.startswith('License :: '): + continue + res.append((index, item.split(' :: ')[-1].lower())) + return res + +LICENCES = _build_licences(_CLASSIFIERS_LIST) + + +class MainProgram: + """Make a project setup configuration file (setup.cfg).""" + + def __init__(self): + self.configparser = None + self.classifiers = set() + self.data = {'name': '', + 'version': '1.0.0', + 'classifier': self.classifiers, + 'packages': [], + 'modules': [], + 'platform': [], + 'resources': [], + 'extra_files': [], + 'scripts': [], + } + self._load_defaults() + + def __call__(self): + setupcfg_defined = False + if self.has_setup_py() and self._prompt_user_for_conversion(): + setupcfg_defined = self.convert_py_to_cfg() + if not setupcfg_defined: + self.define_cfg_values() + self._write_cfg() + + def has_setup_py(self): + """Test for the existence of a setup.py file.""" + return os.path.exists('setup.py') + + def define_cfg_values(self): + self.inspect() + self.query_user() + + def _lookup_option(self, key): + if not self.configparser.has_option('DEFAULT', key): + return None + return self.configparser.get('DEFAULT', key) + + def _load_defaults(self): + # Load default values from a user configuration file + self.configparser = RawConfigParser() + # TODO replace with section in distutils config file + default_cfg = os.path.expanduser(os.path.join('~', _DEFAULT_CFG)) + self.configparser.read(default_cfg) + self.data['author'] = self._lookup_option('author') + self.data['author_email'] = self._lookup_option('author_email') + + def _prompt_user_for_conversion(self): + # Prompt the user about whether they would like to use the setup.py + # conversion utility to generate a setup.cfg or generate the setup.cfg + # from scratch + answer = ask_yn(('A legacy setup.py has been found.\n' + 'Would you like to convert it to a setup.cfg?'), + default="y", + helptext=_helptext['setup.py found']) + return convert_yn_to_bool(answer) + + def _dotted_packages(self, data): + packages = sorted(data) + modified_pkgs = [] + for pkg in packages: + pkg = pkg.lstrip('./') + pkg = pkg.replace('/', '.') + modified_pkgs.append(pkg) + return modified_pkgs + + def _write_cfg(self): + if os.path.exists(_FILENAME): + if os.path.exists('%s.old' % _FILENAME): + message = ("ERROR: %(name)s.old backup exists, please check " + "that current %(name)s is correct and remove " + "%(name)s.old" % {'name': _FILENAME}) + logger.error(message) + return + shutil.move(_FILENAME, '%s.old' % _FILENAME) + + with open(_FILENAME, 'w', encoding='utf-8') as fp: + fp.write('[metadata]\n') + # TODO use metadata module instead of hard-coding field-specific + # behavior here + + # simple string entries + for name in ('name', 'version', 'summary', 'download_url'): + fp.write('%s = %s\n' % (name, self.data.get(name, 'UNKNOWN'))) + + # optional string entries + if 'keywords' in self.data and self.data['keywords']: + # XXX shoud use comma to separate, not space + fp.write('keywords = %s\n' % ' '.join(self.data['keywords'])) + for name in ('home_page', 'author', 'author_email', + 'maintainer', 'maintainer_email', 'description-file'): + if name in self.data and self.data[name]: + fp.write('%s = %s\n' % (name, self.data[name])) + if 'description' in self.data: + fp.write( + 'description = %s\n' + % '\n |'.join(self.data['description'].split('\n'))) + + # multiple use string entries + for name in ('platform', 'supported-platform', 'classifier', + 'requires-dist', 'provides-dist', 'obsoletes-dist', + 'requires-external'): + if not(name in self.data and self.data[name]): + continue + fp.write('%s = ' % name) + fp.write(''.join(' %s\n' % val + for val in self.data[name]).lstrip()) + + fp.write('\n[files]\n') + + for name in ('packages', 'modules', 'scripts', 'extra_files'): + if not(name in self.data and self.data[name]): + continue + fp.write('%s = %s\n' + % (name, '\n '.join(self.data[name]).strip())) + + if self.data.get('package_data'): + fp.write('package_data =\n') + for pkg, spec in sorted(self.data['package_data'].items()): + # put one spec per line, indented under the package name + indent = ' ' * (len(pkg) + 7) + spec = ('\n' + indent).join(spec) + fp.write(' %s = %s\n' % (pkg, spec)) + fp.write('\n') + + if self.data.get('resources'): + fp.write('resources =\n') + for src, dest in self.data['resources']: + fp.write(' %s = %s\n' % (src, dest)) + fp.write('\n') + + os.chmod(_FILENAME, 0o644) + logger.info('Wrote "%s".' % _FILENAME) + + def convert_py_to_cfg(self): + """Generate a setup.cfg from an existing setup.py. + + It only exports the distutils metadata (setuptools specific metadata + is not currently supported). + """ + data = self.data + + def setup_mock(**attrs): + """Mock the setup(**attrs) in order to retrieve metadata.""" + + # TODO use config and metadata instead of Distribution + from distutils.dist import Distribution + dist = Distribution(attrs) + dist.parse_config_files() + + # 1. retrieve metadata fields that are quite similar in + # PEP 314 and PEP 345 + labels = (('name',) * 2, + ('version',) * 2, + ('author',) * 2, + ('author_email',) * 2, + ('maintainer',) * 2, + ('maintainer_email',) * 2, + ('description', 'summary'), + ('long_description', 'description'), + ('url', 'home_page'), + ('platforms', 'platform'), + ('provides', 'provides-dist'), + ('obsoletes', 'obsoletes-dist'), + ('requires', 'requires-dist')) + + get = lambda lab: getattr(dist.metadata, lab.replace('-', '_')) + data.update((new, get(old)) for old, new in labels if get(old)) + + # 2. retrieve data that requires special processing + data['classifier'].update(dist.get_classifiers() or []) + data['scripts'].extend(dist.scripts or []) + data['packages'].extend(dist.packages or []) + data['modules'].extend(dist.py_modules or []) + # 2.1 data_files -> resources + if dist.data_files: + if (len(dist.data_files) < 2 or + isinstance(dist.data_files[1], str)): + dist.data_files = [('', dist.data_files)] + # add tokens in the destination paths + vars = {'distribution.name': data['name']} + path_tokens = sysconfig.get_paths(vars=vars).items() + # sort tokens to use the longest one first + path_tokens = sorted(path_tokens, key=lambda x: len(x[1])) + for dest, srcs in (dist.data_files or []): + dest = os.path.join(sys.prefix, dest) + dest = dest.replace(os.path.sep, '/') + for tok, path in path_tokens: + path = path.replace(os.path.sep, '/') + if not dest.startswith(path): + continue + + dest = ('{%s}' % tok) + dest[len(path):] + files = [('/ '.join(src.rsplit('/', 1)), dest) + for src in srcs] + data['resources'].extend(files) + + # 2.2 package_data + data['package_data'] = dist.package_data.copy() + + # Use README file if its content is the desciption + if "description" in data: + ref = md5(re.sub('\s', '', + self.data['description']).lower().encode()) + ref = ref.digest() + for readme in glob.glob('README*'): + with open(readme, encoding='utf-8') as fp: + contents = fp.read() + contents = re.sub('\s', '', contents.lower()).encode() + val = md5(contents).digest() + if val == ref: + del data['description'] + data['description-file'] = readme + break + + # apply monkey patch to distutils (v1) and setuptools (if needed) + # (abort the feature if distutils v1 has been killed) + try: + from distutils import core + core.setup # make sure it's not d2 maskerading as d1 + except (ImportError, AttributeError): + return + saved_setups = [(core, core.setup)] + core.setup = setup_mock + try: + import setuptools + except ImportError: + pass + else: + saved_setups.append((setuptools, setuptools.setup)) + setuptools.setup = setup_mock + # get metadata by executing the setup.py with the patched setup(...) + success = False # for python < 2.4 + try: + load_setup() + success = True + finally: # revert monkey patches + for patched_module, original_setup in saved_setups: + patched_module.setup = original_setup + if not self.data: + raise ValueError('Unable to load metadata from setup.py') + return success + + def inspect(self): + """Inspect the current working diretory for a name and version. + + This information is harvested in where the directory is named + like [name]-[version]. + """ + dir_name = os.path.basename(os.getcwd()) + self.data['name'] = dir_name + match = re.match(r'(.*)-(\d.+)', dir_name) + if match: + self.data['name'] = match.group(1) + self.data['version'] = match.group(2) + # TODO needs testing! + if not is_valid_version(self.data['version']): + msg = "Invalid version discovered: %s" % self.data['version'] + raise ValueError(msg) + + def query_user(self): + self.data['name'] = ask('Project name', self.data['name'], + _helptext['name']) + + self.data['version'] = ask('Current version number', + self.data.get('version'), _helptext['version']) + self.data['summary'] = ask('Project description summary', + self.data.get('summary'), _helptext['summary'], + lengthy=True) + self.data['author'] = ask('Author name', + self.data.get('author'), _helptext['author']) + self.data['author_email'] = ask('Author email address', + self.data.get('author_email'), _helptext['author_email']) + self.data['home_page'] = ask('Project home page', + self.data.get('home_page'), _helptext['home_page'], + required=False) + + if ask_yn('Do you want me to automatically build the file list ' + 'with everything I can find in the current directory? ' + 'If you say no, you will have to define them manually.') == 'y': + self._find_files() + else: + while ask_yn('Do you want to add a single module?' + ' (you will be able to add full packages next)', + helptext=_helptext['modules']) == 'y': + self._set_multi('Module name', 'modules') + + while ask_yn('Do you want to add a package?', + helptext=_helptext['packages']) == 'y': + self._set_multi('Package name', 'packages') + + while ask_yn('Do you want to add an extra file?', + helptext=_helptext['extra_files']) == 'y': + self._set_multi('Extra file/dir name', 'extra_files') + + if ask_yn('Do you want to set Trove classifiers?', + helptext=_helptext['do_classifier']) == 'y': + self.set_classifier() + + def _find_files(self): + # we are looking for python modules and packages, + # other stuff are added as regular files + pkgs = self.data['packages'] + modules = self.data['modules'] + extra_files = self.data['extra_files'] + + def is_package(path): + return os.path.exists(os.path.join(path, '__init__.py')) + + curdir = os.getcwd() + scanned = [] + _pref = ['lib', 'include', 'dist', 'build', '.', '~'] + _suf = ['.pyc'] + + def to_skip(path): + path = relative(path) + + for pref in _pref: + if path.startswith(pref): + return True + + for suf in _suf: + if path.endswith(suf): + return True + + return False + + def relative(path): + return path[len(curdir) + 1:] + + def dotted(path): + res = relative(path).replace(os.path.sep, '.') + if res.endswith('.py'): + res = res[:-len('.py')] + return res + + # first pass: packages + for root, dirs, files in os.walk(curdir): + if to_skip(root): + continue + for dir_ in sorted(dirs): + if to_skip(dir_): + continue + fullpath = os.path.join(root, dir_) + dotted_name = dotted(fullpath) + if is_package(fullpath) and dotted_name not in pkgs: + pkgs.append(dotted_name) + scanned.append(fullpath) + + # modules and extra files + for root, dirs, files in os.walk(curdir): + if to_skip(root): + continue + + if any(root.startswith(path) for path in scanned): + continue + + for file in sorted(files): + fullpath = os.path.join(root, file) + if to_skip(fullpath): + continue + # single module? + if os.path.splitext(file)[-1] == '.py': + modules.append(dotted(fullpath)) + else: + extra_files.append(relative(fullpath)) + + def _set_multi(self, question, name): + existing_values = self.data[name] + value = ask(question, helptext=_helptext[name]).strip() + if value not in existing_values: + existing_values.append(value) + + def set_classifier(self): + self.set_maturity_status(self.classifiers) + self.set_license(self.classifiers) + self.set_other_classifier(self.classifiers) + + def set_other_classifier(self, classifiers): + if ask_yn('Do you want to set other trove identifiers?', 'n', + _helptext['trove_generic']) != 'y': + return + self.walk_classifiers(classifiers, [CLASSIFIERS], '') + + def walk_classifiers(self, classifiers, trovepath, desc): + trove = trovepath[-1] + + if not trove: + return + + for key in sorted(trove): + if len(trove[key]) == 0: + if ask_yn('Add "%s"' % desc[4:] + ' :: ' + key, 'n') == 'y': + classifiers.add(desc[4:] + ' :: ' + key) + continue + + if ask_yn('Do you want to set items under\n "%s" (%d sub-items)?' + % (key, len(trove[key])), 'n', + _helptext['trove_generic']) == 'y': + self.walk_classifiers(classifiers, trovepath + [trove[key]], + desc + ' :: ' + key) + + def set_license(self, classifiers): + while True: + license = ask('What license do you use?', + helptext=_helptext['trove_license'], required=False) + if not license: + return + + license_words = license.lower().split(' ') + found_list = [] + + for index, licence in LICENCES: + for word in license_words: + if word in licence: + found_list.append(index) + break + + if len(found_list) == 0: + logger.error('Could not find a matching license for "%s"' % + license) + continue + + question = 'Matching licenses:\n\n' + + for index, list_index in enumerate(found_list): + question += ' %s) %s\n' % (index + 1, + _CLASSIFIERS_LIST[list_index]) + + question += ('\nType the number of the license you wish to use or ' + '? to try again:') + choice = ask(question, required=False) + + if choice == '?': + continue + if choice == '': + return + + try: + index = found_list[int(choice) - 1] + except ValueError: + logger.error( + "Invalid selection, type a number from the list above.") + + classifiers.add(_CLASSIFIERS_LIST[index]) + + def set_maturity_status(self, classifiers): + maturity_name = lambda mat: mat.split('- ')[-1] + maturity_question = '''\ + Please select the project status: + + %s + + Status''' % '\n'.join('%s - %s' % (i, maturity_name(n)) + for i, n in enumerate(PROJECT_MATURITY)) + while True: + choice = ask(dedent(maturity_question), required=False) + + if choice: + try: + choice = int(choice) - 1 + key = PROJECT_MATURITY[choice] + classifiers.add(key) + return + except (IndexError, ValueError): + logger.error( + "Invalid selection, type a single digit number.") + + +def main(): + """Main entry point.""" + program = MainProgram() + # # uncomment when implemented + # if not program.load_existing_setup_script(): + # program.inspect_directory() + # program.query_user() + # program.update_config_file() + # program.write_setup_script() + # packaging.util.cfg_to_args() + program() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/database.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/database.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,651 @@ +"""PEP 376 implementation.""" + +import os +import re +import csv +import sys +import zipimport +from io import StringIO +from hashlib import md5 + +from packaging import logger +from packaging.errors import PackagingError +from packaging.version import suggest_normalized_version, VersionPredicate +from packaging.metadata import Metadata + + +__all__ = [ + 'Distribution', 'EggInfoDistribution', 'distinfo_dirname', + 'get_distributions', 'get_distribution', 'get_file_users', + 'provides_distribution', 'obsoletes_distribution', + 'enable_cache', 'disable_cache', 'clear_cache', + # XXX these functions' names look like get_file_users but are not related + 'get_file_path', 'get_file'] + + +# TODO update docs + +DIST_FILES = ('INSTALLER', 'METADATA', 'RECORD', 'REQUESTED', 'RESOURCES') + +# Cache +_cache_name = {} # maps names to Distribution instances +_cache_name_egg = {} # maps names to EggInfoDistribution instances +_cache_path = {} # maps paths to Distribution instances +_cache_path_egg = {} # maps paths to EggInfoDistribution instances +_cache_generated = False # indicates if .dist-info distributions are cached +_cache_generated_egg = False # indicates if .dist-info and .egg are cached +_cache_enabled = True + + +def enable_cache(): + """ + Enables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = True + + +def disable_cache(): + """ + Disables the internal cache. + + Note that this function will not clear the cache in any case, for that + functionality see :func:`clear_cache`. + """ + global _cache_enabled + + _cache_enabled = False + + +def clear_cache(): + """ Clears the internal cache. """ + global _cache_generated, _cache_generated_egg + + _cache_name.clear() + _cache_name_egg.clear() + _cache_path.clear() + _cache_path_egg.clear() + _cache_generated = False + _cache_generated_egg = False + + +def _yield_distributions(include_dist, include_egg, paths): + """ + Yield .dist-info and .egg(-info) distributions, based on the arguments + + :parameter include_dist: yield .dist-info distributions + :parameter include_egg: yield .egg(-info) distributions + """ + for path in paths: + realpath = os.path.realpath(path) + if not os.path.isdir(realpath): + continue + for dir in os.listdir(realpath): + dist_path = os.path.join(realpath, dir) + if include_dist and dir.endswith('.dist-info'): + yield Distribution(dist_path) + elif include_egg and (dir.endswith('.egg-info') or + dir.endswith('.egg')): + yield EggInfoDistribution(dist_path) + + +def _generate_cache(use_egg_info, paths): + global _cache_generated, _cache_generated_egg + + if _cache_generated_egg or (_cache_generated and not use_egg_info): + return + else: + gen_dist = not _cache_generated + gen_egg = use_egg_info + + for dist in _yield_distributions(gen_dist, gen_egg, paths): + if isinstance(dist, Distribution): + _cache_path[dist.path] = dist + if dist.name not in _cache_name: + _cache_name[dist.name] = [] + _cache_name[dist.name].append(dist) + else: + _cache_path_egg[dist.path] = dist + if dist.name not in _cache_name_egg: + _cache_name_egg[dist.name] = [] + _cache_name_egg[dist.name].append(dist) + + if gen_dist: + _cache_generated = True + if gen_egg: + _cache_generated_egg = True + + +class Distribution: + """Created with the *path* of the ``.dist-info`` directory provided to the + constructor. It reads the metadata contained in ``METADATA`` when it is + instantiated.""" + + name = '' + """The name of the distribution.""" + + version = '' + """The version of the distribution.""" + + metadata = None + """A :class:`packaging.metadata.Metadata` instance loaded with + the distribution's ``METADATA`` file.""" + + requested = False + """A boolean that indicates whether the ``REQUESTED`` metadata file is + present (in other words, whether the package was installed by user + request or it was installed as a dependency).""" + + def __init__(self, path): + if _cache_enabled and path in _cache_path: + self.metadata = _cache_path[path].metadata + else: + metadata_path = os.path.join(path, 'METADATA') + self.metadata = Metadata(path=metadata_path) + + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + self.path = path + + if _cache_enabled and path not in _cache_path: + _cache_path[path] = self + + def __repr__(self): + return '' % ( + self.name, self.version, self.path) + + def _get_records(self, local=False): + results = [] + with self.get_distinfo_file('RECORD') as record: + record_reader = csv.reader(record, delimiter=',', + lineterminator='\n') + for row in record_reader: + missing = [None for i in range(len(row), 3)] + path, checksum, size = row + missing + if local: + path = path.replace('/', os.sep) + path = os.path.join(sys.prefix, path) + results.append((path, checksum, size)) + return results + + def get_resource_path(self, relative_path): + with self.get_distinfo_file('RESOURCES') as resources_file: + resources_reader = csv.reader(resources_file, delimiter=',', + lineterminator='\n') + for relative, destination in resources_reader: + if relative == relative_path: + return destination + raise KeyError( + 'no resource file with relative path %r is installed' % + relative_path) + + def list_installed_files(self, local=False): + """ + Iterates over the ``RECORD`` entries and returns a tuple + ``(path, md5, size)`` for each line. If *local* is ``True``, + the returned path is transformed into a local absolute path. + Otherwise the raw value from RECORD is returned. + + A local absolute path is an absolute path in which occurrences of + ``'/'`` have been replaced by the system separator given by ``os.sep``. + + :parameter local: flag to say if the path should be returned as a local + absolute path + + :type local: boolean + :returns: iterator of (path, md5, size) + """ + for result in self._get_records(local): + yield result + + def uses(self, path): + """ + Returns ``True`` if path is listed in ``RECORD``. *path* can be a local + absolute path or a relative ``'/'``-separated path. + + :rtype: boolean + """ + for p, checksum, size in self._get_records(): + local_absolute = os.path.join(sys.prefix, p) + if path == p or path == local_absolute: + return True + return False + + def get_distinfo_file(self, path, binary=False): + """ + Returns a file located under the ``.dist-info`` directory. Returns a + ``file`` instance for the file pointed by *path*. + + :parameter path: a ``'/'``-separated path relative to the + ``.dist-info`` directory or an absolute path; + If *path* is an absolute path and doesn't start + with the ``.dist-info`` directory path, + a :class:`PackagingError` is raised + :type path: string + :parameter binary: If *binary* is ``True``, opens the file in read-only + binary mode (``rb``), otherwise opens it in + read-only mode (``r``). + :rtype: file object + """ + open_flags = 'r' + if binary: + open_flags += 'b' + + # Check if it is an absolute path # XXX use relpath, add tests + if path.find(os.sep) >= 0: + # it's an absolute path? + distinfo_dirname, path = path.split(os.sep)[-2:] + if distinfo_dirname != self.path.split(os.sep)[-1]: + raise PackagingError( + 'dist-info file %r does not belong to the %r %s ' + 'distribution' % (path, self.name, self.version)) + + # The file must be relative + if path not in DIST_FILES: + raise PackagingError('invalid path for a dist-info file: %r' % + path) + + path = os.path.join(self.path, path) + return open(path, open_flags) + + def list_distinfo_files(self, local=False): + """ + Iterates over the ``RECORD`` entries and returns paths for each line if + the path is pointing to a file located in the ``.dist-info`` directory + or one of its subdirectories. + + :parameter local: If *local* is ``True``, each returned path is + transformed into a local absolute path. Otherwise the + raw value from ``RECORD`` is returned. + :type local: boolean + :returns: iterator of paths + """ + for path, checksum, size in self._get_records(local): + # XXX add separator or use real relpath algo + if path.startswith(self.path): + yield path + + def __eq__(self, other): + return isinstance(other, Distribution) and self.path == other.path + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class EggInfoDistribution: + """Created with the *path* of the ``.egg-info`` directory or file provided + to the constructor. It reads the metadata contained in the file itself, or + if the given path happens to be a directory, the metadata is read from the + file ``PKG-INFO`` under that directory.""" + + name = '' + """The name of the distribution.""" + + version = '' + """The version of the distribution.""" + + metadata = None + """A :class:`packaging.metadata.Metadata` instance loaded with + the distribution's ``METADATA`` file.""" + + _REQUIREMENT = re.compile( + r'(?P[-A-Za-z0-9_.]+)\s*' + r'(?P(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)?\s*' + r'(?P(?:\s*,\s*(?:<|<=|!=|==|>=|>)[-A-Za-z0-9_.]+)*)\s*' + r'(?P\[.*\])?') + + def __init__(self, path): + self.path = path + if _cache_enabled and path in _cache_path_egg: + self.metadata = _cache_path_egg[path].metadata + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + return + + # reused from Distribute's pkg_resources + def yield_lines(strs): + """Yield non-empty/non-comment lines of a ``basestring`` + or sequence""" + if isinstance(strs, str): + for s in strs.splitlines(): + s = s.strip() + # skip blank lines/comments + if s and not s.startswith('#'): + yield s + else: + for ss in strs: + for s in yield_lines(ss): + yield s + + requires = None + + if path.endswith('.egg'): + if os.path.isdir(path): + meta_path = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + self.metadata = Metadata(path=meta_path) + try: + req_path = os.path.join(path, 'EGG-INFO', 'requires.txt') + with open(req_path, 'r') as fp: + requires = fp.read() + except IOError: + requires = None + else: + # FIXME handle the case where zipfile is not available + zipf = zipimport.zipimporter(path) + fileobj = StringIO( + zipf.get_data('EGG-INFO/PKG-INFO').decode('utf8')) + self.metadata = Metadata(fileobj=fileobj) + try: + requires = zipf.get_data('EGG-INFO/requires.txt') + except IOError: + requires = None + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + + elif path.endswith('.egg-info'): + if os.path.isdir(path): + path = os.path.join(path, 'PKG-INFO') + try: + with open(os.path.join(path, 'requires.txt'), 'r') as fp: + requires = fp.read() + except IOError: + requires = None + self.metadata = Metadata(path=path) + self.name = self.metadata['Name'] + self.version = self.metadata['Version'] + + else: + raise ValueError('path must end with .egg-info or .egg, got %r' % + path) + + if requires is not None: + if self.metadata['Metadata-Version'] == '1.1': + # we can't have 1.1 metadata *and* Setuptools requires + for field in ('Obsoletes', 'Requires', 'Provides'): + del self.metadata[field] + + reqs = [] + + if requires is not None: + for line in yield_lines(requires): + if line.startswith('['): + logger.warning( + 'extensions in requires.txt are not supported ' + '(used by %r %s)', self.name, self.version) + break + else: + match = self._REQUIREMENT.match(line.strip()) + if not match: + # this happens when we encounter extras; since they + # are written at the end of the file we just exit + break + else: + if match.group('extras'): + msg = ('extra requirements are not supported ' + '(used by %r %s)', self.name, self.version) + logger.warning(msg, self.name) + name = match.group('name') + version = None + if match.group('first'): + version = match.group('first') + if match.group('rest'): + version += match.group('rest') + version = version.replace(' ', '') # trim spaces + if version is None: + reqs.append(name) + else: + reqs.append('%s (%s)' % (name, version)) + + if len(reqs) > 0: + self.metadata['Requires-Dist'] += reqs + + if _cache_enabled: + _cache_path_egg[self.path] = self + + def __repr__(self): + return '' % ( + self.name, self.version, self.path) + + def list_installed_files(self, local=False): + + def _md5(path): + with open(path, 'rb') as f: + content = f.read() + return md5(content).hexdigest() + + def _size(path): + return os.stat(path).st_size + + path = self.path + if local: + path = path.replace('/', os.sep) + + # XXX What about scripts and data files ? + if os.path.isfile(path): + return [(path, _md5(path), _size(path))] + else: + files = [] + for root, dir, files_ in os.walk(path): + for item in files_: + item = os.path.join(root, item) + files.append((item, _md5(item), _size(item))) + return files + + return [] + + def uses(self, path): + return False + + def __eq__(self, other): + return (isinstance(other, EggInfoDistribution) and + self.path == other.path) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +def distinfo_dirname(name, version): + """ + The *name* and *version* parameters are converted into their + filename-escaped form, i.e. any ``'-'`` characters are replaced + with ``'_'`` other than the one in ``'dist-info'`` and the one + separating the name from the version number. + + :parameter name: is converted to a standard distribution name by replacing + any runs of non- alphanumeric characters with a single + ``'-'``. + :type name: string + :parameter version: is converted to a standard version string. Spaces + become dots, and all other non-alphanumeric characters + (except dots) become dashes, with runs of multiple + dashes condensed to a single dash. + :type version: string + :returns: directory name + :rtype: string""" + file_extension = '.dist-info' + name = name.replace('-', '_') + normalized_version = suggest_normalized_version(version) + # Because this is a lookup procedure, something will be returned even if + # it is a version that cannot be normalized + if normalized_version is None: + # Unable to achieve normality? + normalized_version = version + return '-'.join([name, normalized_version]) + file_extension + + +def get_distributions(use_egg_info=False, paths=None): + """ + Provides an iterator that looks for ``.dist-info`` directories in + ``sys.path`` and returns :class:`Distribution` instances for each one of + them. If the parameters *use_egg_info* is ``True``, then the ``.egg-info`` + files and directores are iterated as well. + + :rtype: iterator of :class:`Distribution` and :class:`EggInfoDistribution` + instances + """ + if paths is None: + paths = sys.path + + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info, paths): + yield dist + else: + _generate_cache(use_egg_info, paths) + + for dist in _cache_path.values(): + yield dist + + if use_egg_info: + for dist in _cache_path_egg.values(): + yield dist + + +def get_distribution(name, use_egg_info=False, paths=None): + """ + Scans all elements in ``sys.path`` and looks for all directories + ending with ``.dist-info``. Returns a :class:`Distribution` + corresponding to the ``.dist-info`` directory that contains the + ``METADATA`` that matches *name* for the *name* metadata field. + If no distribution exists with the given *name* and the parameter + *use_egg_info* is set to ``True``, then all files and directories ending + with ``.egg-info`` are scanned. A :class:`EggInfoDistribution` instance is + returned if one is found that has metadata that matches *name* for the + *name* metadata field. + + This function only returns the first result found, as no more than one + value is expected. If the directory is not found, ``None`` is returned. + + :rtype: :class:`Distribution` or :class:`EggInfoDistribution` or None + """ + if paths is None: + paths = sys.path + + if not _cache_enabled: + for dist in _yield_distributions(True, use_egg_info, paths): + if dist.name == name: + return dist + else: + _generate_cache(use_egg_info, paths) + + if name in _cache_name: + return _cache_name[name][0] + elif use_egg_info and name in _cache_name_egg: + return _cache_name_egg[name][0] + else: + return None + + +def obsoletes_distribution(name, version=None, use_egg_info=False): + """ + Iterates over all distributions to find which distributions obsolete + *name*. + + If a *version* is provided, it will be used to filter the results. + If the argument *use_egg_info* is set to ``True``, then ``.egg-info`` + distributions will be considered as well. + + :type name: string + :type version: string + :parameter name: + """ + for dist in get_distributions(use_egg_info): + obsoleted = (dist.metadata['Obsoletes-Dist'] + + dist.metadata['Obsoletes']) + for obs in obsoleted: + o_components = obs.split(' ', 1) + if len(o_components) == 1 or version is None: + if name == o_components[0]: + yield dist + break + else: + try: + predicate = VersionPredicate(obs) + except ValueError: + raise PackagingError( + 'distribution %r has ill-formed obsoletes field: ' + '%r' % (dist.name, obs)) + if name == o_components[0] and predicate.match(version): + yield dist + break + + +def provides_distribution(name, version=None, use_egg_info=False): + """ + Iterates over all distributions to find which distributions provide *name*. + If a *version* is provided, it will be used to filter the results. Scans + all elements in ``sys.path`` and looks for all directories ending with + ``.dist-info``. Returns a :class:`Distribution` corresponding to the + ``.dist-info`` directory that contains a ``METADATA`` that matches *name* + for the name metadata. If the argument *use_egg_info* is set to ``True``, + then all files and directories ending with ``.egg-info`` are considered + as well and returns an :class:`EggInfoDistribution` instance. + + This function only returns the first result found, since no more than + one values are expected. If the directory is not found, returns ``None``. + + :parameter version: a version specifier that indicates the version + required, conforming to the format in ``PEP-345`` + + :type name: string + :type version: string + """ + predicate = None + if not version is None: + try: + predicate = VersionPredicate(name + ' (' + version + ')') + except ValueError: + raise PackagingError('invalid name or version: %r, %r' % + (name, version)) + + for dist in get_distributions(use_egg_info): + provided = dist.metadata['Provides-Dist'] + dist.metadata['Provides'] + + for p in provided: + p_components = p.rsplit(' ', 1) + if len(p_components) == 1 or predicate is None: + if name == p_components[0]: + yield dist + break + else: + p_name, p_ver = p_components + if len(p_ver) < 2 or p_ver[0] != '(' or p_ver[-1] != ')': + raise PackagingError( + 'distribution %r has invalid Provides field: %r' % + (dist.name, p)) + p_ver = p_ver[1:-1] # trim off the parenthesis + if p_name == name and predicate.match(p_ver): + yield dist + break + + +def get_file_users(path): + """ + Iterates over all distributions to find out which distributions use + *path*. + + :parameter path: can be a local absolute path or a relative + ``'/'``-separated path. + :type path: string + :rtype: iterator of :class:`Distribution` instances + """ + for dist in get_distributions(): + if dist.uses(path): + yield dist + + +def get_file_path(distribution_name, relative_path): + """Return the path to a resource file.""" + dist = get_distribution(distribution_name) + if dist is not None: + return dist.get_resource_path(relative_path) + raise LookupError('no distribution named %r found' % distribution_name) + + +def get_file(distribution_name, relative_path, *args, **kwargs): + """Open and return a resource file.""" + return open(get_file_path(distribution_name, relative_path), + *args, **kwargs) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/depgraph.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/depgraph.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,270 @@ +"""Class and functions dealing with dependencies between distributions. + +This module provides a DependencyGraph class to represent the +dependencies between distributions. Auxiliary functions can generate a +graph, find reverse dependencies, and print a graph in DOT format. +""" + +import sys + +from io import StringIO +from packaging.errors import PackagingError +from packaging.version import VersionPredicate, IrrationalVersionError + +__all__ = ['DependencyGraph', 'generate_graph', 'dependent_dists', + 'graph_to_dot'] + + +class DependencyGraph: + """ + Represents a dependency graph between distributions. + + The dependency relationships are stored in an ``adjacency_list`` that maps + distributions to a list of ``(other, label)`` tuples where ``other`` + is a distribution and the edge is labeled with ``label`` (i.e. the version + specifier, if such was provided). Also, for more efficient traversal, for + every distribution ``x``, a list of predecessors is kept in + ``reverse_list[x]``. An edge from distribution ``a`` to + distribution ``b`` means that ``a`` depends on ``b``. If any missing + dependencies are found, they are stored in ``missing``, which is a + dictionary that maps distributions to a list of requirements that were not + provided by any other distributions. + """ + + def __init__(self): + self.adjacency_list = {} + self.reverse_list = {} + self.missing = {} + + def add_distribution(self, distribution): + """Add the *distribution* to the graph. + + :type distribution: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + """ + self.adjacency_list[distribution] = [] + self.reverse_list[distribution] = [] + self.missing[distribution] = [] + + def add_edge(self, x, y, label=None): + """Add an edge from distribution *x* to distribution *y* with the given + *label*. + + :type x: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type y: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type label: ``str`` or ``None`` + """ + self.adjacency_list[x].append((y, label)) + # multiple edges are allowed, so be careful + if x not in self.reverse_list[y]: + self.reverse_list[y].append(x) + + def add_missing(self, distribution, requirement): + """ + Add a missing *requirement* for the given *distribution*. + + :type distribution: :class:`packaging.database.Distribution` or + :class:`packaging.database.EggInfoDistribution` + :type requirement: ``str`` + """ + self.missing[distribution].append(requirement) + + def _repr_dist(self, dist): + return '%r %s' % (dist.name, dist.version) + + def repr_node(self, dist, level=1): + """Prints only a subgraph""" + output = [] + output.append(self._repr_dist(dist)) + for other, label in self.adjacency_list[dist]: + dist = self._repr_dist(other) + if label is not None: + dist = '%s [%s]' % (dist, label) + output.append(' ' * level + str(dist)) + suboutput = self.repr_node(other, level + 1) + subs = suboutput.split('\n') + output.extend(subs[1:]) + return '\n'.join(output) + + def __repr__(self): + """Representation of the graph""" + output = [] + for dist, adjs in self.adjacency_list.items(): + output.append(self.repr_node(dist)) + return '\n'.join(output) + + +def graph_to_dot(graph, f, skip_disconnected=True): + """Writes a DOT output for the graph to the provided file *f*. + + If *skip_disconnected* is set to ``True``, then all distributions + that are not dependent on any other distribution are skipped. + + :type f: has to support ``file``-like operations + :type skip_disconnected: ``bool`` + """ + disconnected = [] + + f.write("digraph dependencies {\n") + for dist, adjs in graph.adjacency_list.items(): + if len(adjs) == 0 and not skip_disconnected: + disconnected.append(dist) + for other, label in adjs: + if not label is None: + f.write('"%s" -> "%s" [label="%s"]\n' % + (dist.name, other.name, label)) + else: + f.write('"%s" -> "%s"\n' % (dist.name, other.name)) + if not skip_disconnected and len(disconnected) > 0: + f.write('subgraph disconnected {\n') + f.write('label = "Disconnected"\n') + f.write('bgcolor = red\n') + + for dist in disconnected: + f.write('"%s"' % dist.name) + f.write('\n') + f.write('}\n') + f.write('}\n') + + +def generate_graph(dists): + """Generates a dependency graph from the given distributions. + + :parameter dists: a list of distributions + :type dists: list of :class:`packaging.database.Distribution` and + :class:`packaging.database.EggInfoDistribution` instances + :rtype: a :class:`DependencyGraph` instance + """ + graph = DependencyGraph() + provided = {} # maps names to lists of (version, dist) tuples + + # first, build the graph and find out the provides + for dist in dists: + graph.add_distribution(dist) + provides = (dist.metadata['Provides-Dist'] + + dist.metadata['Provides'] + + ['%s (%s)' % (dist.name, dist.version)]) + + for p in provides: + comps = p.strip().rsplit(" ", 1) + name = comps[0] + version = None + if len(comps) == 2: + version = comps[1] + if len(version) < 3 or version[0] != '(' or version[-1] != ')': + raise PackagingError('distribution %r has ill-formed' + 'provides field: %r' % (dist.name, p)) + version = version[1:-1] # trim off parenthesis + if name not in provided: + provided[name] = [] + provided[name].append((version, dist)) + + # now make the edges + for dist in dists: + requires = dist.metadata['Requires-Dist'] + dist.metadata['Requires'] + for req in requires: + try: + predicate = VersionPredicate(req) + except IrrationalVersionError: + # XXX compat-mode if cannot read the version + name = req.split()[0] + predicate = VersionPredicate(name) + + name = predicate.name + + if name not in provided: + graph.add_missing(dist, req) + else: + matched = False + for version, provider in provided[name]: + try: + match = predicate.match(version) + except IrrationalVersionError: + # XXX small compat-mode + if version.split(' ') == 1: + match = True + else: + match = False + + if match: + graph.add_edge(dist, provider, req) + matched = True + break + if not matched: + graph.add_missing(dist, req) + return graph + + +def dependent_dists(dists, dist): + """Recursively generate a list of distributions from *dists* that are + dependent on *dist*. + + :param dists: a list of distributions + :param dist: a distribution, member of *dists* for which we are interested + """ + if dist not in dists: + raise ValueError('given distribution %r is not a member of the list' % + dist.name) + graph = generate_graph(dists) + + dep = [dist] # dependent distributions + fringe = graph.reverse_list[dist] # list of nodes we should inspect + + while not len(fringe) == 0: + node = fringe.pop() + dep.append(node) + for prev in graph.reverse_list[node]: + if prev not in dep: + fringe.append(prev) + + dep.pop(0) # remove dist from dep, was there to prevent infinite loops + return dep + + +def main(): + # XXX move to run._graph + from packaging.database import get_distributions + tempout = StringIO() + try: + old = sys.stderr + sys.stderr = tempout + try: + dists = list(get_distributions(use_egg_info=True)) + graph = generate_graph(dists) + finally: + sys.stderr = old + except Exception as e: + tempout.seek(0) + tempout = tempout.read() + print('Could not generate the graph') + print(tempout) + print(e) + sys.exit(1) + + for dist, reqs in graph.missing.items(): + if len(reqs) > 0: + print("Warning: Missing dependencies for %r:" % dist.name, + ", ".join(reqs)) + # XXX replace with argparse + if len(sys.argv) == 1: + print('Dependency graph:') + print(' ', repr(graph).replace('\n', '\n ')) + sys.exit(0) + elif len(sys.argv) > 1 and sys.argv[1] in ('-d', '--dot'): + if len(sys.argv) > 2: + filename = sys.argv[2] + else: + filename = 'depgraph.dot' + + with open(filename, 'w') as f: + graph_to_dot(graph, f, True) + tempout.seek(0) + tempout = tempout.read() + print(tempout) + print('Dot file written at %r' % filename) + sys.exit(0) + else: + print('Supported option: -d [filename]') + sys.exit(1) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/dist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,769 @@ +"""Class representing the project being built/installed/etc.""" + +import os +import re + +from packaging import logger +from packaging.util import strtobool, resolve_name +from packaging.config import Config +from packaging.errors import (PackagingOptionError, PackagingArgError, + PackagingModuleError, PackagingClassError) +from packaging.command import get_command_class, STANDARD_COMMANDS +from packaging.command.cmd import Command +from packaging.metadata import Metadata +from packaging.fancy_getopt import FancyGetopt + +# Regex to define acceptable Packaging command names. This is not *quite* +# the same as a Python name -- leading underscores are not allowed. The fact +# that they're very similar is no coincidence: the default naming scheme is +# to look for a Python module named after the command. +command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') + +USAGE = """\ +usage: %(script)s [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] + or: %(script)s --help [cmd1 cmd2 ...] + or: %(script)s --help-commands + or: %(script)s cmd --help +""" + + +def gen_usage(script_name): + script = os.path.basename(script_name) + return USAGE % {'script': script} + + +class Distribution: + """Class used to represent a project and work with it. + + Most of the work hiding behind 'pysetup run' is really done within a + Distribution instance, which farms the work out to the commands + specified on the command line. + """ + + # 'global_options' describes the command-line options that may be + # supplied to the setup script prior to any actual commands. + # Eg. "pysetup run -n" or "pysetup run --dry-run" both take advantage of + # these global options. This list should be kept to a bare minimum, + # since every global option is also valid as a command option -- and we + # don't want to pollute the commands with too many options that they + # have minimal control over. + global_options = [ + ('dry-run', 'n', "don't actually do anything"), + ('help', 'h', "show detailed help message"), + ('no-user-cfg', None, 'ignore pydistutils.cfg in your home directory'), + ] + + # 'common_usage' is a short (2-3 line) string describing the common + # usage of the setup script. + common_usage = """\ +Common commands: (see '--help-commands' for more) + + pysetup run build will build the project underneath 'build/' + pysetup run install will install the project +""" + + # options that are not propagated to the commands + display_options = [ + ('help-commands', None, + "list all available commands"), + ('use-2to3', None, + "use 2to3 to make source python 3.x compatible"), + ('convert-2to3-doctests', None, + "use 2to3 to convert doctests in separate text files"), + ] + display_option_names = [x[0].replace('-', '_') for x in display_options] + + # negative options are options that exclude other options + negative_opt = {} + + # -- Creation/initialization methods ------------------------------- + def __init__(self, attrs=None): + """Construct a new Distribution instance: initialize all the + attributes of a Distribution, and then use 'attrs' (a dictionary + mapping attribute names to values) to assign some of those + attributes their "real" values. (Any attributes not mentioned in + 'attrs' will be assigned to some null value: 0, None, an empty list + or dictionary, etc.) Most importantly, initialize the + 'command_obj' attribute to the empty dictionary; this will be + filled in with real command objects by 'parse_command_line()'. + """ + + # Default values for our command-line options + self.dry_run = False + self.help = False + for attr in self.display_option_names: + setattr(self, attr, False) + + # Store the configuration + self.config = Config(self) + + # Store the distribution metadata (name, version, author, and so + # forth) in a separate object -- we're getting to have enough + # information here (and enough command-line options) that it's + # worth it. + self.metadata = Metadata() + + # 'cmdclass' maps command names to class objects, so we + # can 1) quickly figure out which class to instantiate when + # we need to create a new command object, and 2) have a way + # for the setup script to override command classes + self.cmdclass = {} + + # 'script_name' and 'script_args' are usually set to sys.argv[0] + # and sys.argv[1:], but they can be overridden when the caller is + # not necessarily a setup script run from the command line. + self.script_name = None + self.script_args = None + + # 'command_options' is where we store command options between + # parsing them (from config files, the command line, etc.) and when + # they are actually needed -- ie. when the command in question is + # instantiated. It is a dictionary of dictionaries of 2-tuples: + # command_options = { command_name : { option : (source, value) } } + self.command_options = {} + + # 'dist_files' is the list of (command, pyversion, file) that + # have been created by any dist commands run so far. This is + # filled regardless of whether the run is dry or not. pyversion + # gives sysconfig.get_python_version() if the dist file is + # specific to a Python version, 'any' if it is good for all + # Python versions on the target platform, and '' for a source + # file. pyversion should not be used to specify minimum or + # maximum required Python versions; use the metainfo for that + # instead. + self.dist_files = [] + + # These options are really the business of various commands, rather + # than of the Distribution itself. We provide aliases for them in + # Distribution as a convenience to the developer. + self.packages = [] + self.package_data = {} + self.package_dir = None + self.py_modules = [] + self.libraries = [] + self.headers = [] + self.ext_modules = [] + self.ext_package = None + self.include_dirs = [] + self.extra_path = None + self.scripts = [] + self.data_files = {} + self.password = '' + self.use_2to3 = False + self.convert_2to3_doctests = [] + self.extra_files = [] + + # And now initialize bookkeeping stuff that can't be supplied by + # the caller at all. 'command_obj' maps command names to + # Command instances -- that's how we enforce that every command + # class is a singleton. + self.command_obj = {} + + # 'have_run' maps command names to boolean values; it keeps track + # of whether we have actually run a particular command, to make it + # cheap to "run" a command whenever we think we might need to -- if + # it's already been done, no need for expensive filesystem + # operations, we just check the 'have_run' dictionary and carry on. + # It's only safe to query 'have_run' for a command class that has + # been instantiated -- a false value will be inserted when the + # command object is created, and replaced with a true value when + # the command is successfully run. Thus it's probably best to use + # '.get()' rather than a straight lookup. + self.have_run = {} + + # Now we'll use the attrs dictionary (ultimately, keyword args from + # the setup script) to possibly override any or all of these + # distribution options. + + if attrs is not None: + # Pull out the set of command options and work on them + # specifically. Note that this order guarantees that aliased + # command options will override any supplied redundantly + # through the general options dictionary. + options = attrs.get('options') + if options is not None: + del attrs['options'] + for command, cmd_options in options.items(): + opt_dict = self.get_option_dict(command) + for opt, val in cmd_options.items(): + opt_dict[opt] = ("setup script", val) + + # Now work on the rest of the attributes. Any attribute that's + # not already defined is invalid! + for key, val in attrs.items(): + if self.metadata.is_metadata_field(key): + self.metadata[key] = val + elif hasattr(self, key): + setattr(self, key, val) + else: + logger.warning( + 'unknown argument given to Distribution: %r', key) + + # no-user-cfg is handled before other command line args + # because other args override the config files, and this + # one is needed before we can load the config files. + # If attrs['script_args'] wasn't passed, assume false. + # + # This also make sure we just look at the global options + self.want_user_cfg = True + + if self.script_args is not None: + for arg in self.script_args: + if not arg.startswith('-'): + break + if arg == '--no-user-cfg': + self.want_user_cfg = False + break + + self.finalize_options() + + def get_option_dict(self, command): + """Get the option dictionary for a given command. If that + command's option dictionary hasn't been created yet, then create it + and return the new dictionary; otherwise, return the existing + option dictionary. + """ + d = self.command_options.get(command) + if d is None: + d = self.command_options[command] = {} + return d + + def get_fullname(self, filesafe=False): + return self.metadata.get_fullname(filesafe) + + def dump_option_dicts(self, header=None, commands=None, indent=""): + from pprint import pformat + + if commands is None: # dump all command option dicts + commands = sorted(self.command_options) + + if header is not None: + logger.info(indent + header) + indent = indent + " " + + if not commands: + logger.info(indent + "no commands known yet") + return + + for cmd_name in commands: + opt_dict = self.command_options.get(cmd_name) + if opt_dict is None: + logger.info(indent + "no option dict for %r command", + cmd_name) + else: + logger.info(indent + "option dict for %r command:", cmd_name) + out = pformat(opt_dict) + for line in out.split('\n'): + logger.info(indent + " " + line) + + # -- Config file finding/parsing methods --------------------------- + # XXX to be removed + def parse_config_files(self, filenames=None): + return self.config.parse_config_files(filenames) + + def find_config_files(self): + return self.config.find_config_files() + + # -- Command-line parsing methods ---------------------------------- + + def parse_command_line(self): + """Parse the setup script's command line, taken from the + 'script_args' instance attribute (which defaults to 'sys.argv[1:]' + -- see 'setup()' in run.py). This list is first processed for + "global options" -- options that set attributes of the Distribution + instance. Then, it is alternately scanned for Packaging commands + and options for that command. Each new command terminates the + options for the previous command. The allowed options for a + command are determined by the 'user_options' attribute of the + command class -- thus, we have to be able to load command classes + in order to parse the command line. Any error in that 'options' + attribute raises PackagingGetoptError; any error on the + command line raises PackagingArgError. If no Packaging commands + were found on the command line, raises PackagingArgError. Return + true if command line was successfully parsed and we should carry + on with executing commands; false if no errors but we shouldn't + execute commands (currently, this only happens if user asks for + help). + """ + # + # We now have enough information to show the Macintosh dialog + # that allows the user to interactively specify the "command line". + # + toplevel_options = self._get_toplevel_options() + + # We have to parse the command line a bit at a time -- global + # options, then the first command, then its options, and so on -- + # because each command will be handled by a different class, and + # the options that are valid for a particular class aren't known + # until we have loaded the command class, which doesn't happen + # until we know what the command is. + + self.commands = [] + parser = FancyGetopt(toplevel_options + self.display_options) + parser.set_negative_aliases(self.negative_opt) + args = parser.getopt(args=self.script_args, object=self) + option_order = parser.get_option_order() + + # for display options we return immediately + if self.handle_display_options(option_order): + return + + while args: + args = self._parse_command_opts(parser, args) + if args is None: # user asked for help (and got it) + return + + # Handle the cases of --help as a "global" option, ie. + # "pysetup run --help" and "pysetup run --help command ...". For the + # former, we show global options (--dry-run, etc.) + # and display-only options (--name, --version, etc.); for the + # latter, we omit the display-only options and show help for + # each command listed on the command line. + if self.help: + self._show_help(parser, + display_options=len(self.commands) == 0, + commands=self.commands) + return + + return True + + def _get_toplevel_options(self): + """Return the non-display options recognized at the top level. + + This includes options that are recognized *only* at the top + level as well as options recognized for commands. + """ + return self.global_options + + def _parse_command_opts(self, parser, args): + """Parse the command-line options for a single command. + 'parser' must be a FancyGetopt instance; 'args' must be the list + of arguments, starting with the current command (whose options + we are about to parse). Returns a new version of 'args' with + the next command at the front of the list; will be the empty + list if there are no more commands on the command line. Returns + None if the user asked for help on this command. + """ + # Pull the current command from the head of the command line + command = args[0] + if not command_re.match(command): + raise SystemExit("invalid command name %r" % command) + self.commands.append(command) + + # Dig up the command class that implements this command, so we + # 1) know that it's a valid command, and 2) know which options + # it takes. + try: + cmd_class = get_command_class(command) + except PackagingModuleError as msg: + raise PackagingArgError(msg) + + # XXX We want to push this in packaging.command + # + # Require that the command class be derived from Command -- want + # to be sure that the basic "command" interface is implemented. + for meth in ('initialize_options', 'finalize_options', 'run'): + if hasattr(cmd_class, meth): + continue + raise PackagingClassError( + 'command %r must implement %r' % (cmd_class, meth)) + + # Also make sure that the command object provides a list of its + # known options. + if not (hasattr(cmd_class, 'user_options') and + isinstance(cmd_class.user_options, list)): + raise PackagingClassError( + "command class %s must provide " + "'user_options' attribute (a list of tuples)" % cmd_class) + + # If the command class has a list of negative alias options, + # merge it in with the global negative aliases. + negative_opt = self.negative_opt + if hasattr(cmd_class, 'negative_opt'): + negative_opt = negative_opt.copy() + negative_opt.update(cmd_class.negative_opt) + + # Check for help_options in command class. They have a different + # format (tuple of four) so we need to preprocess them here. + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_options = cmd_class.help_options[:] + else: + help_options = [] + + # All commands support the global options too, just by adding + # in 'global_options'. + parser.set_option_table(self.global_options + + cmd_class.user_options + + help_options) + parser.set_negative_aliases(negative_opt) + args, opts = parser.getopt(args[1:]) + if hasattr(opts, 'help') and opts.help: + self._show_help(parser, display_options=False, + commands=[cmd_class]) + return + + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_option_found = False + for help_option, short, desc, func in cmd_class.help_options: + if hasattr(opts, help_option.replace('-', '_')): + help_option_found = True + if callable(func): + func() + else: + raise PackagingClassError( + "invalid help function %r for help option %r: " + "must be a callable object (function, etc.)" + % (func, help_option)) + + if help_option_found: + return + + # Put the options from the command line into their official + # holding pen, the 'command_options' dictionary. + opt_dict = self.get_option_dict(command) + for name, value in vars(opts).items(): + opt_dict[name] = ("command line", value) + + return args + + def finalize_options(self): + """Set final values for all the options on the Distribution + instance, analogous to the .finalize_options() method of Command + objects. + """ + if getattr(self, 'convert_2to3_doctests', None): + self.convert_2to3_doctests = [os.path.join(p) + for p in self.convert_2to3_doctests] + else: + self.convert_2to3_doctests = [] + + def _show_help(self, parser, global_options=True, display_options=True, + commands=[]): + """Show help for the setup script command line in the form of + several lists of command-line options. 'parser' should be a + FancyGetopt instance; do not expect it to be returned in the + same state, as its option table will be reset to make it + generate the correct help text. + + If 'global_options' is true, lists the global options: + --dry-run, etc. If 'display_options' is true, lists + the "display-only" options: --help-commands. Finally, + lists per-command help for every command name or command class + in 'commands'. + """ + if global_options: + if display_options: + options = self._get_toplevel_options() + else: + options = self.global_options + parser.set_option_table(options) + parser.print_help(self.common_usage + "\nGlobal options:") + print() + + if display_options: + parser.set_option_table(self.display_options) + parser.print_help( + "Information display options (just display " + + "information, ignore any commands)") + print() + + for command in self.commands: + if isinstance(command, type) and issubclass(command, Command): + cls = command + else: + cls = get_command_class(command) + if (hasattr(cls, 'help_options') and + isinstance(cls.help_options, list)): + parser.set_option_table(cls.user_options + cls.help_options) + else: + parser.set_option_table(cls.user_options) + parser.print_help("Options for %r command:" % cls.__name__) + print() + + print(gen_usage(self.script_name)) + + def handle_display_options(self, option_order): + """If there were any non-global "display-only" options + (--help-commands) on the command line, display the requested info and + return true; else return false. + """ + # User just wants a list of commands -- we'll print it out and stop + # processing now (ie. if they ran "setup --help-commands foo bar", + # we ignore "foo bar"). + if self.help_commands: + self.print_commands() + print() + print(gen_usage(self.script_name)) + return True + + # If user supplied any of the "display metadata" options, then + # display that metadata in the order in which the user supplied the + # metadata options. + any_display_options = False + is_display_option = set() + for option in self.display_options: + is_display_option.add(option[0]) + + for opt, val in option_order: + if val and opt in is_display_option: + opt = opt.replace('-', '_') + value = self.metadata[opt] + if opt in ('keywords', 'platform'): + print(','.join(value)) + elif opt in ('classifier', 'provides', 'requires', + 'obsoletes'): + print('\n'.join(value)) + else: + print(value) + any_display_options = True + + return any_display_options + + def print_command_list(self, commands, header, max_length): + """Print a subset of the list of all commands -- used by + 'print_commands()'. + """ + print(header + ":") + + for cmd in commands: + cls = self.cmdclass.get(cmd) or get_command_class(cmd) + description = getattr(cls, 'description', + '(no description available)') + + print(" %-*s %s" % (max_length, cmd, description)) + + def _get_command_groups(self): + """Helper function to retrieve all the command class names divided + into standard commands (listed in + packaging.command.STANDARD_COMMANDS) and extra commands (given in + self.cmdclass and not standard commands). + """ + extra_commands = [cmd for cmd in self.cmdclass + if cmd not in STANDARD_COMMANDS] + return STANDARD_COMMANDS, extra_commands + + def print_commands(self): + """Print out a help message listing all available commands with a + description of each. The list is divided into standard commands + (listed in packaging.command.STANDARD_COMMANDS) and extra commands + (given in self.cmdclass and not standard commands). The + descriptions come from the command class attribute + 'description'. + """ + std_commands, extra_commands = self._get_command_groups() + max_length = 0 + for cmd in (std_commands + extra_commands): + if len(cmd) > max_length: + max_length = len(cmd) + + self.print_command_list(std_commands, + "Standard commands", + max_length) + if extra_commands: + print() + self.print_command_list(extra_commands, + "Extra commands", + max_length) + + # -- Command class/object methods ---------------------------------- + + def get_command_obj(self, command, create=True): + """Return the command object for 'command'. Normally this object + is cached on a previous call to 'get_command_obj()'; if no command + object for 'command' is in the cache, then we either create and + return it (if 'create' is true) or return None. + """ + cmd_obj = self.command_obj.get(command) + if not cmd_obj and create: + logger.debug("Distribution.get_command_obj(): " + "creating %r command object", command) + + cls = get_command_class(command) + cmd_obj = self.command_obj[command] = cls(self) + self.have_run[command] = 0 + + # Set any options that were supplied in config files or on the + # command line. (XXX support for error reporting is suboptimal + # here: errors aren't reported until finalize_options is called, + # which means we won't report the source of the error.) + options = self.command_options.get(command) + if options: + self._set_command_options(cmd_obj, options) + + return cmd_obj + + def _set_command_options(self, command_obj, option_dict=None): + """Set the options for 'command_obj' from 'option_dict'. Basically + this means copying elements of a dictionary ('option_dict') to + attributes of an instance ('command'). + + 'command_obj' must be a Command instance. If 'option_dict' is not + supplied, uses the standard option dictionary for this command + (from 'self.command_options'). + """ + command_name = command_obj.get_command_name() + if option_dict is None: + option_dict = self.get_option_dict(command_name) + + logger.debug(" setting options for %r command:", command_name) + + for option, (source, value) in option_dict.items(): + logger.debug(" %s = %s (from %s)", option, value, source) + try: + bool_opts = [x.replace('-', '_') + for x in command_obj.boolean_options] + except AttributeError: + bool_opts = [] + try: + neg_opt = command_obj.negative_opt + except AttributeError: + neg_opt = {} + + try: + is_string = isinstance(value, str) + if option in neg_opt and is_string: + setattr(command_obj, neg_opt[option], not strtobool(value)) + elif option in bool_opts and is_string: + setattr(command_obj, option, strtobool(value)) + elif hasattr(command_obj, option): + setattr(command_obj, option, value) + else: + raise PackagingOptionError( + "error in %s: command %r has no such option %r" % + (source, command_name, option)) + except ValueError as msg: + raise PackagingOptionError(msg) + + def reinitialize_command(self, command, reinit_subcommands=False): + """Reinitializes a command to the state it was in when first + returned by 'get_command_obj()': i.e., initialized but not yet + finalized. This provides the opportunity to sneak option + values in programmatically, overriding or supplementing + user-supplied values from the config files and command line. + You'll have to re-finalize the command object (by calling + 'finalize_options()' or 'ensure_finalized()') before using it for + real. + + 'command' should be a command name (string) or command object. If + 'reinit_subcommands' is true, also reinitializes the command's + sub-commands, as declared by the 'sub_commands' class attribute (if + it has one). See the "install_dist" command for an example. Only + reinitializes the sub-commands that actually matter, i.e. those + whose test predicate return true. + + Returns the reinitialized command object. It will be the same + object as the one stored in the self.command_obj attribute. + """ + if not isinstance(command, Command): + command_name = command + command = self.get_command_obj(command_name) + else: + command_name = command.get_command_name() + + if not command.finalized: + return command + + command.initialize_options() + self.have_run[command_name] = 0 + command.finalized = False + self._set_command_options(command) + + if reinit_subcommands: + for sub in command.get_sub_commands(): + self.reinitialize_command(sub, reinit_subcommands) + + return command + + # -- Methods that operate on the Distribution ---------------------- + + def run_commands(self): + """Run each command that was seen on the setup script command line. + Uses the list of commands found and cache of command objects + created by 'get_command_obj()'. + """ + for cmd in self.commands: + self.run_command(cmd) + + # -- Methods that operate on its Commands -------------------------- + + def run_command(self, command, options=None): + """Do whatever it takes to run a command (including nothing at all, + if the command has already been run). Specifically: if we have + already created and run the command named by 'command', return + silently without doing anything. If the command named by 'command' + doesn't even have a command object yet, create one. Then invoke + 'run()' on that command object (or an existing one). + """ + # Already been here, done that? then return silently. + if self.have_run.get(command): + return + + if options is not None: + self.command_options[command] = options + + cmd_obj = self.get_command_obj(command) + cmd_obj.ensure_finalized() + self.run_command_hooks(cmd_obj, 'pre_hook') + logger.info("running %s", command) + cmd_obj.run() + self.run_command_hooks(cmd_obj, 'post_hook') + self.have_run[command] = 1 + + def run_command_hooks(self, cmd_obj, hook_kind): + """Run hooks registered for that command and phase. + + *cmd_obj* is a finalized command object; *hook_kind* is either + 'pre_hook' or 'post_hook'. + """ + if hook_kind not in ('pre_hook', 'post_hook'): + raise ValueError('invalid hook kind: %r' % hook_kind) + + hooks = getattr(cmd_obj, hook_kind, None) + + if hooks is None: + return + + for hook in hooks.values(): + if isinstance(hook, str): + try: + hook_obj = resolve_name(hook) + except ImportError as e: + raise PackagingModuleError(e) + else: + hook_obj = hook + + if not callable(hook_obj): + raise PackagingOptionError('hook %r is not callable' % hook) + + logger.info('running %s %s for command %s', + hook_kind, hook, cmd_obj.get_command_name()) + hook_obj(cmd_obj) + + # -- Distribution query methods ------------------------------------ + def has_pure_modules(self): + return len(self.packages or self.py_modules or []) > 0 + + def has_ext_modules(self): + return self.ext_modules and len(self.ext_modules) > 0 + + def has_c_libraries(self): + return self.libraries and len(self.libraries) > 0 + + def has_modules(self): + return self.has_pure_modules() or self.has_ext_modules() + + def has_headers(self): + return self.headers and len(self.headers) > 0 + + def has_scripts(self): + return self.scripts and len(self.scripts) > 0 + + def has_data_files(self): + return self.data_files and len(self.data_files) > 0 + + def is_pure(self): + return (self.has_pure_modules() and + not self.has_ext_modules() and + not self.has_c_libraries()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/errors.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/errors.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,138 @@ +"""Exceptions used throughout the package. + +Submodules of packaging may raise exceptions defined in this module as +well as standard exceptions; in particular, SystemExit is usually raised +for errors that are obviously the end-user's fault (e.g. bad +command-line arguments). +""" + + +class PackagingError(Exception): + """The root of all Packaging evil.""" + + +class PackagingModuleError(PackagingError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + + +class PackagingClassError(PackagingError): + """Some command class (or possibly distribution class, if anyone + feels a need to subclass Distribution) is found not to be holding + up its end of the bargain, ie. implementing some part of the + "command "interface.""" + + +class PackagingGetoptError(PackagingError): + """The option table provided to 'fancy_getopt()' is bogus.""" + + +class PackagingArgError(PackagingError): + """Raised by fancy_getopt in response to getopt.error -- ie. an + error in the command line usage.""" + + +class PackagingFileError(PackagingError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before IOError or + OSError could be raised.""" + + +class PackagingOptionError(PackagingError): + """Syntactic/semantic errors in command options, such as use of + mutually conflicting options, or inconsistent options, + badly-spelled values, etc. No distinction is made between option + values originating in the setup script, the command line, config + files, or what-have-you -- but if we *know* something originated in + the setup script, we'll raise PackagingSetupError instead.""" + + +class PackagingSetupError(PackagingError): + """For errors that can be definitely blamed on the setup script, + such as invalid keyword arguments to 'setup()'.""" + + +class PackagingPlatformError(PackagingError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + + +class PackagingExecError(PackagingError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + + +class PackagingInternalError(PackagingError): + """Internal inconsistencies or impossibilities (obviously, this + should never be seen if the code is working!).""" + + +class PackagingTemplateError(PackagingError): + """Syntax error in a file list template.""" + + +class PackagingPyPIError(PackagingError): + """Any problem occuring during using the indexes.""" + + +# Exception classes used by the CCompiler implementation classes +class CCompilerError(Exception): + """Some compile/link operation failed.""" + + +class PreprocessError(CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + + +class CompileError(CCompilerError): + """Failure to compile one or more C/C++ source files.""" + + +class LibError(CCompilerError): + """Failure to create a static library from one or more C/C++ object + files.""" + + +class LinkError(CCompilerError): + """Failure to link one or more C/C++ object files into an executable + or shared library file.""" + + +class UnknownFileError(CCompilerError): + """Attempt to process an unknown file type.""" + + +class MetadataMissingError(PackagingError): + """A required metadata is missing""" + + +class MetadataConflictError(PackagingError): + """Attempt to read or write metadata fields that are conflictual.""" + + +class MetadataUnrecognizedVersionError(PackagingError): + """Unknown metadata version number.""" + + +class IrrationalVersionError(Exception): + """This is an irrational version.""" + pass + + +class HugeMajorVersionNumError(IrrationalVersionError): + """An irrational version because the major version number is huge + (often because a year or date was used). + + See `error_on_huge_major_num` option in `NormalizedVersion` for details. + This guard can be disabled by setting that option False. + """ + pass + + +class InstallationException(Exception): + """Base exception for installation scripts""" + + +class InstallationConflict(InstallationException): + """Raised when a conflict is detected""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/fancy_getopt.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/fancy_getopt.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,388 @@ +"""Command line parsing machinery. + +The FancyGetopt class is a Wrapper around the getopt module that +provides the following additional features: + * short and long options are tied together + * options have help strings, so fancy_getopt could potentially + create a complete usage summary + * options set attributes of a passed-in object. + +It is used under the hood by the command classes. Do not use directly. +""" + +import getopt +import re +import sys +import textwrap + +from packaging.errors import PackagingGetoptError, PackagingArgError + +# Much like command_re in packaging.core, this is close to but not quite +# the same as a Python NAME -- except, in the spirit of most GNU +# utilities, we use '-' in place of '_'. (The spirit of LISP lives on!) +# The similarities to NAME are again not a coincidence... +longopt_pat = r'[a-zA-Z](?:[a-zA-Z0-9-]*)' +longopt_re = re.compile(r'^%s$' % longopt_pat) + +# For recognizing "negative alias" options, eg. "quiet=!verbose" +neg_alias_re = re.compile("^(%s)=!(%s)$" % (longopt_pat, longopt_pat)) + + +class FancyGetopt: + """Wrapper around the standard 'getopt()' module that provides some + handy extra functionality: + * short and long options are tied together + * options have help strings, and help text can be assembled + from them + * options set attributes of a passed-in object + * boolean options can have "negative aliases" -- eg. if + --quiet is the "negative alias" of --verbose, then "--quiet" + on the command line sets 'verbose' to false + """ + + def __init__(self, option_table=None): + + # The option table is (currently) a list of tuples. The + # tuples may have 3 or four values: + # (long_option, short_option, help_string [, repeatable]) + # if an option takes an argument, its long_option should have '=' + # appended; short_option should just be a single character, no ':' + # in any case. If a long_option doesn't have a corresponding + # short_option, short_option should be None. All option tuples + # must have long options. + self.option_table = option_table + + # 'option_index' maps long option names to entries in the option + # table (ie. those 3-tuples). + self.option_index = {} + if self.option_table: + self._build_index() + + # 'alias' records (duh) alias options; {'foo': 'bar'} means + # --foo is an alias for --bar + self.alias = {} + + # 'negative_alias' keeps track of options that are the boolean + # opposite of some other option + self.negative_alias = {} + + # These keep track of the information in the option table. We + # don't actually populate these structures until we're ready to + # parse the command line, since the 'option_table' passed in here + # isn't necessarily the final word. + self.short_opts = [] + self.long_opts = [] + self.short2long = {} + self.attr_name = {} + self.takes_arg = {} + + # And 'option_order' is filled up in 'getopt()'; it records the + # original order of options (and their values) on the command line, + # but expands short options, converts aliases, etc. + self.option_order = [] + + def _build_index(self): + self.option_index.clear() + for option in self.option_table: + self.option_index[option[0]] = option + + def set_option_table(self, option_table): + self.option_table = option_table + self._build_index() + + def add_option(self, long_option, short_option=None, help_string=None): + if long_option in self.option_index: + raise PackagingGetoptError( + "option conflict: already an option '%s'" % long_option) + else: + option = (long_option, short_option, help_string) + self.option_table.append(option) + self.option_index[long_option] = option + + def has_option(self, long_option): + """Return true if the option table for this parser has an + option with long name 'long_option'.""" + return long_option in self.option_index + + def _check_alias_dict(self, aliases, what): + assert isinstance(aliases, dict) + for alias, opt in aliases.items(): + if alias not in self.option_index: + raise PackagingGetoptError( + ("invalid %s '%s': " + "option '%s' not defined") % (what, alias, alias)) + if opt not in self.option_index: + raise PackagingGetoptError( + ("invalid %s '%s': " + "aliased option '%s' not defined") % (what, alias, opt)) + + def set_aliases(self, alias): + """Set the aliases for this option parser.""" + self._check_alias_dict(alias, "alias") + self.alias = alias + + def set_negative_aliases(self, negative_alias): + """Set the negative aliases for this option parser. + 'negative_alias' should be a dictionary mapping option names to + option names, both the key and value must already be defined + in the option table.""" + self._check_alias_dict(negative_alias, "negative alias") + self.negative_alias = negative_alias + + def _grok_option_table(self): + """Populate the various data structures that keep tabs on the + option table. Called by 'getopt()' before it can do anything + worthwhile. + """ + self.long_opts = [] + self.short_opts = [] + self.short2long.clear() + self.repeat = {} + + for option in self.option_table: + if len(option) == 3: + longopt, short, help = option + repeat = 0 + elif len(option) == 4: + longopt, short, help, repeat = option + else: + # the option table is part of the code, so simply + # assert that it is correct + raise ValueError("invalid option tuple: %r" % option) + + # Type- and value-check the option names + if not isinstance(longopt, str) or len(longopt) < 2: + raise PackagingGetoptError( + ("invalid long option '%s': " + "must be a string of length >= 2") % longopt) + + if (not ((short is None) or + (isinstance(short, str) and len(short) == 1))): + raise PackagingGetoptError( + ("invalid short option '%s': " + "must be a single character or None") % short) + + self.repeat[longopt] = repeat + self.long_opts.append(longopt) + + if longopt[-1] == '=': # option takes an argument? + if short: + short = short + ':' + longopt = longopt[0:-1] + self.takes_arg[longopt] = 1 + else: + + # Is option is a "negative alias" for some other option (eg. + # "quiet" == "!verbose")? + alias_to = self.negative_alias.get(longopt) + if alias_to is not None: + if self.takes_arg[alias_to]: + raise PackagingGetoptError( + ("invalid negative alias '%s': " + "aliased option '%s' takes a value") % \ + (longopt, alias_to)) + + self.long_opts[-1] = longopt # XXX redundant?! + self.takes_arg[longopt] = 0 + + else: + self.takes_arg[longopt] = 0 + + # If this is an alias option, make sure its "takes arg" flag is + # the same as the option it's aliased to. + alias_to = self.alias.get(longopt) + if alias_to is not None: + if self.takes_arg[longopt] != self.takes_arg[alias_to]: + raise PackagingGetoptError( + ("invalid alias '%s': inconsistent with " + "aliased option '%s' (one of them takes a value, " + "the other doesn't") % (longopt, alias_to)) + + # Now enforce some bondage on the long option name, so we can + # later translate it to an attribute name on some object. Have + # to do this a bit late to make sure we've removed any trailing + # '='. + if not longopt_re.match(longopt): + raise PackagingGetoptError( + ("invalid long option name '%s' " + + "(must be letters, numbers, hyphens only") % longopt) + + self.attr_name[longopt] = longopt.replace('-', '_') + if short: + self.short_opts.append(short) + self.short2long[short[0]] = longopt + + def getopt(self, args=None, object=None): + """Parse command-line options in args. Store as attributes on object. + + If 'args' is None or not supplied, uses 'sys.argv[1:]'. If + 'object' is None or not supplied, creates a new OptionDummy + object, stores option values there, and returns a tuple (args, + object). If 'object' is supplied, it is modified in place and + 'getopt()' just returns 'args'; in both cases, the returned + 'args' is a modified copy of the passed-in 'args' list, which + is left untouched. + """ + if args is None: + args = sys.argv[1:] + if object is None: + object = OptionDummy() + created_object = 1 + else: + created_object = 0 + + self._grok_option_table() + + short_opts = ' '.join(self.short_opts) + + try: + opts, args = getopt.getopt(args, short_opts, self.long_opts) + except getopt.error as msg: + raise PackagingArgError(msg) + + for opt, val in opts: + if len(opt) == 2 and opt[0] == '-': # it's a short option + opt = self.short2long[opt[1]] + else: + assert len(opt) > 2 and opt[:2] == '--' + opt = opt[2:] + + alias = self.alias.get(opt) + if alias: + opt = alias + + if not self.takes_arg[opt]: # boolean option? + assert val == '', "boolean option can't have value" + alias = self.negative_alias.get(opt) + if alias: + opt = alias + val = 0 + else: + val = 1 + + attr = self.attr_name[opt] + # The only repeating option at the moment is 'verbose'. + # It has a negative option -q quiet, which should set verbose = 0. + if val and self.repeat.get(attr) is not None: + val = getattr(object, attr, 0) + 1 + setattr(object, attr, val) + self.option_order.append((opt, val)) + + # for opts + if created_object: + return args, object + else: + return args + + def get_option_order(self): + """Returns the list of (option, value) tuples processed by the + previous run of 'getopt()'. Raises RuntimeError if + 'getopt()' hasn't been called yet. + """ + if self.option_order is None: + raise RuntimeError("'getopt()' hasn't been called yet") + else: + return self.option_order + + return self.option_order + + def generate_help(self, header=None): + """Generate help text (a list of strings, one per suggested line of + output) from the option table for this FancyGetopt object. + """ + # Blithely assume the option table is good: probably wouldn't call + # 'generate_help()' unless you've already called 'getopt()'. + + # First pass: determine maximum length of long option names + max_opt = 0 + for option in self.option_table: + longopt = option[0] + short = option[1] + l = len(longopt) + if longopt[-1] == '=': + l = l - 1 + if short is not None: + l = l + 5 # " (-x)" where short == 'x' + if l > max_opt: + max_opt = l + + opt_width = max_opt + 2 + 2 + 2 # room for indent + dashes + gutter + + # Typical help block looks like this: + # --foo controls foonabulation + # Help block for longest option looks like this: + # --flimflam set the flim-flam level + # and with wrapped text: + # --flimflam set the flim-flam level (must be between + # 0 and 100, except on Tuesdays) + # Options with short names will have the short name shown (but + # it doesn't contribute to max_opt): + # --foo (-f) controls foonabulation + # If adding the short option would make the left column too wide, + # we push the explanation off to the next line + # --flimflam (-l) + # set the flim-flam level + # Important parameters: + # - 2 spaces before option block start lines + # - 2 dashes for each long option name + # - min. 2 spaces between option and explanation (gutter) + # - 5 characters (incl. space) for short option name + + # Now generate lines of help text. (If 80 columns were good enough + # for Jesus, then 78 columns are good enough for me!) + line_width = 78 + text_width = line_width - opt_width + big_indent = ' ' * opt_width + if header: + lines = [header] + else: + lines = ['Option summary:'] + + for option in self.option_table: + longopt, short, help = option[:3] + text = textwrap.wrap(help, text_width) + + # Case 1: no short option at all (makes life easy) + if short is None: + if text: + lines.append(" --%-*s %s" % (max_opt, longopt, text[0])) + else: + lines.append(" --%-*s " % (max_opt, longopt)) + + # Case 2: we have a short option, so we have to include it + # just after the long option + else: + opt_names = "%s (-%s)" % (longopt, short) + if text: + lines.append(" --%-*s %s" % + (max_opt, opt_names, text[0])) + else: + lines.append(" --%-*s" % opt_names) + + for l in text[1:]: + lines.append(big_indent + l) + + return lines + + def print_help(self, header=None, file=None): + if file is None: + file = sys.stdout + for line in self.generate_help(header): + file.write(line + "\n") + + +def fancy_getopt(options, negative_opt, object, args): + parser = FancyGetopt(options) + parser.set_negative_aliases(negative_opt) + return parser.getopt(args, object) + + +class OptionDummy: + """Dummy class just used as a place to hold command-line option + values as instance attributes.""" + + def __init__(self, options=[]): + """Create a new OptionDummy instance. The attributes listed in + 'options' will be initialized to None.""" + for opt in options: + setattr(self, opt, None) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/install.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/install.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,529 @@ +"""Building blocks for installers. + +When used as a script, this module installs a release thanks to info +obtained from an index (e.g. PyPI), with dependencies. + +This is a higher-level module built on packaging.database and +packaging.pypi. +""" +import os +import sys +import stat +import errno +import shutil +import logging +import tempfile +from sysconfig import get_config_var, get_path, is_python_build + +from packaging import logger +from packaging.dist import Distribution +from packaging.util import (_is_archive_file, ask, get_install_method, + egginfo_to_distinfo) +from packaging.pypi import wrapper +from packaging.version import get_version_predicate +from packaging.database import get_distributions, get_distribution +from packaging.depgraph import generate_graph + +from packaging.errors import (PackagingError, InstallationException, + InstallationConflict, CCompilerError) +from packaging.pypi.errors import ProjectNotFound, ReleaseNotFound +from packaging import database + + +__all__ = ['install_dists', 'install_from_infos', 'get_infos', 'remove', + 'install', 'install_local_project'] + + +def _move_files(files, destination): + """Move the list of files in the destination folder, keeping the same + structure. + + Return a list of tuple (old, new) emplacement of files + + :param files: a list of files to move. + :param destination: the destination directory to put on the files. + """ + + for old in files: + filename = os.path.split(old)[-1] + new = os.path.join(destination, filename) + # try to make the paths. + try: + os.makedirs(os.path.dirname(new)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + os.rename(old, new) + yield old, new + + +def _run_distutils_install(path): + # backward compat: using setuptools or plain-distutils + cmd = '%s setup.py install --record=%s' + record_file = os.path.join(path, 'RECORD') + os.system(cmd % (sys.executable, record_file)) + if not os.path.exists(record_file): + raise ValueError('failed to install') + else: + egginfo_to_distinfo(record_file, remove_egginfo=True) + + +def _run_setuptools_install(path): + cmd = '%s setup.py install --record=%s --single-version-externally-managed' + record_file = os.path.join(path, 'RECORD') + + os.system(cmd % (sys.executable, record_file)) + if not os.path.exists(record_file): + raise ValueError('failed to install') + else: + egginfo_to_distinfo(record_file, remove_egginfo=True) + + +def _run_packaging_install(path): + # XXX check for a valid setup.cfg? + dist = Distribution() + dist.parse_config_files() + try: + dist.run_command('install_dist') + name = dist.metadata['Name'] + return database.get_distribution(name) is not None + except (IOError, os.error, PackagingError, CCompilerError) as msg: + raise ValueError("Failed to install, " + str(msg)) + + +def _install_dist(dist, path): + """Install a distribution into a path. + + This: + + * unpack the distribution + * copy the files in "path" + * determine if the distribution is packaging or distutils1. + """ + where = dist.unpack() + + if where is None: + raise ValueError('Cannot locate the unpacked archive') + + return _run_install_from_archive(where) + + +def install_local_project(path): + """Install a distribution from a source directory. + + If the source directory contains a setup.py install using distutils1. + If a setup.cfg is found, install using the install_dist command. + + Returns True on success, False on Failure. + """ + path = os.path.abspath(path) + if os.path.isdir(path): + logger.info('Installing from source directory: %r', path) + return _run_install_from_dir(path) + elif _is_archive_file(path): + logger.info('Installing from archive: %r', path) + _unpacked_dir = tempfile.mkdtemp() + try: + shutil.unpack_archive(path, _unpacked_dir) + return _run_install_from_archive(_unpacked_dir) + finally: + shutil.rmtree(_unpacked_dir) + else: + logger.warning('No project to install.') + return False + + +def _run_install_from_archive(source_dir): + # XXX need a better way + for item in os.listdir(source_dir): + fullpath = os.path.join(source_dir, item) + if os.path.isdir(fullpath): + source_dir = fullpath + break + return _run_install_from_dir(source_dir) + + +install_methods = { + 'packaging': _run_packaging_install, + 'setuptools': _run_setuptools_install, + 'distutils': _run_distutils_install} + + +def _run_install_from_dir(source_dir): + old_dir = os.getcwd() + os.chdir(source_dir) + install_method = get_install_method(source_dir) + func = install_methods[install_method] + try: + func = install_methods[install_method] + try: + func(source_dir) + return True + except ValueError as err: + # failed to install + logger.info(str(err)) + return False + finally: + os.chdir(old_dir) + + +def install_dists(dists, path, paths=None): + """Install all distributions provided in dists, with the given prefix. + + If an error occurs while installing one of the distributions, uninstall all + the installed distribution (in the context if this function). + + Return a list of installed dists. + + :param dists: distributions to install + :param path: base path to install distribution in + :param paths: list of paths (defaults to sys.path) to look for info + """ + + installed_dists = [] + for dist in dists: + logger.info('Installing %r %s...', dist.name, dist.version) + try: + _install_dist(dist, path) + installed_dists.append(dist) + except Exception as e: + logger.info('Failed: %s', e) + + # reverting + for installed_dist in installed_dists: + logger.info('Reverting %r', installed_dist) + remove(installed_dist.name, paths) + raise e + return installed_dists + + +def install_from_infos(install_path=None, install=[], remove=[], conflicts=[], + paths=None): + """Install and remove the given distributions. + + The function signature is made to be compatible with the one of get_infos. + The aim of this script is to povide a way to install/remove what's asked, + and to rollback if needed. + + So, it's not possible to be in an inconsistant state, it could be either + installed, either uninstalled, not half-installed. + + The process follow those steps: + + 1. Move all distributions that will be removed in a temporary location + 2. Install all the distributions that will be installed in a temp. loc. + 3. If the installation fails, rollback (eg. move back) those + distributions, or remove what have been installed. + 4. Else, move the distributions to the right locations, and remove for + real the distributions thats need to be removed. + + :param install_path: the installation path where we want to install the + distributions. + :param install: list of distributions that will be installed; install_path + must be provided if this list is not empty. + :param remove: list of distributions that will be removed. + :param conflicts: list of conflicting distributions, eg. that will be in + conflict once the install and remove distribution will be + processed. + :param paths: list of paths (defaults to sys.path) to look for info + """ + # first of all, if we have conflicts, stop here. + if conflicts: + raise InstallationConflict(conflicts) + + if install and not install_path: + raise ValueError("Distributions are to be installed but `install_path`" + " is not provided.") + + # before removing the files, we will start by moving them away + # then, if any error occurs, we could replace them in the good place. + temp_files = {} # contains lists of {dist: (old, new)} paths + temp_dir = None + if remove: + temp_dir = tempfile.mkdtemp() + for dist in remove: + files = dist.list_installed_files() + temp_files[dist] = _move_files(files, temp_dir) + try: + if install: + install_dists(install, install_path, paths) + except: + # if an error occurs, put back the files in the right place. + for files in temp_files.values(): + for old, new in files: + shutil.move(new, old) + if temp_dir: + shutil.rmtree(temp_dir) + # now re-raising + raise + + # we can remove them for good + for files in temp_files.values(): + for old, new in files: + os.remove(new) + if temp_dir: + shutil.rmtree(temp_dir) + + +def _get_setuptools_deps(release): + # NotImplementedError + pass + + +def get_infos(requirements, index=None, installed=None, prefer_final=True): + """Return the informations on what's going to be installed and upgraded. + + :param requirements: is a *string* containing the requirements for this + project (for instance "FooBar 1.1" or "BarBaz (<1.2)") + :param index: If an index is specified, use this one, otherwise, use + :class index.ClientWrapper: to get project metadatas. + :param installed: a list of already installed distributions. + :param prefer_final: when picking up the releases, prefer a "final" one + over a beta/alpha/etc one. + + The results are returned in a dict, containing all the operations + needed to install the given requirements:: + + >>> get_install_info("FooBar (<=1.2)") + {'install': [], 'remove': [], 'conflict': []} + + Conflict contains all the conflicting distributions, if there is a + conflict. + """ + # this function does several things: + # 1. get a release specified by the requirements + # 2. gather its metadata, using setuptools compatibility if needed + # 3. compare this tree with what is currently installed on the system, + # return the requirements of what is missing + # 4. do that recursively and merge back the results + # 5. return a dict containing information about what is needed to install + # or remove + + if not installed: + logger.debug('Reading installed distributions') + installed = list(get_distributions(use_egg_info=True)) + + infos = {'install': [], 'remove': [], 'conflict': []} + # Is a compatible version of the project already installed ? + predicate = get_version_predicate(requirements) + found = False + + # check that the project isn't already installed + for installed_project in installed: + # is it a compatible project ? + if predicate.name.lower() != installed_project.name.lower(): + continue + found = True + logger.info('Found %r %s', installed_project.name, + installed_project.version) + + # if we already have something installed, check it matches the + # requirements + if predicate.match(installed_project.version): + return infos + break + + if not found: + logger.debug('Project not installed') + + if not index: + index = wrapper.ClientWrapper() + + if not installed: + installed = get_distributions(use_egg_info=True) + + # Get all the releases that match the requirements + try: + release = index.get_release(requirements) + except (ReleaseNotFound, ProjectNotFound): + raise InstallationException('Release not found: %r' % requirements) + + if release is None: + logger.info('Could not find a matching project') + return infos + + metadata = release.fetch_metadata() + + # we need to build setuptools deps if any + if 'requires_dist' not in metadata: + metadata['requires_dist'] = _get_setuptools_deps(release) + + # build the dependency graph with local and required dependencies + dists = list(installed) + dists.append(release) + depgraph = generate_graph(dists) + + # Get what the missing deps are + dists = depgraph.missing[release] + if dists: + logger.info("Missing dependencies found, retrieving metadata") + # we have missing deps + for dist in dists: + _update_infos(infos, get_infos(dist, index, installed)) + + # Fill in the infos + existing = [d for d in installed if d.name == release.name] + if existing: + infos['remove'].append(existing[0]) + infos['conflict'].extend(depgraph.reverse_list[existing[0]]) + infos['install'].append(release) + return infos + + +def _update_infos(infos, new_infos): + """extends the lists contained in the `info` dict with those contained + in the `new_info` one + """ + for key, value in infos.items(): + if key in new_infos: + infos[key].extend(new_infos[key]) + + +def remove(project_name, paths=None, auto_confirm=True): + """Removes a single project from the installation. + + Returns True on success + """ + dist = get_distribution(project_name, use_egg_info=True, paths=paths) + if dist is None: + raise PackagingError('Distribution %r not found' % project_name) + files = dist.list_installed_files(local=True) + rmdirs = [] + rmfiles = [] + tmp = tempfile.mkdtemp(prefix=project_name + '-uninstall') + + def _move_file(source, target): + try: + os.rename(source, target) + except OSError as err: + return err + return None + + success = True + error = None + try: + for file_, md5, size in files: + if os.path.isfile(file_): + dirname, filename = os.path.split(file_) + tmpfile = os.path.join(tmp, filename) + try: + error = _move_file(file_, tmpfile) + if error is not None: + success = False + break + finally: + if not os.path.isfile(file_): + os.rename(tmpfile, file_) + if file_ not in rmfiles: + rmfiles.append(file_) + if dirname not in rmdirs: + rmdirs.append(dirname) + finally: + shutil.rmtree(tmp) + + if not success: + logger.info('%r cannot be removed.', project_name) + logger.info('Error: %s', error) + return False + + logger.info('Removing %r: ', project_name) + + for file_ in rmfiles: + logger.info(' %s', file_) + + # Taken from the pip project + if auto_confirm: + response = 'y' + else: + response = ask('Proceed (y/n)? ', ('y', 'n')) + + if response == 'y': + file_count = 0 + for file_ in rmfiles: + os.remove(file_) + file_count += 1 + + dir_count = 0 + for dirname in rmdirs: + if not os.path.exists(dirname): + # could + continue + + files_count = 0 + for root, dir, files in os.walk(dirname): + files_count += len(files) + + if files_count > 0: + # XXX Warning + continue + + # empty dirs with only empty dirs + if os.stat(dirname).st_mode & stat.S_IWUSR: + # XXX Add a callable in shutil.rmtree to count + # the number of deleted elements + shutil.rmtree(dirname) + dir_count += 1 + + # removing the top path + # XXX count it ? + if os.path.exists(dist.path): + shutil.rmtree(dist.path) + + logger.info('Success: removed %d files and %d dirs', + file_count, dir_count) + + return True + + +def install(project): + """Installs a project. + + Returns True on success, False on failure + """ + if is_python_build(): + # Python would try to install into the site-packages directory under + # $PREFIX, but when running from an uninstalled code checkout we don't + # want to create directories under the installation root + message = ('installing third-party projects from an uninstalled ' + 'Python is not supported') + logger.error(message) + return False + + logger.info('Checking the installation location...') + purelib_path = get_path('purelib') + + # trying to write a file there + try: + with tempfile.NamedTemporaryFile(suffix=project, + dir=purelib_path) as testfile: + testfile.write(b'test') + except OSError: + # FIXME this should check the errno, or be removed altogether (race + # condition: the directory permissions could be changed between here + # and the actual install) + logger.info('Unable to write in "%s". Do you have the permissions ?' + % purelib_path) + return False + + logger.info('Getting information about %r...', project) + try: + info = get_infos(project) + except InstallationException: + logger.info('Cound not find %r', project) + return False + + if info['install'] == []: + logger.info('Nothing to install') + return False + + install_path = get_config_var('base') + try: + install_from_infos(install_path, + info['install'], info['remove'], info['conflict']) + + except InstallationConflict as e: + if logger.isEnabledFor(logging.INFO): + projects = ('%r %s' % (p.name, p.version) for p in e.args[0]) + logger.info('%r conflicts with %s', project, ','.join(projects)) + + return True diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/manifest.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/manifest.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,381 @@ +"""Class representing the list of files in a distribution. + +The Manifest class can be used to: + + - read or write a MANIFEST file + - read a template file and find out the file list +""" +# XXX todo: document + add tests +import re +import os +import fnmatch + +from packaging import logger +from packaging.util import write_file, convert_path +from packaging.errors import (PackagingTemplateError, + PackagingInternalError) + +__all__ = ['Manifest'] + +# a \ followed by some spaces + EOL +_COLLAPSE_PATTERN = re.compile('\\\w*\n', re.M) +_COMMENTED_LINE = re.compile('#.*?(?=\n)|\n(?=$)', re.M | re.S) + + +class Manifest(object): + """A list of files built by on exploring the filesystem and filtered by + applying various patterns to what we find there. + """ + + def __init__(self): + self.allfiles = None + self.files = [] + + # + # Public API + # + + def findall(self, dir=os.curdir): + self.allfiles = _findall(dir) + + def append(self, item): + self.files.append(item) + + def extend(self, items): + self.files.extend(items) + + def sort(self): + # Not a strict lexical sort! + self.files = [os.path.join(*path_tuple) for path_tuple in + sorted(os.path.split(path) for path in self.files)] + + def clear(self): + """Clear all collected files.""" + self.files = [] + if self.allfiles is not None: + self.allfiles = [] + + def remove_duplicates(self): + # Assumes list has been sorted! + for i in range(len(self.files) - 1, 0, -1): + if self.files[i] == self.files[i - 1]: + del self.files[i] + + def read_template(self, path_or_file): + """Read and parse a manifest template file. + 'path' can be a path or a file-like object. + + Updates the list accordingly. + """ + if isinstance(path_or_file, str): + f = open(path_or_file) + else: + f = path_or_file + + try: + content = f.read() + # first, let's unwrap collapsed lines + content = _COLLAPSE_PATTERN.sub('', content) + # next, let's remove commented lines and empty lines + content = _COMMENTED_LINE.sub('', content) + + # now we have our cleaned up lines + lines = [line.strip() for line in content.split('\n')] + finally: + f.close() + + for line in lines: + if line == '': + continue + try: + self._process_template_line(line) + except PackagingTemplateError as msg: + logger.warning("%s, %s", path_or_file, msg) + + def write(self, path): + """Write the file list in 'self.filelist' (presumably as filled in + by 'add_defaults()' and 'read_template()') to the manifest file + named by 'self.manifest'. + """ + if os.path.isfile(path): + with open(path) as fp: + first_line = fp.readline() + + if first_line != '# file GENERATED by packaging, do NOT edit\n': + logger.info("not writing to manually maintained " + "manifest file %r", path) + return + + self.sort() + self.remove_duplicates() + content = self.files[:] + content.insert(0, '# file GENERATED by packaging, do NOT edit') + logger.info("writing manifest file %r", path) + write_file(path, content) + + def read(self, path): + """Read the manifest file (named by 'self.manifest') and use it to + fill in 'self.filelist', the list of files to include in the source + distribution. + """ + logger.info("reading manifest file %r", path) + with open(path) as manifest: + for line in manifest.readlines(): + self.append(line) + + def exclude_pattern(self, pattern, anchor=True, prefix=None, + is_regex=False): + """Remove strings (presumably filenames) from 'files' that match + 'pattern'. + + Other parameters are the same as for 'include_pattern()', above. + The list 'self.files' is modified in place. Return True if files are + found. + """ + files_found = False + pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) + for i in range(len(self.files) - 1, -1, -1): + if pattern_re.search(self.files[i]): + del self.files[i] + files_found = True + + return files_found + + # + # Private API + # + + def _parse_template_line(self, line): + words = line.split() + if len(words) == 1 and words[0] not in ( + 'include', 'exclude', 'global-include', 'global-exclude', + 'recursive-include', 'recursive-exclude', 'graft', 'prune'): + # no action given, let's use the default 'include' + words.insert(0, 'include') + + action = words[0] + patterns = dir = dir_pattern = None + + if action in ('include', 'exclude', + 'global-include', 'global-exclude'): + if len(words) < 2: + raise PackagingTemplateError( + "%r expects ..." % action) + + patterns = [convert_path(word) for word in words[1:]] + + elif action in ('recursive-include', 'recursive-exclude'): + if len(words) < 3: + raise PackagingTemplateError( + "%r expects ..." % action) + + dir = convert_path(words[1]) + patterns = [convert_path(word) for word in words[2:]] + + elif action in ('graft', 'prune'): + if len(words) != 2: + raise PackagingTemplateError( + "%r expects a single " % action) + + dir_pattern = convert_path(words[1]) + + else: + raise PackagingTemplateError("unknown action %r" % action) + + return action, patterns, dir, dir_pattern + + def _process_template_line(self, line): + # Parse the line: split it up, make sure the right number of words + # is there, and return the relevant words. 'action' is always + # defined: it's the first word of the line. Which of the other + # three are defined depends on the action; it'll be either + # patterns, (dir and patterns), or (dir_pattern). + action, patterns, dir, dir_pattern = self._parse_template_line(line) + + # OK, now we know that the action is valid and we have the + # right number of words on the line for that action -- so we + # can proceed with minimal error-checking. + if action == 'include': + for pattern in patterns: + if not self._include_pattern(pattern, anchor=True): + logger.warning("no files found matching %r", pattern) + + elif action == 'exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, anchor=True): + logger.warning("no previously-included files " + "found matching %r", pattern) + + elif action == 'global-include': + for pattern in patterns: + if not self._include_pattern(pattern, anchor=False): + logger.warning("no files found matching %r " + "anywhere in distribution", pattern) + + elif action == 'global-exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, anchor=False): + logger.warning("no previously-included files " + "matching %r found anywhere in " + "distribution", pattern) + + elif action == 'recursive-include': + for pattern in patterns: + if not self._include_pattern(pattern, prefix=dir): + logger.warning("no files found matching %r " + "under directory %r", pattern, dir) + + elif action == 'recursive-exclude': + for pattern in patterns: + if not self.exclude_pattern(pattern, prefix=dir): + logger.warning("no previously-included files " + "matching %r found under directory %r", + pattern, dir) + + elif action == 'graft': + if not self._include_pattern(None, prefix=dir_pattern): + logger.warning("no directories found matching %r", + dir_pattern) + + elif action == 'prune': + if not self.exclude_pattern(None, prefix=dir_pattern): + logger.warning("no previously-included directories found " + "matching %r", dir_pattern) + else: + raise PackagingInternalError( + "this cannot happen: invalid action %r" % action) + + def _include_pattern(self, pattern, anchor=True, prefix=None, + is_regex=False): + """Select strings (presumably filenames) from 'self.files' that + match 'pattern', a Unix-style wildcard (glob) pattern. + + Patterns are not quite the same as implemented by the 'fnmatch' + module: '*' and '?' match non-special characters, where "special" + is platform-dependent: slash on Unix; colon, slash, and backslash on + DOS/Windows; and colon on Mac OS. + + If 'anchor' is true (the default), then the pattern match is more + stringent: "*.py" will match "foo.py" but not "foo/bar.py". If + 'anchor' is false, both of these will match. + + If 'prefix' is supplied, then only filenames starting with 'prefix' + (itself a pattern) and ending with 'pattern', with anything in between + them, will match. 'anchor' is ignored in this case. + + If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and + 'pattern' is assumed to be either a string containing a regex or a + regex object -- no translation is done, the regex is just compiled + and used as-is. + + Selected strings will be added to self.files. + + Return True if files are found. + """ + # XXX docstring lying about what the special chars are? + files_found = False + pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex) + + # delayed loading of allfiles list + if self.allfiles is None: + self.findall() + + for name in self.allfiles: + if pattern_re.search(name): + self.files.append(name) + files_found = True + + return files_found + + +# +# Utility functions +# +def _findall(dir=os.curdir): + """Find all files under 'dir' and return the list of full filenames + (relative to 'dir'). + """ + from stat import S_ISREG, S_ISDIR, S_ISLNK + + list = [] + stack = [dir] + pop = stack.pop + push = stack.append + + while stack: + dir = pop() + names = os.listdir(dir) + + for name in names: + if dir != os.curdir: # avoid the dreaded "./" syndrome + fullname = os.path.join(dir, name) + else: + fullname = name + + # Avoid excess stat calls -- just one will do, thank you! + stat = os.stat(fullname) + mode = stat.st_mode + if S_ISREG(mode): + list.append(fullname) + elif S_ISDIR(mode) and not S_ISLNK(mode): + push(fullname) + + return list + + +def _glob_to_re(pattern): + """Translate a shell-like glob pattern to a regular expression. + + Return a string containing the regex. Differs from + 'fnmatch.translate()' in that '*' does not match "special characters" + (which are platform-specific). + """ + pattern_re = fnmatch.translate(pattern) + + # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which + # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, + # and by extension they shouldn't match such "special characters" under + # any OS. So change all non-escaped dots in the RE to match any + # character except the special characters (currently: just os.sep). + sep = os.sep + if os.sep == '\\': + # we're using a regex to manipulate a regex, so we need + # to escape the backslash twice + sep = r'\\\\' + escaped = r'\1[^%s]' % sep + pattern_re = re.sub(r'((?': lambda x, y: x > y, + '>=': lambda x, y: x >= y, + '<': lambda x, y: x < y, + '<=': lambda x, y: x <= y, + 'in': lambda x, y: x in y, + 'not in': lambda x, y: x not in y} + + +def _operate(operation, x, y): + return _OPERATORS[operation](x, y) + + +# restricted set of variables +_VARS = {'sys.platform': sys.platform, + 'python_version': '%s.%s' % sys.version_info[:2], + # FIXME parsing sys.platform is not reliable, but there is no other + # way to get e.g. 2.7.2+, and the PEP is defined with sys.version + 'python_full_version': sys.version.split(' ', 1)[0], + 'os.name': os.name, + 'platform.version': platform.version(), + 'platform.machine': platform.machine(), + 'platform.python_implementation': platform.python_implementation(), + } + + +class _Operation: + + def __init__(self, execution_context=None): + self.left = None + self.op = None + self.right = None + if execution_context is None: + execution_context = {} + self.execution_context = execution_context + + def _get_var(self, name): + if name in self.execution_context: + return self.execution_context[name] + return _VARS[name] + + def __repr__(self): + return '%s %s %s' % (self.left, self.op, self.right) + + def _is_string(self, value): + if value is None or len(value) < 2: + return False + for delimiter in '"\'': + if value[0] == value[-1] == delimiter: + return True + return False + + def _is_name(self, value): + return value in _VARS + + def _convert(self, value): + if value in _VARS: + return self._get_var(value) + return value.strip('"\'') + + def _check_name(self, value): + if value not in _VARS: + raise NameError(value) + + def _nonsense_op(self): + msg = 'This operation is not supported : "%s"' % self + raise SyntaxError(msg) + + def __call__(self): + # make sure we do something useful + if self._is_string(self.left): + if self._is_string(self.right): + self._nonsense_op() + self._check_name(self.right) + else: + if not self._is_string(self.right): + self._nonsense_op() + self._check_name(self.left) + + if self.op not in _OPERATORS: + raise TypeError('Operator not supported "%s"' % self.op) + + left = self._convert(self.left) + right = self._convert(self.right) + return _operate(self.op, left, right) + + +class _OR: + def __init__(self, left, right=None): + self.left = left + self.right = right + + def filled(self): + return self.right is not None + + def __repr__(self): + return 'OR(%r, %r)' % (self.left, self.right) + + def __call__(self): + return self.left() or self.right() + + +class _AND: + def __init__(self, left, right=None): + self.left = left + self.right = right + + def filled(self): + return self.right is not None + + def __repr__(self): + return 'AND(%r, %r)' % (self.left, self.right) + + def __call__(self): + return self.left() and self.right() + + +def interpret(marker, execution_context=None): + """Interpret a marker and return a result depending on environment.""" + marker = marker.strip().encode() + ops = [] + op_starting = True + for token in tokenize(BytesIO(marker).readline): + # Unpack token + toktype, tokval, rowcol, line, logical_line = token + if toktype not in (NAME, OP, STRING, ENDMARKER, ENCODING): + raise SyntaxError('Type not supported "%s"' % tokval) + + if op_starting: + op = _Operation(execution_context) + if len(ops) > 0: + last = ops[-1] + if isinstance(last, (_OR, _AND)) and not last.filled(): + last.right = op + else: + ops.append(op) + else: + ops.append(op) + op_starting = False + else: + op = ops[-1] + + if (toktype == ENDMARKER or + (toktype == NAME and tokval in ('and', 'or'))): + if toktype == NAME and tokval == 'and': + ops.append(_AND(ops.pop())) + elif toktype == NAME and tokval == 'or': + ops.append(_OR(ops.pop())) + op_starting = True + continue + + if isinstance(op, (_OR, _AND)) and op.right is not None: + op = op.right + + if ((toktype in (NAME, STRING) and tokval not in ('in', 'not')) + or (toktype == OP and tokval == '.')): + if op.op is None: + if op.left is None: + op.left = tokval + else: + op.left += tokval + else: + if op.right is None: + op.right = tokval + else: + op.right += tokval + elif toktype == OP or tokval in ('in', 'not'): + if tokval == 'in' and op.op == 'not': + op.op = 'not in' + else: + op.op = tokval + + for op in ops: + if not op(): + return False + return True diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/metadata.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/metadata.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,570 @@ +"""Implementation of the Metadata for Python packages PEPs. + +Supports all metadata formats (1.0, 1.1, 1.2). +""" + +import re +import logging + +from io import StringIO +from email import message_from_file +from packaging import logger +from packaging.markers import interpret +from packaging.version import (is_valid_predicate, is_valid_version, + is_valid_versions) +from packaging.errors import (MetadataMissingError, + MetadataConflictError, + MetadataUnrecognizedVersionError) + +try: + # docutils is installed + from docutils.utils import Reporter + from docutils.parsers.rst import Parser + from docutils import frontend + from docutils import nodes + + class SilentReporter(Reporter): + + def __init__(self, source, report_level, halt_level, stream=None, + debug=0, encoding='ascii', error_handler='replace'): + self.messages = [] + super(SilentReporter, self).__init__( + source, report_level, halt_level, stream, + debug, encoding, error_handler) + + def system_message(self, level, message, *children, **kwargs): + self.messages.append((level, message, children, kwargs)) + + _HAS_DOCUTILS = True +except ImportError: + # docutils is not installed + _HAS_DOCUTILS = False + +# public API of this module +__all__ = ['Metadata', 'PKG_INFO_ENCODING', 'PKG_INFO_PREFERRED_VERSION'] + +# Encoding used for the PKG-INFO files +PKG_INFO_ENCODING = 'utf-8' + +# preferred version. Hopefully will be changed +# to 1.2 once PEP 345 is supported everywhere +PKG_INFO_PREFERRED_VERSION = '1.0' + +_LINE_PREFIX = re.compile('\n \|') +_241_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'License') + +_314_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'License', 'Classifier', 'Download-URL', 'Obsoletes', + 'Provides', 'Requires') + +_314_MARKERS = ('Obsoletes', 'Provides', 'Requires', 'Classifier', + 'Download-URL') + +_345_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', + 'Supported-Platform', 'Summary', 'Description', + 'Keywords', 'Home-page', 'Author', 'Author-email', + 'Maintainer', 'Maintainer-email', 'License', + 'Classifier', 'Download-URL', 'Obsoletes-Dist', + 'Project-URL', 'Provides-Dist', 'Requires-Dist', + 'Requires-Python', 'Requires-External') + +_345_MARKERS = ('Provides-Dist', 'Requires-Dist', 'Requires-Python', + 'Obsoletes-Dist', 'Requires-External', 'Maintainer', + 'Maintainer-email', 'Project-URL') + +_ALL_FIELDS = set() +_ALL_FIELDS.update(_241_FIELDS) +_ALL_FIELDS.update(_314_FIELDS) +_ALL_FIELDS.update(_345_FIELDS) + + +def _version2fieldlist(version): + if version == '1.0': + return _241_FIELDS + elif version == '1.1': + return _314_FIELDS + elif version == '1.2': + return _345_FIELDS + raise MetadataUnrecognizedVersionError(version) + + +def _best_version(fields): + """Detect the best version depending on the fields used.""" + def _has_marker(keys, markers): + for marker in markers: + if marker in keys: + return True + return False + + keys = list(fields) + possible_versions = ['1.0', '1.1', '1.2'] + + # first let's try to see if a field is not part of one of the version + for key in keys: + if key not in _241_FIELDS and '1.0' in possible_versions: + possible_versions.remove('1.0') + if key not in _314_FIELDS and '1.1' in possible_versions: + possible_versions.remove('1.1') + if key not in _345_FIELDS and '1.2' in possible_versions: + possible_versions.remove('1.2') + + # possible_version contains qualified versions + if len(possible_versions) == 1: + return possible_versions[0] # found ! + elif len(possible_versions) == 0: + raise MetadataConflictError('Unknown metadata set') + + # let's see if one unique marker is found + is_1_1 = '1.1' in possible_versions and _has_marker(keys, _314_MARKERS) + is_1_2 = '1.2' in possible_versions and _has_marker(keys, _345_MARKERS) + if is_1_1 and is_1_2: + raise MetadataConflictError('You used incompatible 1.1 and 1.2 fields') + + # we have the choice, either 1.0, or 1.2 + # - 1.0 has a broken Summary field but works with all tools + # - 1.1 is to avoid + # - 1.2 fixes Summary but is not widespread yet + if not is_1_1 and not is_1_2: + # we couldn't find any specific marker + if PKG_INFO_PREFERRED_VERSION in possible_versions: + return PKG_INFO_PREFERRED_VERSION + if is_1_1: + return '1.1' + + # default marker when 1.0 is disqualified + return '1.2' + + +_ATTR2FIELD = { + 'metadata_version': 'Metadata-Version', + 'name': 'Name', + 'version': 'Version', + 'platform': 'Platform', + 'supported_platform': 'Supported-Platform', + 'summary': 'Summary', + 'description': 'Description', + 'keywords': 'Keywords', + 'home_page': 'Home-page', + 'author': 'Author', + 'author_email': 'Author-email', + 'maintainer': 'Maintainer', + 'maintainer_email': 'Maintainer-email', + 'license': 'License', + 'classifier': 'Classifier', + 'download_url': 'Download-URL', + 'obsoletes_dist': 'Obsoletes-Dist', + 'provides_dist': 'Provides-Dist', + 'requires_dist': 'Requires-Dist', + 'requires_python': 'Requires-Python', + 'requires_external': 'Requires-External', + 'requires': 'Requires', + 'provides': 'Provides', + 'obsoletes': 'Obsoletes', + 'project_url': 'Project-URL', +} + +_PREDICATE_FIELDS = ('Requires-Dist', 'Obsoletes-Dist', 'Provides-Dist') +_VERSIONS_FIELDS = ('Requires-Python',) +_VERSION_FIELDS = ('Version',) +_LISTFIELDS = ('Platform', 'Classifier', 'Obsoletes', + 'Requires', 'Provides', 'Obsoletes-Dist', + 'Provides-Dist', 'Requires-Dist', 'Requires-External', + 'Project-URL', 'Supported-Platform') +_LISTTUPLEFIELDS = ('Project-URL',) + +_ELEMENTSFIELD = ('Keywords',) + +_UNICODEFIELDS = ('Author', 'Maintainer', 'Summary', 'Description') + +_MISSING = object() + +_FILESAFE = re.compile('[^A-Za-z0-9.]+') + + +class Metadata: + """The metadata of a release. + + Supports versions 1.0, 1.1 and 1.2 (auto-detected). You can + instantiate the class with one of these arguments (or none): + - *path*, the path to a METADATA file + - *fileobj* give a file-like object with METADATA as content + - *mapping* is a dict-like object + """ + # TODO document that execution_context and platform_dependent are used + # to filter on query, not when setting a key + # also document the mapping API and UNKNOWN default key + + def __init__(self, path=None, platform_dependent=False, + execution_context=None, fileobj=None, mapping=None): + self._fields = {} + self.requires_files = [] + self.docutils_support = _HAS_DOCUTILS + self.platform_dependent = platform_dependent + self.execution_context = execution_context + if [path, fileobj, mapping].count(None) < 2: + raise TypeError('path, fileobj and mapping are exclusive') + if path is not None: + self.read(path) + elif fileobj is not None: + self.read_file(fileobj) + elif mapping is not None: + self.update(mapping) + + def _set_best_version(self): + self._fields['Metadata-Version'] = _best_version(self._fields) + + def _write_field(self, file, name, value): + file.write('%s: %s\n' % (name, value)) + + def __getitem__(self, name): + return self.get(name) + + def __setitem__(self, name, value): + return self.set(name, value) + + def __delitem__(self, name): + field_name = self._convert_name(name) + try: + del self._fields[field_name] + except KeyError: + raise KeyError(name) + self._set_best_version() + + def __contains__(self, name): + return (name in self._fields or + self._convert_name(name) in self._fields) + + def _convert_name(self, name): + if name in _ALL_FIELDS: + return name + name = name.replace('-', '_').lower() + return _ATTR2FIELD.get(name, name) + + def _default_value(self, name): + if name in _LISTFIELDS or name in _ELEMENTSFIELD: + return [] + return 'UNKNOWN' + + def _check_rst_data(self, data): + """Return warnings when the provided data has syntax errors.""" + source_path = StringIO() + parser = Parser() + settings = frontend.OptionParser().get_default_values() + settings.tab_width = 4 + settings.pep_references = None + settings.rfc_references = None + reporter = SilentReporter(source_path, + settings.report_level, + settings.halt_level, + stream=settings.warning_stream, + debug=settings.debug, + encoding=settings.error_encoding, + error_handler=settings.error_encoding_error_handler) + + document = nodes.document(settings, reporter, source=source_path) + document.note_source(source_path, -1) + try: + parser.parse(data, document) + except AttributeError: + reporter.messages.append((-1, 'Could not finish the parsing.', + '', {})) + + return reporter.messages + + def _platform(self, value): + if not self.platform_dependent or ';' not in value: + return True, value + value, marker = value.split(';') + return interpret(marker, self.execution_context), value + + def _remove_line_prefix(self, value): + return _LINE_PREFIX.sub('\n', value) + + # + # Public API + # + def get_fullname(self, filesafe=False): + """Return the distribution name with version. + + If filesafe is true, return a filename-escaped form.""" + name, version = self['Name'], self['Version'] + if filesafe: + # For both name and version any runs of non-alphanumeric or '.' + # characters are replaced with a single '-'. Additionally any + # spaces in the version string become '.' + name = _FILESAFE.sub('-', name) + version = _FILESAFE.sub('-', version.replace(' ', '.')) + return '%s-%s' % (name, version) + + def is_metadata_field(self, name): + """return True if name is a valid metadata key""" + name = self._convert_name(name) + return name in _ALL_FIELDS + + def is_multi_field(self, name): + name = self._convert_name(name) + return name in _LISTFIELDS + + def read(self, filepath): + """Read the metadata values from a file path.""" + with open(filepath, 'r', encoding='utf-8') as fp: + self.read_file(fp) + + def read_file(self, fileob): + """Read the metadata values from a file object.""" + msg = message_from_file(fileob) + self._fields['Metadata-Version'] = msg['metadata-version'] + + for field in _version2fieldlist(self['Metadata-Version']): + if field in _LISTFIELDS: + # we can have multiple lines + values = msg.get_all(field) + if field in _LISTTUPLEFIELDS and values is not None: + values = [tuple(value.split(',')) for value in values] + self.set(field, values) + else: + # single line + value = msg[field] + if value is not None and value != 'UNKNOWN': + self.set(field, value) + + def write(self, filepath): + """Write the metadata fields to filepath.""" + with open(filepath, 'w', encoding='utf-8') as fp: + self.write_file(fp) + + def write_file(self, fileobject): + """Write the PKG-INFO format data to a file object.""" + self._set_best_version() + for field in _version2fieldlist(self['Metadata-Version']): + values = self.get(field) + if field in _ELEMENTSFIELD: + self._write_field(fileobject, field, ','.join(values)) + continue + if field not in _LISTFIELDS: + if field == 'Description': + values = values.replace('\n', '\n |') + values = [values] + + if field in _LISTTUPLEFIELDS: + values = [','.join(value) for value in values] + + for value in values: + self._write_field(fileobject, field, value) + + def update(self, other=None, **kwargs): + """Set metadata values from the given iterable `other` and kwargs. + + Behavior is like `dict.update`: If `other` has a ``keys`` method, + they are looped over and ``self[key]`` is assigned ``other[key]``. + Else, ``other`` is an iterable of ``(key, value)`` iterables. + + Keys that don't match a metadata field or that have an empty value are + dropped. + """ + # XXX the code should just use self.set, which does tbe same checks and + # conversions already, but that would break packaging.pypi: it uses the + # update method, which does not call _set_best_version (which set + # does), and thus allows having a Metadata object (as long as you don't + # modify or write it) with extra fields from PyPI that are not fields + # defined in Metadata PEPs. to solve it, the best_version system + # should be reworked so that it's called only for writing, or in a new + # strict mode, or with a new, more lax Metadata subclass in p7g.pypi + def _set(key, value): + if key in _ATTR2FIELD and value: + self.set(self._convert_name(key), value) + + if not other: + # other is None or empty container + pass + elif hasattr(other, 'keys'): + for k in other.keys(): + _set(k, other[k]) + else: + for k, v in other: + _set(k, v) + + if kwargs: + for k, v in kwargs.items(): + _set(k, v) + + def set(self, name, value): + """Control then set a metadata field.""" + name = self._convert_name(name) + + if ((name in _ELEMENTSFIELD or name == 'Platform') and + not isinstance(value, (list, tuple))): + if isinstance(value, str): + value = [v.strip() for v in value.split(',')] + else: + value = [] + elif (name in _LISTFIELDS and + not isinstance(value, (list, tuple))): + if isinstance(value, str): + value = [value] + else: + value = [] + + if logger.isEnabledFor(logging.WARNING): + project_name = self['Name'] + + if name in _PREDICATE_FIELDS and value is not None: + for v in value: + # check that the values are valid predicates + if not is_valid_predicate(v.split(';')[0]): + logger.warning( + '%r: %r is not a valid predicate (field %r)', + project_name, v, name) + # FIXME this rejects UNKNOWN, is that right? + elif name in _VERSIONS_FIELDS and value is not None: + if not is_valid_versions(value): + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) + elif name in _VERSION_FIELDS and value is not None: + if not is_valid_version(value): + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) + + if name in _UNICODEFIELDS: + if name == 'Description': + value = self._remove_line_prefix(value) + + self._fields[name] = value + self._set_best_version() + + def get(self, name, default=_MISSING): + """Get a metadata field.""" + name = self._convert_name(name) + if name not in self._fields: + if default is _MISSING: + default = self._default_value(name) + return default + if name in _UNICODEFIELDS: + value = self._fields[name] + return value + elif name in _LISTFIELDS: + value = self._fields[name] + if value is None: + return [] + res = [] + for val in value: + valid, val = self._platform(val) + if not valid: + continue + if name not in _LISTTUPLEFIELDS: + res.append(val) + else: + # That's for Project-URL + res.append((val[0], val[1])) + return res + + elif name in _ELEMENTSFIELD: + valid, value = self._platform(self._fields[name]) + if not valid: + return [] + if isinstance(value, str): + return value.split(',') + valid, value = self._platform(self._fields[name]) + if not valid: + return None + return value + + def check(self, strict=False, restructuredtext=False): + """Check if the metadata is compliant. If strict is False then raise if + no Name or Version are provided""" + # XXX should check the versions (if the file was loaded) + missing, warnings = [], [] + + for attr in ('Name', 'Version'): # required by PEP 345 + if attr not in self: + missing.append(attr) + + if strict and missing != []: + msg = 'missing required metadata: %s' % ', '.join(missing) + raise MetadataMissingError(msg) + + for attr in ('Home-page', 'Author'): + if attr not in self: + missing.append(attr) + + if _HAS_DOCUTILS and restructuredtext: + warnings.extend(self._check_rst_data(self['Description'])) + + # checking metadata 1.2 (XXX needs to check 1.1, 1.0) + if self['Metadata-Version'] != '1.2': + return missing, warnings + + def is_valid_predicates(value): + for v in value: + if not is_valid_predicate(v.split(';')[0]): + return False + return True + + for fields, controller in ((_PREDICATE_FIELDS, is_valid_predicates), + (_VERSIONS_FIELDS, is_valid_versions), + (_VERSION_FIELDS, is_valid_version)): + for field in fields: + value = self.get(field, None) + if value is not None and not controller(value): + warnings.append('Wrong value for %r: %s' % (field, value)) + + return missing, warnings + + def todict(self): + """Return fields as a dict. + + Field names will be converted to use the underscore-lowercase style + instead of hyphen-mixed case (i.e. home_page instead of Home-page). + """ + data = { + 'metadata_version': self['Metadata-Version'], + 'name': self['Name'], + 'version': self['Version'], + 'summary': self['Summary'], + 'home_page': self['Home-page'], + 'author': self['Author'], + 'author_email': self['Author-email'], + 'license': self['License'], + 'description': self['Description'], + 'keywords': self['Keywords'], + 'platform': self['Platform'], + 'classifier': self['Classifier'], + 'download_url': self['Download-URL'], + } + + if self['Metadata-Version'] == '1.2': + data['requires_dist'] = self['Requires-Dist'] + data['requires_python'] = self['Requires-Python'] + data['requires_external'] = self['Requires-External'] + data['provides_dist'] = self['Provides-Dist'] + data['obsoletes_dist'] = self['Obsoletes-Dist'] + data['project_url'] = [','.join(url) for url in + self['Project-URL']] + + elif self['Metadata-Version'] == '1.1': + data['provides'] = self['Provides'] + data['requires'] = self['Requires'] + data['obsoletes'] = self['Obsoletes'] + + return data + + # Mapping API + # XXX these methods should return views or sets in 3.x + + def keys(self): + return list(_version2fieldlist(self['Metadata-Version'])) + + def __iter__(self): + for key in self.keys(): + yield key + + def values(self): + return [self[key] for key in self.keys()] + + def items(self): + return [(key, self[key]) for key in self.keys()] diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,9 @@ +"""Low-level and high-level APIs to interact with project indexes.""" + +__all__ = ['simple', + 'xmlrpc', + 'dist', + 'errors', + 'mirrors'] + +from packaging.pypi.dist import ReleaseInfo, ReleasesList, DistInfo diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/base.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/base.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,48 @@ +"""Base class for index crawlers.""" + +from packaging.pypi.dist import ReleasesList + + +class BaseClient: + """Base class containing common methods for the index crawlers/clients""" + + def __init__(self, prefer_final, prefer_source): + self._prefer_final = prefer_final + self._prefer_source = prefer_source + self._index = self + + def _get_prefer_final(self, prefer_final=None): + """Return the prefer_final internal parameter or the specified one if + provided""" + if prefer_final: + return prefer_final + else: + return self._prefer_final + + def _get_prefer_source(self, prefer_source=None): + """Return the prefer_source internal parameter or the specified one if + provided""" + if prefer_source: + return prefer_source + else: + return self._prefer_source + + def _get_project(self, project_name): + """Return an project instance, create it if necessary""" + return self._projects.setdefault(project_name.lower(), + ReleasesList(project_name, index=self._index)) + + def download_distribution(self, requirements, temp_path=None, + prefer_source=None, prefer_final=None): + """Download a distribution from the last release according to the + requirements. + + If temp_path is provided, download to this path, otherwise, create a + temporary location for the download and return it. + """ + prefer_final = self._get_prefer_final(prefer_final) + prefer_source = self._get_prefer_source(prefer_source) + release = self.get_release(requirements, prefer_final) + if release: + dist = release.get_distribution(prefer_source=prefer_source) + return dist.download(temp_path) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/dist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,544 @@ +"""Classes representing releases and distributions retrieved from indexes. + +A project (= unique name) can have several releases (= versions) and +each release can have several distributions (= sdist and bdists). + +Release objects contain metadata-related information (see PEP 376); +distribution objects contain download-related information. +""" + +import re +import hashlib +import tempfile +import urllib.request +import urllib.parse +import urllib.error +import urllib.parse +from shutil import unpack_archive + +from packaging.errors import IrrationalVersionError +from packaging.version import (suggest_normalized_version, NormalizedVersion, + get_version_predicate) +from packaging.metadata import Metadata +from packaging.pypi.errors import (HashDoesNotMatch, UnsupportedHashName, + CantParseArchiveName) + + +__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url'] + +EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz .egg".split() +MD5_HASH = re.compile(r'^.*#md5=([a-f0-9]+)$') +DIST_TYPES = ['bdist', 'sdist'] + + +class IndexReference: + """Mixin used to store the index reference""" + def set_index(self, index=None): + self._index = index + + +class ReleaseInfo(IndexReference): + """Represent a release of a project (a project with a specific version). + The release contain the _metadata informations related to this specific + version, and is also a container for distribution related informations. + + See the DistInfo class for more information about distributions. + """ + + def __init__(self, name, version, metadata=None, hidden=False, + index=None, **kwargs): + """ + :param name: the name of the distribution + :param version: the version of the distribution + :param metadata: the metadata fields of the release. + :type metadata: dict + :param kwargs: optional arguments for a new distribution. + """ + self.set_index(index) + self.name = name + self._version = None + self.version = version + if metadata: + self.metadata = Metadata(mapping=metadata) + else: + self.metadata = None + self.dists = {} + self.hidden = hidden + + if 'dist_type' in kwargs: + dist_type = kwargs.pop('dist_type') + self.add_distribution(dist_type, **kwargs) + + def set_version(self, version): + try: + self._version = NormalizedVersion(version) + except IrrationalVersionError: + suggestion = suggest_normalized_version(version) + if suggestion: + self.version = suggestion + else: + raise IrrationalVersionError(version) + + def get_version(self): + return self._version + + version = property(get_version, set_version) + + def fetch_metadata(self): + """If the metadata is not set, use the indexes to get it""" + if not self.metadata: + self._index.get_metadata(self.name, str(self.version)) + return self.metadata + + @property + def is_final(self): + """proxy to version.is_final""" + return self.version.is_final + + def fetch_distributions(self): + if self.dists is None: + self._index.get_distributions(self.name, str(self.version)) + if self.dists is None: + self.dists = {} + return self.dists + + def add_distribution(self, dist_type='sdist', python_version=None, + **params): + """Add distribution informations to this release. + If distribution information is already set for this distribution type, + add the given url paths to the distribution. This can be useful while + some of them fails to download. + + :param dist_type: the distribution type (eg. "sdist", "bdist", etc.) + :param params: the fields to be passed to the distribution object + (see the :class:DistInfo constructor). + """ + if dist_type not in DIST_TYPES: + raise ValueError(dist_type) + if dist_type in self.dists: + self.dists[dist_type].add_url(**params) + else: + self.dists[dist_type] = DistInfo(self, dist_type, + index=self._index, **params) + if python_version: + self.dists[dist_type].python_version = python_version + + def get_distribution(self, dist_type=None, prefer_source=True): + """Return a distribution. + + If dist_type is set, find first for this distribution type, and just + act as an alias of __get_item__. + + If prefer_source is True, search first for source distribution, and if + not return one existing distribution. + """ + if len(self.dists) == 0: + raise LookupError + if dist_type: + return self[dist_type] + if prefer_source: + if "sdist" in self.dists: + dist = self["sdist"] + else: + dist = next(self.dists.values()) + return dist + + def unpack(self, path=None, prefer_source=True): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + return self.get_distribution(prefer_source=prefer_source)\ + .unpack(path=path) + + def download(self, temp_path=None, prefer_source=True): + """Download the distribution, using the requirements. + + If more than one distribution match the requirements, use the last + version. + Download the distribution, and put it in the temp_path. If no temp_path + is given, creates and return one. + + Returns the complete absolute path to the downloaded archive. + """ + return self.get_distribution(prefer_source=prefer_source)\ + .download(path=temp_path) + + def set_metadata(self, metadata): + if not self.metadata: + self.metadata = Metadata() + self.metadata.update(metadata) + + def __getitem__(self, item): + """distributions are available using release["sdist"]""" + return self.dists[item] + + def _check_is_comparable(self, other): + if not isinstance(other, ReleaseInfo): + raise TypeError("cannot compare %s and %s" + % (type(self).__name__, type(other).__name__)) + elif self.name != other.name: + raise TypeError("cannot compare %s and %s" + % (self.name, other.name)) + + def __repr__(self): + return "<%s %s>" % (self.name, self.version) + + def __eq__(self, other): + self._check_is_comparable(other) + return self.version == other.version + + def __lt__(self, other): + self._check_is_comparable(other) + return self.version < other.version + + def __ne__(self, other): + return not self.__eq__(other) + + def __gt__(self, other): + return not (self.__lt__(other) or self.__eq__(other)) + + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other): + return self.__eq__(other) or self.__gt__(other) + + # See http://docs.python.org/reference/datamodel#object.__hash__ + __hash__ = object.__hash__ + + +class DistInfo(IndexReference): + """Represents a distribution retrieved from an index (sdist, bdist, ...) + """ + + def __init__(self, release, dist_type=None, url=None, hashname=None, + hashval=None, is_external=True, python_version=None, + index=None): + """Create a new instance of DistInfo. + + :param release: a DistInfo class is relative to a release. + :param dist_type: the type of the dist (eg. source, bin-*, etc.) + :param url: URL where we found this distribution + :param hashname: the name of the hash we want to use. Refer to the + hashlib.new documentation for more information. + :param hashval: the hash value. + :param is_external: we need to know if the provided url comes from + an index browsing, or from an external resource. + + """ + self.set_index(index) + self.release = release + self.dist_type = dist_type + self.python_version = python_version + self._unpacked_dir = None + # set the downloaded path to None by default. The goal here + # is to not download distributions multiple times + self.downloaded_location = None + # We store urls in dict, because we need to have a bit more infos + # than the simple URL. It will be used later to find the good url to + # use. + # We have two _url* attributes: _url and urls. urls contains a list + # of dict for the different urls, and _url contains the choosen url, in + # order to dont make the selection process multiple times. + self.urls = [] + self._url = None + self.add_url(url, hashname, hashval, is_external) + + def add_url(self, url=None, hashname=None, hashval=None, is_external=True): + """Add a new url to the list of urls""" + if hashname is not None: + try: + hashlib.new(hashname) + except ValueError: + raise UnsupportedHashName(hashname) + if url not in [u['url'] for u in self.urls]: + self.urls.append({ + 'url': url, + 'hashname': hashname, + 'hashval': hashval, + 'is_external': is_external, + }) + # reset the url selection process + self._url = None + + @property + def url(self): + """Pick up the right url for the list of urls in self.urls""" + # We return internal urls over externals. + # If there is more than one internal or external, return the first + # one. + if self._url is None: + if len(self.urls) > 1: + internals_urls = [u for u in self.urls \ + if u['is_external'] == False] + if len(internals_urls) >= 1: + self._url = internals_urls[0] + if self._url is None: + self._url = self.urls[0] + return self._url + + @property + def is_source(self): + """return if the distribution is a source one or not""" + return self.dist_type == 'sdist' + + def download(self, path=None): + """Download the distribution to a path, and return it. + + If the path is given in path, use this, otherwise, generates a new one + Return the download location. + """ + if path is None: + path = tempfile.mkdtemp() + + # if we do not have downloaded it yet, do it. + if self.downloaded_location is None: + url = self.url['url'] + archive_name = urllib.parse.urlparse(url)[2].split('/')[-1] + filename, headers = urllib.request.urlretrieve(url, + path + "/" + archive_name) + self.downloaded_location = filename + self._check_md5(filename) + return self.downloaded_location + + def unpack(self, path=None): + """Unpack the distribution to the given path. + + If not destination is given, creates a temporary location. + + Returns the location of the extracted files (root). + """ + if not self._unpacked_dir: + if path is None: + path = tempfile.mkdtemp() + + filename = self.download(path) + unpack_archive(filename, path) + self._unpacked_dir = path + + return path + + def _check_md5(self, filename): + """Check that the md5 checksum of the given file matches the one in + url param""" + hashname = self.url['hashname'] + expected_hashval = self.url['hashval'] + if None not in (expected_hashval, hashname): + with open(filename, 'rb') as f: + hashval = hashlib.new(hashname) + hashval.update(f.read()) + + if hashval.hexdigest() != expected_hashval: + raise HashDoesNotMatch("got %s instead of %s" + % (hashval.hexdigest(), expected_hashval)) + + def __repr__(self): + if self.release is None: + return "" % self.dist_type + + return "<%s %s %s>" % ( + self.release.name, self.release.version, self.dist_type or "") + + +class ReleasesList(IndexReference): + """A container of Release. + + Provides useful methods and facilities to sort and filter releases. + """ + def __init__(self, name, releases=None, contains_hidden=False, index=None): + self.set_index(index) + self.releases = [] + self.name = name + self.contains_hidden = contains_hidden + if releases: + self.add_releases(releases) + + def fetch_releases(self): + self._index.get_releases(self.name) + return self.releases + + def filter(self, predicate): + """Filter and return a subset of releases matching the given predicate. + """ + return ReleasesList(self.name, [release for release in self.releases + if predicate.match(release.version)], + index=self._index) + + def get_last(self, requirements, prefer_final=None): + """Return the "last" release, that satisfy the given predicates. + + "last" is defined by the version number of the releases, you also could + set prefer_final parameter to True or False to change the order results + """ + predicate = get_version_predicate(requirements) + releases = self.filter(predicate) + if len(releases) == 0: + return None + releases.sort_releases(prefer_final, reverse=True) + return releases[0] + + def add_releases(self, releases): + """Add releases in the release list. + + :param: releases is a list of ReleaseInfo objects. + """ + for r in releases: + self.add_release(release=r) + + def add_release(self, version=None, dist_type='sdist', release=None, + **dist_args): + """Add a release to the list. + + The release can be passed in the `release` parameter, and in this case, + it will be crawled to extract the useful informations if necessary, or + the release informations can be directly passed in the `version` and + `dist_type` arguments. + + Other keywords arguments can be provided, and will be forwarded to the + distribution creation (eg. the arguments of the DistInfo constructor). + """ + if release: + if release.name.lower() != self.name.lower(): + raise ValueError("%s is not the same project as %s" % + (release.name, self.name)) + version = str(release.version) + + if version not in self.get_versions(): + # append only if not already exists + self.releases.append(release) + for dist in release.dists.values(): + for url in dist.urls: + self.add_release(version, dist.dist_type, **url) + else: + matches = [r for r in self.releases + if str(r.version) == version and r.name == self.name] + if not matches: + release = ReleaseInfo(self.name, version, index=self._index) + self.releases.append(release) + else: + release = matches[0] + + release.add_distribution(dist_type=dist_type, **dist_args) + + def sort_releases(self, prefer_final=False, reverse=True, *args, **kwargs): + """Sort the results with the given properties. + + The `prefer_final` argument can be used to specify if final + distributions (eg. not dev, beta or alpha) would be preferred or not. + + Results can be inverted by using `reverse`. + + Any other parameter provided will be forwarded to the sorted call. You + cannot redefine the key argument of "sorted" here, as it is used + internally to sort the releases. + """ + + sort_by = [] + if prefer_final: + sort_by.append("is_final") + sort_by.append("version") + + self.releases.sort( + key=lambda i: tuple(getattr(i, arg) for arg in sort_by), + reverse=reverse, *args, **kwargs) + + def get_release(self, version): + """Return a release from its version.""" + matches = [r for r in self.releases if str(r.version) == version] + if len(matches) != 1: + raise KeyError(version) + return matches[0] + + def get_versions(self): + """Return a list of releases versions contained""" + return [str(r.version) for r in self.releases] + + def __getitem__(self, key): + return self.releases[key] + + def __len__(self): + return len(self.releases) + + def __repr__(self): + string = 'Project "%s"' % self.name + if self.get_versions(): + string += ' versions: %s' % ', '.join(self.get_versions()) + return '<%s>' % string + + +def get_infos_from_url(url, probable_dist_name=None, is_external=True): + """Get useful informations from an URL. + + Return a dict of (name, version, url, hashtype, hash, is_external) + + :param url: complete url of the distribution + :param probable_dist_name: A probable name of the project. + :param is_external: Tell if the url commes from an index or from + an external URL. + """ + # if the url contains a md5 hash, get it. + md5_hash = None + match = MD5_HASH.match(url) + if match is not None: + md5_hash = match.group(1) + # remove the hash + url = url.replace("#md5=%s" % md5_hash, "") + + # parse the archive name to find dist name and version + archive_name = urllib.parse.urlparse(url)[2].split('/')[-1] + extension_matched = False + # remove the extension from the name + for ext in EXTENSIONS: + if archive_name.endswith(ext): + archive_name = archive_name[:-len(ext)] + extension_matched = True + + name, version = split_archive_name(archive_name) + if extension_matched is True: + return {'name': name, + 'version': version, + 'url': url, + 'hashname': "md5", + 'hashval': md5_hash, + 'is_external': is_external, + 'dist_type': 'sdist'} + + +def split_archive_name(archive_name, probable_name=None): + """Split an archive name into two parts: name and version. + + Return the tuple (name, version) + """ + # Try to determine wich part is the name and wich is the version using the + # "-" separator. Take the larger part to be the version number then reduce + # if this not works. + def eager_split(str, maxsplit=2): + # split using the "-" separator + splits = str.rsplit("-", maxsplit) + name = splits[0] + version = "-".join(splits[1:]) + if version.startswith("-"): + version = version[1:] + if suggest_normalized_version(version) is None and maxsplit >= 0: + # we dont get a good version number: recurse ! + return eager_split(str, maxsplit - 1) + else: + return name, version + if probable_name is not None: + probable_name = probable_name.lower() + name = None + if probable_name is not None and probable_name in archive_name: + # we get the name from probable_name, if given. + name = probable_name + version = archive_name.lstrip(name) + else: + name, version = eager_split(archive_name) + + version = suggest_normalized_version(version) + if version is not None and name != "": + return name.lower(), version + else: + raise CantParseArchiveName(archive_name) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/errors.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/errors.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,39 @@ +"""Exceptions raised by packaging.pypi code.""" + +from packaging.errors import PackagingPyPIError + + +class ProjectNotFound(PackagingPyPIError): + """Project has not been found""" + + +class DistributionNotFound(PackagingPyPIError): + """The release has not been found""" + + +class ReleaseNotFound(PackagingPyPIError): + """The release has not been found""" + + +class CantParseArchiveName(PackagingPyPIError): + """An archive name can't be parsed to find distribution name and version""" + + +class DownloadError(PackagingPyPIError): + """An error has occurs while downloading""" + + +class HashDoesNotMatch(DownloadError): + """Compared hashes does not match""" + + +class UnsupportedHashName(PackagingPyPIError): + """A unsupported hashname has been used""" + + +class UnableToDownload(PackagingPyPIError): + """All mirrors have been tried, without success""" + + +class InvalidSearchField(PackagingPyPIError): + """An invalid search field has been used""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/mirrors.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/mirrors.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,52 @@ +"""Utilities related to the mirror infrastructure defined in PEP 381.""" + +from string import ascii_lowercase +import socket + +DEFAULT_MIRROR_URL = "last.pypi.python.org" + + +def get_mirrors(hostname=None): + """Return the list of mirrors from the last record found on the DNS + entry:: + + >>> from packaging.pypi.mirrors import get_mirrors + >>> get_mirrors() + ['a.pypi.python.org', 'b.pypi.python.org', 'c.pypi.python.org', + 'd.pypi.python.org'] + + """ + if hostname is None: + hostname = DEFAULT_MIRROR_URL + + # return the last mirror registered on PyPI. + try: + hostname = socket.gethostbyname_ex(hostname)[0] + except socket.gaierror: + return [] + end_letter = hostname.split(".", 1) + + # determine the list from the last one. + return ["%s.%s" % (s, end_letter[1]) for s in string_range(end_letter[0])] + + +def string_range(last): + """Compute the range of string between "a" and last. + + This works for simple "a to z" lists, but also for "a to zz" lists. + """ + for k in range(len(last)): + for x in product(ascii_lowercase, repeat=(k + 1)): + result = ''.join(x) + yield result + if result == last: + return + + +def product(*args, **kwds): + pools = [tuple(arg) for arg in args] * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x + [y] for x in result for y in pool] + for prod in result: + yield tuple(prod) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/simple.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/simple.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,462 @@ +"""Spider using the screen-scraping "simple" PyPI API. + +This module contains the class Crawler, a simple spider that +can be used to find and retrieve distributions from a project index +(like the Python Package Index), using its so-called simple API (see +reference implementation available at http://pypi.python.org/simple/). +""" + +import http.client +import re +import socket +import sys +import urllib.request +import urllib.parse +import urllib.error +import os + +from fnmatch import translate +from functools import wraps +from packaging import logger +from packaging.metadata import Metadata +from packaging.version import get_version_predicate +from packaging import __version__ as packaging_version +from packaging.pypi.base import BaseClient +from packaging.pypi.dist import (ReleasesList, EXTENSIONS, + get_infos_from_url, MD5_HASH) +from packaging.pypi.errors import (PackagingPyPIError, DownloadError, + UnableToDownload, CantParseArchiveName, + ReleaseNotFound, ProjectNotFound) +from packaging.pypi.mirrors import get_mirrors + +__all__ = ['Crawler', 'DEFAULT_SIMPLE_INDEX_URL'] + +# -- Constants ----------------------------------------------- +DEFAULT_SIMPLE_INDEX_URL = "http://a.pypi.python.org/simple/" +DEFAULT_HOSTS = ("*",) +SOCKET_TIMEOUT = 15 +USER_AGENT = "Python-urllib/%s.%s packaging/%s" % ( + sys.version_info[0], sys.version_info[1], packaging_version) + +# -- Regexps ------------------------------------------------- +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') +HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match + +# This pattern matches a character entity reference (a decimal numeric +# references, a hexadecimal numeric reference, or a named reference). +ENTITY_SUB = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub +REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) + + +def socket_timeout(timeout=SOCKET_TIMEOUT): + """Decorator to add a socket timeout when requesting pages on PyPI. + """ + def wrapper(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + old_timeout = socket.getdefaulttimeout() + if hasattr(self, "_timeout"): + timeout = self._timeout + socket.setdefaulttimeout(timeout) + try: + return func(self, *args, **kwargs) + finally: + socket.setdefaulttimeout(old_timeout) + return wrapped + return wrapper + + +def with_mirror_support(): + """Decorator that makes the mirroring support easier""" + def wrapper(func): + @wraps(func) + def wrapped(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except DownloadError: + # if an error occurs, try with the next index_url + if self._mirrors_tries >= self._mirrors_max_tries: + try: + self._switch_to_next_mirror() + except KeyError: + raise UnableToDownload("Tried all mirrors") + else: + self._mirrors_tries += 1 + self._projects.clear() + return wrapped(self, *args, **kwargs) + return wrapped + return wrapper + + +class Crawler(BaseClient): + """Provides useful tools to request the Python Package Index simple API. + + You can specify both mirrors and mirrors_url, but mirrors_url will only be + used if mirrors is set to None. + + :param index_url: the url of the simple index to search on. + :param prefer_final: if the version is not mentioned, and the last + version is not a "final" one (alpha, beta, etc.), + pick up the last final version. + :param prefer_source: if the distribution type is not mentioned, pick up + the source one if available. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param hosts: a list of hosts allowed to be processed while using + follow_externals=True. Default behavior is to follow all + hosts. + :param follow_externals: tell if following external links is needed or + not. Default is False. + :param mirrors_url: the url to look on for DNS records giving mirror + addresses. + :param mirrors: a list of mirrors (see PEP 381). + :param timeout: time in seconds to consider a url has timeouted. + :param mirrors_max_tries": number of times to try requesting informations + on mirrors before switching. + """ + + def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False, + prefer_source=True, hosts=DEFAULT_HOSTS, + follow_externals=False, mirrors_url=None, mirrors=None, + timeout=SOCKET_TIMEOUT, mirrors_max_tries=0): + super(Crawler, self).__init__(prefer_final, prefer_source) + self.follow_externals = follow_externals + + # mirroring attributes. + parsed = urllib.parse.urlparse(index_url) + self.scheme = parsed[0] + if self.scheme == 'file': + ender = os.path.sep + else: + ender = '/' + if not index_url.endswith(ender): + index_url += ender + # if no mirrors are defined, use the method described in PEP 381. + if mirrors is None: + mirrors = get_mirrors(mirrors_url) + self._mirrors = set(mirrors) + self._mirrors_used = set() + self.index_url = index_url + self._mirrors_max_tries = mirrors_max_tries + self._mirrors_tries = 0 + self._timeout = timeout + + # create a regexp to match all given hosts + self._allowed_hosts = re.compile('|'.join(map(translate, hosts))).match + + # we keep an index of pages we have processed, in order to avoid + # scanning them multple time (eg. if there is multiple pages pointing + # on one) + self._processed_urls = [] + self._projects = {} + + @with_mirror_support() + def search_projects(self, name=None, **kwargs): + """Search the index for projects containing the given name. + + Return a list of names. + """ + if '*' in name: + name.replace('*', '.*') + else: + name = "%s%s%s" % ('*.?', name, '*.?') + name = name.replace('*', '[^<]*') # avoid matching end tag + pattern = (']*>(%s)' % name).encode('utf-8') + projectname = re.compile(pattern, re.I) + matching_projects = [] + + with self._open_url(self.index_url) as index: + index_content = index.read() + + for match in projectname.finditer(index_content): + project_name = match.group(1).decode('utf-8') + matching_projects.append(self._get_project(project_name)) + return matching_projects + + def get_releases(self, requirements, prefer_final=None, + force_update=False): + """Search for releases and return a ReleasesList object containing + the results. + """ + predicate = get_version_predicate(requirements) + if predicate.name.lower() in self._projects and not force_update: + return self._projects.get(predicate.name.lower()) + prefer_final = self._get_prefer_final(prefer_final) + logger.debug('Reading info on PyPI about %s', predicate.name) + self._process_index_page(predicate.name) + + if predicate.name.lower() not in self._projects: + raise ProjectNotFound + + releases = self._projects.get(predicate.name.lower()) + releases.sort_releases(prefer_final=prefer_final) + return releases + + def get_release(self, requirements, prefer_final=None): + """Return only one release that fulfill the given requirements""" + predicate = get_version_predicate(requirements) + release = self.get_releases(predicate, prefer_final)\ + .get_last(predicate) + if not release: + raise ReleaseNotFound("No release matches the given criterias") + return release + + def get_distributions(self, project_name, version): + """Return the distributions found on the index for the specific given + release""" + # as the default behavior of get_release is to return a release + # containing the distributions, just alias it. + return self.get_release("%s (%s)" % (project_name, version)) + + def get_metadata(self, project_name, version): + """Return the metadatas from the simple index. + + Currently, download one archive, extract it and use the PKG-INFO file. + """ + release = self.get_distributions(project_name, version) + if not release.metadata: + location = release.get_distribution().unpack() + pkg_info = os.path.join(location, 'PKG-INFO') + release.metadata = Metadata(pkg_info) + return release + + def _switch_to_next_mirror(self): + """Switch to the next mirror (eg. point self.index_url to the next + mirror url. + + Raise a KeyError if all mirrors have been tried. + """ + self._mirrors_used.add(self.index_url) + index_url = self._mirrors.pop() + # XXX use urllib.parse for a real check of missing scheme part + if not index_url.startswith(("http://", "https://", "file://")): + index_url = "http://%s" % index_url + + if not index_url.endswith("/simple"): + index_url = "%s/simple/" % index_url + + self.index_url = index_url + + def _is_browsable(self, url): + """Tell if the given URL can be browsed or not. + + It uses the follow_externals and the hosts list to tell if the given + url is browsable or not. + """ + # if _index_url is contained in the given URL, we are browsing the + # index, and it's always "browsable". + # local files are always considered browable resources + if self.index_url in url or urllib.parse.urlparse(url)[0] == "file": + return True + elif self.follow_externals: + if self._allowed_hosts(urllib.parse.urlparse(url)[1]): # 1 is netloc + return True + else: + return False + return False + + def _is_distribution(self, link): + """Tell if the given URL matches to a distribution name or not. + """ + #XXX find a better way to check that links are distributions + # Using a regexp ? + for ext in EXTENSIONS: + if ext in link: + return True + return False + + def _register_release(self, release=None, release_info={}): + """Register a new release. + + Both a release or a dict of release_info can be provided, the preferred + way (eg. the quicker) is the dict one. + + Return the list of existing releases for the given project. + """ + # Check if the project already has a list of releases (refering to + # the project name). If not, create a new release list. + # Then, add the release to the list. + if release: + name = release.name + else: + name = release_info['name'] + if name.lower() not in self._projects: + self._projects[name.lower()] = ReleasesList(name, index=self._index) + + if release: + self._projects[name.lower()].add_release(release=release) + else: + name = release_info.pop('name') + version = release_info.pop('version') + dist_type = release_info.pop('dist_type') + self._projects[name.lower()].add_release(version, dist_type, + **release_info) + return self._projects[name.lower()] + + def _process_url(self, url, project_name=None, follow_links=True): + """Process an url and search for distributions packages. + + For each URL found, if it's a download, creates a PyPIdistribution + object. If it's a homepage and we can follow links, process it too. + + :param url: the url to process + :param project_name: the project name we are searching for. + :param follow_links: Do not want to follow links more than from one + level. This parameter tells if we want to follow + the links we find (eg. run recursively this + method on it) + """ + with self._open_url(url) as f: + base_url = f.url + if url not in self._processed_urls: + self._processed_urls.append(url) + link_matcher = self._get_link_matcher(url) + for link, is_download in link_matcher(f.read().decode(), base_url): + if link not in self._processed_urls: + if self._is_distribution(link) or is_download: + self._processed_urls.append(link) + # it's a distribution, so create a dist object + try: + infos = get_infos_from_url(link, project_name, + is_external=self.index_url not in url) + except CantParseArchiveName as e: + logger.warning( + "version has not been parsed: %s", e) + else: + self._register_release(release_info=infos) + else: + if self._is_browsable(link) and follow_links: + self._process_url(link, project_name, + follow_links=False) + + def _get_link_matcher(self, url): + """Returns the right link matcher function of the given url + """ + if self.index_url in url: + return self._simple_link_matcher + else: + return self._default_link_matcher + + def _get_full_url(self, url, base_url): + return urllib.parse.urljoin(base_url, self._htmldecode(url)) + + def _simple_link_matcher(self, content, base_url): + """Yield all links with a rel="download" or rel="homepage". + + This matches the simple index requirements for matching links. + If follow_externals is set to False, dont yeld the external + urls. + + :param content: the content of the page we want to parse + :param base_url: the url of this page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if MD5_HASH.match(url): + yield (url, True) + + for match in REL.finditer(content): + # search for rel links. + tag, rel = match.groups() + rels = [s.strip() for s in rel.lower().split(',')] + if 'homepage' in rels or 'download' in rels: + for match in HREF.finditer(tag): + url = self._get_full_url(match.group(1), base_url) + if 'download' in rels or self._is_browsable(url): + # yield a list of (url, is_download) + yield (url, 'download' in rels) + + def _default_link_matcher(self, content, base_url): + """Yield all links found on the page. + """ + for match in HREF.finditer(content): + url = self._get_full_url(match.group(1), base_url) + if self._is_browsable(url): + yield (url, False) + + @with_mirror_support() + def _process_index_page(self, name): + """Find and process a PyPI page for the given project name. + + :param name: the name of the project to find the page + """ + # Browse and index the content of the given PyPI page. + if self.scheme == 'file': + ender = os.path.sep + else: + ender = '/' + url = self.index_url + name + ender + self._process_url(url, name) + + @socket_timeout() + def _open_url(self, url): + """Open a urllib2 request, handling HTTP authentication, and local + files support. + + """ + scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) + + # authentication stuff + if scheme in ('http', 'https'): + auth, host = urllib.parse.splituser(netloc) + else: + auth = None + + # add index.html automatically for filesystem paths + if scheme == 'file': + if url.endswith(os.path.sep): + url += "index.html" + + # add authorization headers if auth is provided + if auth: + auth = "Basic " + \ + urllib.parse.unquote(auth).encode('base64').strip() + new_url = urllib.parse.urlunparse(( + scheme, host, path, params, query, frag)) + request = urllib.request.Request(new_url) + request.add_header("Authorization", auth) + else: + request = urllib.request.Request(url) + request.add_header('User-Agent', USER_AGENT) + try: + fp = urllib.request.urlopen(request) + except (ValueError, http.client.InvalidURL) as v: + msg = ' '.join([str(arg) for arg in v.args]) + raise PackagingPyPIError('%s %s' % (url, msg)) + except urllib.error.HTTPError as v: + return v + except urllib.error.URLError as v: + raise DownloadError("Download error for %s: %s" % (url, v.reason)) + except http.client.BadStatusLine as v: + raise DownloadError('%s returned a bad status line. ' + 'The server might be down, %s' % (url, v.line)) + except http.client.HTTPException as v: + raise DownloadError("Download error for %s: %s" % (url, v)) + except socket.timeout: + raise DownloadError("The server timeouted") + + if auth: + # Put authentication info back into request URL if same host, + # so that links found on the page will work + s2, h2, path2, param2, query2, frag2 = \ + urllib.parse.urlparse(fp.url) + if s2 == scheme and h2 == host: + fp.url = urllib.parse.urlunparse( + (s2, netloc, path2, param2, query2, frag2)) + return fp + + def _decode_entity(self, match): + what = match.group(1) + if what.startswith('#x'): + what = int(what[2:], 16) + elif what.startswith('#'): + what = int(what[1:]) + else: + from html.entities import name2codepoint + what = name2codepoint.get(what, match.group(0)) + return chr(what) + + def _htmldecode(self, text): + """Decode HTML entities in the given text.""" + return ENTITY_SUB(self._decode_entity, text) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/wrapper.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/wrapper.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,99 @@ +"""Convenient client for all PyPI APIs. + +This module provides a ClientWrapper class which will use the "simple" +or XML-RPC API to request information or files from an index. +""" + +from packaging.pypi import simple, xmlrpc + +_WRAPPER_MAPPINGS = {'get_release': 'simple', + 'get_releases': 'simple', + 'search_projects': 'simple', + 'get_metadata': 'xmlrpc', + 'get_distributions': 'simple'} + +_WRAPPER_INDEXES = {'xmlrpc': xmlrpc.Client, + 'simple': simple.Crawler} + + +def switch_index_if_fails(func, wrapper): + """Decorator that switch of index (for instance from xmlrpc to simple) + if the first mirror return an empty list or raises an exception. + """ + def decorator(*args, **kwargs): + retry = True + exception = None + methods = [func] + for f in wrapper._indexes.values(): + if f != func.__self__ and hasattr(f, func.__name__): + methods.append(getattr(f, func.__name__)) + for method in methods: + try: + response = method(*args, **kwargs) + retry = False + except Exception as e: + exception = e + if not retry: + break + if retry and exception: + raise exception + else: + return response + return decorator + + +class ClientWrapper: + """Wrapper around simple and xmlrpc clients, + + Choose the best implementation to use depending the needs, using the given + mappings. + If one of the indexes returns an error, tries to use others indexes. + + :param index: tell which index to rely on by default. + :param index_classes: a dict of name:class to use as indexes. + :param indexes: a dict of name:index already instantiated + :param mappings: the mappings to use for this wrapper + """ + + def __init__(self, default_index='simple', index_classes=_WRAPPER_INDEXES, + indexes={}, mappings=_WRAPPER_MAPPINGS): + self._projects = {} + self._mappings = mappings + self._indexes = indexes + self._default_index = default_index + + # instantiate the classes and set their _project attribute to the one + # of the wrapper. + for name, cls in index_classes.items(): + obj = self._indexes.setdefault(name, cls()) + obj._projects = self._projects + obj._index = self + + def __getattr__(self, method_name): + """When asking for methods of the wrapper, return the implementation of + the wrapped classes, depending the mapping. + + Decorate the methods to switch of implementation if an error occurs + """ + real_method = None + if method_name in _WRAPPER_MAPPINGS: + obj = self._indexes[_WRAPPER_MAPPINGS[method_name]] + real_method = getattr(obj, method_name) + else: + # the method is not defined in the mappings, so we try first to get + # it via the default index, and rely on others if needed. + try: + real_method = getattr(self._indexes[self._default_index], + method_name) + except AttributeError: + other_indexes = [i for i in self._indexes + if i != self._default_index] + for index in other_indexes: + real_method = getattr(self._indexes[index], method_name, + None) + if real_method: + break + if real_method: + return switch_index_if_fails(real_method, self) + else: + raise AttributeError("No index have attribute '%s'" % method_name) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/pypi/xmlrpc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/pypi/xmlrpc.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,200 @@ +"""Spider using the XML-RPC PyPI API. + +This module contains the class Client, a spider that can be used to find +and retrieve distributions from a project index (like the Python Package +Index), using its XML-RPC API (see documentation of the reference +implementation at http://wiki.python.org/moin/PyPiXmlRpc). +""" + +import xmlrpc.client + +from packaging import logger +from packaging.errors import IrrationalVersionError +from packaging.version import get_version_predicate +from packaging.pypi.base import BaseClient +from packaging.pypi.errors import (ProjectNotFound, InvalidSearchField, + ReleaseNotFound) +from packaging.pypi.dist import ReleaseInfo + +__all__ = ['Client', 'DEFAULT_XMLRPC_INDEX_URL'] + +DEFAULT_XMLRPC_INDEX_URL = 'http://python.org/pypi' + +_SEARCH_FIELDS = ['name', 'version', 'author', 'author_email', 'maintainer', + 'maintainer_email', 'home_page', 'license', 'summary', + 'description', 'keywords', 'platform', 'download_url'] + + +class Client(BaseClient): + """Client to query indexes using XML-RPC method calls. + + If no server_url is specified, use the default PyPI XML-RPC URL, + defined in the DEFAULT_XMLRPC_INDEX_URL constant:: + + >>> client = Client() + >>> client.server_url == DEFAULT_XMLRPC_INDEX_URL + True + + >>> client = Client("http://someurl/") + >>> client.server_url + 'http://someurl/' + """ + + def __init__(self, server_url=DEFAULT_XMLRPC_INDEX_URL, prefer_final=False, + prefer_source=True): + super(Client, self).__init__(prefer_final, prefer_source) + self.server_url = server_url + self._projects = {} + + def get_release(self, requirements, prefer_final=False): + """Return a release with all complete metadata and distribution + related informations. + """ + prefer_final = self._get_prefer_final(prefer_final) + predicate = get_version_predicate(requirements) + releases = self.get_releases(predicate.name) + release = releases.get_last(predicate, prefer_final) + self.get_metadata(release.name, str(release.version)) + self.get_distributions(release.name, str(release.version)) + return release + + def get_releases(self, requirements, prefer_final=None, show_hidden=True, + force_update=False): + """Return the list of existing releases for a specific project. + + Cache the results from one call to another. + + If show_hidden is True, return the hidden releases too. + If force_update is True, reprocess the index to update the + informations (eg. make a new XML-RPC call). + :: + + >>> client = Client() + >>> client.get_releases('Foo') + ['1.1', '1.2', '1.3'] + + If no such project exists, raise a ProjectNotFound exception:: + + >>> client.get_project_versions('UnexistingProject') + ProjectNotFound: UnexistingProject + + """ + def get_versions(project_name, show_hidden): + return self.proxy.package_releases(project_name, show_hidden) + + predicate = get_version_predicate(requirements) + prefer_final = self._get_prefer_final(prefer_final) + project_name = predicate.name + if not force_update and (project_name.lower() in self._projects): + project = self._projects[project_name.lower()] + if not project.contains_hidden and show_hidden: + # if hidden releases are requested, and have an existing + # list of releases that does not contains hidden ones + all_versions = get_versions(project_name, show_hidden) + existing_versions = project.get_versions() + hidden_versions = set(all_versions) - set(existing_versions) + for version in hidden_versions: + project.add_release(release=ReleaseInfo(project_name, + version, index=self._index)) + else: + versions = get_versions(project_name, show_hidden) + if not versions: + raise ProjectNotFound(project_name) + project = self._get_project(project_name) + project.add_releases([ReleaseInfo(project_name, version, + index=self._index) + for version in versions]) + project = project.filter(predicate) + if len(project) == 0: + raise ReleaseNotFound("%s" % predicate) + project.sort_releases(prefer_final) + return project + + + def get_distributions(self, project_name, version): + """Grab informations about distributions from XML-RPC. + + Return a ReleaseInfo object, with distribution-related informations + filled in. + """ + url_infos = self.proxy.release_urls(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + for info in url_infos: + packagetype = info['packagetype'] + dist_infos = {'url': info['url'], + 'hashval': info['md5_digest'], + 'hashname': 'md5', + 'is_external': False, + 'python_version': info['python_version']} + release.add_distribution(packagetype, **dist_infos) + return release + + def get_metadata(self, project_name, version): + """Retrieve project metadata. + + Return a ReleaseInfo object, with metadata informations filled in. + """ + # to be case-insensitive, get the informations from the XMLRPC API + projects = [d['name'] for d in + self.proxy.search({'name': project_name}) + if d['name'].lower() == project_name] + if len(projects) > 0: + project_name = projects[0] + + metadata = self.proxy.release_data(project_name, version) + project = self._get_project(project_name) + if version not in project.get_versions(): + project.add_release(release=ReleaseInfo(project_name, version, + index=self._index)) + release = project.get_release(version) + release.set_metadata(metadata) + return release + + def search_projects(self, name=None, operator="or", **kwargs): + """Find using the keys provided in kwargs. + + You can set operator to "and" or "or". + """ + for key in kwargs: + if key not in _SEARCH_FIELDS: + raise InvalidSearchField(key) + if name: + kwargs["name"] = name + projects = self.proxy.search(kwargs, operator) + for p in projects: + project = self._get_project(p['name']) + try: + project.add_release(release=ReleaseInfo(p['name'], + p['version'], metadata={'summary': p['summary']}, + index=self._index)) + except IrrationalVersionError as e: + logger.warning("Irrational version error found: %s", e) + return [self._projects[p['name'].lower()] for p in projects] + + def get_all_projects(self): + """Return the list of all projects registered in the package index""" + projects = self.proxy.list_packages() + for name in projects: + self.get_releases(name, show_hidden=True) + + return [self._projects[name.lower()] for name in set(projects)] + + @property + def proxy(self): + """Property used to return the XMLRPC server proxy. + + If no server proxy is defined yet, creates a new one:: + + >>> client = Client() + >>> client.proxy() + + + """ + if not hasattr(self, '_server_proxy'): + self._server_proxy = xmlrpc.client.ServerProxy(self.server_url) + + return self._server_proxy diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/run.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/run.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,663 @@ +"""Main command line parser. Implements the pysetup script.""" + +import os +import re +import sys +import getopt +import logging + +from packaging import logger +from packaging.dist import Distribution +from packaging.util import _is_archive_file, generate_setup_py +from packaging.command import get_command_class, STANDARD_COMMANDS +from packaging.install import install, install_local_project, remove +from packaging.database import get_distribution, get_distributions +from packaging.depgraph import generate_graph +from packaging.fancy_getopt import FancyGetopt +from packaging.errors import (PackagingArgError, PackagingError, + PackagingModuleError, PackagingClassError, + CCompilerError) + + +command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') + +common_usage = """\ +Actions: +%(actions)s + +To get more help on an action, use: + + pysetup action --help +""" + +global_options = [ + # The fourth entry for verbose means that it can be repeated. + ('verbose', 'v', "run verbosely (default)", True), + ('quiet', 'q', "run quietly (turns verbosity off)"), + ('dry-run', 'n', "don't actually do anything"), + ('help', 'h', "show detailed help message"), + ('no-user-cfg', None, 'ignore pydistutils.cfg in your home directory'), + ('version', None, 'Display the version'), +] + +negative_opt = {'quiet': 'verbose'} + +display_options = [ + ('help-commands', None, "list all available commands"), +] + +display_option_names = [x[0].replace('-', '_') for x in display_options] + + +def _parse_args(args, options, long_options): + """Transform sys.argv input into a dict. + + :param args: the args to parse (i.e sys.argv) + :param options: the list of options to pass to getopt + :param long_options: the list of string with the names of the long options + to be passed to getopt. + + The function returns a dict with options/long_options as keys and matching + values as values. + """ + optlist, args = getopt.gnu_getopt(args, options, long_options) + optdict = {} + optdict['args'] = args + for k, v in optlist: + k = k.lstrip('-') + if k not in optdict: + optdict[k] = [] + if v: + optdict[k].append(v) + else: + optdict[k].append(v) + return optdict + + +class action_help: + """Prints a help message when the standard help flags: -h and --help + are used on the commandline. + """ + + def __init__(self, help_msg): + self.help_msg = help_msg + + def __call__(self, f): + def wrapper(*args, **kwargs): + f_args = args[1] + if '--help' in f_args or '-h' in f_args: + print(self.help_msg) + return + return f(*args, **kwargs) + return wrapper + + +@action_help("""\ +Usage: pysetup create + or: pysetup create --help + +Create a new Python project. +""") +def _create(distpatcher, args, **kw): + from packaging.create import main + return main() + + +@action_help("""\ +Usage: pysetup generate-setup + or: pysetup generate-setup --help + +Generate a setup.py script for backward-compatibility purposes. +""") +def _generate(distpatcher, args, **kw): + generate_setup_py() + logger.info('The setup.py was generated') + + +@action_help("""\ +Usage: pysetup graph dist + or: pysetup graph --help + +Print dependency graph for the distribution. + +positional arguments: + dist installed distribution name +""") +def _graph(dispatcher, args, **kw): + name = args[1] + dist = get_distribution(name, use_egg_info=True) + if dist is None: + logger.warning('Distribution not found.') + return 1 + else: + dists = get_distributions(use_egg_info=True) + graph = generate_graph(dists) + print(graph.repr_node(dist)) + + +@action_help("""\ +Usage: pysetup install [dist] + or: pysetup install [archive] + or: pysetup install [src_dir] + or: pysetup install --help + +Install a Python distribution from the indexes, source directory, or sdist. + +positional arguments: + archive path to source distribution (zip, tar.gz) + dist distribution name to install from the indexes + scr_dir path to source directory +""") +def _install(dispatcher, args, **kw): + # first check if we are in a source directory + if len(args) < 2: + # are we inside a project dir? + if os.path.isfile('setup.cfg') or os.path.isfile('setup.py'): + args.insert(1, os.getcwd()) + else: + logger.warning('No project to install.') + return 1 + + target = args[1] + # installing from a source dir or archive file? + if os.path.isdir(target) or _is_archive_file(target): + return not install_local_project(target) + else: + # download from PyPI + return not install(target) + + +@action_help("""\ +Usage: pysetup metadata [dist] + or: pysetup metadata [dist] [-f field ...] + or: pysetup metadata --help + +Print metadata for the distribution. + +positional arguments: + dist installed distribution name + +optional arguments: + -f metadata field to print; omit to get all fields +""") +def _metadata(dispatcher, args, **kw): + opts = _parse_args(args[1:], 'f:', []) + if opts['args']: + name = opts['args'][0] + dist = get_distribution(name, use_egg_info=True) + if dist is None: + logger.warning('%r not installed', name) + return 1 + elif os.path.isfile('setup.cfg'): + logger.info('searching local dir for metadata') + dist = Distribution() # XXX use config module + dist.parse_config_files() + else: + logger.warning('no argument given and no local setup.cfg found') + return 1 + + metadata = dist.metadata + + if 'f' in opts: + keys = (k for k in opts['f'] if k in metadata) + else: + keys = metadata.keys() + + for key in keys: + if key in metadata: + print(metadata._convert_name(key) + ':') + value = metadata[key] + if isinstance(value, list): + for v in value: + print(' ', v) + else: + print(' ', value.replace('\n', '\n ')) + + +@action_help("""\ +Usage: pysetup remove dist [-y] + or: pysetup remove --help + +Uninstall a Python distribution. + +positional arguments: + dist installed distribution name + +optional arguments: + -y auto confirm distribution removal +""") +def _remove(distpatcher, args, **kw): + opts = _parse_args(args[1:], 'y', []) + if 'y' in opts: + auto_confirm = True + else: + auto_confirm = False + + retcode = 0 + for dist in set(opts['args']): + try: + remove(dist, auto_confirm=auto_confirm) + except PackagingError: + logger.warning('%r not installed', dist) + retcode = 1 + + return retcode + + +@action_help("""\ +Usage: pysetup run [global_opts] cmd1 [cmd1_opts] [cmd2 [cmd2_opts] ...] + or: pysetup run --help + or: pysetup run --list-commands + or: pysetup run cmd --help +""") +def _run(dispatcher, args, **kw): + parser = dispatcher.parser + args = args[1:] + + commands = STANDARD_COMMANDS # FIXME display extra commands + + if args == ['--list-commands']: + print('List of available commands:') + for cmd in commands: + cls = dispatcher.cmdclass.get(cmd) or get_command_class(cmd) + desc = getattr(cls, 'description', '(no description available)') + print(' %s: %s' % (cmd, desc)) + return + + while args: + args = dispatcher._parse_command_opts(parser, args) + if args is None: + return + + # create the Distribution class + # need to feed setup.cfg here ! + dist = Distribution() + + # Find and parse the config file(s): they will override options from + # the setup script, but be overridden by the command line. + + # XXX still need to be extracted from Distribution + dist.parse_config_files() + + for cmd in dispatcher.commands: + # FIXME need to catch MetadataMissingError here (from the check command + # e.g.)--or catch any exception, print an error message and exit with 1 + dist.run_command(cmd, dispatcher.command_options[cmd]) + + return 0 + + +@action_help("""\ +Usage: pysetup list [dist ...] + or: pysetup list --help + +Print name, version and location for the matching installed distributions. + +positional arguments: + dist installed distribution name; omit to get all distributions +""") +def _list(dispatcher, args, **kw): + opts = _parse_args(args[1:], '', []) + dists = get_distributions(use_egg_info=True) + if opts['args']: + results = (d for d in dists if d.name.lower() in opts['args']) + listall = False + else: + results = dists + listall = True + + number = 0 + for dist in results: + print('%r %s (from %r)' % (dist.name, dist.version, dist.path)) + number += 1 + + if number == 0: + if listall: + logger.info('Nothing seems to be installed.') + else: + logger.warning('No matching distribution found.') + return 1 + else: + logger.info('Found %d projects installed.', number) + + +@action_help("""\ +Usage: pysetup search [project] [--simple [url]] [--xmlrpc [url] [--fieldname value ...] --operator or|and] + or: pysetup search --help + +Search the indexes for the matching projects. + +positional arguments: + project the project pattern to search for + +optional arguments: + --xmlrpc [url] whether to use the xmlrpc index or not. If an url is + specified, it will be used rather than the default one. + + --simple [url] whether to use the simple index or not. If an url is + specified, it will be used rather than the default one. + + --fieldname value Make a search on this field. Can only be used if + --xmlrpc has been selected or is the default index. + + --operator or|and Defines what is the operator to use when doing xmlrpc + searchs with multiple fieldnames. Can only be used if + --xmlrpc has been selected or is the default index. +""") +def _search(dispatcher, args, **kw): + """The search action. + + It is able to search for a specific index (specified with --index), using + the simple or xmlrpc index types (with --type xmlrpc / --type simple) + """ + #opts = _parse_args(args[1:], '', ['simple', 'xmlrpc']) + # 1. what kind of index is requested ? (xmlrpc / simple) + logger.error('not implemented') + return 1 + + +actions = [ + ('run', 'Run one or several commands', _run), + ('metadata', 'Display the metadata of a project', _metadata), + ('install', 'Install a project', _install), + ('remove', 'Remove a project', _remove), + ('search', 'Search for a project in the indexes', _search), + ('list', 'List installed projects', _list), + ('graph', 'Display a graph', _graph), + ('create', 'Create a project', _create), + ('generate-setup', 'Generate a backward-compatible setup.py', _generate), +] + + +class Dispatcher: + """Reads the command-line options + """ + def __init__(self, args=None): + self.verbose = 1 + self.dry_run = False + self.help = False + self.cmdclass = {} + self.commands = [] + self.command_options = {} + + for attr in display_option_names: + setattr(self, attr, False) + + self.parser = FancyGetopt(global_options + display_options) + self.parser.set_negative_aliases(negative_opt) + # FIXME this parses everything, including command options (e.g. "run + # build -i" errors with "option -i not recognized") + args = self.parser.getopt(args=args, object=self) + + # if first arg is "run", we have some commands + if len(args) == 0: + self.action = None + else: + self.action = args[0] + + allowed = [action[0] for action in actions] + [None] + if self.action not in allowed: + msg = 'Unrecognized action "%s"' % self.action + raise PackagingArgError(msg) + + self._set_logger() + self.args = args + + # for display options we return immediately + if self.help or self.action is None: + self._show_help(self.parser, display_options_=False) + + def _set_logger(self): + # setting up the logging level from the command-line options + # -q gets warning, error and critical + if self.verbose == 0: + level = logging.WARNING + # default level or -v gets info too + # XXX there's a bug somewhere: the help text says that -v is default + # (and verbose is set to 1 above), but when the user explicitly gives + # -v on the command line, self.verbose is incremented to 2! Here we + # compensate for that (I tested manually). On a related note, I think + # it's a good thing to use -q/nothing/-v/-vv on the command line + # instead of logging constants; it will be easy to add support for + # logging configuration in setup.cfg for advanced users. --merwok + elif self.verbose in (1, 2): + level = logging.INFO + else: # -vv and more for debug + level = logging.DEBUG + + # setting up the stream handler + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(level) + logger.addHandler(handler) + logger.setLevel(level) + + def _parse_command_opts(self, parser, args): + # Pull the current command from the head of the command line + command = args[0] + if not command_re.match(command): + raise SystemExit("invalid command name %r" % (command,)) + self.commands.append(command) + + # Dig up the command class that implements this command, so we + # 1) know that it's a valid command, and 2) know which options + # it takes. + try: + cmd_class = get_command_class(command) + except PackagingModuleError as msg: + raise PackagingArgError(msg) + + # XXX We want to push this in packaging.command + # + # Require that the command class be derived from Command -- want + # to be sure that the basic "command" interface is implemented. + for meth in ('initialize_options', 'finalize_options', 'run'): + if hasattr(cmd_class, meth): + continue + raise PackagingClassError( + 'command %r must implement %r' % (cmd_class, meth)) + + # Also make sure that the command object provides a list of its + # known options. + if not (hasattr(cmd_class, 'user_options') and + isinstance(cmd_class.user_options, list)): + raise PackagingClassError( + "command class %s must provide " + "'user_options' attribute (a list of tuples)" % cmd_class) + + # If the command class has a list of negative alias options, + # merge it in with the global negative aliases. + _negative_opt = negative_opt.copy() + + if hasattr(cmd_class, 'negative_opt'): + _negative_opt.update(cmd_class.negative_opt) + + # Check for help_options in command class. They have a different + # format (tuple of four) so we need to preprocess them here. + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_options = cmd_class.help_options[:] + else: + help_options = [] + + # All commands support the global options too, just by adding + # in 'global_options'. + parser.set_option_table(global_options + + cmd_class.user_options + + help_options) + parser.set_negative_aliases(_negative_opt) + args, opts = parser.getopt(args[1:]) + + if hasattr(opts, 'help') and opts.help: + self._show_command_help(cmd_class) + return + + if (hasattr(cmd_class, 'help_options') and + isinstance(cmd_class.help_options, list)): + help_option_found = False + for help_option, short, desc, func in cmd_class.help_options: + if hasattr(opts, help_option.replace('-', '_')): + help_option_found = True + if callable(func): + func() + else: + raise PackagingClassError( + "invalid help function %r for help option %r: " + "must be a callable object (function, etc.)" + % (func, help_option)) + + if help_option_found: + return + + # Put the options from the command line into their official + # holding pen, the 'command_options' dictionary. + opt_dict = self.get_option_dict(command) + for name, value in vars(opts).items(): + opt_dict[name] = ("command line", value) + + return args + + def get_option_dict(self, command): + """Get the option dictionary for a given command. If that + command's option dictionary hasn't been created yet, then create it + and return the new dictionary; otherwise, return the existing + option dictionary. + """ + d = self.command_options.get(command) + if d is None: + d = self.command_options[command] = {} + return d + + def show_help(self): + self._show_help(self.parser) + + def print_usage(self, parser): + parser.set_option_table(global_options) + + actions_ = [' %s: %s' % (name, desc) for name, desc, __ in actions] + usage = common_usage % {'actions': '\n'.join(actions_)} + + parser.print_help(usage + "\nGlobal options:") + + def _show_help(self, parser, global_options_=True, display_options_=True, + commands=[]): + # late import because of mutual dependence between these modules + from packaging.command.cmd import Command + + print('Usage: pysetup [options] action [action_options]') + print() + if global_options_: + self.print_usage(self.parser) + print() + + if display_options_: + parser.set_option_table(display_options) + parser.print_help( + "Information display options (just display " + + "information, ignore any commands)") + print() + + for command in commands: + if isinstance(command, type) and issubclass(command, Command): + cls = command + else: + cls = get_command_class(command) + if (hasattr(cls, 'help_options') and + isinstance(cls.help_options, list)): + parser.set_option_table(cls.user_options + cls.help_options) + else: + parser.set_option_table(cls.user_options) + + parser.print_help("Options for %r command:" % cls.__name__) + print() + + def _show_command_help(self, command): + if isinstance(command, str): + command = get_command_class(command) + + desc = getattr(command, 'description', '(no description available)') + print('Description:', desc) + print() + + if (hasattr(command, 'help_options') and + isinstance(command.help_options, list)): + self.parser.set_option_table(command.user_options + + command.help_options) + else: + self.parser.set_option_table(command.user_options) + + self.parser.print_help("Options:") + print() + + def _get_command_groups(self): + """Helper function to retrieve all the command class names divided + into standard commands (listed in + packaging.command.STANDARD_COMMANDS) and extra commands (given in + self.cmdclass and not standard commands). + """ + extra_commands = [cmd for cmd in self.cmdclass + if cmd not in STANDARD_COMMANDS] + return STANDARD_COMMANDS, extra_commands + + def print_commands(self): + """Print out a help message listing all available commands with a + description of each. The list is divided into standard commands + (listed in packaging.command.STANDARD_COMMANDS) and extra commands + (given in self.cmdclass and not standard commands). The + descriptions come from the command class attribute + 'description'. + """ + std_commands, extra_commands = self._get_command_groups() + max_length = max(len(command) + for commands in (std_commands, extra_commands) + for command in commands) + + self.print_command_list(std_commands, "Standard commands", max_length) + if extra_commands: + print() + self.print_command_list(extra_commands, "Extra commands", + max_length) + + def print_command_list(self, commands, header, max_length): + """Print a subset of the list of all commands -- used by + 'print_commands()'. + """ + print(header + ":") + + for cmd in commands: + cls = self.cmdclass.get(cmd) or get_command_class(cmd) + description = getattr(cls, 'description', + '(no description available)') + + print(" %-*s %s" % (max_length, cmd, description)) + + def __call__(self): + if self.action is None: + return + + for action, desc, func in actions: + if action == self.action: + return func(self, self.args) + return -1 + + +def main(args=None): + old_level = logger.level + old_handlers = list(logger.handlers) + try: + dispatcher = Dispatcher(args) + if dispatcher.action is None: + return + return dispatcher() + except KeyboardInterrupt: + logger.info('interrupted') + return 1 + except (IOError, os.error, PackagingError, CCompilerError) as exc: + logger.exception(exc) + return 1 + finally: + logger.setLevel(old_level) + logger.handlers[:] = old_handlers + + +if __name__ == '__main__': + sys.exit(main()) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/LONG_DESC.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/LONG_DESC.txt Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,44 @@ +CLVault +======= + +CLVault uses Keyring to provide a command-line utility to safely store +and retrieve passwords. + +Install it using pip or the setup.py script:: + + $ python setup.py install + + $ pip install clvault + +Once it's installed, you will have three scripts installed in your +Python scripts folder, you can use to list, store and retrieve passwords:: + + $ clvault-set blog + Set your password: + Set the associated username (can be blank): tarek + Set a description (can be blank): My blog password + Password set. + + $ clvault-get blog + The username is "tarek" + The password has been copied in your clipboard + + $ clvault-list + Registered services: + blog My blog password + + +*clvault-set* takes a service name then prompt you for a password, and some +optional information about your service. The password is safely stored in +a keyring while the description is saved in a ``.clvault`` file in your +home directory. This file is created automatically the first time the command +is used. + +*clvault-get* copies the password for a given service in your clipboard, and +displays the associated user if any. + +*clvault-list* lists all registered services, with their description when +given. + + +Project page: http://bitbucket.org/tarek/clvault diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/PKG-INFO --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/PKG-INFO Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,57 @@ +Metadata-Version: 1.2 +Name: CLVault +Version: 0.5 +Summary: Command-Line utility to store and retrieve passwords +Home-page: http://bitbucket.org/tarek/clvault +Author: Tarek Ziade +Author-email: tarek@ziade.org +License: PSF +Keywords: keyring,password,crypt +Requires-Dist: foo; sys.platform == 'okook' +Requires-Dist: bar; sys.platform == '%s' +Platform: UNKNOWN +Description: CLVault + |======= + | + |CLVault uses Keyring to provide a command-line utility to safely store + |and retrieve passwords. + | + |Install it using pip or the setup.py script:: + | + | $ python setup.py install + | + | $ pip install clvault + | + |Once it's installed, you will have three scripts installed in your + |Python scripts folder, you can use to list, store and retrieve passwords:: + | + | $ clvault-set blog + | Set your password: + | Set the associated username (can be blank): tarek + | Set a description (can be blank): My blog password + | Password set. + | + | $ clvault-get blog + | The username is "tarek" + | The password has been copied in your clipboard + | + | $ clvault-list + | Registered services: + | blog My blog password + | + | + |*clvault-set* takes a service name then prompt you for a password, and some + |optional information about your service. The password is safely stored in + |a keyring while the description is saved in a ``.clvault`` file in your + |home directory. This file is created automatically the first time the command + |is used. + | + |*clvault-get* copies the password for a given service in your clipboard, and + |displays the associated user if any. + | + |*clvault-list* lists all registered services, with their description when + |given. + | + | + |Project page: http://bitbucket.org/tarek/clvault + | diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/SETUPTOOLS-PKG-INFO --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,182 @@ +Metadata-Version: 1.0 +Name: setuptools +Version: 0.6c9 +Summary: Download, build, install, upgrade, and uninstall Python packages -- easily! +Home-page: http://pypi.python.org/pypi/setuptools +Author: Phillip J. Eby +Author-email: distutils-sig@python.org +License: PSF or ZPL +Description: =============================== + Installing and Using Setuptools + =============================== + + .. contents:: **Table of Contents** + + + ------------------------- + Installation Instructions + ------------------------- + + Windows + ======= + + Install setuptools using the provided ``.exe`` installer. If you've previously + installed older versions of setuptools, please delete all ``setuptools*.egg`` + and ``setuptools.pth`` files from your system's ``site-packages`` directory + (and any other ``sys.path`` directories) FIRST. + + If you are upgrading a previous version of setuptools that was installed using + an ``.exe`` installer, please be sure to also *uninstall that older version* + via your system's "Add/Remove Programs" feature, BEFORE installing the newer + version. + + Once installation is complete, you will find an ``easy_install.exe`` program in + your Python ``Scripts`` subdirectory. Be sure to add this directory to your + ``PATH`` environment variable, if you haven't already done so. + + + RPM-Based Systems + ================= + + Install setuptools using the provided source RPM. The included ``.spec`` file + assumes you are installing using the default ``python`` executable, and is not + specific to a particular Python version. The ``easy_install`` executable will + be installed to a system ``bin`` directory such as ``/usr/bin``. + + If you wish to install to a location other than the default Python + installation's default ``site-packages`` directory (and ``$prefix/bin`` for + scripts), please use the ``.egg``-based installation approach described in the + following section. + + + Cygwin, Mac OS X, Linux, Other + ============================== + + 1. Download the appropriate egg for your version of Python (e.g. + ``setuptools-0.6c9-py2.4.egg``). Do NOT rename it. + + 2. Run it as if it were a shell script, e.g. ``sh setuptools-0.6c9-py2.4.egg``. + Setuptools will install itself using the matching version of Python (e.g. + ``python2.4``), and will place the ``easy_install`` executable in the + default location for installing Python scripts (as determined by the + standard distutils configuration files, or by the Python installation). + + If you want to install setuptools to somewhere other than ``site-packages`` or + your default distutils installation locations for libraries and scripts, you + may include EasyInstall command-line options such as ``--prefix``, + ``--install-dir``, and so on, following the ``.egg`` filename on the same + command line. For example:: + + sh setuptools-0.6c9-py2.4.egg --prefix=~ + + You can use ``--help`` to get a full options list, but we recommend consulting + the `EasyInstall manual`_ for detailed instructions, especially `the section + on custom installation locations`_. + + .. _EasyInstall manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _the section on custom installation locations: http://peak.telecommunity.com/DevCenter/EasyInstall#custom-installation-locations + + + Cygwin Note + ----------- + + If you are trying to install setuptools for the **Windows** version of Python + (as opposed to the Cygwin version that lives in ``/usr/bin``), you must make + sure that an appropriate executable (``python2.3``, ``python2.4``, or + ``python2.5``) is on your **Cygwin** ``PATH`` when invoking the egg. For + example, doing the following at a Cygwin bash prompt will install setuptools + for the **Windows** Python found at ``C:\\Python24``:: + + ln -s /cygdrive/c/Python24/python.exe python2.4 + PATH=.:$PATH sh setuptools-0.6c9-py2.4.egg + rm python2.4 + + + Downloads + ========= + + All setuptools downloads can be found at `the project's home page in the Python + Package Index`_. Scroll to the very bottom of the page to find the links. + + .. _the project's home page in the Python Package Index: http://pypi.python.org/pypi/setuptools + + In addition to the PyPI downloads, the development version of ``setuptools`` + is available from the `Python SVN sandbox`_, and in-development versions of the + `0.6 branch`_ are available as well. + + .. _0.6 branch: http://svn.python.org/projects/sandbox/branches/setuptools-0.6/#egg=setuptools-dev06 + + .. _Python SVN sandbox: http://svn.python.org/projects/sandbox/trunk/setuptools/#egg=setuptools-dev + + -------------------------------- + Using Setuptools and EasyInstall + -------------------------------- + + Here are some of the available manuals, tutorials, and other resources for + learning about Setuptools, Python Eggs, and EasyInstall: + + * `The EasyInstall user's guide and reference manual`_ + * `The setuptools Developer's Guide`_ + * `The pkg_resources API reference`_ + * `Package Compatibility Notes`_ (user-maintained) + * `The Internal Structure of Python Eggs`_ + + Questions, comments, and bug reports should be directed to the `distutils-sig + mailing list`_. If you have written (or know of) any tutorials, documentation, + plug-ins, or other resources for setuptools users, please let us know about + them there, so this reference list can be updated. If you have working, + *tested* patches to correct problems or add features, you may submit them to + the `setuptools bug tracker`_. + + .. _setuptools bug tracker: http://bugs.python.org/setuptools/ + .. _Package Compatibility Notes: http://peak.telecommunity.com/DevCenter/PackageNotes + .. _The Internal Structure of Python Eggs: http://peak.telecommunity.com/DevCenter/EggFormats + .. _The setuptools Developer's Guide: http://peak.telecommunity.com/DevCenter/setuptools + .. _The pkg_resources API reference: http://peak.telecommunity.com/DevCenter/PkgResources + .. _The EasyInstall user's guide and reference manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ + + + ------- + Credits + ------- + + * The original design for the ``.egg`` format and the ``pkg_resources`` API was + co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first + version of ``pkg_resources``, and supplied the OS X operating system version + compatibility algorithm. + + * Ian Bicking implemented many early "creature comfort" features of + easy_install, including support for downloading via Sourceforge and + Subversion repositories. Ian's comments on the Web-SIG about WSGI + application deployment also inspired the concept of "entry points" in eggs, + and he has given talks at PyCon and elsewhere to inform and educate the + community about eggs and setuptools. + + * Jim Fulton contributed time and effort to build automated tests of various + aspects of ``easy_install``, and supplied the doctests for the command-line + ``.exe`` wrappers on Windows. + + * Phillip J. Eby is the principal author and maintainer of setuptools, and + first proposed the idea of an importable binary distribution format for + Python application plug-ins. + + * Significant parts of the implementation of setuptools were funded by the Open + Source Applications Foundation, to provide a plug-in infrastructure for the + Chandler PIM application. In addition, many OSAF staffers (such as Mike + "Code Bear" Taylor) contributed their time and stress as guinea pigs for the + use of eggs and setuptools, even before eggs were "cool". (Thanks, guys!) + + +Keywords: CPAN PyPI distutils eggs package management +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: License :: OSI Approved :: Zope Public License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/SETUPTOOLS-PKG-INFO2 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/SETUPTOOLS-PKG-INFO2 Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,183 @@ +Metadata-Version: 1.1 +Name: setuptools +Version: 0.6c9 +Summary: Download, build, install, upgrade, and uninstall Python packages -- easily! +Home-page: http://pypi.python.org/pypi/setuptools +Author: Phillip J. Eby +Author-email: distutils-sig@python.org +License: PSF or ZPL +Description: =============================== + Installing and Using Setuptools + =============================== + + .. contents:: **Table of Contents** + + + ------------------------- + Installation Instructions + ------------------------- + + Windows + ======= + + Install setuptools using the provided ``.exe`` installer. If you've previously + installed older versions of setuptools, please delete all ``setuptools*.egg`` + and ``setuptools.pth`` files from your system's ``site-packages`` directory + (and any other ``sys.path`` directories) FIRST. + + If you are upgrading a previous version of setuptools that was installed using + an ``.exe`` installer, please be sure to also *uninstall that older version* + via your system's "Add/Remove Programs" feature, BEFORE installing the newer + version. + + Once installation is complete, you will find an ``easy_install.exe`` program in + your Python ``Scripts`` subdirectory. Be sure to add this directory to your + ``PATH`` environment variable, if you haven't already done so. + + + RPM-Based Systems + ================= + + Install setuptools using the provided source RPM. The included ``.spec`` file + assumes you are installing using the default ``python`` executable, and is not + specific to a particular Python version. The ``easy_install`` executable will + be installed to a system ``bin`` directory such as ``/usr/bin``. + + If you wish to install to a location other than the default Python + installation's default ``site-packages`` directory (and ``$prefix/bin`` for + scripts), please use the ``.egg``-based installation approach described in the + following section. + + + Cygwin, Mac OS X, Linux, Other + ============================== + + 1. Download the appropriate egg for your version of Python (e.g. + ``setuptools-0.6c9-py2.4.egg``). Do NOT rename it. + + 2. Run it as if it were a shell script, e.g. ``sh setuptools-0.6c9-py2.4.egg``. + Setuptools will install itself using the matching version of Python (e.g. + ``python2.4``), and will place the ``easy_install`` executable in the + default location for installing Python scripts (as determined by the + standard distutils configuration files, or by the Python installation). + + If you want to install setuptools to somewhere other than ``site-packages`` or + your default distutils installation locations for libraries and scripts, you + may include EasyInstall command-line options such as ``--prefix``, + ``--install-dir``, and so on, following the ``.egg`` filename on the same + command line. For example:: + + sh setuptools-0.6c9-py2.4.egg --prefix=~ + + You can use ``--help`` to get a full options list, but we recommend consulting + the `EasyInstall manual`_ for detailed instructions, especially `the section + on custom installation locations`_. + + .. _EasyInstall manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _the section on custom installation locations: http://peak.telecommunity.com/DevCenter/EasyInstall#custom-installation-locations + + + Cygwin Note + ----------- + + If you are trying to install setuptools for the **Windows** version of Python + (as opposed to the Cygwin version that lives in ``/usr/bin``), you must make + sure that an appropriate executable (``python2.3``, ``python2.4``, or + ``python2.5``) is on your **Cygwin** ``PATH`` when invoking the egg. For + example, doing the following at a Cygwin bash prompt will install setuptools + for the **Windows** Python found at ``C:\\Python24``:: + + ln -s /cygdrive/c/Python24/python.exe python2.4 + PATH=.:$PATH sh setuptools-0.6c9-py2.4.egg + rm python2.4 + + + Downloads + ========= + + All setuptools downloads can be found at `the project's home page in the Python + Package Index`_. Scroll to the very bottom of the page to find the links. + + .. _the project's home page in the Python Package Index: http://pypi.python.org/pypi/setuptools + + In addition to the PyPI downloads, the development version of ``setuptools`` + is available from the `Python SVN sandbox`_, and in-development versions of the + `0.6 branch`_ are available as well. + + .. _0.6 branch: http://svn.python.org/projects/sandbox/branches/setuptools-0.6/#egg=setuptools-dev06 + + .. _Python SVN sandbox: http://svn.python.org/projects/sandbox/trunk/setuptools/#egg=setuptools-dev + + -------------------------------- + Using Setuptools and EasyInstall + -------------------------------- + + Here are some of the available manuals, tutorials, and other resources for + learning about Setuptools, Python Eggs, and EasyInstall: + + * `The EasyInstall user's guide and reference manual`_ + * `The setuptools Developer's Guide`_ + * `The pkg_resources API reference`_ + * `Package Compatibility Notes`_ (user-maintained) + * `The Internal Structure of Python Eggs`_ + + Questions, comments, and bug reports should be directed to the `distutils-sig + mailing list`_. If you have written (or know of) any tutorials, documentation, + plug-ins, or other resources for setuptools users, please let us know about + them there, so this reference list can be updated. If you have working, + *tested* patches to correct problems or add features, you may submit them to + the `setuptools bug tracker`_. + + .. _setuptools bug tracker: http://bugs.python.org/setuptools/ + .. _Package Compatibility Notes: http://peak.telecommunity.com/DevCenter/PackageNotes + .. _The Internal Structure of Python Eggs: http://peak.telecommunity.com/DevCenter/EggFormats + .. _The setuptools Developer's Guide: http://peak.telecommunity.com/DevCenter/setuptools + .. _The pkg_resources API reference: http://peak.telecommunity.com/DevCenter/PkgResources + .. _The EasyInstall user's guide and reference manual: http://peak.telecommunity.com/DevCenter/EasyInstall + .. _distutils-sig mailing list: http://mail.python.org/pipermail/distutils-sig/ + + + ------- + Credits + ------- + + * The original design for the ``.egg`` format and the ``pkg_resources`` API was + co-created by Phillip Eby and Bob Ippolito. Bob also implemented the first + version of ``pkg_resources``, and supplied the OS X operating system version + compatibility algorithm. + + * Ian Bicking implemented many early "creature comfort" features of + easy_install, including support for downloading via Sourceforge and + Subversion repositories. Ian's comments on the Web-SIG about WSGI + application deployment also inspired the concept of "entry points" in eggs, + and he has given talks at PyCon and elsewhere to inform and educate the + community about eggs and setuptools. + + * Jim Fulton contributed time and effort to build automated tests of various + aspects of ``easy_install``, and supplied the doctests for the command-line + ``.exe`` wrappers on Windows. + + * Phillip J. Eby is the principal author and maintainer of setuptools, and + first proposed the idea of an importable binary distribution format for + Python application plug-ins. + + * Significant parts of the implementation of setuptools were funded by the Open + Source Applications Foundation, to provide a plug-in infrastructure for the + Chandler PIM application. In addition, many OSAF staffers (such as Mike + "Code Bear" Taylor) contributed their time and stress as guinea pigs for the + use of eggs and setuptools, even before eggs were "cool". (Thanks, guys!) + + +Keywords: CPAN PyPI distutils eggs package management +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: License :: OSI Approved :: Zope Public License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System :: Archiving :: Packaging +Classifier: Topic :: System :: Systems Administration +Classifier: Topic :: Utilities +Requires: Foo diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,28 @@ +"""Test suite for packaging. + +This test suite consists of a collection of test modules in the +packaging.tests package. Each test module has a name starting with +'test' and contains a function test_suite(). The function is expected +to return an initialized unittest.TestSuite instance. + +Utility code is included in packaging.tests.support. + +Always import unittest from this module: it will be unittest from the +standard library for packaging tests and unittest2 for distutils2 tests. +""" + +import os +import sys +import unittest + + +def test_suite(): + suite = unittest.TestSuite() + here = os.path.dirname(__file__) or os.curdir + for fn in os.listdir(here): + if fn.startswith("test") and fn.endswith(".py"): + modname = "packaging.tests." + fn[:-3] + __import__(modname) + module = sys.modules[modname] + suite.addTest(module.test_suite()) + return suite diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/__main__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/__main__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,24 @@ +"""Packaging test suite runner.""" + +# Ripped from importlib tests, thanks Brett! + +import os +import unittest +from test.support import run_unittest, reap_children, reap_threads + + +@reap_threads +def test_main(): + try: + start_dir = os.path.dirname(__file__) + top_dir = os.path.dirname(os.path.dirname(start_dir)) + test_loader = unittest.TestLoader() + # XXX find out how to use unittest.main, to get command-line options + # (failfast, catch, etc.) + run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir)) + finally: + reap_children() + + +if __name__ == '__main__': + test_main() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/METADATA Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,4 @@ +Metadata-version: 1.2 +Name: babar +Version: 0.1 +Author: FELD Boris \ No newline at end of file diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/babar-0.1.dist-info/RESOURCES Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,2 @@ +babar.png,babar.png +babar.cfg,babar.cfg \ No newline at end of file diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/babar.cfg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/babar.cfg Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +Config \ No newline at end of file diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/bacon-0.1.egg-info/PKG-INFO Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +Metadata-Version: 1.2 +Name: bacon +Version: 0.1 +Provides-Dist: truffles (2.0) +Provides-Dist: bacon (0.1) +Obsoletes-Dist: truffles (>=0.9,<=1.5) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/PKG-INFO Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,18 @@ +Metadata-Version: 1.0 +Name: banana +Version: 0.4 +Summary: A yellow fruit +Home-page: http://en.wikipedia.org/wiki/Banana +Author: Josip Djolonga +Author-email: foo@nbar.com +License: BSD +Description: A fruit +Keywords: foo bar +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Scientific/Engineering :: GIS diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/dependency_links.txt Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/entry_points.txt Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ + + # -*- Entry points: -*- + \ No newline at end of file diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/not-zip-safe Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/banana-0.4.egg/EGG-INFO/requires.txt Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +# this should be ignored + +strawberry >=0.5 + +[section ignored] +foo ==0.5 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/cheese-2.0.2.egg-info Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: cheese +Version: 2.0.2 +Provides-Dist: truffles (1.0.2) +Obsoletes-Dist: truffles (!=1.2,<=2.0) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,9 @@ +Metadata-Version: 1.2 +Name: choxie +Version: 2.0.0.9 +Summary: Chocolate with a kick! +Requires-Dist: towel-stuff (0.1) +Requires-Dist: nut +Provides-Dist: truffles (1.0) +Obsoletes-Dist: truffles (<=0.8,>=0.5) +Obsoletes-Dist: truffles (<=0.9,>=0.6) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +# -*- coding: utf-8 -*- diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from towel_stuff import Towel + +class Chocolate(object): + """A piece of chocolate.""" + + def wrap_with_towel(self): + towel = Towel() + towel.wrap(self) + return towel diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/choxie-2.0.0.9/truffles.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +from choxie.chocolate import Chocolate + +class Truffle(Chocolate): + """A truffle.""" diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/coconuts-aster-10.3.egg-info/PKG-INFO Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: coconuts-aster +Version: 10.3 +Provides-Dist: strawberry (0.6) +Provides-Dist: banana (0.4) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4.dist-info/METADATA Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +Metadata-Version: 1.2 +Name: grammar +Version: 1.0a4 +Requires-Dist: truffles (>=1.2) +Author: Sherlock Holmes diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +# -*- coding: utf-8 -*- diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/grammar-1.0a4/grammar/utils.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from random import randint + +def is_valid_grammar(sentence): + if randint(0, 10) < 2: + return False + else: + return True diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/nut-funkyversion.egg-info Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: nut +Version: funkyversion diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/strawberry-0.6.egg Binary file Lib/packaging/tests/fake_dists/strawberry-0.6.egg has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1.dist-info/METADATA Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,7 @@ +Metadata-Version: 1.2 +Name: towel-stuff +Version: 0.1 +Provides-Dist: truffles (1.1.2) +Provides-Dist: towel-stuff (0.1) +Obsoletes-Dist: truffles (!=0.8,<1.0) +Requires-Dist: bacon (<=0.2) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/towel_stuff-0.1/towel_stuff/__init__.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +class Towel(object): + """A towel, that one should never be without.""" + + def __init__(self, color='tie-dye'): + self.color = color + self.wrapped_obj = None + + def wrap(self, obj): + """Wrap an object up in our towel.""" + self.wrapped_obj = obj + + def unwrap(self): + """Unwrap whatever is in our towel and return whatever it is.""" + obj = self.wrapped_obj + self.wrapped_obj = None + return obj diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fake_dists/truffles-5.0.egg-info --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fake_dists/truffles-5.0.egg-info Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ +Metadata-Version: 1.2 +Name: truffles +Version: 5.0 diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fixer/fix_echo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fixer/fix_echo.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho(fixer_base.BaseFix): + + BM_compatible = True + PATTERN = """ + power< name='echo' trailer< '(' [any] ')' > any* > + """ + + def transform(self, node, results): + name = results['name'] + name.replace(Name('print', prefix=name.prefix)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/fixer/fix_echo2.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/fixer/fix_echo2.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho2(fixer_base.BaseFix): + + BM_compatible = True + PATTERN = """ + power< name='echo2' trailer< '(' [any] ')' > any* > + """ + + def transform(self, node, results): + name = results['name'] + name.replace(Name('print', prefix=name.prefix)) diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypi_server.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypi_server.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,449 @@ +"""Mock PyPI Server implementation, to use in tests. + +This module also provides a simple test case to extend if you need to use +the PyPIServer all along your test case. Be sure to read the documentation +before any use. + +XXX TODO: + +The mock server can handle simple HTTP request (to simulate a simple index) or +XMLRPC requests, over HTTP. Both does not have the same intergface to deal +with, and I think it's a pain. + +A good idea could be to re-think a bit the way dstributions are handled in the +mock server. As it should return malformed HTML pages, we need to keep the +static behavior. + +I think of something like that: + + >>> server = PyPIMockServer() + >>> server.startHTTP() + >>> server.startXMLRPC() + +Then, the server must have only one port to rely on, eg. + + >>> server.fulladdress() + "http://ip:port/" + +It could be simple to have one HTTP server, relaying the requests to the two +implementations (static HTTP and XMLRPC over HTTP). +""" + +import os +import queue +import select +import threading +from functools import wraps +from http.server import HTTPServer, SimpleHTTPRequestHandler +from xmlrpc.server import SimpleXMLRPCServer + +from packaging.tests import unittest + + +PYPI_DEFAULT_STATIC_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'pypiserver') + + +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + + +def use_pypi_server(*server_args, **server_kwargs): + """Decorator to make use of the PyPIServer for test methods, + just when needed, and not for the entire duration of the testcase. + """ + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + server = PyPIServer(*server_args, **server_kwargs) + server.start() + try: + func(server=server, *args, **kwargs) + finally: + server.stop() + return wrapped + return wrapper + + +class PyPIServerTestCase(unittest.TestCase): + + def setUp(self): + super(PyPIServerTestCase, self).setUp() + self.pypi = PyPIServer() + self.pypi.start() + self.addCleanup(self.pypi.stop) + + +class PyPIServer(threading.Thread): + """PyPI Mocked server. + Provides a mocked version of the PyPI API's, to ease tests. + + Support serving static content and serving previously given text. + """ + + def __init__(self, test_static_path=None, + static_filesystem_paths=None, + static_uri_paths=["simple", "packages"], serve_xmlrpc=False): + """Initialize the server. + + Default behavior is to start the HTTP server. You can either start the + xmlrpc server by setting xmlrpc to True. Caution: Only one server will + be started. + + static_uri_paths and static_base_path are parameters used to provides + respectively the http_paths to serve statically, and where to find the + matching files on the filesystem. + """ + # we want to launch the server in a new dedicated thread, to not freeze + # tests. + super(PyPIServer, self).__init__() + self._run = True + self._serve_xmlrpc = serve_xmlrpc + if static_filesystem_paths is None: + static_filesystem_paths = ["default"] + + #TODO allow to serve XMLRPC and HTTP static files at the same time. + if not self._serve_xmlrpc: + self.server = HTTPServer(('127.0.0.1', 0), PyPIRequestHandler) + self.server.RequestHandlerClass.pypi_server = self + + self.request_queue = queue.Queue() + self._requests = [] + self.default_response_status = 404 + self.default_response_headers = [('Content-type', 'text/plain')] + self.default_response_data = "The page does not exists" + + # initialize static paths / filesystems + self.static_uri_paths = static_uri_paths + + # append the static paths defined locally + if test_static_path is not None: + static_filesystem_paths.append(test_static_path) + self.static_filesystem_paths = [ + PYPI_DEFAULT_STATIC_PATH + "/" + path + for path in static_filesystem_paths] + else: + # XMLRPC server + self.server = PyPIXMLRPCServer(('127.0.0.1', 0)) + self.xmlrpc = XMLRPCMockIndex() + # register the xmlrpc methods + self.server.register_introspection_functions() + self.server.register_instance(self.xmlrpc) + + self.address = ('127.0.0.1', self.server.server_port) + # to not have unwanted outputs. + self.server.RequestHandlerClass.log_request = lambda *_: None + + def run(self): + # loop because we can't stop it otherwise, for python < 2.6 + while self._run: + r, w, e = select.select([self.server], [], [], 0.5) + if r: + self.server.handle_request() + + def stop(self): + """self shutdown is not supported for python < 2.6""" + self._run = False + if self.is_alive(): + self.join() + self.server.server_close() + + def get_next_response(self): + return (self.default_response_status, + self.default_response_headers, + self.default_response_data) + + @property + def requests(self): + """Use this property to get all requests that have been made + to the server + """ + while True: + try: + self._requests.append(self.request_queue.get_nowait()) + except queue.Empty: + break + return self._requests + + @property + def full_address(self): + return "http://%s:%s" % self.address + + +class PyPIRequestHandler(SimpleHTTPRequestHandler): + # we need to access the pypi server while serving the content + pypi_server = None + + def serve_request(self): + """Serve the content. + + Also record the requests to be accessed later. If trying to access an + url matching a static uri, serve static content, otherwise serve + what is provided by the `get_next_response` method. + + If nothing is defined there, return a 404 header. + """ + # record the request. Read the input only on PUT or POST requests + if self.command in ("PUT", "POST"): + if 'content-length' in self.headers: + request_data = self.rfile.read( + int(self.headers['content-length'])) + else: + request_data = self.rfile.read() + + elif self.command in ("GET", "DELETE"): + request_data = '' + + self.pypi_server.request_queue.put((self, request_data)) + + # serve the content from local disc if we request an URL beginning + # by a pattern defined in `static_paths` + url_parts = self.path.split("/") + if (len(url_parts) > 1 and + url_parts[1] in self.pypi_server.static_uri_paths): + data = None + # always take the last first. + fs_paths = [] + fs_paths.extend(self.pypi_server.static_filesystem_paths) + fs_paths.reverse() + relative_path = self.path + for fs_path in fs_paths: + try: + if self.path.endswith("/"): + relative_path += "index.html" + + if relative_path.endswith('.tar.gz'): + with open(fs_path + relative_path, 'rb') as file: + data = file.read() + headers = [('Content-type', 'application/x-gtar')] + else: + with open(fs_path + relative_path) as file: + data = file.read().encode() + headers = [('Content-type', 'text/html')] + + headers.append(('Content-Length', len(data))) + self.make_response(data, headers=headers) + + except IOError: + pass + + if data is None: + self.make_response("Not found", 404) + + # otherwise serve the content from get_next_response + else: + # send back a response + status, headers, data = self.pypi_server.get_next_response() + self.make_response(data, status, headers) + + do_POST = do_GET = do_DELETE = do_PUT = serve_request + + def make_response(self, data, status=200, + headers=[('Content-type', 'text/html')]): + """Send the response to the HTTP client""" + if not isinstance(status, int): + try: + status = int(status) + except ValueError: + # we probably got something like YYY Codename. + # Just get the first 3 digits + status = int(status[:3]) + + self.send_response(status) + for header, value in headers: + self.send_header(header, value) + self.end_headers() + + if isinstance(data, str): + data = data.encode('utf-8') + + self.wfile.write(data) + + +class PyPIXMLRPCServer(SimpleXMLRPCServer): + def server_bind(self): + """Override server_bind to store the server name.""" + super(PyPIXMLRPCServer, self).server_bind() + host, port = self.socket.getsockname()[:2] + self.server_port = port + + +class MockDist: + """Fake distribution, used in the Mock PyPI Server""" + + def __init__(self, name, version="1.0", hidden=False, url="http://url/", + type="sdist", filename="", size=10000, + digest="123456", downloads=7, has_sig=False, + python_version="source", comment="comment", + author="John Doe", author_email="john@doe.name", + maintainer="Main Tayner", maintainer_email="maintainer_mail", + project_url="http://project_url/", homepage="http://homepage/", + keywords="", platform="UNKNOWN", classifiers=[], licence="", + description="Description", summary="Summary", stable_version="", + ordering="", documentation_id="", code_kwalitee_id="", + installability_id="", obsoletes=[], obsoletes_dist=[], + provides=[], provides_dist=[], requires=[], requires_dist=[], + requires_external=[], requires_python=""): + + # basic fields + self.name = name + self.version = version + self.hidden = hidden + + # URL infos + self.url = url + self.digest = digest + self.downloads = downloads + self.has_sig = has_sig + self.python_version = python_version + self.comment = comment + self.type = type + + # metadata + self.author = author + self.author_email = author_email + self.maintainer = maintainer + self.maintainer_email = maintainer_email + self.project_url = project_url + self.homepage = homepage + self.keywords = keywords + self.platform = platform + self.classifiers = classifiers + self.licence = licence + self.description = description + self.summary = summary + self.stable_version = stable_version + self.ordering = ordering + self.cheesecake_documentation_id = documentation_id + self.cheesecake_code_kwalitee_id = code_kwalitee_id + self.cheesecake_installability_id = installability_id + + self.obsoletes = obsoletes + self.obsoletes_dist = obsoletes_dist + self.provides = provides + self.provides_dist = provides_dist + self.requires = requires + self.requires_dist = requires_dist + self.requires_external = requires_external + self.requires_python = requires_python + + def url_infos(self): + return { + 'url': self.url, + 'packagetype': self.type, + 'filename': 'filename.tar.gz', + 'size': '6000', + 'md5_digest': self.digest, + 'downloads': self.downloads, + 'has_sig': self.has_sig, + 'python_version': self.python_version, + 'comment_text': self.comment, + } + + def metadata(self): + return { + 'maintainer': self.maintainer, + 'project_url': [self.project_url], + 'maintainer_email': self.maintainer_email, + 'cheesecake_code_kwalitee_id': self.cheesecake_code_kwalitee_id, + 'keywords': self.keywords, + 'obsoletes_dist': self.obsoletes_dist, + 'requires_external': self.requires_external, + 'author': self.author, + 'author_email': self.author_email, + 'download_url': self.url, + 'platform': self.platform, + 'version': self.version, + 'obsoletes': self.obsoletes, + 'provides': self.provides, + 'cheesecake_documentation_id': self.cheesecake_documentation_id, + '_pypi_hidden': self.hidden, + 'description': self.description, + '_pypi_ordering': 19, + 'requires_dist': self.requires_dist, + 'requires_python': self.requires_python, + 'classifiers': [], + 'name': self.name, + 'licence': self.licence, # XXX licence or license? + 'summary': self.summary, + 'home_page': self.homepage, + 'stable_version': self.stable_version, + # FIXME doesn't that reproduce the bug from 6527d3106e9f? + 'provides_dist': (self.provides_dist or + "%s (%s)" % (self.name, self.version)), + 'requires': self.requires, + 'cheesecake_installability_id': self.cheesecake_installability_id, + } + + def search_result(self): + return { + '_pypi_ordering': 0, + 'version': self.version, + 'name': self.name, + 'summary': self.summary, + } + + +class XMLRPCMockIndex: + """Mock XMLRPC server""" + + def __init__(self, dists=[]): + self._dists = dists + self._search_result = [] + + def add_distributions(self, dists): + for dist in dists: + self._dists.append(MockDist(**dist)) + + def set_distributions(self, dists): + self._dists = [] + self.add_distributions(dists) + + def set_search_result(self, result): + """set a predefined search result""" + self._search_result = result + + def _get_search_results(self): + results = [] + for name in self._search_result: + found_dist = [d for d in self._dists if d.name == name] + if found_dist: + results.append(found_dist[0]) + else: + dist = MockDist(name) + results.append(dist) + self._dists.append(dist) + return [r.search_result() for r in results] + + def list_packages(self): + return [d.name for d in self._dists] + + def package_releases(self, package_name, show_hidden=False): + if show_hidden: + # return all + return [d.version for d in self._dists if d.name == package_name] + else: + # return only un-hidden + return [d.version for d in self._dists if d.name == package_name + and not d.hidden] + + def release_urls(self, package_name, version): + return [d.url_infos() for d in self._dists + if d.name == package_name and d.version == version] + + def release_data(self, package_name, version): + release = [d for d in self._dists + if d.name == package_name and d.version == version] + if release: + return release[0].metadata() + else: + return {} + + def search(self, spec, operator="and"): + return self._get_search_results() diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypi_test_server.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypi_test_server.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,59 @@ +"""Test PyPI Server implementation at testpypi.python.org, to use in tests. + +This is a drop-in replacement for the mock pypi server for testing against a +real pypi server hosted by python.org especially for testing against. +""" + +import unittest + +PYPI_DEFAULT_STATIC_PATH = None + + +def use_xmlrpc_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = True + return use_pypi_server(*server_args, **server_kwargs) + + +def use_http_server(*server_args, **server_kwargs): + server_kwargs['serve_xmlrpc'] = False + return use_pypi_server(*server_args, **server_kwargs) + + +def use_pypi_server(*server_args, **server_kwargs): + """Decorator to make use of the PyPIServer for test methods, + just when needed, and not for the entire duration of the testcase. + """ + def wrapper(func): + def wrapped(*args, **kwargs): + server = PyPIServer(*server_args, **server_kwargs) + func(server=server, *args, **kwargs) + return wrapped + return wrapper + + +class PyPIServerTestCase(unittest.TestCase): + + def setUp(self): + super(PyPIServerTestCase, self).setUp() + self.pypi = PyPIServer() + self.pypi.start() + self.addCleanup(self.pypi.stop) + + +class PyPIServer: + """Shim to access testpypi.python.org, for testing a real server.""" + + def __init__(self, test_static_path=None, + static_filesystem_paths=["default"], + static_uri_paths=["simple"], serve_xmlrpc=False): + self.address = ('testpypi.python.org', '80') + + def start(self): + pass + + def stop(self): + pass + + @property + def full_address(self): + return "http://%s:%s" % self.address diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gz Binary file Lib/packaging/tests/pypiserver/downloads_with_md5/packages/source/f/foobar/foobar-0.1.tar.gz has changed diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/badmd5/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ + +badmd5-0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/foobar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ + +foobar-0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/downloads_with_md5/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,2 @@ +foobar/ +badmd5/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/bar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +Links for bar

    Links for bar

    +bar-1.0.tar.gz
    +bar-1.0.1.tar.gz
    +bar-2.0.tar.gz
    +bar-2.0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/baz/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +Links for baz

    Links for baz

    +baz-1.0.tar.gz
    +baz-1.0.1.tar.gz
    +baz-2.0.tar.gz
    +baz-2.0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/foo/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +Links for foo

    Links for foo

    +foo-1.0.tar.gz
    +foo-1.0.1.tar.gz
    +foo-2.0.tar.gz
    +foo-2.0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/foo_bar_baz/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ +foo/ +bar/ +baz/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/project_list/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/project_list/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,5 @@ +FooBar-bar +Foobar-baz +Baz-FooBar +Baz +Foo diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/test_found_links/simple/foobar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ +Links for Foobar

    Links for Foobar

    +Foobar-1.0.tar.gz
    +Foobar-1.0.1.tar.gz
    +Foobar-2.0.tar.gz
    +Foobar-2.0.1.tar.gz
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/test_found_links/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/test_found_links/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +foobar/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/test_pypi_server/external/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +index.html from external server diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/test_pypi_server/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +Yeah diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_externals/external/external.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_externals/external/external.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,3 @@ + +bad old link + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_externals/simple/foobar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,4 @@ + +foobar-0.1.tar.gz
    +external homepage
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_externals/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_externals/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +foobar/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_norel_links/external/homepage.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,7 @@ + + +

    a rel=homepage HTML page

    +foobar 2.0 + + + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_norel_links/external/nonrel.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +A page linked without rel="download" or rel="homepage" link. diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_norel_links/simple/foobar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,6 @@ + +foobar-0.1.tar.gz
    +external homepage
    +unrelated link
    +unrelated download
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_norel_links/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +foobar/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_real_externals/simple/foobar/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,4 @@ + +foobar-0.1.tar.gz
    +external homepage
    + diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/pypiserver/with_real_externals/simple/index.html Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,1 @@ +foobar/ diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/support.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/support.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,400 @@ +"""Support code for packaging test cases. + +*This module should not be considered public: its content and API may +change in incompatible ways.* + +A few helper classes are provided: LoggingCatcher, TempdirManager and +EnvironRestorer. They are written to be used as mixins:: + + from packaging.tests import unittest + from packaging.tests.support import LoggingCatcher + + class SomeTestCase(LoggingCatcher, unittest.TestCase): + ... + +If you need to define a setUp method on your test class, you have to +call the mixin class' setUp method or it won't work (same thing for +tearDown): + + def setUp(self): + super(SomeTestCase, self).setUp() + ... # other setup code + +Also provided is a DummyCommand class, useful to mock commands in the +tests of another command that needs them, for example to fake +compilation in build_ext (this requires that the mock build_ext command +be injected into the distribution object's command_obj dictionary). + +For tests that need to compile an extension module, use the +copy_xxmodule_c and fixup_build_ext functions. + +Each class or function has a docstring to explain its purpose and usage. +Existing tests should also be used as examples. +""" + +import os +import sys +import shutil +import logging +import weakref +import tempfile +import sysconfig + +from packaging.dist import Distribution +from packaging.util import resolve_name +from packaging.command import set_command, _COMMANDS + +from packaging.tests import unittest +from test.support import requires_zlib, unlink + +# define __all__ to make pydoc more useful +__all__ = [ + # TestCase mixins + 'LoggingCatcher', 'TempdirManager', 'EnvironRestorer', + # mocks + 'DummyCommand', 'TestDistribution', 'Inputs', + # misc. functions and decorators + 'fake_dec', 'create_distribution', 'use_command', + 'copy_xxmodule_c', 'fixup_build_ext', + 'skip_2to3_optimize', + # imported from this module for backport purposes + 'unittest', 'requires_zlib', 'skip_unless_symlink', +] + + +logger = logging.getLogger('packaging') +logger2to3 = logging.getLogger('RefactoringTool') + + +class _TestHandler(logging.handlers.BufferingHandler): + # stolen and adapted from test.support + + def __init__(self): + super(_TestHandler, self).__init__(0) + self.setLevel(logging.DEBUG) + + def shouldFlush(self): + return False + + def emit(self, record): + self.buffer.append(record) + + +class LoggingCatcher: + """TestCase-compatible mixin to receive logging calls. + + Upon setUp, instances of this classes get a BufferingHandler that's + configured to record all messages logged to the 'packaging' logger. + + Use get_logs to retrieve messages and self.loghandler.flush to discard + them. get_logs automatically flushes the logs, unless you pass + *flush=False*, for example to make multiple calls to the method with + different level arguments. If your test calls some code that generates + logging message and then you don't call get_logs, you will need to flush + manually before testing other code in the same test_* method, otherwise + get_logs in the next lines will see messages from the previous lines. + See example in test_command_check. + """ + + def setUp(self): + super(LoggingCatcher, self).setUp() + self.loghandler = handler = _TestHandler() + self._old_levels = logger.level, logger2to3.level + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) # we want all messages + logger2to3.setLevel(logging.CRITICAL) # we don't want 2to3 messages + + def tearDown(self): + handler = self.loghandler + # All this is necessary to properly shut down the logging system and + # avoid a regrtest complaint. Thanks to Vinay Sajip for the help. + handler.close() + logger.removeHandler(handler) + for ref in weakref.getweakrefs(handler): + logging._removeHandlerRef(ref) + del self.loghandler + logger.setLevel(self._old_levels[0]) + logger2to3.setLevel(self._old_levels[1]) + super(LoggingCatcher, self).tearDown() + + def get_logs(self, level=logging.WARNING, flush=True): + """Return all log messages with given level. + + *level* defaults to logging.WARNING. + + For log calls with arguments (i.e. logger.info('bla bla %r', arg)), + the messages will be formatted before being returned (e.g. "bla bla + 'thing'"). + + Returns a list. Automatically flushes the loghandler after being + called, unless *flush* is False (this is useful to get e.g. all + warnings then all info messages). + """ + messages = [log.getMessage() for log in self.loghandler.buffer + if log.levelno == level] + if flush: + self.loghandler.flush() + return messages + + +class TempdirManager: + """TestCase-compatible mixin to create temporary directories and files. + + Directories and files created in a test_* method will be removed after it + has run. + """ + + def setUp(self): + super(TempdirManager, self).setUp() + self._olddir = os.getcwd() + self._basetempdir = tempfile.mkdtemp() + self._files = [] + + def tearDown(self): + for handle, name in self._files: + handle.close() + unlink(name) + + os.chdir(self._olddir) + shutil.rmtree(self._basetempdir) + super(TempdirManager, self).tearDown() + + def mktempfile(self): + """Create a read-write temporary file and return it.""" + fd, fn = tempfile.mkstemp(dir=self._basetempdir) + os.close(fd) + fp = open(fn, 'w+') + self._files.append((fp, fn)) + return fp + + def mkdtemp(self): + """Create a temporary directory and return its path.""" + d = tempfile.mkdtemp(dir=self._basetempdir) + return d + + def write_file(self, path, content='xxx', encoding=None): + """Write a file at the given path. + + path can be a string, a tuple or a list; if it's a tuple or list, + os.path.join will be used to produce a path. + """ + if isinstance(path, (list, tuple)): + path = os.path.join(*path) + with open(path, 'w', encoding=encoding) as f: + f.write(content) + + def create_dist(self, **kw): + """Create a stub distribution object and files. + + This function creates a Distribution instance (use keyword arguments + to customize it) and a temporary directory with a project structure + (currently an empty directory). + + It returns the path to the directory and the Distribution instance. + You can use self.write_file to write any file in that + directory, e.g. setup scripts or Python modules. + """ + if 'name' not in kw: + kw['name'] = 'foo' + tmp_dir = self.mkdtemp() + project_dir = os.path.join(tmp_dir, kw['name']) + os.mkdir(project_dir) + dist = Distribution(attrs=kw) + return project_dir, dist + + def assertIsFile(self, *args): + path = os.path.join(*args) + dirname = os.path.dirname(path) + file = os.path.basename(path) + if os.path.isdir(dirname): + files = os.listdir(dirname) + msg = "%s not found in %s: %s" % (file, dirname, files) + assert os.path.isfile(path), msg + else: + raise AssertionError( + '%s not found. %s does not exist' % (file, dirname)) + + def assertIsNotFile(self, *args): + path = os.path.join(*args) + self.assertFalse(os.path.isfile(path), "%r exists" % path) + + +class EnvironRestorer: + """TestCase-compatible mixin to restore or delete environment variables. + + The variables to restore (or delete if they were not originally present) + must be explicitly listed in self.restore_environ. It's better to be + aware of what we're modifying instead of saving and restoring the whole + environment. + """ + + def setUp(self): + super(EnvironRestorer, self).setUp() + self._saved = [] + self._added = [] + for key in self.restore_environ: + if key in os.environ: + self._saved.append((key, os.environ[key])) + else: + self._added.append(key) + + def tearDown(self): + for key, value in self._saved: + os.environ[key] = value + for key in self._added: + os.environ.pop(key, None) + super(EnvironRestorer, self).tearDown() + + +class DummyCommand: + """Class to store options for retrieval via set_undefined_options(). + + Useful for mocking one dependency command in the tests for another + command, see e.g. the dummy build command in test_build_scripts. + """ + # XXX does not work with dist.reinitialize_command, which typechecks + # and wants a finalized attribute + + def __init__(self, **kwargs): + for kw, val in kwargs.items(): + setattr(self, kw, val) + + def ensure_finalized(self): + pass + + +class TestDistribution(Distribution): + """Distribution subclasses that avoids the default search for + configuration files. + + The ._config_files attribute must be set before + .parse_config_files() is called. + """ + + def find_config_files(self): + return self._config_files + + +class Inputs: + """Fakes user inputs.""" + # TODO document usage + # TODO use context manager or something for auto cleanup + + def __init__(self, *answers): + self.answers = answers + self.index = 0 + + def __call__(self, prompt=''): + try: + return self.answers[self.index] + finally: + self.index += 1 + + +def create_distribution(configfiles=()): + """Prepares a distribution with given config files parsed.""" + d = TestDistribution() + d.config.find_config_files = d.find_config_files + d._config_files = configfiles + d.parse_config_files() + d.parse_command_line() + return d + + +def use_command(testcase, fullname): + """Register command at *fullname* for the duration of a test.""" + set_command(fullname) + # XXX maybe set_command should return the class object + name = resolve_name(fullname).get_command_name() + # XXX maybe we need a public API to remove commands + testcase.addCleanup(_COMMANDS.__delitem__, name) + + +def fake_dec(*args, **kw): + """Fake decorator""" + def _wrap(func): + def __wrap(*args, **kw): + return func(*args, **kw) + return __wrap + return _wrap + + +def copy_xxmodule_c(directory): + """Helper for tests that need the xxmodule.c source file. + + Example use: + + def test_compile(self): + copy_xxmodule_c(self.tmpdir) + self.assertIn('xxmodule.c', os.listdir(self.tmpdir)) + + If the source file can be found, it will be copied to *directory*. If not, + the test will be skipped. Errors during copy are not caught. + """ + filename = _get_xxmodule_path() + if filename is None: + raise unittest.SkipTest('cannot find xxmodule.c') + shutil.copy(filename, directory) + + +def _get_xxmodule_path(): + if sysconfig.is_python_build(): + srcdir = sysconfig.get_config_var('projectbase') + path = os.path.join(os.getcwd(), srcdir, 'Modules', 'xxmodule.c') + else: + path = os.path.join(os.path.dirname(__file__), 'xxmodule.c') + if os.path.exists(path): + return path + + +def fixup_build_ext(cmd): + """Function needed to make build_ext tests pass. + + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. (Mac OS X embeds absolute paths + to shared libraries into executables, so the fixup is a no-op on that + platform.) + + When Python was built with in debug mode on Windows, build_ext commands + need their debug attribute set, and it is not done automatically for + some reason. + + This function handles both of these things, and also fixes + cmd.distribution.include_dirs if the running Python is an uninstalled + build. Example use: + + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + """ + if os.name == 'nt': + cmd.debug = sys.executable.endswith('_d.exe') + elif sysconfig.get_config_var('Py_ENABLE_SHARED'): + # To further add to the shared builds fun on Unix, we can't just add + # library_dirs to the Extension() instance because that doesn't get + # plumbed through to the final compiler command. + runshared = sysconfig.get_config_var('RUNSHARED') + if runshared is None: + cmd.library_dirs = ['.'] + else: + if sys.platform == 'darwin': + cmd.library_dirs = [] + else: + name, equals, value = runshared.partition('=') + cmd.library_dirs = value.split(os.pathsep) + + # Allow tests to run with an uninstalled Python + if sysconfig.is_python_build(): + pysrcdir = sysconfig.get_config_var('projectbase') + cmd.distribution.include_dirs.append(os.path.join(pysrcdir, 'Include')) + + +try: + from test.support import skip_unless_symlink +except ImportError: + skip_unless_symlink = unittest.skip( + 'requires test.support.skip_unless_symlink') + +skip_2to3_optimize = unittest.skipIf(sys.flags.optimize, + "2to3 doesn't work under -O") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_ccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_ccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,15 @@ +"""Tests for distutils.compiler.ccompiler.""" + +from packaging.compiler import ccompiler +from packaging.tests import unittest, support + + +class CCompilerTestCase(unittest.TestCase): + pass # XXX need some tests on CCompiler + + +def test_suite(): + return unittest.makeSuite(CCompilerTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_bdist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_bdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,61 @@ +"""Tests for distutils.command.bdist.""" +import os +from test.support import captured_stdout +from packaging.command.bdist import bdist, show_formats +from packaging.tests import unittest, support + + +class BuildTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_formats(self): + # let's create a command and make sure + # we can set the format + dist = self.create_dist()[1] + cmd = bdist(dist) + cmd.formats = ['msi'] + cmd.ensure_finalized() + self.assertEqual(cmd.formats, ['msi']) + + # what formats does bdist offer? + # XXX hard-coded lists are not the best way to find available bdist_* + # commands; we should add a registry + formats = ['bztar', 'gztar', 'msi', 'tar', 'wininst', 'zip'] + found = sorted(cmd.format_command) + self.assertEqual(found, formats) + + def test_skip_build(self): + # bug #10946: bdist --skip-build should trickle down to subcommands + dist = self.create_dist()[1] + cmd = bdist(dist) + cmd.skip_build = True + cmd.ensure_finalized() + dist.command_obj['bdist'] = cmd + + names = ['bdist_dumb', 'bdist_wininst'] + if os.name == 'nt': + names.append('bdist_msi') + + for name in names: + subcmd = cmd.get_finalized_command(name) + self.assertTrue(subcmd.skip_build, + '%s should take --skip-build from bdist' % name) + + def test_show_formats(self): + with captured_stdout() as stdout: + show_formats() + stdout = stdout.getvalue() + + # the output should be a header line + one line per format + num_formats = len(bdist.format_commands) + output = [line for line in stdout.split('\n') + if line.strip().startswith('--formats=')] + self.assertEqual(len(output), num_formats) + + +def test_suite(): + return unittest.makeSuite(BuildTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_bdist_dumb.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_bdist_dumb.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,91 @@ +"""Tests for distutils.command.bdist_dumb.""" + +import os +import imp +import sys +import zipfile +import packaging.util + +from packaging.dist import Distribution +from packaging.command.bdist_dumb import bdist_dumb +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +class BuildDumbTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(BuildDumbTestCase, self).setUp() + self.old_location = os.getcwd() + + def tearDown(self): + os.chdir(self.old_location) + packaging.util._path_created.clear() + super(BuildDumbTestCase, self).tearDown() + + @requires_zlib + def test_simple_built(self): + + # let's create a simple package + tmp_dir = self.mkdtemp() + pkg_dir = os.path.join(tmp_dir, 'foo') + os.mkdir(pkg_dir) + self.write_file((pkg_dir, 'foo.py'), '#') + self.write_file((pkg_dir, 'MANIFEST.in'), 'include foo.py') + self.write_file((pkg_dir, 'README'), '') + + dist = Distribution({'name': 'foo', 'version': '0.1', + 'py_modules': ['foo'], + 'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx'}) + os.chdir(pkg_dir) + cmd = bdist_dumb(dist) + + # so the output is the same no matter + # what is the platform + cmd.format = 'zip' + + cmd.ensure_finalized() + cmd.run() + + # see what we have + dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) + base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name) + if os.name == 'os2': + base = base.replace(':', '-') + + self.assertEqual(dist_created, [base]) + + # now let's check what we have in the zip file + with zipfile.ZipFile(os.path.join('dist', base)) as fp: + contents = fp.namelist() + + contents = sorted(os.path.basename(fn) for fn in contents) + wanted = ['foo.py', + 'foo.%s.pyc' % imp.get_tag(), + 'METADATA', 'INSTALLER', 'REQUESTED', 'RECORD'] + self.assertEqual(contents, sorted(wanted)) + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + os.chdir(pkg_dir) + cmd = bdist_dumb(dist) + self.assertEqual(cmd.bdist_dir, None) + cmd.finalize_options() + + # bdist_dir is initialized to bdist_base/dumb if not set + base = cmd.get_finalized_command('bdist').bdist_base + self.assertEqual(cmd.bdist_dir, os.path.join(base, 'dumb')) + + # the format is set to a default value depending on the os.name + default = cmd.default_format[os.name] + self.assertEqual(cmd.format, default) + + +def test_suite(): + return unittest.makeSuite(BuildDumbTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_bdist_msi.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_bdist_msi.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,25 @@ +"""Tests for distutils.command.bdist_msi.""" +import sys + +from packaging.tests import unittest, support + + +@unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows') +class BDistMSITestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_minimal(self): + # minimal test XXX need more tests + from packaging.command.bdist_msi import bdist_msi + project_dir, dist = self.create_dist() + cmd = bdist_msi(dist) + cmd.ensure_finalized() + + +def test_suite(): + return unittest.makeSuite(BDistMSITestCase) + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_bdist_wininst.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_bdist_wininst.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,32 @@ +"""Tests for distutils.command.bdist_wininst.""" + +from packaging.command.bdist_wininst import bdist_wininst +from packaging.tests import unittest, support + + +class BuildWinInstTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_get_exe_bytes(self): + + # issue5731: command was broken on non-windows platforms + # this test makes sure it works now for every platform + # let's create a command + pkg_pth, dist = self.create_dist() + cmd = bdist_wininst(dist) + cmd.ensure_finalized() + + # let's run the code that finds the right wininst*.exe file + # and make sure it finds it and returns its content + # no matter what platform we have + exe_file = cmd.get_exe_bytes() + self.assertGreater(len(exe_file), 10) + + +def test_suite(): + return unittest.makeSuite(BuildWinInstTestCase) + + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_build.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_build.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,56 @@ +"""Tests for distutils.command.build.""" +import os +import sys + +from packaging.command.build import build +from sysconfig import get_platform +from packaging.tests import unittest, support + + +class BuildTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + cmd = build(dist) + cmd.finalize_options() + + # if not specified, plat_name gets the current platform + self.assertEqual(cmd.plat_name, get_platform()) + + # build_purelib is build + lib + wanted = os.path.join(cmd.build_base, 'lib') + self.assertEqual(cmd.build_purelib, wanted) + + # build_platlib is 'build/lib.platform-x.x[-pydebug]' + # examples: + # build/lib.macosx-10.3-i386-2.7 + pyversion = '%s.%s' % sys.version_info[:2] + plat_spec = '.%s-%s' % (cmd.plat_name, pyversion) + if hasattr(sys, 'gettotalrefcount'): + self.assertTrue(cmd.build_platlib.endswith('-pydebug')) + plat_spec += '-pydebug' + wanted = os.path.join(cmd.build_base, 'lib' + plat_spec) + self.assertEqual(cmd.build_platlib, wanted) + + # by default, build_lib = build_purelib + self.assertEqual(cmd.build_lib, cmd.build_purelib) + + # build_temp is build/temp. + wanted = os.path.join(cmd.build_base, 'temp' + plat_spec) + self.assertEqual(cmd.build_temp, wanted) + + # build_scripts is build/scripts-x.x + wanted = os.path.join(cmd.build_base, 'scripts-' + pyversion) + self.assertEqual(cmd.build_scripts, wanted) + + # executable is os.path.normpath(sys.executable) + self.assertEqual(cmd.executable, os.path.normpath(sys.executable)) + + +def test_suite(): + return unittest.makeSuite(BuildTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_build_clib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_build_clib.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,141 @@ +"""Tests for distutils.command.build_clib.""" +import os +import sys + +from packaging.util import find_executable +from packaging.command.build_clib import build_clib +from packaging.errors import PackagingSetupError +from packaging.tests import unittest, support + + +class BuildCLibTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_check_library_dist(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + # 'libraries' option must be a list + self.assertRaises(PackagingSetupError, cmd.check_library_list, 'foo') + + # each element of 'libraries' must a 2-tuple + self.assertRaises(PackagingSetupError, cmd.check_library_list, + ['foo1', 'foo2']) + + # first element of each tuple in 'libraries' + # must be a string (the library name) + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [(1, 'foo1'), ('name', 'foo2')]) + + # library name may not contain directory separators + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [('name', 'foo1'), + ('another/name', 'foo2')]) + + # second element of each tuple must be a dictionary (build info) + self.assertRaises(PackagingSetupError, cmd.check_library_list, + [('name', {}), + ('another', 'foo2')]) + + # those work + libs = [('name', {}), ('name', {'ok': 'good'})] + cmd.check_library_list(libs) + + def test_get_source_files(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + # "in 'libraries' option 'sources' must be present and must be + # a list of source filenames + cmd.libraries = [('name', {})] + self.assertRaises(PackagingSetupError, cmd.get_source_files) + + cmd.libraries = [('name', {'sources': 1})] + self.assertRaises(PackagingSetupError, cmd.get_source_files) + + cmd.libraries = [('name', {'sources': ['a', 'b']})] + self.assertEqual(cmd.get_source_files(), ['a', 'b']) + + cmd.libraries = [('name', {'sources': ('a', 'b')})] + self.assertEqual(cmd.get_source_files(), ['a', 'b']) + + cmd.libraries = [('name', {'sources': ('a', 'b')}), + ('name2', {'sources': ['c', 'd']})] + self.assertEqual(cmd.get_source_files(), ['a', 'b', 'c', 'd']) + + def test_build_libraries(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + class FakeCompiler: + def compile(*args, **kw): + pass + create_static_lib = compile + + cmd.compiler = FakeCompiler() + + # build_libraries is also doing a bit of type checking + lib = [('name', {'sources': 'notvalid'})] + self.assertRaises(PackagingSetupError, cmd.build_libraries, lib) + + lib = [('name', {'sources': []})] + cmd.build_libraries(lib) + + lib = [('name', {'sources': ()})] + cmd.build_libraries(lib) + + def test_finalize_options(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + cmd.include_dirs = 'one-dir' + cmd.finalize_options() + self.assertEqual(cmd.include_dirs, ['one-dir']) + + cmd.include_dirs = None + cmd.finalize_options() + self.assertEqual(cmd.include_dirs, []) + + cmd.distribution.libraries = 'WONTWORK' + self.assertRaises(PackagingSetupError, cmd.finalize_options) + + @unittest.skipIf(sys.platform == 'win32', 'disabled on win32') + def test_run(self): + pkg_dir, dist = self.create_dist() + cmd = build_clib(dist) + + foo_c = os.path.join(pkg_dir, 'foo.c') + self.write_file(foo_c, 'int main(void) { return 1;}\n') + cmd.libraries = [('foo', {'sources': [foo_c]})] + + build_temp = os.path.join(pkg_dir, 'build') + os.mkdir(build_temp) + cmd.build_temp = build_temp + cmd.build_clib = build_temp + + # before we run the command, we want to make sure + # all commands are present on the system + # by creating a compiler and checking its executables + from packaging.compiler import new_compiler, customize_compiler + + compiler = new_compiler() + customize_compiler(compiler) + for ccmd in compiler.executables.values(): + if ccmd is None: + continue + if find_executable(ccmd[0]) is None: + raise unittest.SkipTest("can't test") + + # this should work + cmd.run() + + # let's check the result + self.assertIn('libfoo.a', os.listdir(build_temp)) + + +def test_suite(): + return unittest.makeSuite(BuildCLibTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_build_ext.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_build_ext.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,394 @@ +import os +import sys +import site +import sysconfig +import textwrap +from packaging.dist import Distribution +from packaging.errors import (UnknownFileError, CompileError, + PackagingPlatformError) +from packaging.command.build_ext import build_ext +from packaging.compiler.extension import Extension + +from test.script_helper import assert_python_ok +from packaging.tests import support, unittest + + +class BuildExtTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + def setUp(self): + super(BuildExtTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.old_user_base = site.USER_BASE + site.USER_BASE = self.mkdtemp() + + def tearDown(self): + site.USER_BASE = self.old_user_base + super(BuildExtTestCase, self).tearDown() + + def test_build_ext(self): + support.copy_xxmodule_c(self.tmp_dir) + xx_c = os.path.join(self.tmp_dir, 'xxmodule.c') + xx_ext = Extension('xx', [xx_c]) + dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]}) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + cmd.ensure_finalized() + cmd.run() + + code = textwrap.dedent("""\ + import sys + sys.path.insert(0, %r) + + import xx + + for attr in ('error', 'foo', 'new', 'roj'): + assert hasattr(xx, attr) + + assert xx.foo(2, 5) == 7 + assert xx.foo(13, 15) == 28 + assert xx.new().demo() is None + doc = 'This is a template module just for instruction.' + assert xx.__doc__ == doc + assert isinstance(xx.Null(), xx.Null) + assert isinstance(xx.Str(), xx.Str) + """) + code = code % self.tmp_dir + assert_python_ok('-c', code) + + def test_solaris_enable_shared(self): + dist = Distribution({'name': 'xx'}) + cmd = build_ext(dist) + old = sys.platform + + sys.platform = 'sunos' # fooling finalize_options + + old_var = sysconfig.get_config_var('Py_ENABLE_SHARED') + sysconfig._CONFIG_VARS['Py_ENABLE_SHARED'] = 1 + try: + cmd.ensure_finalized() + finally: + sys.platform = old + if old_var is None: + del sysconfig._CONFIG_VARS['Py_ENABLE_SHARED'] + else: + sysconfig._CONFIG_VARS['Py_ENABLE_SHARED'] = old_var + + # make sure we get some library dirs under solaris + self.assertGreater(len(cmd.library_dirs), 0) + + def test_user_site(self): + dist = Distribution({'name': 'xx'}) + cmd = build_ext(dist) + + # making sure the user option is there + options = [name for name, short, label in + cmd.user_options] + self.assertIn('user', options) + + # setting a value + cmd.user = True + + # setting user based lib and include + lib = os.path.join(site.USER_BASE, 'lib') + incl = os.path.join(site.USER_BASE, 'include') + os.mkdir(lib) + os.mkdir(incl) + + # let's run finalize + cmd.ensure_finalized() + + # see if include_dirs and library_dirs + # were set + self.assertIn(lib, cmd.library_dirs) + self.assertIn(lib, cmd.rpath) + self.assertIn(incl, cmd.include_dirs) + + def test_optional_extension(self): + + # this extension will fail, but let's ignore this failure + # with the optional argument. + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + self.assertRaises((UnknownFileError, CompileError), + cmd.run) # should raise an error + + modules = [Extension('foo', ['xxx'], optional=True)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + cmd.run() # should pass + + def test_finalize_options(self): + # Make sure Python's include directories (for Python.h, pyconfig.h, + # etc.) are in the include search path. + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.finalize_options() + + py_include = sysconfig.get_path('include') + self.assertIn(py_include, cmd.include_dirs) + + plat_py_include = sysconfig.get_path('platinclude') + self.assertIn(plat_py_include, cmd.include_dirs) + + # make sure cmd.libraries is turned into a list + # if it's a string + cmd = build_ext(dist) + cmd.libraries = 'my_lib, other_lib lastlib' + cmd.finalize_options() + self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib']) + + # make sure cmd.library_dirs is turned into a list + # if it's a string + cmd = build_ext(dist) + cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep + cmd.finalize_options() + self.assertIn('my_lib_dir', cmd.library_dirs) + self.assertIn('other_lib_dir', cmd.library_dirs) + + # make sure rpath is turned into a list + # if it's a string + cmd = build_ext(dist) + cmd.rpath = 'one%stwo' % os.pathsep + cmd.finalize_options() + self.assertEqual(cmd.rpath, ['one', 'two']) + + # XXX more tests to perform for win32 + + # make sure define is turned into 2-tuples + # strings if they are ','-separated strings + cmd = build_ext(dist) + cmd.define = 'one,two' + cmd.finalize_options() + self.assertEqual(cmd.define, [('one', '1'), ('two', '1')]) + + # make sure undef is turned into a list of + # strings if they are ','-separated strings + cmd = build_ext(dist) + cmd.undef = 'one,two' + cmd.finalize_options() + self.assertEqual(cmd.undef, ['one', 'two']) + + # make sure swig_opts is turned into a list + cmd = build_ext(dist) + cmd.swig_opts = None + cmd.finalize_options() + self.assertEqual(cmd.swig_opts, []) + + cmd = build_ext(dist) + cmd.swig_opts = '1 2' + cmd.finalize_options() + self.assertEqual(cmd.swig_opts, ['1', '2']) + + def test_get_source_files(self): + modules = [Extension('foo', ['xxx'], optional=False)] + dist = Distribution({'name': 'xx', 'ext_modules': modules}) + cmd = build_ext(dist) + cmd.ensure_finalized() + self.assertEqual(cmd.get_source_files(), ['xxx']) + + def test_compiler_option(self): + # cmd.compiler is an option and + # should not be overriden by a compiler instance + # when the command is run + dist = Distribution() + cmd = build_ext(dist) + cmd.compiler = 'unix' + cmd.ensure_finalized() + cmd.run() + self.assertEqual(cmd.compiler, 'unix') + + def test_get_outputs(self): + tmp_dir = self.mkdtemp() + c_file = os.path.join(tmp_dir, 'foo.c') + self.write_file(c_file, 'void PyInit_foo(void) {}\n') + ext = Extension('foo', [c_file], optional=False) + dist = Distribution({'name': 'xx', + 'ext_modules': [ext]}) + cmd = build_ext(dist) + support.fixup_build_ext(cmd) + cmd.ensure_finalized() + self.assertEqual(len(cmd.get_outputs()), 1) + + cmd.build_lib = os.path.join(self.tmp_dir, 'build') + cmd.build_temp = os.path.join(self.tmp_dir, 'tempt') + + # issue #5977 : distutils build_ext.get_outputs + # returns wrong result with --inplace + other_tmp_dir = os.path.realpath(self.mkdtemp()) + old_wd = os.getcwd() + os.chdir(other_tmp_dir) + try: + cmd.inplace = True + cmd.run() + so_file = cmd.get_outputs()[0] + finally: + os.chdir(old_wd) + self.assertTrue(os.path.exists(so_file)) + so_ext = sysconfig.get_config_var('SO') + self.assertTrue(so_file.endswith(so_ext)) + so_dir = os.path.dirname(so_file) + self.assertEqual(so_dir, other_tmp_dir) + + cmd.inplace = False + cmd.run() + so_file = cmd.get_outputs()[0] + self.assertTrue(os.path.exists(so_file)) + self.assertTrue(so_file.endswith(so_ext)) + so_dir = os.path.dirname(so_file) + self.assertEqual(so_dir, cmd.build_lib) + + # inplace = False, cmd.package = 'bar' + build_py = cmd.get_finalized_command('build_py') + build_py.package_dir = 'bar' + path = cmd.get_ext_fullpath('foo') + # checking that the last directory is the build_dir + path = os.path.split(path)[0] + self.assertEqual(path, cmd.build_lib) + + # inplace = True, cmd.package = 'bar' + cmd.inplace = True + other_tmp_dir = os.path.realpath(self.mkdtemp()) + old_wd = os.getcwd() + os.chdir(other_tmp_dir) + try: + path = cmd.get_ext_fullpath('foo') + finally: + os.chdir(old_wd) + # checking that the last directory is bar + path = os.path.split(path)[0] + lastdir = os.path.split(path)[-1] + self.assertEqual(lastdir, 'bar') + + def test_ext_fullpath(self): + ext = sysconfig.get_config_vars()['SO'] + # building lxml.etree inplace + #etree_c = os.path.join(self.tmp_dir, 'lxml.etree.c') + #etree_ext = Extension('lxml.etree', [etree_c]) + #dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]}) + dist = Distribution() + cmd = build_ext(dist) + cmd.inplace = True + cmd.distribution.package_dir = 'src' + cmd.distribution.packages = ['lxml', 'lxml.html'] + curdir = os.getcwd() + wanted = os.path.join(curdir, 'src', 'lxml', 'etree' + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + + # building lxml.etree not inplace + cmd.inplace = False + cmd.build_lib = os.path.join(curdir, 'tmpdir') + wanted = os.path.join(curdir, 'tmpdir', 'lxml', 'etree' + ext) + path = cmd.get_ext_fullpath('lxml.etree') + self.assertEqual(wanted, path) + + # building twisted.runner.portmap not inplace + build_py = cmd.get_finalized_command('build_py') + build_py.package_dir = None + cmd.distribution.packages = ['twisted', 'twisted.runner.portmap'] + path = cmd.get_ext_fullpath('twisted.runner.portmap') + wanted = os.path.join(curdir, 'tmpdir', 'twisted', 'runner', + 'portmap' + ext) + self.assertEqual(wanted, path) + + # building twisted.runner.portmap inplace + cmd.inplace = True + path = cmd.get_ext_fullpath('twisted.runner.portmap') + wanted = os.path.join(curdir, 'twisted', 'runner', 'portmap' + ext) + self.assertEqual(wanted, path) + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_default(self): + # Issue 9516: Test that, in the absence of the environment variable, + # an extension module is compiled with the same deployment target as + # the interpreter. + self._try_compile_deployment_target('==', None) + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_too_low(self): + # Issue 9516: Test that an extension module is not allowed to be + # compiled with a deployment target less than that of the interpreter. + self.assertRaises(PackagingPlatformError, + self._try_compile_deployment_target, '>', '10.1') + + @unittest.skipUnless(sys.platform == 'darwin', + 'test only relevant for Mac OS X') + def test_deployment_target_higher_ok(self): + # Issue 9516: Test that an extension module can be compiled with a + # deployment target higher than that of the interpreter: the ext + # module may depend on some newer OS feature. + deptarget = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if deptarget: + # increment the minor version number (i.e. 10.6 -> 10.7) + deptarget = [int(x) for x in deptarget.split('.')] + deptarget[-1] += 1 + deptarget = '.'.join(str(i) for i in deptarget) + self._try_compile_deployment_target('<', deptarget) + + def _try_compile_deployment_target(self, operator, target): + orig_environ = os.environ + os.environ = orig_environ.copy() + self.addCleanup(setattr, os, 'environ', orig_environ) + + if target is None: + if os.environ.get('MACOSX_DEPLOYMENT_TARGET'): + del os.environ['MACOSX_DEPLOYMENT_TARGET'] + else: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = target + + deptarget_c = os.path.join(self.tmp_dir, 'deptargetmodule.c') + + with open(deptarget_c, 'w') as fp: + fp.write(textwrap.dedent('''\ + #include + + int dummy; + + #if TARGET %s MAC_OS_X_VERSION_MIN_REQUIRED + #else + #error "Unexpected target" + #endif + + ''' % operator)) + + # get the deployment target that the interpreter was built with + target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + target = tuple(map(int, target.split('.'))) + target = '%02d%01d0' % target + + deptarget_ext = Extension( + 'deptarget', + [deptarget_c], + extra_compile_args=['-DTARGET=%s' % (target,)], + ) + dist = Distribution({ + 'name': 'deptarget', + 'ext_modules': [deptarget_ext], + }) + dist.package_dir = self.tmp_dir + cmd = build_ext(dist) + cmd.build_lib = self.tmp_dir + cmd.build_temp = self.tmp_dir + + try: + cmd.ensure_finalized() + cmd.run() + except CompileError: + self.fail("Wrong deployment target during compilation") + + +def test_suite(): + return unittest.makeSuite(BuildExtTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_build_py.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_build_py.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,146 @@ +"""Tests for distutils.command.build_py.""" + +import os +import sys +import imp + +from packaging.command.build_py import build_py +from packaging.dist import Distribution +from packaging.errors import PackagingFileError + +from packaging.tests import unittest, support + + +class BuildPyTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_package_data(self): + sources = self.mkdtemp() + pkg_dir = os.path.join(sources, 'pkg') + os.mkdir(pkg_dir) + f = open(os.path.join(pkg_dir, "__init__.py"), "w") + try: + f.write("# Pretend this is a package.") + finally: + f.close() + # let's have two files to make sure globbing works + f = open(os.path.join(pkg_dir, "README.txt"), "w") + try: + f.write("Info about this package") + finally: + f.close() + f = open(os.path.join(pkg_dir, "HACKING.txt"), "w") + try: + f.write("How to contribute") + finally: + f.close() + + destination = self.mkdtemp() + + dist = Distribution({"packages": ["pkg"], + "package_dir": sources}) + + dist.command_obj["build"] = support.DummyCommand( + force=False, + build_lib=destination, + use_2to3_fixers=None, + convert_2to3_doctests=None, + use_2to3=False) + dist.packages = ["pkg"] + dist.package_data = {"pkg": ["*.txt"]} + dist.package_dir = sources + + cmd = build_py(dist) + cmd.compile = True + cmd.ensure_finalized() + self.assertEqual(cmd.package_data, dist.package_data) + + cmd.run() + + # This makes sure the list of outputs includes byte-compiled + # files for Python modules but not for package data files + # (there shouldn't *be* byte-code files for those!). + # FIXME the test below is not doing what the comment above says, and + # if it did it would show a code bug: if we add a demo.py file to + # package_data, it gets byte-compiled! + outputs = cmd.get_outputs() + self.assertEqual(len(outputs), 4, outputs) + pkgdest = os.path.join(destination, "pkg") + files = os.listdir(pkgdest) + pycache_dir = os.path.join(pkgdest, "__pycache__") + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + self.assertIn("HACKING.txt", files) + pyc_files = os.listdir(pycache_dir) + self.assertEqual(["__init__.%s.pyc" % imp.get_tag()], pyc_files) + + def test_empty_package_dir(self): + # See SF 1668596/1720897. + # create the distribution files. + sources = self.mkdtemp() + pkg = os.path.join(sources, 'pkg') + os.mkdir(pkg) + open(os.path.join(pkg, "__init__.py"), "wb").close() + testdir = os.path.join(pkg, "doc") + os.mkdir(testdir) + open(os.path.join(testdir, "testfile"), "wb").close() + + os.chdir(sources) + dist = Distribution({"packages": ["pkg"], + "package_dir": sources, + "package_data": {"pkg": ["doc/*"]}}) + dist.script_args = ["build"] + dist.parse_command_line() + + try: + dist.run_commands() + except PackagingFileError: + self.fail("failed package_data test when package_dir is ''") + + def test_byte_compile(self): + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + os.chdir(project_dir) + self.write_file('boiledeggs.py', 'import antigravity') + cmd = build_py(dist) + cmd.compile = True + cmd.build_lib = 'here' + cmd.finalize_options() + cmd.run() + + found = os.listdir(cmd.build_lib) + self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) + found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) + self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()]) + + def test_byte_compile_optimized(self): + project_dir, dist = self.create_dist(py_modules=['boiledeggs']) + os.chdir(project_dir) + self.write_file('boiledeggs.py', 'import antigravity') + cmd = build_py(dist) + cmd.compile = True + cmd.optimize = 1 + cmd.build_lib = 'here' + cmd.finalize_options() + cmd.run() + + found = os.listdir(cmd.build_lib) + self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py']) + found = os.listdir(os.path.join(cmd.build_lib, '__pycache__')) + self.assertEqual(sorted(found), ['boiledeggs.%s.pyc' % imp.get_tag(), + 'boiledeggs.%s.pyo' % imp.get_tag()]) + + def test_byte_compile_under_B(self): + # make sure byte compilation works under -B (dont_write_bytecode) + self.addCleanup(setattr, sys, 'dont_write_bytecode', + sys.dont_write_bytecode) + sys.dont_write_bytecode = True + self.test_byte_compile() + self.test_byte_compile_optimized() + + +def test_suite(): + return unittest.makeSuite(BuildPyTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_build_scripts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_build_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,109 @@ +"""Tests for distutils.command.build_scripts.""" + +import os +import sys +import sysconfig +from packaging.dist import Distribution +from packaging.command.build_scripts import build_scripts + +from packaging.tests import unittest, support + + +class BuildScriptsTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_default_settings(self): + cmd = self.get_build_scripts_cmd("/foo/bar", []) + self.assertFalse(cmd.force) + self.assertIs(cmd.build_dir, None) + + cmd.finalize_options() + + self.assertTrue(cmd.force) + self.assertEqual(cmd.build_dir, "/foo/bar") + + def test_build(self): + source = self.mkdtemp() + target = self.mkdtemp() + expected = self.write_sample_scripts(source) + + cmd = self.get_build_scripts_cmd(target, + [os.path.join(source, fn) + for fn in expected]) + cmd.finalize_options() + cmd.run() + + built = os.listdir(target) + for name in expected: + self.assertIn(name, built) + + def get_build_scripts_cmd(self, target, scripts): + dist = Distribution() + dist.scripts = scripts + dist.command_obj["build"] = support.DummyCommand( + build_scripts=target, + force=True, + executable=sys.executable, + use_2to3=False, + use_2to3_fixers=None, + convert_2to3_doctests=None + ) + return build_scripts(dist) + + def write_sample_scripts(self, dir): + expected = [] + expected.append("script1.py") + self.write_script(dir, "script1.py", + ("#! /usr/bin/env python2.3\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + expected.append("script2.py") + self.write_script(dir, "script2.py", + ("#!/usr/bin/python\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + expected.append("shell.sh") + self.write_script(dir, "shell.sh", + ("#!/bin/sh\n" + "# bogus shell script w/ sh-bang\n" + "exit 0\n")) + return expected + + def write_script(self, dir, name, text): + with open(os.path.join(dir, name), "w") as f: + f.write(text) + + def test_version_int(self): + source = self.mkdtemp() + target = self.mkdtemp() + expected = self.write_sample_scripts(source) + + + cmd = self.get_build_scripts_cmd(target, + [os.path.join(source, fn) + for fn in expected]) + cmd.finalize_options() + + # http://bugs.python.org/issue4524 + # + # On linux-g++-32 with command line `./configure --enable-ipv6 + # --with-suffix=3`, python is compiled okay but the build scripts + # failed when writing the name of the executable + old = sysconfig.get_config_vars().get('VERSION') + sysconfig._CONFIG_VARS['VERSION'] = 4 + try: + cmd.run() + finally: + if old is not None: + sysconfig._CONFIG_VARS['VERSION'] = old + + built = os.listdir(target) + for name in expected: + self.assertIn(name, built) + +def test_suite(): + return unittest.makeSuite(BuildScriptsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_check.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_check.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,161 @@ +"""Tests for distutils.command.check.""" + +from packaging.command.check import check +from packaging.metadata import _HAS_DOCUTILS +from packaging.errors import PackagingSetupError, MetadataMissingError +from packaging.tests import unittest, support + + +class CheckTestCase(support.LoggingCatcher, + support.TempdirManager, + unittest.TestCase): + + def _run(self, metadata=None, **options): + if metadata is None: + metadata = {'name': 'xxx', 'version': '1.2'} + pkg_info, dist = self.create_dist(**metadata) + cmd = check(dist) + cmd.initialize_options() + for name, value in options.items(): + setattr(cmd, name, value) + cmd.ensure_finalized() + cmd.run() + return cmd + + def test_check_metadata(self): + # let's run the command with no metadata at all + # by default, check is checking the metadata + # should have some warnings + self._run() + # trick: using assertNotEqual with an empty list will give us a more + # useful error message than assertGreater(.., 0) when the code change + # and the test fails + self.assertNotEqual(self.get_logs(), []) + + # now let's add the required fields + # and run it again, to make sure we don't get + # any warning anymore + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': '4.2', + } + self._run(metadata) + self.assertEqual(self.get_logs(), []) + + # now with the strict mode, we should + # get an error if there are missing metadata + self.assertRaises(MetadataMissingError, self._run, {}, **{'strict': 1}) + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1}) + + # clear warnings from the previous calls + self.loghandler.flush() + + # and of course, no error when all metadata fields are present + self._run(metadata, strict=True) + self.assertEqual(self.get_logs(), []) + + # now a test with non-ASCII characters + metadata = {'home_page': 'xxx', 'author': '\u00c9ric', + 'author_email': 'xxx', 'name': 'xxx', + 'version': '1.2', + 'summary': 'Something about esszet \u00df', + 'description': 'More things about esszet \u00df'} + self._run(metadata) + self.assertEqual(self.get_logs(), []) + + def test_check_metadata_1_2(self): + # let's run the command with no metadata at all + # by default, check is checking the metadata + # should have some warnings + self._run() + self.assertNotEqual(self.get_logs(), []) + + # now let's add the required fields and run it again, to make sure we + # don't get any warning anymore let's use requires_python as a marker + # to enforce Metadata-Version 1.2 + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': '4.2', + 'requires_python': '2.4', + } + self._run(metadata) + self.assertEqual(self.get_logs(), []) + + # now with the strict mode, we should + # get an error if there are missing metadata + self.assertRaises(MetadataMissingError, self._run, {}, **{'strict': 1}) + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1}) + + # complain about version format + metadata['version'] = 'xxx' + self.assertRaises(PackagingSetupError, self._run, metadata, + **{'strict': 1}) + + # clear warnings from the previous calls + self.loghandler.flush() + + # now with correct version format again + metadata['version'] = '4.2' + self._run(metadata, strict=True) + self.assertEqual(self.get_logs(), []) + + @unittest.skipUnless(_HAS_DOCUTILS, "requires docutils") + def test_check_restructuredtext(self): + # let's see if it detects broken rest in description + broken_rest = 'title\n===\n\ntest' + pkg_info, dist = self.create_dist(description=broken_rest) + cmd = check(dist) + cmd.check_restructuredtext() + self.assertEqual(len(self.get_logs()), 1) + + # let's see if we have an error with strict=1 + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': '1.2', + 'description': broken_rest} + self.assertRaises(PackagingSetupError, self._run, metadata, + strict=True, all=True) + self.loghandler.flush() + + # and non-broken rest, including a non-ASCII character to test #12114 + dist = self.create_dist(description='title\n=====\n\ntest \u00df')[1] + cmd = check(dist) + cmd.check_restructuredtext() + self.assertEqual(self.get_logs(), []) + + def test_check_all(self): + self.assertRaises(PackagingSetupError, self._run, + {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1, + 'all': 1}) + self.assertRaises(MetadataMissingError, self._run, + {}, **{'strict': 1, + 'all': 1}) + + def test_check_hooks(self): + pkg_info, dist = self.create_dist() + dist.command_options['install_dist'] = { + 'pre_hook': ('file', {"a": 'some.nonextistant.hook.ghrrraarrhll'}), + } + cmd = check(dist) + cmd.check_hooks_resolvable() + self.assertEqual(len(self.get_logs()), 1) + + def test_warn(self): + _, dist = self.create_dist() + cmd = check(dist) + self.assertEqual(self.get_logs(), []) + cmd.warn('hello') + self.assertEqual(self.get_logs(), ['check: hello']) + cmd.warn('hello %s', 'world') + self.assertEqual(self.get_logs(), ['check: hello world']) + cmd.warn('hello %s %s', 'beautiful', 'world') + self.assertEqual(self.get_logs(), ['check: hello beautiful world']) + + +def test_suite(): + return unittest.makeSuite(CheckTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_clean.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_clean.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,46 @@ +"""Tests for distutils.command.clean.""" +import os + +from packaging.command.clean import clean +from packaging.tests import unittest, support + + +class cleanTestCase(support.TempdirManager, support.LoggingCatcher, + unittest.TestCase): + + def test_simple_run(self): + pkg_dir, dist = self.create_dist() + cmd = clean(dist) + + # let's add some elements clean should remove + dirs = [(d, os.path.join(pkg_dir, d)) + for d in ('build_temp', 'build_lib', 'bdist_base', + 'build_scripts', 'build_base')] + + for name, path in dirs: + os.mkdir(path) + setattr(cmd, name, path) + if name == 'build_base': + continue + for f in ('one', 'two', 'three'): + self.write_file((path, f)) + + # let's run the command + cmd.all = True + cmd.ensure_finalized() + cmd.run() + + # make sure the files where removed + for name, path in dirs: + self.assertFalse(os.path.exists(path), + '%r was not removed' % path) + + # let's run the command again (should spit warnings but succeed) + cmd.run() + + +def test_suite(): + return unittest.makeSuite(cleanTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_cmd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_cmd.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,102 @@ +"""Tests for distutils.cmd.""" +import os +import logging + +from packaging.command.cmd import Command +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError +from packaging.tests import support, unittest + + +class MyCmd(Command): + def initialize_options(self): + pass + + +class CommandTestCase(support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(CommandTestCase, self).setUp() + dist = Distribution() + self.cmd = MyCmd(dist) + + def test_make_file(self): + cmd = self.cmd + + # making sure it raises when infiles is not a string or a list/tuple + self.assertRaises(TypeError, cmd.make_file, + infiles=1, outfile='', func='func', args=()) + + # making sure execute gets called properly + def _execute(func, args, exec_msg, level): + self.assertEqual(exec_msg, 'generating out from in') + cmd.force = True + cmd.execute = _execute + cmd.make_file(infiles='in', outfile='out', func='func', args=()) + + def test_dump_options(self): + cmd = self.cmd + cmd.option1 = 1 + cmd.option2 = 1 + cmd.user_options = [('option1', '', ''), ('option2', '', '')] + cmd.dump_options() + + wanted = ["command options for 'MyCmd':", ' option1 = 1', + ' option2 = 1'] + msgs = self.get_logs(logging.INFO) + self.assertEqual(msgs, wanted) + + def test_ensure_string(self): + cmd = self.cmd + cmd.option1 = 'ok' + cmd.ensure_string('option1') + + cmd.option2 = None + cmd.ensure_string('option2', 'xxx') + self.assertTrue(hasattr(cmd, 'option2')) + + cmd.option3 = 1 + self.assertRaises(PackagingOptionError, cmd.ensure_string, 'option3') + + def test_ensure_string_list(self): + cmd = self.cmd + cmd.option1 = 'ok,dok' + cmd.ensure_string_list('option1') + self.assertEqual(cmd.option1, ['ok', 'dok']) + + cmd.yes_string_list = ['one', 'two', 'three'] + cmd.yes_string_list2 = 'ok' + cmd.ensure_string_list('yes_string_list') + cmd.ensure_string_list('yes_string_list2') + self.assertEqual(cmd.yes_string_list, ['one', 'two', 'three']) + self.assertEqual(cmd.yes_string_list2, ['ok']) + + cmd.not_string_list = ['one', 2, 'three'] + cmd.not_string_list2 = object() + self.assertRaises(PackagingOptionError, + cmd.ensure_string_list, 'not_string_list') + + self.assertRaises(PackagingOptionError, + cmd.ensure_string_list, 'not_string_list2') + + def test_ensure_filename(self): + cmd = self.cmd + cmd.option1 = __file__ + cmd.ensure_filename('option1') + cmd.option2 = 'xxx' + self.assertRaises(PackagingOptionError, cmd.ensure_filename, 'option2') + + def test_ensure_dirname(self): + cmd = self.cmd + cmd.option1 = os.path.dirname(__file__) or os.curdir + cmd.ensure_dirname('option1') + cmd.option2 = 'xxx' + self.assertRaises(PackagingOptionError, cmd.ensure_dirname, 'option2') + + +def test_suite(): + return unittest.makeSuite(CommandTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_config.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,76 @@ +"""Tests for distutils.command.config.""" +import os +import sys +import logging + +from packaging.command.config import dump_file, config +from packaging.tests import unittest, support + + +class ConfigTestCase(support.LoggingCatcher, + support.TempdirManager, + unittest.TestCase): + + def test_dump_file(self): + this_file = __file__.rstrip('co') + with open(this_file) as f: + numlines = len(f.readlines()) + + dump_file(this_file, 'I am the header') + + logs = [] + for log in self.get_logs(logging.INFO): + logs.extend(line for line in log.split('\n')) + self.assertEqual(len(logs), numlines + 2) + + @unittest.skipIf(sys.platform == 'win32', 'disabled on win32') + def test_search_cpp(self): + pkg_dir, dist = self.create_dist() + cmd = config(dist) + + # simple pattern searches + match = cmd.search_cpp(pattern='xxx', body='/* xxx */') + self.assertEqual(match, 0) + + match = cmd.search_cpp(pattern='_configtest', body='/* xxx */') + self.assertEqual(match, 1) + + def test_finalize_options(self): + # finalize_options does a bit of transformation + # on options + pkg_dir, dist = self.create_dist() + cmd = config(dist) + cmd.include_dirs = 'one%stwo' % os.pathsep + cmd.libraries = 'one' + cmd.library_dirs = 'three%sfour' % os.pathsep + cmd.ensure_finalized() + + self.assertEqual(cmd.include_dirs, ['one', 'two']) + self.assertEqual(cmd.libraries, ['one']) + self.assertEqual(cmd.library_dirs, ['three', 'four']) + + def test_clean(self): + # _clean removes files + tmp_dir = self.mkdtemp() + f1 = os.path.join(tmp_dir, 'one') + f2 = os.path.join(tmp_dir, 'two') + + self.write_file(f1, 'xxx') + self.write_file(f2, 'xxx') + + for f in (f1, f2): + self.assertTrue(os.path.exists(f)) + + pkg_dir, dist = self.create_dist() + cmd = config(dist) + cmd._clean(f1, f2) + + for f in (f1, f2): + self.assertFalse(os.path.exists(f)) + + +def test_suite(): + return unittest.makeSuite(ConfigTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_data.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_data.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,148 @@ +"""Tests for packaging.command.install_data.""" +import os +import sys +import sysconfig +import packaging.database +from sysconfig import _get_default_scheme +from packaging.tests import unittest, support +from packaging.command.install_data import install_data +from packaging.command.install_dist import install_dist +from packaging.command.install_distinfo import install_distinfo + + +class InstallDataTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def setUp(self): + super(InstallDataTestCase, self).setUp() + scheme = _get_default_scheme() + old_items = sysconfig._SCHEMES.items(scheme) + + def restore(): + sysconfig._SCHEMES.remove_section(scheme) + sysconfig._SCHEMES.add_section(scheme) + for option, value in old_items: + sysconfig._SCHEMES.set(scheme, option, value) + + self.addCleanup(restore) + + def test_simple_run(self): + pkg_dir, dist = self.create_dist() + cmd = install_data(dist) + cmd.install_dir = inst = os.path.join(pkg_dir, 'inst') + scheme = _get_default_scheme() + + sysconfig._SCHEMES.set(scheme, 'inst', + os.path.join(pkg_dir, 'inst')) + sysconfig._SCHEMES.set(scheme, 'inst2', + os.path.join(pkg_dir, 'inst2')) + + one = os.path.join(pkg_dir, 'one') + self.write_file(one, 'xxx') + inst2 = os.path.join(pkg_dir, 'inst2') + two = os.path.join(pkg_dir, 'two') + self.write_file(two, 'xxx') + + # FIXME this creates a literal \{inst2\} directory! + cmd.data_files = {one: '{inst}/one', two: '{inst2}/two'} + self.assertCountEqual(cmd.get_inputs(), [one, two]) + + # let's run the command + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 2) + rtwo = os.path.split(two)[-1] + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + rone = os.path.split(one)[-1] + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + cmd.outfiles = [] + + # let's try with warn_dir one + cmd.warn_dir = True + cmd.finalized = False + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 2) + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + cmd.outfiles = [] + + # now using root and empty dir + cmd.root = os.path.join(pkg_dir, 'root') + three = os.path.join(cmd.install_dir, 'three') + self.write_file(three, 'xx') + + sysconfig._SCHEMES.set(scheme, 'inst3', cmd.install_dir) + + cmd.data_files = {one: '{inst}/one', two: '{inst2}/two', + three: '{inst3}/three'} + cmd.finalized = False + cmd.ensure_finalized() + cmd.run() + + # let's check the result + self.assertEqual(len(cmd.get_outputs()), 3) + self.assertTrue(os.path.exists(os.path.join(inst2, rtwo))) + self.assertTrue(os.path.exists(os.path.join(inst, rone))) + + def test_resources(self): + install_dir = self.mkdtemp() + scripts_dir = self.mkdtemp() + project_dir, dist = self.create_dist( + name='Spamlib', version='0.1', + data_files={'spamd': '{scripts}/spamd'}) + + os.chdir(project_dir) + self.write_file('spamd', '# Python script') + sysconfig._SCHEMES.set(_get_default_scheme(), 'scripts', scripts_dir) + sys.path.insert(0, install_dir) + packaging.database.disable_cache() + self.addCleanup(sys.path.remove, install_dir) + self.addCleanup(packaging.database.enable_cache) + + cmd = install_dist(dist) + cmd.outputs = ['spamd'] + cmd.install_lib = install_dir + dist.command_obj['install_dist'] = cmd + + cmd = install_data(dist) + cmd.install_dir = install_dir + cmd.ensure_finalized() + dist.command_obj['install_data'] = cmd + cmd.run() + + cmd = install_distinfo(dist) + cmd.ensure_finalized() + dist.command_obj['install_distinfo'] = cmd + cmd.run() + + # first a few sanity checks + self.assertEqual(os.listdir(scripts_dir), ['spamd']) + self.assertEqual(os.listdir(install_dir), ['Spamlib-0.1.dist-info']) + + # now the real test + fn = os.path.join(install_dir, 'Spamlib-0.1.dist-info', 'RESOURCES') + with open(fn, encoding='utf-8') as fp: + content = fp.read().strip() + + expected = 'spamd,%s' % os.path.join(scripts_dir, 'spamd') + self.assertEqual(content, expected) + + # just to be sure, we also test that get_file works here, even though + # packaging.database has its own test file + with packaging.database.get_file('Spamlib', 'spamd') as fp: + content = fp.read() + + self.assertEqual('# Python script', content) + + +def test_suite(): + return unittest.makeSuite(InstallDataTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_dist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_dist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,241 @@ +"""Tests for packaging.command.install.""" + +import os +import imp +import sys +from sysconfig import (get_scheme_names, get_config_vars, + _SCHEMES, get_config_var, get_path) + +from packaging.command.build_ext import build_ext +from packaging.command.install_dist import install_dist +from packaging.compiler.extension import Extension +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError + +from packaging.tests import unittest, support + + +_CONFIG_VARS = get_config_vars() + + +def _make_ext_name(modname): + if os.name == 'nt' and sys.executable.endswith('_d.exe'): + modname += '_d' + return modname + get_config_var('SO') + + +class InstallTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_home_installation_scheme(self): + # This ensure two things: + # - that --home generates the desired set of directory names + # - test --home is supported on all platforms + builddir = self.mkdtemp() + destination = os.path.join(builddir, "installation") + + dist = Distribution({"name": "foopkg"}) + dist.command_obj["build"] = support.DummyCommand( + build_base=builddir, + build_lib=os.path.join(builddir, "lib"), + ) + + old_posix_prefix = _SCHEMES.get('posix_prefix', 'platinclude') + old_posix_home = _SCHEMES.get('posix_home', 'platinclude') + + new_path = '{platbase}/include/python{py_version_short}' + _SCHEMES.set('posix_prefix', 'platinclude', new_path) + _SCHEMES.set('posix_home', 'platinclude', '{platbase}/include/python') + + try: + cmd = install_dist(dist) + cmd.home = destination + cmd.ensure_finalized() + finally: + _SCHEMES.set('posix_prefix', 'platinclude', old_posix_prefix) + _SCHEMES.set('posix_home', 'platinclude', old_posix_home) + + self.assertEqual(cmd.install_base, destination) + self.assertEqual(cmd.install_platbase, destination) + + def check_path(got, expected): + got = os.path.normpath(got) + expected = os.path.normpath(expected) + self.assertEqual(got, expected) + + libdir = os.path.join(destination, "lib", "python") + check_path(cmd.install_lib, libdir) + check_path(cmd.install_platlib, libdir) + check_path(cmd.install_purelib, libdir) + check_path(cmd.install_headers, + os.path.join(destination, "include", "python", "foopkg")) + check_path(cmd.install_scripts, os.path.join(destination, "bin")) + check_path(cmd.install_data, destination) + + def test_user_site(self): + # test install with --user + # preparing the environment for the test + self.old_user_base = get_config_var('userbase') + self.old_user_site = get_path('purelib', '%s_user' % os.name) + self.tmpdir = self.mkdtemp() + self.user_base = os.path.join(self.tmpdir, 'B') + self.user_site = os.path.join(self.tmpdir, 'S') + _CONFIG_VARS['userbase'] = self.user_base + scheme = '%s_user' % os.name + _SCHEMES.set(scheme, 'purelib', self.user_site) + + def _expanduser(path): + if path[0] == '~': + path = os.path.normpath(self.tmpdir) + path[1:] + return path + + self.old_expand = os.path.expanduser + os.path.expanduser = _expanduser + + def cleanup(): + _CONFIG_VARS['userbase'] = self.old_user_base + _SCHEMES.set(scheme, 'purelib', self.old_user_site) + os.path.expanduser = self.old_expand + + self.addCleanup(cleanup) + + schemes = get_scheme_names() + for key in ('nt_user', 'posix_user', 'os2_home'): + self.assertIn(key, schemes) + + dist = Distribution({'name': 'xx'}) + cmd = install_dist(dist) + + # making sure the user option is there + options = [name for name, short, lable in + cmd.user_options] + self.assertIn('user', options) + + # setting a value + cmd.user = True + + # user base and site shouldn't be created yet + self.assertFalse(os.path.exists(self.user_base)) + self.assertFalse(os.path.exists(self.user_site)) + + # let's run finalize + cmd.ensure_finalized() + + # now they should + self.assertTrue(os.path.exists(self.user_base)) + self.assertTrue(os.path.exists(self.user_site)) + + self.assertIn('userbase', cmd.config_vars) + self.assertIn('usersite', cmd.config_vars) + + def test_handle_extra_path(self): + dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'}) + cmd = install_dist(dist) + + # two elements + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, ['path', 'dirs']) + self.assertEqual(cmd.extra_dirs, 'dirs') + self.assertEqual(cmd.path_file, 'path') + + # one element + cmd.extra_path = ['path'] + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, ['path']) + self.assertEqual(cmd.extra_dirs, 'path') + self.assertEqual(cmd.path_file, 'path') + + # none + dist.extra_path = cmd.extra_path = None + cmd.handle_extra_path() + self.assertEqual(cmd.extra_path, None) + self.assertEqual(cmd.extra_dirs, '') + self.assertEqual(cmd.path_file, None) + + # three elements (no way !) + cmd.extra_path = 'path,dirs,again' + self.assertRaises(PackagingOptionError, cmd.handle_extra_path) + + def test_finalize_options(self): + dist = Distribution({'name': 'xx'}) + cmd = install_dist(dist) + + # must supply either prefix/exec-prefix/home or + # install-base/install-platbase -- not both + cmd.prefix = 'prefix' + cmd.install_base = 'base' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + # must supply either home or prefix/exec-prefix -- not both + cmd.install_base = None + cmd.home = 'home' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + # can't combine user with with prefix/exec_prefix/home or + # install_(plat)base + cmd.prefix = None + cmd.user = 'user' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + def test_old_record(self): + # test pre-PEP 376 --record option (outside dist-info dir) + install_dir = self.mkdtemp() + project_dir, dist = self.create_dist(py_modules=['hello'], + scripts=['sayhi']) + os.chdir(project_dir) + self.write_file('hello.py', "def main(): print('o hai')") + self.write_file('sayhi', 'from hello import main; main()') + + cmd = install_dist(dist) + dist.command_obj['install_dist'] = cmd + cmd.root = install_dir + cmd.record = os.path.join(project_dir, 'filelist') + cmd.ensure_finalized() + cmd.run() + + with open(cmd.record) as f: + content = f.read() + + found = [os.path.basename(line) for line in content.splitlines()] + expected = ['hello.py', 'hello.%s.pyc' % imp.get_tag(), 'sayhi', + 'METADATA', 'INSTALLER', 'REQUESTED', 'RECORD'] + self.assertEqual(sorted(found), sorted(expected)) + + # XXX test that fancy_getopt is okay with options named + # record and no-record but unrelated + + def test_old_record_extensions(self): + # test pre-PEP 376 --record option with ext modules + install_dir = self.mkdtemp() + project_dir, dist = self.create_dist(ext_modules=[ + Extension('xx', ['xxmodule.c'])]) + os.chdir(project_dir) + support.copy_xxmodule_c(project_dir) + + buildextcmd = build_ext(dist) + support.fixup_build_ext(buildextcmd) + buildextcmd.ensure_finalized() + + cmd = install_dist(dist) + dist.command_obj['install_dist'] = cmd + dist.command_obj['build_ext'] = buildextcmd + cmd.root = install_dir + cmd.record = os.path.join(project_dir, 'filelist') + cmd.ensure_finalized() + cmd.run() + + with open(cmd.record) as f: + content = f.read() + + found = [os.path.basename(line) for line in content.splitlines()] + expected = [_make_ext_name('xx'), + 'METADATA', 'INSTALLER', 'REQUESTED', 'RECORD'] + self.assertEqual(found, expected) + + +def test_suite(): + return unittest.makeSuite(InstallTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_distinfo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_distinfo.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,252 @@ +"""Tests for ``packaging.command.install_distinfo``. + +Writing of the RESOURCES file is tested in test_command_install_data. +""" + +import os +import csv +import hashlib +import sysconfig + +from packaging.command.install_distinfo import install_distinfo +from packaging.command.cmd import Command +from packaging.compiler.extension import Extension +from packaging.metadata import Metadata +from packaging.tests import unittest, support + + +class DummyInstallCmd(Command): + + def __init__(self, dist=None): + self.outputs = [] + self.distribution = dist + + def __getattr__(self, name): + return None + + def ensure_finalized(self): + pass + + def get_outputs(self): + return (self.outputs + + self.get_finalized_command('install_distinfo').get_outputs()) + + +class InstallDistinfoTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + checkLists = lambda self, x, y: self.assertListEqual(sorted(x), sorted(y)) + + def test_empty_install(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.install_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + self.checkLists(os.listdir(install_dir), ['foo-1.0.dist-info']) + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'REQUESTED', 'INSTALLER']) + with open(os.path.join(dist_info, 'INSTALLER')) as fp: + self.assertEqual(fp.read(), 'distutils') + with open(os.path.join(dist_info, 'REQUESTED')) as fp: + self.assertEqual(fp.read(), '') + meta_path = os.path.join(dist_info, 'METADATA') + self.assertTrue(Metadata(path=meta_path).check()) + + def test_installer(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.install_dir = install_dir + cmd.installer = 'bacon-python' + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + with open(os.path.join(dist_info, 'INSTALLER')) as fp: + self.assertEqual(fp.read(), 'bacon-python') + + def test_requested(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.install_dir = install_dir + cmd.requested = False + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'RECORD', 'INSTALLER']) + + def test_no_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.install_dir = install_dir + cmd.no_record = True + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + self.checkLists(os.listdir(dist_info), + ['METADATA', 'REQUESTED', 'INSTALLER']) + + def test_record_basic(self): + install_dir = self.mkdtemp() + modules_dest = os.path.join(install_dir, 'lib') + scripts_dest = os.path.join(install_dir, 'bin') + project_dir, dist = self.create_dist( + name='Spamlib', version='0.1', + py_modules=['spam'], scripts=['spamd'], + ext_modules=[Extension('_speedspam', ['_speedspam.c'])]) + + # using a real install_dist command is too painful, so we use a mock + # class that's only a holder for options to be used by install_distinfo + # and we create placeholder files manually instead of using build_*. + # the install_* commands will still be consulted by install_distinfo. + os.chdir(project_dir) + self.write_file('spam', '# Python module') + self.write_file('spamd', '# Python script') + extmod = '_speedspam' + sysconfig.get_config_var('SO') + self.write_file(extmod, '') + + install = DummyInstallCmd(dist) + install.outputs = ['spam', 'spamd', extmod] + install.install_lib = modules_dest + install.install_scripts = scripts_dest + dist.command_obj['install_dist'] = install + + cmd = install_distinfo(dist) + cmd.ensure_finalized() + dist.command_obj['install_distinfo'] = cmd + cmd.run() + + # checksum and size are not hard-coded for METADATA as it is + # platform-dependent (line endings) + metadata = os.path.join(modules_dest, 'Spamlib-0.1.dist-info', + 'METADATA') + with open(metadata, 'rb') as fp: + content = fp.read() + + metadata_size = str(len(content)) + metadata_md5 = hashlib.md5(content).hexdigest() + + record = os.path.join(modules_dest, 'Spamlib-0.1.dist-info', 'RECORD') + with open(record, encoding='utf-8') as fp: + content = fp.read() + + found = [] + for line in content.splitlines(): + filename, checksum, size = line.split(',') + filename = os.path.basename(filename) + found.append((filename, checksum, size)) + + expected = [ + ('spam', '6ab2f288ef2545868effe68757448b45', '15'), + ('spamd', 'd13e6156ce78919a981e424b2fdcd974', '15'), + (extmod, 'd41d8cd98f00b204e9800998ecf8427e', '0'), + ('METADATA', metadata_md5, metadata_size), + ('INSTALLER', '44e3fde05f3f537ed85831969acf396d', '9'), + ('REQUESTED', 'd41d8cd98f00b204e9800998ecf8427e', '0'), + ('RECORD', '', ''), + ] + self.assertEqual(found, expected) + + def test_record(self): + pkg_dir, dist = self.create_dist(name='foo', + version='1.0') + install_dir = self.mkdtemp() + + install = DummyInstallCmd(dist) + dist.command_obj['install_dist'] = install + + fake_dists = os.path.join(os.path.dirname(__file__), 'fake_dists') + fake_dists = os.path.realpath(fake_dists) + + # for testing, we simply add all files from _backport's fake_dists + dirs = [] + for dir in os.listdir(fake_dists): + full_path = os.path.join(fake_dists, dir) + if (not dir.endswith('.egg') or dir.endswith('.egg-info') or + dir.endswith('.dist-info')) and os.path.isdir(full_path): + dirs.append(full_path) + + for dir in dirs: + for path, subdirs, files in os.walk(dir): + install.outputs += [os.path.join(path, f) for f in files] + install.outputs += [os.path.join('path', f + 'c') + for f in files if f.endswith('.py')] + + cmd = install_distinfo(dist) + dist.command_obj['install_distinfo'] = cmd + + cmd.install_dir = install_dir + cmd.ensure_finalized() + cmd.run() + + dist_info = os.path.join(install_dir, 'foo-1.0.dist-info') + + expected = [] + for f in install.get_outputs(): + if (f.endswith(('.pyc', '.pyo')) or f == os.path.join( + install_dir, 'foo-1.0.dist-info', 'RECORD')): + expected.append([f, '', '']) + else: + size = os.path.getsize(f) + md5 = hashlib.md5() + with open(f, 'rb') as fp: + md5.update(fp.read()) + hash = md5.hexdigest() + expected.append([f, hash, str(size)]) + + parsed = [] + with open(os.path.join(dist_info, 'RECORD'), 'r') as f: + reader = csv.reader(f, delimiter=',', + lineterminator=os.linesep, + quotechar='"') + parsed = list(reader) + + self.maxDiff = None + self.checkLists(parsed, expected) + + +def test_suite(): + return unittest.makeSuite(InstallDistinfoTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_headers.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_headers.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,38 @@ +"""Tests for packaging.command.install_headers.""" +import os + +from packaging.command.install_headers import install_headers +from packaging.tests import unittest, support + + +class InstallHeadersTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_simple_run(self): + # we have two headers + header_list = self.mkdtemp() + header1 = os.path.join(header_list, 'header1') + header2 = os.path.join(header_list, 'header2') + self.write_file(header1) + self.write_file(header2) + headers = [header1, header2] + + pkg_dir, dist = self.create_dist(headers=headers) + cmd = install_headers(dist) + self.assertEqual(cmd.get_inputs(), headers) + + # let's run the command + cmd.install_dir = os.path.join(pkg_dir, 'inst') + cmd.ensure_finalized() + cmd.run() + + # let's check the results + self.assertEqual(len(cmd.get_outputs()), 2) + + +def test_suite(): + return unittest.makeSuite(InstallHeadersTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_lib.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_lib.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,110 @@ +"""Tests for packaging.command.install_data.""" +import os +import sys +import imp + +from packaging.tests import unittest, support +from packaging.command.install_lib import install_lib +from packaging.compiler.extension import Extension +from packaging.errors import PackagingOptionError + + +class InstallLibTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['PYTHONPATH'] + + def test_finalize_options(self): + dist = self.create_dist()[1] + cmd = install_lib(dist) + + cmd.finalize_options() + self.assertTrue(cmd.compile) + self.assertEqual(cmd.optimize, 0) + + # optimize must be 0, 1, or 2 + cmd.optimize = 'foo' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + cmd.optimize = '4' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + cmd.optimize = '2' + cmd.finalize_options() + self.assertEqual(cmd.optimize, 2) + + def test_byte_compile(self): + project_dir, dist = self.create_dist() + os.chdir(project_dir) + cmd = install_lib(dist) + cmd.compile = True + cmd.optimize = 1 + + f = os.path.join(project_dir, 'foo.py') + self.write_file(f, '# python file') + cmd.byte_compile([f]) + pyc_file = imp.cache_from_source('foo.py', True) + pyo_file = imp.cache_from_source('foo.py', False) + self.assertTrue(os.path.exists(pyc_file)) + self.assertTrue(os.path.exists(pyo_file)) + + def test_byte_compile_under_B(self): + # make sure byte compilation works under -B (dont_write_bytecode) + self.addCleanup(setattr, sys, 'dont_write_bytecode', + sys.dont_write_bytecode) + sys.dont_write_bytecode = True + self.test_byte_compile() + + def test_get_outputs(self): + project_dir, dist = self.create_dist() + os.chdir(project_dir) + os.mkdir('spam') + cmd = install_lib(dist) + + # setting up a dist environment + cmd.compile = True + cmd.optimize = 1 + cmd.install_dir = self.mkdtemp() + f = os.path.join(project_dir, 'spam', '__init__.py') + self.write_file(f, '# python package') + cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] + cmd.distribution.packages = ['spam'] + + # make sure the build_lib is set the temp dir # XXX what? this is not + # needed in the same distutils test and should work without manual + # intervention + build_dir = os.path.split(project_dir)[0] + cmd.get_finalized_command('build_py').build_lib = build_dir + + # get_outputs should return 4 elements: spam/__init__.py, .pyc and + # .pyo, foo.import-tag-abiflags.so / foo.pyd + outputs = cmd.get_outputs() + self.assertEqual(len(outputs), 4, outputs) + + def test_get_inputs(self): + project_dir, dist = self.create_dist() + os.chdir(project_dir) + os.mkdir('spam') + cmd = install_lib(dist) + + # setting up a dist environment + cmd.compile = True + cmd.optimize = 1 + cmd.install_dir = self.mkdtemp() + f = os.path.join(project_dir, 'spam', '__init__.py') + self.write_file(f, '# python package') + cmd.distribution.ext_modules = [Extension('foo', ['xxx'])] + cmd.distribution.packages = ['spam'] + + # get_inputs should return 2 elements: spam/__init__.py and + # foo.import-tag-abiflags.so / foo.pyd + inputs = cmd.get_inputs() + self.assertEqual(len(inputs), 2, inputs) + + +def test_suite(): + return unittest.makeSuite(InstallLibTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_install_scripts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_install_scripts.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,75 @@ +"""Tests for packaging.command.install_scripts.""" +import os + +from packaging.tests import unittest, support +from packaging.command.install_scripts import install_scripts +from packaging.dist import Distribution + + +class InstallScriptsTestCase(support.TempdirManager, + support.LoggingCatcher, + unittest.TestCase): + + def test_default_settings(self): + dist = Distribution() + dist.command_obj["build"] = support.DummyCommand( + build_scripts="/foo/bar") + dist.command_obj["install_dist"] = support.DummyCommand( + install_scripts="/splat/funk", + force=True, + skip_build=True, + ) + cmd = install_scripts(dist) + self.assertFalse(cmd.force) + self.assertFalse(cmd.skip_build) + self.assertIs(cmd.build_dir, None) + self.assertIs(cmd.install_dir, None) + + cmd.finalize_options() + + self.assertTrue(cmd.force) + self.assertTrue(cmd.skip_build) + self.assertEqual(cmd.build_dir, "/foo/bar") + self.assertEqual(cmd.install_dir, "/splat/funk") + + def test_installation(self): + source = self.mkdtemp() + expected = [] + + def write_script(name, text): + expected.append(name) + with open(os.path.join(source, name), "w") as f: + f.write(text) + + write_script("script1.py", ("#! /usr/bin/env python2.3\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + write_script("script2.py", ("#!/usr/bin/python\n" + "# bogus script w/ Python sh-bang\n" + "pass\n")) + write_script("shell.sh", ("#!/bin/sh\n" + "# bogus shell script w/ sh-bang\n" + "exit 0\n")) + + target = self.mkdtemp() + dist = Distribution() + dist.command_obj["build"] = support.DummyCommand(build_scripts=source) + dist.command_obj["install_dist"] = support.DummyCommand( + install_scripts=target, + force=True, + skip_build=True, + ) + cmd = install_scripts(dist) + cmd.finalize_options() + cmd.run() + + installed = os.listdir(target) + for name in expected: + self.assertIn(name, installed) + + +def test_suite(): + return unittest.makeSuite(InstallScriptsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_register.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_register.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,260 @@ +"""Tests for packaging.command.register.""" +import os +import getpass +import urllib.request +import urllib.error +import urllib.parse + +try: + import docutils + DOCUTILS_SUPPORT = True +except ImportError: + DOCUTILS_SUPPORT = False + +from packaging.tests import unittest, support +from packaging.tests.support import Inputs +from packaging.command import register as register_module +from packaging.command.register import register +from packaging.errors import PackagingSetupError + + +PYPIRC_NOPASSWORD = """\ +[distutils] + +index-servers = + server1 + +[server1] +username:me +""" + +WANTED_PYPIRC = """\ +[distutils] +index-servers = + pypi + +[pypi] +username:tarek +password:password +""" + + +class FakeOpener: + """Fakes a PyPI server""" + def __init__(self): + self.reqs = [] + + def __call__(self, *args): + return self + + def open(self, req): + self.reqs.append(req) + return self + + def read(self): + return 'xxx' + + +class RegisterTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + unittest.TestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(RegisterTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + + # patching the password prompt + self._old_getpass = getpass.getpass + + def _getpass(prompt): + return 'password' + + getpass.getpass = _getpass + self.old_opener = urllib.request.build_opener + self.conn = urllib.request.build_opener = FakeOpener() + + def tearDown(self): + getpass.getpass = self._old_getpass + urllib.request.build_opener = self.old_opener + if hasattr(register_module, 'input'): + del register_module.input + super(RegisterTestCase, self).tearDown() + + def _get_cmd(self, metadata=None): + if metadata is None: + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': 'xxx'} + pkg_info, dist = self.create_dist(**metadata) + return register(dist) + + def test_create_pypirc(self): + # this test makes sure a .pypirc file + # is created when requested. + + # let's create a register instance + cmd = self._get_cmd() + + # we shouldn't have a .pypirc file yet + self.assertFalse(os.path.exists(self.rc)) + + # patching input and getpass.getpass + # so register gets happy + # Here's what we are faking : + # use your existing login (choice 1.) + # Username : 'tarek' + # Password : 'password' + # Save your login (y/N)? : 'y' + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # we should have a brand new .pypirc file + self.assertTrue(os.path.exists(self.rc)) + + # with the content similar to WANTED_PYPIRC + with open(self.rc) as fp: + content = fp.read() + self.assertEqual(content, WANTED_PYPIRC) + + # now let's make sure the .pypirc file generated + # really works : we shouldn't be asked anything + # if we run the command again + def _no_way(prompt=''): + raise AssertionError(prompt) + + register_module.input = _no_way + cmd.show_response = True + cmd.finalized = False + cmd.ensure_finalized() + cmd.run() + + # let's see what the server received : we should + # have 2 similar requests + self.assertEqual(len(self.conn.reqs), 2) + req1 = dict(self.conn.reqs[0].headers) + req2 = dict(self.conn.reqs[1].headers) + self.assertEqual(req2['Content-length'], req1['Content-length']) + self.assertIn(b'xxx', self.conn.reqs[1].data) + + def test_password_not_in_file(self): + + self.write_file(self.rc, PYPIRC_NOPASSWORD) + cmd = self._get_cmd() + cmd.finalize_options() + cmd._set_config() + cmd.send_metadata() + + # dist.password should be set + # therefore used afterwards by other commands + self.assertEqual(cmd.distribution.password, 'password') + + def test_registration(self): + # this test runs choice 2 + cmd = self._get_cmd() + inputs = Inputs('2', 'tarek', 'tarek@ziade.org') + register_module.input = inputs + # let's run the command + # FIXME does this send a real request? use a mock server + cmd.ensure_finalized() + cmd.run() + + # we should have send a request + self.assertEqual(len(self.conn.reqs), 1) + req = self.conn.reqs[0] + headers = dict(req.headers) + self.assertEqual(headers['Content-length'], '628') + self.assertIn(b'tarek', req.data) + + def test_password_reset(self): + # this test runs choice 3 + cmd = self._get_cmd() + inputs = Inputs('3', 'tarek@ziade.org') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # we should have send a request + self.assertEqual(len(self.conn.reqs), 1) + req = self.conn.reqs[0] + headers = dict(req.headers) + self.assertEqual(headers['Content-length'], '298') + self.assertIn(b'tarek', req.data) + + @unittest.skipUnless(DOCUTILS_SUPPORT, 'needs docutils') + def test_strict(self): + # testing the strict option: when on, the register command stops if the + # metadata is incomplete or if description contains bad reST + + # empty metadata # XXX this is not really empty.. + cmd = self._get_cmd({'name': 'xxx', 'version': 'xxx'}) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + self.assertRaises(PackagingSetupError, cmd.run) + + # metadata is OK but description is broken + metadata = {'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'éxéxé', + 'name': 'xxx', 'version': '4.2', + 'description': 'title\n==\n\ntext'} + + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + self.assertRaises(PackagingSetupError, cmd.run) + + # now something that works + metadata['description'] = 'title\n=====\n\ntext' + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # strict is not by default + cmd = self._get_cmd() + cmd.ensure_finalized() + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + # and finally a Unicode test (bug #12114) + metadata = {'home_page': 'xxx', 'author': '\u00c9ric', + 'author_email': 'xxx', 'name': 'xxx', + 'version': 'xxx', + 'summary': 'Something about esszet \u00df', + 'description': 'More things about esszet \u00df'} + + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = Inputs('1', 'tarek', 'y') + register_module.input = inputs + cmd.ensure_finalized() + cmd.run() + + def test_register_pep345(self): + cmd = self._get_cmd({}) + cmd.ensure_finalized() + cmd.distribution.metadata['Requires-Dist'] = ['lxml'] + data = cmd.build_post_data('submit') + self.assertEqual(data['metadata_version'], '1.2') + self.assertEqual(data['requires_dist'], ['lxml']) + + +def test_suite(): + return unittest.makeSuite(RegisterTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_sdist.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_sdist.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,394 @@ +"""Tests for packaging.command.sdist.""" +import os +import tarfile +import zipfile + +try: + import grp + import pwd + UID_GID_SUPPORT = True +except ImportError: + UID_GID_SUPPORT = False + +from shutil import get_archive_formats +from os.path import join +from packaging.dist import Distribution +from packaging.util import find_executable +from packaging.errors import PackagingOptionError +from packaging.command.sdist import sdist, show_formats + +from test.support import captured_stdout +from packaging.tests import support, unittest +from packaging.tests.support import requires_zlib + + +MANIFEST = """\ +# file GENERATED by packaging, do NOT edit +inroot.txt +setup.cfg +data%(sep)sdata.dt +scripts%(sep)sscript.py +some%(sep)sfile.txt +some%(sep)sother_file.txt +somecode%(sep)s__init__.py +somecode%(sep)sdoc.dat +somecode%(sep)sdoc.txt +""" + + +def builder(dist, filelist): + filelist.append('bah') + + +class SDistTestCase(support.TempdirManager, + support.LoggingCatcher, + support.EnvironRestorer, + unittest.TestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(SDistTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + os.environ['HOME'] = self.tmp_dir + # setting up an environment + self.old_path = os.getcwd() + os.mkdir(join(self.tmp_dir, 'somecode')) + os.mkdir(join(self.tmp_dir, 'dist')) + # a package, and a README + self.write_file((self.tmp_dir, 'README'), 'xxx') + self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#') + os.chdir(self.tmp_dir) + + def tearDown(self): + # back to normal + os.chdir(self.old_path) + super(SDistTestCase, self).tearDown() + + def get_cmd(self, metadata=None): + """Returns a cmd""" + if metadata is None: + metadata = {'name': 'fake', 'version': '1.0', + 'home_page': 'xxx', 'author': 'xxx', + 'author_email': 'xxx'} + dist = Distribution(metadata) + dist.packages = ['somecode'] + cmd = sdist(dist) + cmd.dist_dir = 'dist' + return dist, cmd + + @requires_zlib + def test_prune_file_list(self): + # this test creates a package with some vcs dirs in it + # and launch sdist to make sure they get pruned + # on all systems + + # creating VCS directories with some files in them + os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) + self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx') + + os.mkdir(join(self.tmp_dir, 'somecode', '.hg')) + self.write_file((self.tmp_dir, 'somecode', '.hg', + 'ok'), 'xxx') + + os.mkdir(join(self.tmp_dir, 'somecode', '.git')) + self.write_file((self.tmp_dir, 'somecode', '.git', + 'ok'), 'xxx') + + # now building a sdist + dist, cmd = self.get_cmd() + + # zip is available universally + # (tar might not be installed under win32) + cmd.formats = ['zip'] + + cmd.ensure_finalized() + cmd.run() + + # now let's check what we have + dist_folder = join(self.tmp_dir, 'dist') + files = os.listdir(dist_folder) + self.assertEqual(files, ['fake-1.0.zip']) + + with zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip')) as zip_file: + content = zip_file.namelist() + + # making sure everything has been pruned correctly + self.assertEqual(len(content), 2) + + @requires_zlib + @unittest.skipIf(find_executable('tar') is None or + find_executable('gzip') is None, + 'requires tar and gzip programs') + def test_make_distribution(self): + # building a sdist + dist, cmd = self.get_cmd() + + # creating a gztar then a tar + cmd.formats = ['gztar', 'tar'] + cmd.ensure_finalized() + cmd.run() + + # making sure we have two files + dist_folder = join(self.tmp_dir, 'dist') + result = sorted(os.listdir(dist_folder)) + self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz']) + + os.remove(join(dist_folder, 'fake-1.0.tar')) + os.remove(join(dist_folder, 'fake-1.0.tar.gz')) + + # now trying a tar then a gztar + cmd.formats = ['tar', 'gztar'] + cmd.finalized = False + cmd.ensure_finalized() + cmd.run() + + result = sorted(os.listdir(dist_folder)) + self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz']) + + @requires_zlib + def test_add_defaults(self): + + # http://bugs.python.org/issue2279 + + # add_default should also include + # data_files and package_data + dist, cmd = self.get_cmd() + + # filling data_files by pointing files + # in package_data + dist.package_data = {'': ['*.cfg', '*.dat'], + 'somecode': ['*.txt']} + self.write_file((self.tmp_dir, 'setup.cfg'), '#') + self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#') + self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#') + + # adding some data in data_files + data_dir = join(self.tmp_dir, 'data') + os.mkdir(data_dir) + self.write_file((data_dir, 'data.dt'), '#') + some_dir = join(self.tmp_dir, 'some') + os.mkdir(some_dir) + self.write_file((self.tmp_dir, 'inroot.txt'), '#') + self.write_file((some_dir, 'file.txt'), '#') + self.write_file((some_dir, 'other_file.txt'), '#') + + dist.data_files = {'data/data.dt': '{appdata}/data.dt', + 'inroot.txt': '{appdata}/inroot.txt', + 'some/file.txt': '{appdata}/file.txt', + 'some/other_file.txt': '{appdata}/other_file.txt'} + + # adding a script + script_dir = join(self.tmp_dir, 'scripts') + os.mkdir(script_dir) + self.write_file((script_dir, 'script.py'), '#') + dist.scripts = [join('scripts', 'script.py')] + + cmd.formats = ['zip'] + cmd.use_defaults = True + + cmd.ensure_finalized() + cmd.run() + + # now let's check what we have + dist_folder = join(self.tmp_dir, 'dist') + files = os.listdir(dist_folder) + self.assertEqual(files, ['fake-1.0.zip']) + + with zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip')) as zip_file: + content = zip_file.namelist() + + # Making sure everything was added. This includes 8 code and data + # files in addition to PKG-INFO and setup.cfg + self.assertEqual(len(content), 10) + + # Checking the MANIFEST + with open(join(self.tmp_dir, 'MANIFEST')) as fp: + manifest = fp.read() + self.assertEqual(manifest, MANIFEST % {'sep': os.sep}) + + @requires_zlib + def test_metadata_check_option(self): + # testing the `check-metadata` option + dist, cmd = self.get_cmd(metadata={'name': 'xxx', 'version': 'xxx'}) + + # this should cause the check subcommand to log two warnings: + # version is invalid, home-page and author are missing + cmd.ensure_finalized() + cmd.run() + warnings = self.get_logs() + check_warnings = [msg for msg in warnings if + not msg.startswith('sdist:')] + self.assertEqual(len(check_warnings), 2, warnings) + + # trying with a complete set of metadata + self.loghandler.flush() + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + cmd.metadata_check = False + cmd.run() + warnings = self.get_logs() + self.assertEqual(len(warnings), 2) + self.assertIn('using default file list', warnings[0]) + self.assertIn("'setup.cfg' file not found", warnings[1]) + + def test_show_formats(self): + with captured_stdout() as stdout: + show_formats() + stdout = stdout.getvalue() + + # the output should be a header line + one line per format + num_formats = len(get_archive_formats()) + output = [line for line in stdout.split('\n') + if line.strip().startswith('--formats=')] + self.assertEqual(len(output), num_formats) + + def test_finalize_options(self): + dist, cmd = self.get_cmd() + cmd.finalize_options() + + # default options set by finalize + self.assertEqual(cmd.manifest, 'MANIFEST') + self.assertEqual(cmd.dist_dir, 'dist') + + # formats has to be a string splitable on (' ', ',') or + # a stringlist + cmd.formats = 1 + self.assertRaises(PackagingOptionError, cmd.finalize_options) + cmd.formats = ['zip'] + cmd.finalize_options() + + # formats has to be known + cmd.formats = 'supazipa' + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + @requires_zlib + def test_template(self): + dist, cmd = self.get_cmd() + dist.extra_files = ['include yeah'] + cmd.ensure_finalized() + self.write_file((self.tmp_dir, 'yeah'), 'xxx') + cmd.run() + with open(cmd.manifest) as f: + content = f.read() + + self.assertIn('yeah', content) + + @requires_zlib + @unittest.skipUnless(UID_GID_SUPPORT, "requires grp and pwd support") + @unittest.skipIf(find_executable('tar') is None or + find_executable('gzip') is None, + 'requires tar and gzip programs') + def test_make_distribution_owner_group(self): + # building a sdist + dist, cmd = self.get_cmd() + + # creating a gztar and specifying the owner+group + cmd.formats = ['gztar'] + cmd.owner = pwd.getpwuid(0)[0] + cmd.group = grp.getgrgid(0)[0] + cmd.ensure_finalized() + cmd.run() + + # making sure we have the good rights + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + with tarfile.open(archive_name) as archive: + for member in archive.getmembers(): + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + + # building a sdist again + dist, cmd = self.get_cmd() + + # creating a gztar + cmd.formats = ['gztar'] + cmd.ensure_finalized() + cmd.run() + + # making sure we have the good rights + archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz') + with tarfile.open(archive_name) as archive: + + # note that we are not testing the group ownership here + # because, depending on the platforms and the container + # rights (see #7408) + for member in archive.getmembers(): + self.assertEqual(member.uid, os.getuid()) + + @requires_zlib + def test_get_file_list(self): + # make sure MANIFEST is recalculated + dist, cmd = self.get_cmd() + # filling data_files by pointing files in package_data + dist.package_data = {'somecode': ['*.txt']} + self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#') + cmd.ensure_finalized() + cmd.run() + + # Should produce four lines. Those lines are one comment, one default + # (README) and two package files. + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + self.assertEqual(len(manifest), 3) + + # Adding a file + self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') + + # make sure build_py is reinitialized, like a fresh run + build_py = dist.get_command_obj('build_py') + build_py.finalized = False + build_py.ensure_finalized() + + cmd.run() + + with open(cmd.manifest) as f: + manifest2 = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + # Do we have the new file in MANIFEST? + self.assertEqual(len(manifest2), 4) + self.assertIn('doc2.txt', manifest2[-1]) + + @requires_zlib + def test_manifest_marker(self): + # check that autogenerated MANIFESTs have a marker + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + cmd.run() + + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + self.assertEqual(manifest[0], + '# file GENERATED by packaging, do NOT edit') + + @requires_zlib + def test_manual_manifest(self): + # check that a MANIFEST without a marker is left alone + dist, cmd = self.get_cmd() + cmd.ensure_finalized() + self.write_file((self.tmp_dir, cmd.manifest), 'README.manual') + cmd.run() + + with open(cmd.manifest) as f: + manifest = [line.strip() for line in f.read().split('\n') + if line.strip() != ''] + + self.assertEqual(manifest, ['README.manual']) + + @requires_zlib + def test_manifest_builder(self): + dist, cmd = self.get_cmd() + cmd.manifest_builders = 'packaging.tests.test_command_sdist.builder' + cmd.ensure_finalized() + cmd.run() + self.assertIn('bah', cmd.filelist.files) + + +def test_suite(): + return unittest.makeSuite(SDistTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_test.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_test.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,224 @@ +import os +import re +import sys +import shutil +import unittest as ut1 +import packaging.database + +from os.path import join +from operator import getitem, setitem, delitem +from packaging.command.build import build +from packaging.tests import unittest +from packaging.tests.support import (TempdirManager, EnvironRestorer, + LoggingCatcher) +from packaging.command.test import test +from packaging.command import set_command +from packaging.dist import Distribution + + +EXPECTED_OUTPUT_RE = r'''FAIL: test_blah \(myowntestmodule.SomeTest\) +---------------------------------------------------------------------- +Traceback \(most recent call last\): + File ".+/myowntestmodule.py", line \d+, in test_blah + self.fail\("horribly"\) +AssertionError: horribly +''' + +here = os.path.dirname(os.path.abspath(__file__)) + + +class MockBuildCmd(build): + build_lib = "mock build lib" + command_name = 'build' + plat_name = 'whatever' + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + self._record.append("build has run") + + +class TestTest(TempdirManager, + EnvironRestorer, + LoggingCatcher, + unittest.TestCase): + + restore_environ = ['PYTHONPATH'] + + def setUp(self): + super(TestTest, self).setUp() + self.addCleanup(packaging.database.clear_cache) + new_pythonpath = os.path.dirname(os.path.dirname(here)) + pythonpath = os.environ.get('PYTHONPATH') + if pythonpath is not None: + new_pythonpath = os.pathsep.join((new_pythonpath, pythonpath)) + os.environ['PYTHONPATH'] = new_pythonpath + + def assert_re_match(self, pattern, string): + def quote(s): + lines = ['## ' + line for line in s.split('\n')] + sep = ["#" * 60] + return [''] + sep + lines + sep + msg = quote(pattern) + ["didn't match"] + quote(string) + msg = "\n".join(msg) + if not re.search(pattern, string): + self.fail(msg) + + def prepare_dist(self, dist_name): + pkg_dir = join(os.path.dirname(__file__), "dists", dist_name) + temp_pkg_dir = join(self.mkdtemp(), dist_name) + shutil.copytree(pkg_dir, temp_pkg_dir) + return temp_pkg_dir + + def safely_replace(self, obj, attr, + new_val=None, delete=False, dictionary=False): + """Replace a object's attribute returning to its original state at the + end of the test run. Creates the attribute if not present before + (deleting afterwards). When delete=True, makes sure the value is del'd + for the test run. If dictionary is set to True, operates of its items + rather than attributes.""" + if dictionary: + _setattr, _getattr, _delattr = setitem, getitem, delitem + + def _hasattr(_dict, value): + return value in _dict + else: + _setattr, _getattr, _delattr, _hasattr = (setattr, getattr, + delattr, hasattr) + + orig_has_attr = _hasattr(obj, attr) + if orig_has_attr: + orig_val = _getattr(obj, attr) + + if delete is False: + _setattr(obj, attr, new_val) + elif orig_has_attr: + _delattr(obj, attr) + + def do_cleanup(): + if orig_has_attr: + _setattr(obj, attr, orig_val) + elif _hasattr(obj, attr): + _delattr(obj, attr) + + self.addCleanup(do_cleanup) + + def test_runs_unittest(self): + module_name, a_module = self.prepare_a_module() + record = [] + a_module.recorder = lambda *args: record.append("suite") + + class MockTextTestRunner: + def __init__(*_, **__): + pass + + def run(_self, suite): + record.append("run") + + self.safely_replace(ut1, "TextTestRunner", MockTextTestRunner) + + dist = Distribution() + cmd = test(dist) + cmd.suite = "%s.recorder" % module_name + cmd.run() + self.assertEqual(record, ["suite", "run"]) + + def test_builds_before_running_tests(self): + self.addCleanup(set_command, 'packaging.command.build.build') + set_command('packaging.tests.test_command_test.MockBuildCmd') + + dist = Distribution() + dist.get_command_obj('build')._record = record = [] + cmd = test(dist) + cmd.runner = self.prepare_named_function(lambda: None) + cmd.ensure_finalized() + cmd.run() + self.assertEqual(['build has run'], record) + + @unittest.skip('needs to be written') + def test_works_with_2to3(self): + pass + + def test_checks_requires(self): + dist = Distribution() + cmd = test(dist) + phony_project = 'ohno_ohno-impossible_1234-name_stop-that!' + cmd.tests_require = [phony_project] + cmd.ensure_finalized() + logs = self.get_logs() + self.assertIn(phony_project, logs[-1]) + + def prepare_a_module(self): + tmp_dir = self.mkdtemp() + sys.path.append(tmp_dir) + self.addCleanup(sys.path.remove, tmp_dir) + + self.write_file((tmp_dir, 'packaging_tests_a.py'), '') + import packaging_tests_a as a_module + return "packaging_tests_a", a_module + + def prepare_named_function(self, func): + module_name, a_module = self.prepare_a_module() + a_module.recorder = func + return "%s.recorder" % module_name + + def test_custom_runner(self): + dist = Distribution() + cmd = test(dist) + record = [] + cmd.runner = self.prepare_named_function( + lambda: record.append("runner called")) + cmd.ensure_finalized() + cmd.run() + self.assertEqual(["runner called"], record) + + def prepare_mock_ut2(self): + class MockUTClass: + def __init__(*_, **__): + pass + + def discover(self): + pass + + def run(self, _): + pass + + class MockUTModule: + TestLoader = MockUTClass + TextTestRunner = MockUTClass + + mock_ut2 = MockUTModule() + self.safely_replace(sys.modules, "unittest2", + mock_ut2, dictionary=True) + return mock_ut2 + + def test_gets_unittest_discovery(self): + mock_ut2 = self.prepare_mock_ut2() + dist = Distribution() + cmd = test(dist) + self.safely_replace(ut1.TestLoader, "discover", lambda: None) + self.assertEqual(cmd.get_ut_with_discovery(), ut1) + + del ut1.TestLoader.discover + self.assertEqual(cmd.get_ut_with_discovery(), mock_ut2) + + def test_calls_discover(self): + self.safely_replace(ut1.TestLoader, "discover", delete=True) + mock_ut2 = self.prepare_mock_ut2() + record = [] + mock_ut2.TestLoader.discover = lambda self, path: record.append(path) + dist = Distribution() + cmd = test(dist) + cmd.run() + self.assertEqual([os.curdir], record) + + +def test_suite(): + return unittest.makeSuite(TestTest) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_upload.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_upload.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,159 @@ +"""Tests for packaging.command.upload.""" +import os + +from packaging.command.upload import upload +from packaging.dist import Distribution +from packaging.errors import PackagingOptionError + +from packaging.tests import unittest, support +try: + import threading + from packaging.tests.pypi_server import PyPIServerTestCase +except ImportError: + threading = None + PyPIServerTestCase = unittest.TestCase + + +PYPIRC_NOPASSWORD = """\ +[distutils] + +index-servers = + server1 + +[server1] +username:me +""" + +PYPIRC = """\ +[distutils] + +index-servers = + server1 + server2 + +[server1] +username:me +password:secret + +[server2] +username:meagain +password: secret +realm:acme +repository:http://another.pypi/ +""" + + +@unittest.skipIf(threading is None, 'needs threading') +class UploadTestCase(support.TempdirManager, support.EnvironRestorer, + support.LoggingCatcher, PyPIServerTestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(UploadTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + + def test_finalize_options(self): + # new format + self.write_file(self.rc, PYPIRC) + dist = Distribution() + cmd = upload(dist) + cmd.finalize_options() + for attr, expected in (('username', 'me'), ('password', 'secret'), + ('realm', 'pypi'), + ('repository', 'http://pypi.python.org/pypi')): + self.assertEqual(getattr(cmd, attr), expected) + + def test_finalize_options_unsigned_identity_raises_exception(self): + self.write_file(self.rc, PYPIRC) + dist = Distribution() + cmd = upload(dist) + cmd.identity = True + cmd.sign = False + self.assertRaises(PackagingOptionError, cmd.finalize_options) + + def test_saved_password(self): + # file with no password + self.write_file(self.rc, PYPIRC_NOPASSWORD) + + # make sure it passes + dist = Distribution() + cmd = upload(dist) + cmd.ensure_finalized() + self.assertEqual(cmd.password, None) + + # make sure we get it as well, if another command + # initialized it at the dist level + dist.password = 'xxx' + cmd = upload(dist) + cmd.finalize_options() + self.assertEqual(cmd.password, 'xxx') + + def test_upload_without_files_raises_exception(self): + dist = Distribution() + cmd = upload(dist) + self.assertRaises(PackagingOptionError, cmd.run) + + def test_upload(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '3.3', path + dist_files = [(command, pyversion, filename)] + + # let's run it + dist = self.create_dist(dist_files=dist_files, author='dédé')[1] + cmd = upload(dist) + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + cmd.run() + + # what did we send? + handler, request_data = self.pypi.requests[-1] + headers = handler.headers + self.assertIn('dédé'.encode('utf-8'), request_data) + self.assertIn(b'xxx', request_data) + + self.assertEqual(int(headers['content-length']), len(request_data)) + self.assertLess(int(headers['content-length']), 2500) + self.assertTrue(headers['content-type'].startswith( + 'multipart/form-data')) + self.assertEqual(handler.command, 'POST') + self.assertNotIn('\n', headers['authorization']) + + def test_upload_docs(self): + path = os.path.join(self.tmp_dir, 'xxx') + self.write_file(path) + command, pyversion, filename = 'xxx', '3.3', path + dist_files = [(command, pyversion, filename)] + docs_path = os.path.join(self.tmp_dir, "build", "docs") + os.makedirs(docs_path) + self.write_file((docs_path, "index.html"), "yellow") + self.write_file(self.rc, PYPIRC) + + # let's run it + dist = self.create_dist(dist_files=dist_files, author='dédé')[1] + + cmd = upload(dist) + cmd.get_finalized_command("build").run() + cmd.upload_docs = True + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + os.chdir(self.tmp_dir) + cmd.run() + + handler, request_data = self.pypi.requests[-1] + action, name, content = request_data.split( + "----------------GHSKFJDLGDS7543FJKLFHRE75642756743254" + .encode())[1:4] + + self.assertIn(b'name=":action"', action) + self.assertIn(b'doc_upload', action) + + +def test_suite(): + return unittest.makeSuite(UploadTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_command_upload_docs.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_command_upload_docs.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,186 @@ +"""Tests for packaging.command.upload_docs.""" +import os +import shutil +import logging +import zipfile +try: + import _ssl +except ImportError: + _ssl = None + +from packaging.command import upload_docs as upload_docs_mod +from packaging.command.upload_docs import upload_docs, zip_dir +from packaging.dist import Distribution +from packaging.errors import PackagingFileError, PackagingOptionError + +from packaging.tests import unittest, support +try: + import threading + from packaging.tests.pypi_server import PyPIServerTestCase +except ImportError: + threading = None + PyPIServerTestCase = unittest.TestCase + + +PYPIRC = """\ +[distutils] +index-servers = server1 + +[server1] +repository = %s +username = real_slim_shady +password = long_island +""" + + +@unittest.skipIf(threading is None, "Needs threading") +class UploadDocsTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + PyPIServerTestCase): + + restore_environ = ['HOME'] + + def setUp(self): + super(UploadDocsTestCase, self).setUp() + self.tmp_dir = self.mkdtemp() + self.rc = os.path.join(self.tmp_dir, '.pypirc') + os.environ['HOME'] = self.tmp_dir + self.dist = Distribution() + self.dist.metadata['Name'] = "distr-name" + self.cmd = upload_docs(self.dist) + + def test_default_uploaddir(self): + sandbox = self.mkdtemp() + os.chdir(sandbox) + os.mkdir("build") + self.prepare_sample_dir("build") + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.upload_dir, os.path.join("build", "docs")) + + def test_default_uploaddir_looks_for_doc_also(self): + sandbox = self.mkdtemp() + os.chdir(sandbox) + os.mkdir("build") + self.prepare_sample_dir("build") + os.rename(os.path.join("build", "docs"), os.path.join("build", "doc")) + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.upload_dir, os.path.join("build", "doc")) + + def prepare_sample_dir(self, sample_dir=None): + if sample_dir is None: + sample_dir = self.mkdtemp() + os.mkdir(os.path.join(sample_dir, "docs")) + self.write_file((sample_dir, "docs", "index.html"), "Ce mortel ennui") + self.write_file((sample_dir, "index.html"), "Oh la la") + return sample_dir + + def test_zip_dir(self): + source_dir = self.prepare_sample_dir() + compressed = zip_dir(source_dir) + + zip_f = zipfile.ZipFile(compressed) + self.assertEqual(zip_f.namelist(), ['index.html', 'docs/index.html']) + + def prepare_command(self): + self.cmd.upload_dir = self.prepare_sample_dir() + self.cmd.ensure_finalized() + self.cmd.repository = self.pypi.full_address + self.cmd.username = "username" + self.cmd.password = "password" + + def test_upload(self): + self.prepare_command() + self.cmd.run() + + self.assertEqual(len(self.pypi.requests), 1) + handler, request_data = self.pypi.requests[-1] + self.assertIn(b"content", request_data) + self.assertIn("Basic", handler.headers['authorization']) + self.assertTrue(handler.headers['content-type'] + .startswith('multipart/form-data;')) + + action, name, version, content = request_data.split( + b'----------------GHSKFJDLGDS7543FJKLFHRE75642756743254')[1:5] + + # check that we picked the right chunks + self.assertIn(b'name=":action"', action) + self.assertIn(b'name="name"', name) + self.assertIn(b'name="version"', version) + self.assertIn(b'name="content"', content) + + # check their contents + self.assertIn(b'doc_upload', action) + self.assertIn(b'distr-name', name) + self.assertIn(b'docs/index.html', content) + self.assertIn(b'Ce mortel ennui', content) + + @unittest.skipIf(_ssl is None, 'Needs SSL support') + def test_https_connection(self): + self.https_called = False + self.addCleanup( + setattr, upload_docs_mod.http.client, 'HTTPSConnection', + upload_docs_mod.http.client.HTTPSConnection) + + def https_conn_wrapper(*args): + self.https_called = True + # the testing server is http + return upload_docs_mod.http.client.HTTPConnection(*args) + + upload_docs_mod.http.client.HTTPSConnection = https_conn_wrapper + + self.prepare_command() + self.cmd.run() + self.assertFalse(self.https_called) + + self.cmd.repository = self.cmd.repository.replace("http", "https") + self.cmd.run() + self.assertTrue(self.https_called) + + def test_handling_response(self): + self.pypi.default_response_status = '403 Forbidden' + self.prepare_command() + self.cmd.run() + errors = self.get_logs(logging.ERROR) + self.assertEqual(len(errors), 1) + self.assertIn('Upload failed (403): Forbidden', errors[0]) + + self.pypi.default_response_status = '301 Moved Permanently' + self.pypi.default_response_headers.append( + ("Location", "brand_new_location")) + self.cmd.run() + lastlog = self.get_logs(logging.INFO)[-1] + self.assertIn('brand_new_location', lastlog) + + def test_reads_pypirc_data(self): + self.write_file(self.rc, PYPIRC % self.pypi.full_address) + self.cmd.repository = self.pypi.full_address + self.cmd.upload_dir = self.prepare_sample_dir() + self.cmd.ensure_finalized() + self.assertEqual(self.cmd.username, "real_slim_shady") + self.assertEqual(self.cmd.password, "long_island") + + def test_checks_index_html_presence(self): + self.cmd.upload_dir = self.prepare_sample_dir() + os.remove(os.path.join(self.cmd.upload_dir, "index.html")) + self.assertRaises(PackagingFileError, self.cmd.ensure_finalized) + + def test_checks_upload_dir(self): + self.cmd.upload_dir = self.prepare_sample_dir() + shutil.rmtree(os.path.join(self.cmd.upload_dir)) + self.assertRaises(PackagingOptionError, self.cmd.ensure_finalized) + + def test_show_response(self): + self.prepare_command() + self.cmd.show_response = True + self.cmd.run() + record = self.get_logs(logging.INFO)[-1] + self.assertTrue(record, "should report the response") + self.assertIn(self.pypi.default_response_data, record) + + +def test_suite(): + return unittest.makeSuite(UploadDocsTestCase) + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_compiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_compiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,66 @@ +"""Tests for distutils.compiler.""" +import os + +from packaging.compiler import (get_default_compiler, customize_compiler, + gen_lib_options) +from packaging.tests import unittest, support + + +class FakeCompiler: + + name = 'fake' + description = 'Fake' + + def library_dir_option(self, dir): + return "-L" + dir + + def runtime_library_dir_option(self, dir): + return ["-cool", "-R" + dir] + + def find_library_file(self, dirs, lib, debug=False): + return 'found' + + def library_option(self, lib): + return "-l" + lib + + +class CompilerTestCase(support.EnvironRestorer, unittest.TestCase): + + restore_environ = ['AR', 'ARFLAGS'] + + @unittest.skipUnless(get_default_compiler() == 'unix', + 'irrelevant if default compiler is not unix') + def test_customize_compiler(self): + + os.environ['AR'] = 'my_ar' + os.environ['ARFLAGS'] = '-arflags' + + # make sure AR gets caught + class compiler: + name = 'unix' + + def set_executables(self, **kw): + self.exes = kw + + comp = compiler() + customize_compiler(comp) + self.assertEqual(comp.exes['archiver'], 'my_ar -arflags') + + def test_gen_lib_options(self): + compiler = FakeCompiler() + libdirs = ['lib1', 'lib2'] + runlibdirs = ['runlib1'] + libs = [os.path.join('dir', 'name'), 'name2'] + + opts = gen_lib_options(compiler, libdirs, runlibdirs, libs) + wanted = ['-Llib1', '-Llib2', '-cool', '-Rrunlib1', 'found', + '-lname2'] + self.assertEqual(opts, wanted) + + +def test_suite(): + return unittest.makeSuite(CompilerTestCase) + + +if __name__ == "__main__": + unittest.main(defaultTest="test_suite") diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_config.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,519 @@ +"""Tests for packaging.config.""" +import os +import sys + +from packaging import command +from packaging.dist import Distribution +from packaging.errors import PackagingFileError, PackagingOptionError +from packaging.compiler import new_compiler, _COMPILERS +from packaging.command.sdist import sdist + +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +SETUP_CFG = """ +[metadata] +name = RestingParrot +version = 0.6.4 +author = Carl Meyer +author_email = carl@oddbird.net +maintainer = Éric Araujo +maintainer_email = merwok@netwok.org +summary = A sample project demonstrating packaging +description-file = %(description-file)s +keywords = packaging, sample project + +classifier = + Development Status :: 4 - Beta + Environment :: Console (Text Based) + Environment :: X11 Applications :: GTK; python_version < '3' + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + +requires_python = >=2.4, <3.2 + +requires_dist = + PetShoppe + MichaelPalin (> 1.1) + pywin32; sys.platform == 'win32' + pysqlite2; python_version < '2.5' + inotify (0.0.1); sys.platform == 'linux2' + +requires_external = libxml2 + +provides_dist = packaging-sample-project (0.2) + unittest2-sample-project + +project_url = + Main repository, http://bitbucket.org/carljm/sample-distutils2-project + Fork in progress, http://bitbucket.org/Merwok/sample-distutils2-project + +[files] +packages_root = src + +packages = one + two + three + +modules = haven + +scripts = + script1.py + scripts/find-coconuts + bin/taunt + +package_data = + cheese = data/templates/* doc/* + doc/images/*.png + + +extra_files = %(extra-files)s + +# Replaces MANIFEST.in +# FIXME no, it's extra_files +# (but sdist_extra is a better name, should use it) +sdist_extra = + include THANKS HACKING + recursive-include examples *.txt *.py + prune examples/sample?/build + +resources= + bm/ {b1,b2}.gif = {icon} + Cf*/ *.CFG = {config}/baBar/ + init_script = {script}/JunGle/ + +[global] +commands = + packaging.tests.test_config.FooBarBazTest + +compilers = + packaging.tests.test_config.DCompiler + +setup_hooks = %(setup-hooks)s + + + +[install_dist] +sub_commands = foo +""" + +SETUP_CFG_PKGDATA_BUGGY_1 = """ +[files] +package_data = foo.* +""" + +SETUP_CFG_PKGDATA_BUGGY_2 = """ +[files] +package_data = + foo.* +""" + +# Can not be merged with SETUP_CFG else install_dist +# command will fail when trying to compile C sources +# TODO use a DummyCommand to mock build_ext +EXT_SETUP_CFG = """ +[files] +packages = one + two + parent.undeclared + +[extension:one.speed_coconuts] +sources = c_src/speed_coconuts.c +extra_link_args = "`gcc -print-file-name=libgcc.a`" -shared +define_macros = HAVE_CAIRO HAVE_GTK2 +libraries = gecodeint gecodekernel -- sys.platform != 'win32' + GecodeInt GecodeKernel -- sys.platform == 'win32' + +[extension: two.fast_taunt] +sources = cxx_src/utils_taunt.cxx + cxx_src/python_module.cxx +include_dirs = /usr/include/gecode + /usr/include/blitz +extra_compile_args = -fPIC -O2 + -DGECODE_VERSION=$(./gecode_version) -- sys.platform != 'win32' + /DGECODE_VERSION=win32 -- sys.platform == 'win32' +language = cxx + +# corner case: if the parent package of an extension is declared but +# not its grandparent, it's legal +[extension: parent.undeclared._speed] +sources = parent/undeclared/_speed.c +""" + +EXT_SETUP_CFG_BUGGY_1 = """ +[extension: realname] +name = crash_here +""" + +EXT_SETUP_CFG_BUGGY_2 = """ +[files] +packages = ham + +[extension: spam.eggs] +""" + +EXT_SETUP_CFG_BUGGY_3 = """ +[files] +packages = ok + ok.works + +[extension: ok.works.breaks._ext] +""" + +HOOKS_MODULE = """ +import logging + +logger = logging.getLogger('packaging') + +def logging_hook(config): + logger.warning('logging_hook called') +""" + + +class DCompiler: + name = 'd' + description = 'D Compiler' + + def __init__(self, *args): + pass + + +def version_hook(config): + config['metadata']['version'] += '.dev1' + + +def first_hook(config): + config['files']['modules'] += '\n first' + + +def third_hook(config): + config['files']['modules'] += '\n third' + + +class FooBarBazTest: + + def __init__(self, dist): + self.distribution = dist + self._record = [] + + @classmethod + def get_command_name(cls): + return 'foo' + + def run(self): + self._record.append('foo has run') + + def nothing(self): + pass + + def get_source_files(self): + return [] + + ensure_finalized = finalize_options = initialize_options = nothing + + +class ConfigTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + unittest.TestCase): + + restore_environ = ['PLAT'] + + def setUp(self): + super(ConfigTestCase, self).setUp() + tempdir = self.mkdtemp() + self.working_dir = os.getcwd() + os.chdir(tempdir) + self.tempdir = tempdir + + def write_setup(self, kwargs=None): + opts = {'description-file': 'README', 'extra-files': '', + 'setup-hooks': 'packaging.tests.test_config.version_hook'} + if kwargs: + opts.update(kwargs) + self.write_file('setup.cfg', SETUP_CFG % opts, encoding='utf-8') + + def get_dist(self): + dist = Distribution() + dist.parse_config_files() + return dist + + def test_config(self): + self.write_setup() + self.write_file('README', 'yeah') + os.mkdir('bm') + self.write_file(('bm', 'b1.gif'), '') + self.write_file(('bm', 'b2.gif'), '') + os.mkdir('Cfg') + self.write_file(('Cfg', 'data.CFG'), '') + self.write_file('init_script', '') + + # try to load the metadata now + dist = self.get_dist() + + # check what was done + self.assertEqual(dist.metadata['Author'], 'Carl Meyer') + self.assertEqual(dist.metadata['Author-Email'], 'carl@oddbird.net') + + # the hook adds .dev1 + self.assertEqual(dist.metadata['Version'], '0.6.4.dev1') + + wanted = [ + 'Development Status :: 4 - Beta', + 'Environment :: Console (Text Based)', + "Environment :: X11 Applications :: GTK; python_version < '3'", + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3'] + self.assertEqual(dist.metadata['Classifier'], wanted) + + wanted = ['packaging', 'sample project'] + self.assertEqual(dist.metadata['Keywords'], wanted) + + self.assertEqual(dist.metadata['Requires-Python'], '>=2.4, <3.2') + + wanted = ['PetShoppe', + 'MichaelPalin (> 1.1)', + "pywin32; sys.platform == 'win32'", + "pysqlite2; python_version < '2.5'", + "inotify (0.0.1); sys.platform == 'linux2'"] + + self.assertEqual(dist.metadata['Requires-Dist'], wanted) + urls = [('Main repository', + 'http://bitbucket.org/carljm/sample-distutils2-project'), + ('Fork in progress', + 'http://bitbucket.org/Merwok/sample-distutils2-project')] + self.assertEqual(dist.metadata['Project-Url'], urls) + + self.assertEqual(dist.packages, ['one', 'two', 'three']) + self.assertEqual(dist.py_modules, ['haven']) + self.assertEqual(dist.package_data, + {'cheese': ['data/templates/*', 'doc/*', + 'doc/images/*.png']}) + self.assertEqual(dist.data_files, + {'bm/b1.gif': '{icon}/b1.gif', + 'bm/b2.gif': '{icon}/b2.gif', + 'Cfg/data.CFG': '{config}/baBar/data.CFG', + 'init_script': '{script}/JunGle/init_script'}) + + self.assertEqual(dist.package_dir, 'src') + + # Make sure we get the foo command loaded. We use a string comparison + # instead of assertIsInstance because the class is not the same when + # this test is run directly: foo is packaging.tests.test_config.Foo + # because get_command_class uses the full name, but a bare "Foo" in + # this file would be __main__.Foo when run as "python test_config.py". + # The name FooBarBazTest should be unique enough to prevent + # collisions. + self.assertEqual(dist.get_command_obj('foo').__class__.__name__, + 'FooBarBazTest') + + # did the README got loaded ? + self.assertEqual(dist.metadata['description'], 'yeah') + + # do we have the D Compiler enabled ? + self.assertIn('d', _COMPILERS) + d = new_compiler(compiler='d') + self.assertEqual(d.description, 'D Compiler') + + # check error reporting for invalid package_data value + self.write_file('setup.cfg', SETUP_CFG_PKGDATA_BUGGY_1) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', SETUP_CFG_PKGDATA_BUGGY_2) + self.assertRaises(PackagingOptionError, self.get_dist) + + def test_multiple_description_file(self): + self.write_setup({'description-file': 'README CHANGES'}) + self.write_file('README', 'yeah') + self.write_file('CHANGES', 'changelog2') + dist = self.get_dist() + self.assertEqual(dist.metadata.requires_files, ['README', 'CHANGES']) + + def test_multiline_description_file(self): + self.write_setup({'description-file': 'README\n CHANGES'}) + self.write_file('README', 'yeah') + self.write_file('CHANGES', 'changelog') + dist = self.get_dist() + self.assertEqual(dist.metadata['description'], 'yeah\nchangelog') + self.assertEqual(dist.metadata.requires_files, ['README', 'CHANGES']) + + def test_parse_extensions_in_config(self): + self.write_file('setup.cfg', EXT_SETUP_CFG) + dist = self.get_dist() + + ext_modules = dict((mod.name, mod) for mod in dist.ext_modules) + self.assertEqual(len(ext_modules), 3) + ext = ext_modules.get('one.speed_coconuts') + self.assertEqual(ext.sources, ['c_src/speed_coconuts.c']) + self.assertEqual(ext.define_macros, ['HAVE_CAIRO', 'HAVE_GTK2']) + libs = ['gecodeint', 'gecodekernel'] + if sys.platform == 'win32': + libs = ['GecodeInt', 'GecodeKernel'] + self.assertEqual(ext.libraries, libs) + self.assertEqual(ext.extra_link_args, + ['`gcc -print-file-name=libgcc.a`', '-shared']) + + ext = ext_modules.get('two.fast_taunt') + self.assertEqual(ext.sources, + ['cxx_src/utils_taunt.cxx', 'cxx_src/python_module.cxx']) + self.assertEqual(ext.include_dirs, + ['/usr/include/gecode', '/usr/include/blitz']) + cargs = ['-fPIC', '-O2'] + if sys.platform == 'win32': + cargs.append("/DGECODE_VERSION=win32") + else: + cargs.append('-DGECODE_VERSION=$(./gecode_version)') + self.assertEqual(ext.extra_compile_args, cargs) + self.assertEqual(ext.language, 'cxx') + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_1) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_2) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_3) + self.assertRaises(PackagingOptionError, self.get_dist) + + def test_project_setup_hook_works(self): + # Bug #11637: ensure the project directory is on sys.path to allow + # project-specific hooks + self.write_setup({'setup-hooks': 'hooks.logging_hook'}) + self.write_file('README', 'yeah') + self.write_file('hooks.py', HOOKS_MODULE) + self.get_dist() + self.assertEqual(['logging_hook called'], self.get_logs()) + self.assertIn('hooks', sys.modules) + + def test_missing_setup_hook_warns(self): + self.write_setup({'setup-hooks': 'does._not.exist'}) + self.write_file('README', 'yeah') + self.get_dist() + logs = self.get_logs() + self.assertEqual(1, len(logs)) + self.assertIn('cannot find setup hook', logs[0]) + + def test_multiple_setup_hooks(self): + self.write_setup({ + 'setup-hooks': '\n packaging.tests.test_config.first_hook' + '\n packaging.tests.test_config.missing_hook' + '\n packaging.tests.test_config.third_hook', + }) + self.write_file('README', 'yeah') + dist = self.get_dist() + + self.assertEqual(['haven', 'first', 'third'], dist.py_modules) + logs = self.get_logs() + self.assertEqual(1, len(logs)) + self.assertIn('cannot find setup hook', logs[0]) + + def test_metadata_requires_description_files_missing(self): + self.write_setup({'description-file': 'README README2'}) + self.write_file('README', 'yeah') + self.write_file('README2', 'yeah') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + dist = self.get_dist() + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + self.assertRaises(PackagingFileError, cmd.make_distribution) + + @requires_zlib + def test_metadata_requires_description_files(self): + # Create the following file structure: + # README + # README2 + # script1.py + # scripts/ + # find-coconuts + # bin/ + # taunt + # src/ + # haven.py + # one/__init__.py + # two/__init__.py + # three/__init__.py + + self.write_setup({'description-file': 'README\n README2', + 'extra-files': '\n README3'}) + self.write_file('README', 'yeah 1') + self.write_file('README2', 'yeah 2') + self.write_file('README3', 'yeah 3') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + dist = self.get_dist() + self.assertIn('yeah 1\nyeah 2', dist.metadata['description']) + + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + self.assertRaises(PackagingFileError, cmd.make_distribution) + + self.write_setup({'description-file': 'README\n README2', + 'extra-files': '\n README2\n README'}) + dist = self.get_dist() + cmd = sdist(dist) + cmd.finalize_options() + cmd.get_file_list() + cmd.make_distribution() + with open('MANIFEST') as fp: + self.assertIn('README\nREADME2\n', fp.read()) + + def test_sub_commands(self): + self.write_setup() + self.write_file('README', 'yeah') + os.mkdir('src') + self.write_file(('src', 'haven.py'), '#') + self.write_file('script1.py', '#') + os.mkdir('scripts') + self.write_file(('scripts', 'find-coconuts'), '#') + os.mkdir('bin') + self.write_file(('bin', 'taunt'), '#') + + for pkg in ('one', 'two', 'three'): + pkg = os.path.join('src', pkg) + os.mkdir(pkg) + self.write_file((pkg, '__init__.py'), '#') + + # try to run the install command to see if foo is called + self.addCleanup(command._COMMANDS.__delitem__, 'foo') + dist = self.get_dist() + dist.run_command('install_dist') + cmd = dist.get_command_obj('foo') + self.assertEqual(cmd.__class__.__name__, 'FooBarBazTest') + self.assertEqual(cmd._record, ['foo has run']) + + +def test_suite(): + return unittest.makeSuite(ConfigTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_create.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_create.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,233 @@ +"""Tests for packaging.create.""" +import os +import sys +import sysconfig +from textwrap import dedent +from packaging import create +from packaging.create import MainProgram, ask_yn, ask, main + +from packaging.tests import support, unittest +from packaging.tests.support import Inputs + + +class CreateTestCase(support.TempdirManager, + support.EnvironRestorer, + support.LoggingCatcher, + unittest.TestCase): + + maxDiff = None + restore_environ = ['PLAT'] + + def setUp(self): + super(CreateTestCase, self).setUp() + self.wdir = self.mkdtemp() + os.chdir(self.wdir) + # patch sysconfig + self._old_get_paths = sysconfig.get_paths + sysconfig.get_paths = lambda *args, **kwargs: { + 'man': sys.prefix + '/share/man', + 'doc': sys.prefix + '/share/doc/pyxfoil', } + + def tearDown(self): + sysconfig.get_paths = self._old_get_paths + if hasattr(create, 'input'): + del create.input + super(CreateTestCase, self).tearDown() + + def test_ask_yn(self): + create.input = Inputs('y') + self.assertEqual('y', ask_yn('is this a test')) + + def test_ask(self): + create.input = Inputs('a', 'b') + self.assertEqual('a', ask('is this a test')) + self.assertEqual('b', ask(str(list(range(0, 70))), default='c', + lengthy=True)) + + def test_set_multi(self): + mainprogram = MainProgram() + create.input = Inputs('aaaaa') + mainprogram.data['author'] = [] + mainprogram._set_multi('_set_multi test', 'author') + self.assertEqual(['aaaaa'], mainprogram.data['author']) + + def test_find_files(self): + # making sure we scan a project dir correctly + mainprogram = MainProgram() + + # building the structure + tempdir = self.wdir + dirs = ['pkg1', 'data', 'pkg2', 'pkg2/sub'] + files = [ + 'README', + 'data/data1', + 'foo.py', + 'pkg1/__init__.py', + 'pkg1/bar.py', + 'pkg2/__init__.py', + 'pkg2/sub/__init__.py', + ] + + for dir_ in dirs: + os.mkdir(os.path.join(tempdir, dir_)) + + for file_ in files: + self.write_file((tempdir, file_), 'xxx') + + mainprogram._find_files() + mainprogram.data['packages'].sort() + + # do we have what we want? + self.assertEqual(mainprogram.data['packages'], + ['pkg1', 'pkg2', 'pkg2.sub']) + self.assertEqual(mainprogram.data['modules'], ['foo']) + data_fn = os.path.join('data', 'data1') + self.assertEqual(mainprogram.data['extra_files'], + ['README', data_fn]) + + def test_convert_setup_py_to_cfg(self): + self.write_file((self.wdir, 'setup.py'), + dedent(""" + # coding: utf-8 + from distutils.core import setup + + long_description = '''My super Death-scription + barbar is now on the public domain, + ho, baby !''' + + setup(name='pyxfoil', + version='0.2', + description='Python bindings for the Xfoil engine', + long_description=long_description, + maintainer='André Espaze', + maintainer_email='andre.espaze@logilab.fr', + url='http://www.python-science.org/project/pyxfoil', + license='GPLv2', + packages=['pyxfoil', 'babar', 'me'], + data_files=[ + ('share/doc/pyxfoil', ['README.rst']), + ('share/man', ['pyxfoil.1']), + ], + py_modules=['my_lib', 'mymodule'], + package_dir={ + 'babar': '', + 'me': 'Martinique/Lamentin', + }, + package_data={ + 'babar': ['Pom', 'Flora', 'Alexander'], + 'me': ['dady', 'mumy', 'sys', 'bro'], + 'pyxfoil': ['fengine.so'], + }, + scripts=['my_script', 'bin/run'], + ) + """), encoding='utf-8') + create.input = Inputs('y') + main() + + path = os.path.join(self.wdir, 'setup.cfg') + with open(path, encoding='utf-8') as fp: + contents = fp.read() + + self.assertEqual(contents, dedent("""\ + [metadata] + name = pyxfoil + version = 0.2 + summary = Python bindings for the Xfoil engine + download_url = UNKNOWN + home_page = http://www.python-science.org/project/pyxfoil + maintainer = André Espaze + maintainer_email = andre.espaze@logilab.fr + description = My super Death-scription + |barbar is now on the public domain, + |ho, baby ! + + [files] + packages = pyxfoil + babar + me + modules = my_lib + mymodule + scripts = my_script + bin/run + package_data = + babar = Pom + Flora + Alexander + me = dady + mumy + sys + bro + pyxfoil = fengine.so + + resources = + README.rst = {doc} + pyxfoil.1 = {man} + + """)) + + def test_convert_setup_py_to_cfg_with_description_in_readme(self): + self.write_file((self.wdir, 'setup.py'), + dedent(""" + # coding: utf-8 + from distutils.core import setup + with open('README.txt') as fp: + long_description = fp.read() + + setup(name='pyxfoil', + version='0.2', + description='Python bindings for the Xfoil engine', + long_description=long_description, + maintainer='André Espaze', + maintainer_email='andre.espaze@logilab.fr', + url='http://www.python-science.org/project/pyxfoil', + license='GPLv2', + packages=['pyxfoil'], + package_data={'pyxfoil': ['fengine.so', 'babar.so']}, + data_files=[ + ('share/doc/pyxfoil', ['README.rst']), + ('share/man', ['pyxfoil.1']), + ], + ) + """), encoding='utf-8') + self.write_file((self.wdir, 'README.txt'), + dedent(''' +My super Death-scription +barbar is now in the public domain, +ho, baby! + ''')) + create.input = Inputs('y') + main() + + path = os.path.join(self.wdir, 'setup.cfg') + with open(path, encoding='utf-8') as fp: + contents = fp.read() + + self.assertEqual(contents, dedent("""\ + [metadata] + name = pyxfoil + version = 0.2 + summary = Python bindings for the Xfoil engine + download_url = UNKNOWN + home_page = http://www.python-science.org/project/pyxfoil + maintainer = André Espaze + maintainer_email = andre.espaze@logilab.fr + description-file = README.txt + + [files] + packages = pyxfoil + package_data = + pyxfoil = fengine.so + babar.so + + resources = + README.rst = {doc} + pyxfoil.1 = {man} + + """)) + + +def test_suite(): + return unittest.makeSuite(CreateTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_cygwinccompiler.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_cygwinccompiler.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,88 @@ +"""Tests for packaging.cygwinccompiler.""" +import os +import sys +import sysconfig +from packaging.compiler.cygwinccompiler import ( + check_config_h, get_msvcr, + CONFIG_H_OK, CONFIG_H_NOTOK, CONFIG_H_UNCERTAIN) + +from packaging.tests import unittest, support + + +class CygwinCCompilerTestCase(support.TempdirManager, + unittest.TestCase): + + def setUp(self): + super(CygwinCCompilerTestCase, self).setUp() + self.version = sys.version + self.python_h = os.path.join(self.mkdtemp(), 'python.h') + self.old_get_config_h_filename = sysconfig.get_config_h_filename + sysconfig.get_config_h_filename = self._get_config_h_filename + + def tearDown(self): + sys.version = self.version + sysconfig.get_config_h_filename = self.old_get_config_h_filename + super(CygwinCCompilerTestCase, self).tearDown() + + def _get_config_h_filename(self): + return self.python_h + + def test_check_config_h(self): + # check_config_h looks for "GCC" in sys.version first + # returns CONFIG_H_OK if found + sys.version = ('2.6.1 (r261:67515, Dec 6 2008, 16:42:21) \n[GCC ' + '4.0.1 (Apple Computer, Inc. build 5370)]') + + self.assertEqual(check_config_h()[0], CONFIG_H_OK) + + # then it tries to see if it can find "__GNUC__" in pyconfig.h + sys.version = 'something without the *CC word' + + # if the file doesn't exist it returns CONFIG_H_UNCERTAIN + self.assertEqual(check_config_h()[0], CONFIG_H_UNCERTAIN) + + # if it exists but does not contain __GNUC__, it returns CONFIG_H_NOTOK + self.write_file(self.python_h, 'xxx') + self.assertEqual(check_config_h()[0], CONFIG_H_NOTOK) + + # and CONFIG_H_OK if __GNUC__ is found + self.write_file(self.python_h, 'xxx __GNUC__ xxx') + self.assertEqual(check_config_h()[0], CONFIG_H_OK) + + def test_get_msvcr(self): + # none + sys.version = ('2.6.1 (r261:67515, Dec 6 2008, 16:42:21) ' + '\n[GCC 4.0.1 (Apple Computer, Inc. build 5370)]') + self.assertEqual(get_msvcr(), None) + + # MSVC 7.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1300 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr70']) + + # MSVC 7.1 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1310 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr71']) + + # VS2005 / MSVC 8.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1400 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr80']) + + # VS2008 / MSVC 9.0 + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1500 32 bits (Intel)]') + self.assertEqual(get_msvcr(), ['msvcr90']) + + # unknown + sys.version = ('2.5.1 (r251:54863, Apr 18 2007, 08:51:08) ' + '[MSC v.1999 32 bits (Intel)]') + self.assertRaises(ValueError, get_msvcr) + + +def test_suite(): + return unittest.makeSuite(CygwinCCompilerTestCase) + +if __name__ == '__main__': + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_database.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_database.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,686 @@ +import os +import io +import csv +import sys +import shutil +import tempfile +from hashlib import md5 +from textwrap import dedent + +from packaging.tests.test_util import GlobTestCaseBase +from packaging.tests.support import requires_zlib + +import packaging.database +from packaging.config import get_resources_dests +from packaging.errors import PackagingError +from packaging.metadata import Metadata +from packaging.tests import unittest, support +from packaging.database import ( + Distribution, EggInfoDistribution, get_distribution, get_distributions, + provides_distribution, obsoletes_distribution, get_file_users, + enable_cache, disable_cache, distinfo_dirname, _yield_distributions, + get_file, get_file_path) + +# TODO Add a test for getting a distribution provided by another distribution +# TODO Add a test for absolute pathed RECORD items (e.g. /etc/myapp/config.ini) +# TODO Add tests from the former pep376 project (zipped site-packages, etc.) + + +def get_hexdigest(filename): + with open(filename, 'rb') as file: + checksum = md5(file.read()) + return checksum.hexdigest() + + +def record_pieces(path): + path = os.path.join(*path) + digest = get_hexdigest(path) + size = os.path.getsize(path) + return path, digest, size + + +class FakeDistsMixin: + + def setUp(self): + super(FakeDistsMixin, self).setUp() + self.addCleanup(enable_cache) + disable_cache() + + # make a copy that we can write into for our fake installed + # distributions + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + self.fake_dists_path = os.path.realpath( + os.path.join(tmpdir, 'fake_dists')) + fake_dists_src = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fake_dists')) + shutil.copytree(fake_dists_src, self.fake_dists_path) + # XXX ugly workaround: revert copystat calls done by shutil behind our + # back (to avoid getting a read-only copy of a read-only file). we + # could pass a custom copy_function to change the mode of files, but + # shutil gives no control over the mode of directories :( + # see http://bugs.python.org/issue1666318 + for root, dirs, files in os.walk(self.fake_dists_path): + os.chmod(root, 0o755) + for f in files: + os.chmod(os.path.join(root, f), 0o644) + for d in dirs: + os.chmod(os.path.join(root, d), 0o755) + + +class CommonDistributionTests(FakeDistsMixin): + """Mixin used to test the interface common to both Distribution classes. + + Derived classes define cls, sample_dist, dirs and records. These + attributes are used in test methods. See source code for details. + """ + + def test_instantiation(self): + # check that useful attributes are here + name, version, distdir = self.sample_dist + here = os.path.abspath(os.path.dirname(__file__)) + dist_path = os.path.join(here, 'fake_dists', distdir) + + dist = self.dist = self.cls(dist_path) + self.assertEqual(dist.path, dist_path) + self.assertEqual(dist.name, name) + self.assertEqual(dist.metadata['Name'], name) + self.assertIsInstance(dist.metadata, Metadata) + self.assertEqual(dist.version, version) + self.assertEqual(dist.metadata['Version'], version) + + @requires_zlib + def test_repr(self): + dist = self.cls(self.dirs[0]) + # just check that the class name is in the repr + self.assertIn(self.cls.__name__, repr(dist)) + + @requires_zlib + def test_comparison(self): + # tests for __eq__ and __hash__ + dist = self.cls(self.dirs[0]) + dist2 = self.cls(self.dirs[0]) + dist3 = self.cls(self.dirs[1]) + self.assertIn(dist, {dist: True}) + self.assertEqual(dist, dist) + + self.assertIsNot(dist, dist2) + self.assertEqual(dist, dist2) + self.assertNotEqual(dist, dist3) + self.assertNotEqual(dist, ()) + + def test_list_installed_files(self): + for dir_ in self.dirs: + dist = self.cls(dir_) + for path, md5_, size in dist.list_installed_files(): + record_data = self.records[dist.path] + self.assertIn(path, record_data) + self.assertEqual(md5_, record_data[path][0]) + self.assertEqual(size, record_data[path][1]) + + +class TestDistribution(CommonDistributionTests, unittest.TestCase): + + cls = Distribution + sample_dist = 'choxie', '2.0.0.9', 'choxie-2.0.0.9.dist-info' + + def setUp(self): + super(TestDistribution, self).setUp() + self.dirs = [os.path.join(self.fake_dists_path, f) + for f in os.listdir(self.fake_dists_path) + if f.endswith('.dist-info')] + + self.records = {} + for distinfo_dir in self.dirs: + + record_file = os.path.join(distinfo_dir, 'RECORD') + with open(record_file, 'w') as file: + record_writer = csv.writer( + file, delimiter=',', quoting=csv.QUOTE_NONE, + lineterminator='\n') + + dist_location = distinfo_dir.replace('.dist-info', '') + + for path, dirs, files in os.walk(dist_location): + for f in files: + record_writer.writerow(record_pieces((path, f))) + for file in ('INSTALLER', 'METADATA', 'REQUESTED'): + record_writer.writerow(record_pieces((distinfo_dir, file))) + record_writer.writerow([record_file]) + + with open(record_file) as file: + record_reader = csv.reader(file, lineterminator='\n') + record_data = {} + for row in record_reader: + if row == []: + continue + path, md5_, size = (row[:] + + [None for i in range(len(row), 3)]) + record_data[path] = md5_, size + self.records[distinfo_dir] = record_data + + def test_instantiation(self): + super(TestDistribution, self).test_instantiation() + self.assertIsInstance(self.dist.requested, bool) + + def test_uses(self): + # Test to determine if a distribution uses a specified file. + # Criteria to test against + distinfo_name = 'grammar-1.0a4' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + true_path = [self.fake_dists_path, distinfo_name, + 'grammar', 'utils.py'] + true_path = os.path.join(*true_path) + false_path = [self.fake_dists_path, 'towel_stuff-0.1', 'towel_stuff', + '__init__.py'] + false_path = os.path.join(*false_path) + + # Test if the distribution uses the file in question + dist = Distribution(distinfo_dir) + self.assertTrue(dist.uses(true_path), 'dist %r is supposed to use %r' % + (dist, true_path)) + self.assertFalse(dist.uses(false_path), 'dist %r is not supposed to ' + 'use %r' % (dist, true_path)) + + def test_get_distinfo_file(self): + # Test the retrieval of dist-info file objects. + distinfo_name = 'choxie-2.0.0.9' + other_distinfo_name = 'grammar-1.0a4' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + # Test for known good file matches + distinfo_files = [ + # Relative paths + 'INSTALLER', 'METADATA', + # Absolute paths + os.path.join(distinfo_dir, 'RECORD'), + os.path.join(distinfo_dir, 'REQUESTED'), + ] + + for distfile in distinfo_files: + with dist.get_distinfo_file(distfile) as value: + self.assertIsInstance(value, io.TextIOWrapper) + # Is it the correct file? + self.assertEqual(value.name, + os.path.join(distinfo_dir, distfile)) + + # Test an absolute path that is part of another distributions dist-info + other_distinfo_file = os.path.join( + self.fake_dists_path, other_distinfo_name + '.dist-info', + 'REQUESTED') + self.assertRaises(PackagingError, dist.get_distinfo_file, + other_distinfo_file) + # Test for a file that should not exist + self.assertRaises(PackagingError, dist.get_distinfo_file, + 'MAGICFILE') + + def test_list_distinfo_files(self): + distinfo_name = 'towel_stuff-0.1' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + # Test for the iteration of the raw path + distinfo_files = [os.path.join(distinfo_dir, filename) for filename in + os.listdir(distinfo_dir)] + found = dist.list_distinfo_files() + self.assertEqual(sorted(found), sorted(distinfo_files)) + # Test for the iteration of local absolute paths + distinfo_files = [os.path.join(sys.prefix, distinfo_dir, path) for + path in distinfo_files] + found = sorted(dist.list_distinfo_files(local=True)) + if os.sep != '/': + self.assertNotIn('/', found[0]) + self.assertIn(os.sep, found[0]) + self.assertEqual(found, sorted(distinfo_files)) + + def test_get_resources_path(self): + distinfo_name = 'babar-0.1' + distinfo_dir = os.path.join(self.fake_dists_path, + distinfo_name + '.dist-info') + dist = Distribution(distinfo_dir) + resource_path = dist.get_resource_path('babar.png') + self.assertEqual(resource_path, 'babar.png') + self.assertRaises(KeyError, dist.get_resource_path, 'notexist') + + +class TestEggInfoDistribution(CommonDistributionTests, + support.LoggingCatcher, + unittest.TestCase): + + cls = EggInfoDistribution + sample_dist = 'bacon', '0.1', 'bacon-0.1.egg-info' + + def setUp(self): + super(TestEggInfoDistribution, self).setUp() + + self.dirs = [os.path.join(self.fake_dists_path, f) + for f in os.listdir(self.fake_dists_path) + if f.endswith('.egg') or f.endswith('.egg-info')] + + self.records = {} + + @unittest.skip('not implemented yet') + def test_list_installed_files(self): + # EggInfoDistribution defines list_installed_files but there is no + # test for it yet; someone with setuptools expertise needs to add a + # file with the list of installed files for one of the egg fake dists + # and write the support code to populate self.records (and then delete + # this method) + pass + + +class TestDatabase(support.LoggingCatcher, + FakeDistsMixin, + unittest.TestCase): + + def setUp(self): + super(TestDatabase, self).setUp() + sys.path.insert(0, self.fake_dists_path) + self.addCleanup(sys.path.remove, self.fake_dists_path) + + def test_caches(self): + # sanity check for internal caches + for name in ('_cache_name', '_cache_name_egg', + '_cache_path', '_cache_path_egg'): + self.assertEqual(getattr(packaging.database, name), {}) + + def test_distinfo_dirname(self): + # Given a name and a version, we expect the distinfo_dirname function + # to return a standard distribution information directory name. + + items = [ + # (name, version, standard_dirname) + # Test for a very simple single word name and decimal version + # number + ('docutils', '0.5', 'docutils-0.5.dist-info'), + # Test for another except this time with a '-' in the name, which + # needs to be transformed during the name lookup + ('python-ldap', '2.5', 'python_ldap-2.5.dist-info'), + # Test for both '-' in the name and a funky version number + ('python-ldap', '2.5 a---5', 'python_ldap-2.5 a---5.dist-info'), + ] + + # Loop through the items to validate the results + for name, version, standard_dirname in items: + dirname = distinfo_dirname(name, version) + self.assertEqual(dirname, standard_dirname) + + @requires_zlib + def test_get_distributions(self): + # Lookup all distributions found in the ``sys.path``. + # This test could potentially pick up other installed distributions + fake_dists = [('grammar', '1.0a4'), ('choxie', '2.0.0.9'), + ('towel-stuff', '0.1'), ('babar', '0.1')] + found_dists = [] + + # Verify the fake dists have been found. + dists = [dist for dist in get_distributions()] + for dist in dists: + self.assertIsInstance(dist, Distribution) + if (dist.name in dict(fake_dists) and + dist.path.startswith(self.fake_dists_path)): + found_dists.append((dist.name, dist.version)) + else: + # check that it doesn't find anything more than this + self.assertFalse(dist.path.startswith(self.fake_dists_path)) + # otherwise we don't care what other distributions are found + + # Finally, test that we found all that we were looking for + self.assertEqual(sorted(found_dists), sorted(fake_dists)) + + # Now, test if the egg-info distributions are found correctly as well + fake_dists += [('bacon', '0.1'), ('cheese', '2.0.2'), + ('coconuts-aster', '10.3'), + ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('nut', 'funkyversion')] + found_dists = [] + + dists = [dist for dist in get_distributions(use_egg_info=True)] + for dist in dists: + self.assertIsInstance(dist, (Distribution, EggInfoDistribution)) + if (dist.name in dict(fake_dists) and + dist.path.startswith(self.fake_dists_path)): + found_dists.append((dist.name, dist.version)) + else: + self.assertFalse(dist.path.startswith(self.fake_dists_path)) + + self.assertEqual(sorted(fake_dists), sorted(found_dists)) + + @requires_zlib + def test_get_distribution(self): + # Test for looking up a distribution by name. + # Test the lookup of the towel-stuff distribution + name = 'towel-stuff' # Note: This is different from the directory name + + # Lookup the distribution + dist = get_distribution(name) + self.assertIsInstance(dist, Distribution) + self.assertEqual(dist.name, name) + + # Verify that an unknown distribution returns None + self.assertIsNone(get_distribution('bogus')) + + # Verify partial name matching doesn't work + self.assertIsNone(get_distribution('towel')) + + # Verify that it does not find egg-info distributions, when not + # instructed to + self.assertIsNone(get_distribution('bacon')) + self.assertIsNone(get_distribution('cheese')) + self.assertIsNone(get_distribution('strawberry')) + self.assertIsNone(get_distribution('banana')) + + # Now check that it works well in both situations, when egg-info + # is a file and directory respectively. + dist = get_distribution('cheese', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'cheese') + + dist = get_distribution('bacon', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'bacon') + + dist = get_distribution('banana', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'banana') + + dist = get_distribution('strawberry', use_egg_info=True) + self.assertIsInstance(dist, EggInfoDistribution) + self.assertEqual(dist.name, 'strawberry') + + def test_get_file_users(self): + # Test the iteration of distributions that use a file. + name = 'towel_stuff-0.1' + path = os.path.join(self.fake_dists_path, name, + 'towel_stuff', '__init__.py') + for dist in get_file_users(path): + self.assertIsInstance(dist, Distribution) + self.assertEqual(dist.name, name) + + @requires_zlib + def test_provides(self): + # Test for looking up distributions by what they provide + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + l = [dist.name for dist in provides_distribution('truffles')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '1.0')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in provides_distribution('truffles', '1.0', + use_egg_info=True)] + checkLists(l, ['choxie', 'cheese']) + + l = [dist.name for dist in provides_distribution('truffles', '1.1.2')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '1.1')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', + '!=1.1,<=2.0')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in provides_distribution('truffles', + '!=1.1,<=2.0', + use_egg_info=True)] + checkLists(l, ['choxie', 'bacon', 'cheese']) + + l = [dist.name for dist in provides_distribution('truffles', '>1.0')] + checkLists(l, ['towel-stuff']) + + l = [dist.name for dist in provides_distribution('truffles', '>1.5')] + checkLists(l, []) + + l = [dist.name for dist in provides_distribution('truffles', '>1.5', + use_egg_info=True)] + checkLists(l, ['bacon']) + + l = [dist.name for dist in provides_distribution('truffles', '>=1.0')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in provides_distribution('strawberry', '0.6', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('strawberry', '>=0.5', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('strawberry', '>0.6', + use_egg_info=True)] + checkLists(l, []) + + l = [dist.name for dist in provides_distribution('banana', '0.4', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('banana', '>=0.3', + use_egg_info=True)] + checkLists(l, ['coconuts-aster']) + + l = [dist.name for dist in provides_distribution('banana', '!=0.4', + use_egg_info=True)] + checkLists(l, []) + + @requires_zlib + def test_obsoletes(self): + # Test looking for distributions based on what they obsolete + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + l = [dist.name for dist in obsoletes_distribution('truffles', '1.0')] + checkLists(l, []) + + l = [dist.name for dist in obsoletes_distribution('truffles', '1.0', + use_egg_info=True)] + checkLists(l, ['cheese', 'bacon']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.8')] + checkLists(l, ['choxie']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.8', + use_egg_info=True)] + checkLists(l, ['choxie', 'cheese']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.9.6')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in obsoletes_distribution('truffles', + '0.5.2.3')] + checkLists(l, ['choxie', 'towel-stuff']) + + l = [dist.name for dist in obsoletes_distribution('truffles', '0.2')] + checkLists(l, ['towel-stuff']) + + @requires_zlib + def test_yield_distribution(self): + # tests the internal function _yield_distributions + checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y)) + + eggs = [('bacon', '0.1'), ('banana', '0.4'), ('strawberry', '0.6'), + ('truffles', '5.0'), ('cheese', '2.0.2'), + ('coconuts-aster', '10.3'), ('nut', 'funkyversion')] + dists = [('choxie', '2.0.0.9'), ('grammar', '1.0a4'), + ('towel-stuff', '0.1'), ('babar', '0.1')] + + checkLists([], _yield_distributions(False, False, sys.path)) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(False, True, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(eggs, found) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(True, False, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists, found) + + found = [(dist.name, dist.version) + for dist in _yield_distributions(True, True, sys.path) + if dist.path.startswith(self.fake_dists_path)] + checkLists(dists + eggs, found) + + +class DataFilesTestCase(GlobTestCaseBase): + + def assertRulesMatch(self, rules, spec): + tempdir = self.build_files_tree(spec) + expected = self.clean_tree(spec) + result = get_resources_dests(tempdir, rules) + self.assertEqual(expected, result) + + def clean_tree(self, spec): + files = {} + for path, value in spec.items(): + if value is not None: + files[path] = value + return files + + def test_simple_glob(self): + rules = [('', '*.tpl', '{data}')] + spec = {'coucou.tpl': '{data}/coucou.tpl', + 'Donotwant': None} + self.assertRulesMatch(rules, spec) + + def test_multiple_match(self): + rules = [('scripts', '*.bin', '{appdata}'), + ('scripts', '*', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match(self): + rules = [('scripts', '*.{bin,sh}', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'scripts/babar.sh': '{appscript}/babar.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_multiple(self): + rules = [('scripts', 'script{s,}.{bin,sh}', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': '{appscript}/script.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_exclude(self): + rules = [('scripts', '*', '{appscript}'), + ('', os.path.join('**', '*.sh'), None)] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': None, + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_glob_in_base(self): + rules = [('scrip*', '*.bin', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripouille/babar.bin': '{appscript}/babar.bin', + 'scriptortu/lotus.bin': '{appscript}/lotus.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_recursive_glob(self): + rules = [('', os.path.join('**', '*.bin'), '{binary}')] + spec = {'binary0.bin': '{binary}/binary0.bin', + 'scripts/binary1.bin': '{binary}/scripts/binary1.bin', + 'scripts/bin/binary2.bin': '{binary}/scripts/bin/binary2.bin', + 'you/kill/pandabear.guy': None} + self.assertRulesMatch(rules, spec) + + def test_final_exemple_glob(self): + rules = [ + ('mailman/database/schemas/', '*', '{appdata}/schemas'), + ('', os.path.join('**', '*.tpl'), '{appdata}/templates'), + ('', os.path.join('developer-docs', '**', '*.txt'), '{doc}'), + ('', 'README', '{doc}'), + ('mailman/etc/', '*', '{config}'), + ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), + '{config}/baz'), + ('mailman/foo/', os.path.join('**', '*.cfg'), '{config}/hmm'), + ('', 'some-new-semantic.sns', '{funky-crazy-category}'), + ] + spec = { + 'README': '{doc}/README', + 'some.tpl': '{appdata}/templates/some.tpl', + 'some-new-semantic.sns': + '{funky-crazy-category}/some-new-semantic.sns', + 'mailman/database/mailman.db': None, + 'mailman/database/schemas/blah.schema': + '{appdata}/schemas/blah.schema', + 'mailman/etc/my.cnf': '{config}/my.cnf', + 'mailman/foo/some/path/bar/my.cfg': + '{config}/hmm/some/path/bar/my.cfg', + 'mailman/foo/some/path/other.cfg': + '{config}/hmm/some/path/other.cfg', + 'developer-docs/index.txt': '{doc}/developer-docs/index.txt', + 'developer-docs/api/toc.txt': '{doc}/developer-docs/api/toc.txt', + } + self.maxDiff = None + self.assertRulesMatch(rules, spec) + + def test_get_file(self): + # Create a fake dist + temp_site_packages = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, temp_site_packages) + + dist_name = 'test' + dist_info = os.path.join(temp_site_packages, 'test-0.1.dist-info') + os.mkdir(dist_info) + + metadata_path = os.path.join(dist_info, 'METADATA') + resources_path = os.path.join(dist_info, 'RESOURCES') + + with open(metadata_path, 'w') as fp: + fp.write(dedent("""\ + Metadata-Version: 1.2 + Name: test + Version: 0.1 + Summary: test + Author: me + """)) + + test_path = 'test.cfg' + + fd, test_resource_path = tempfile.mkstemp() + os.close(fd) + self.addCleanup(os.remove, test_resource_path) + + with open(test_resource_path, 'w') as fp: + fp.write('Config') + + with open(resources_path, 'w') as fp: + fp.write('%s,%s' % (test_path, test_resource_path)) + + # Add fake site-packages to sys.path to retrieve fake dist + self.addCleanup(sys.path.remove, temp_site_packages) + sys.path.insert(0, temp_site_packages) + + # Force packaging.database to rescan the sys.path + self.addCleanup(enable_cache) + disable_cache() + + # Try to retrieve resources paths and files + self.assertEqual(get_file_path(dist_name, test_path), + test_resource_path) + self.assertRaises(KeyError, get_file_path, dist_name, 'i-dont-exist') + + with get_file(dist_name, test_path) as fp: + self.assertEqual(fp.read(), 'Config') + self.assertRaises(KeyError, get_file, dist_name, 'i-dont-exist') + + +def test_suite(): + suite = unittest.TestSuite() + load = unittest.defaultTestLoader.loadTestsFromTestCase + suite.addTest(load(TestDistribution)) + suite.addTest(load(TestEggInfoDistribution)) + suite.addTest(load(TestDatabase)) + suite.addTest(load(DataFilesTestCase)) + return suite + + +if __name__ == "__main__": + unittest.main(defaultTest='test_suite') diff -r 6db40a9955dc -r 0d413f60cc23 Lib/packaging/tests/test_depgraph.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/packaging/tests/test_depgraph.py Mon Jan 25 17:05:13 2016 +0100 @@ -0,0 +1,310 @@ +"""Tests for packaging.depgraph """ +import os +import re +import sys +from io import StringIO + +from packaging import depgraph +from packaging.database import get_distribution, enable_cache, disable_cache + +from packaging.tests import unittest, support +from packaging.tests.support import requires_zlib + + +class DepGraphTestCase(support.LoggingCatcher, + unittest.TestCase): + + DISTROS_DIST = ('choxie', 'grammar', 'towel-stuff') + DISTROS_EGG = ('bacon', 'banana', 'strawberry', 'cheese') + BAD_EGGS = ('nut',) + + EDGE = re.compile( + r'"(?P.*)" -> "(?P.*)" \[label="(?P